From 2f195d4f023280c9e3b2b1058a8bf7d56453ac1b Mon Sep 17 00:00:00 2001 From: winlogon Date: Sat, 6 Jun 2026 11:18:28 +0200 Subject: [PATCH 1/4] refactor(parser): replace pest expression parser with Pratt parser The pest-based expression parser ran out of stack on Windows. The recursive descent consumed one boxed `Pair` per grammar level, and the grammar had grown deep enough (struct inits, array indexing, range literals) that 1MB of stack was no longer enough. Replace it with a hand-written Pratt parser that operates on a token stream instead, so recursion depth tracks expression nesting rather than grammar-tree depth. Range literal (a...b) support moves into the Pratt infix layer with binding power 2/2 (looser than assignment, tighter than logical or) and produces Expr::RangeLiteral, matching the pest grammar's range_expr structure. Summary of changes: - Re-add logos to dependencies - Introduce Tok, SpannedTok, tokenize, and tok_name in lexer.rs - Add kitlang/src/codegen/parser/diagnostics.rs - Add kitlang/src/codegen/parser/expr_pratt.rs, containing Pratt core - Wire Parser::parse_expr to PestExpr -> ExprParser pipeline - Add Tok::Ellipsis as a Pratt infix op building Expr::RangeLiteral - Move struct-init / field-init semantics into the Pratt path - Delete kitlang/src/codegen/parser/expr.rs (pest-based parse_expr) - Drop dead parse_bool_literal and unescape helpers from parser/mod.rs - Add range-literal unit tests: * range_literal_simple * range_literal_with_expressions - Mark diagnostic seams with #[allow(dead_code)] for the follow-up that will wire them in --- Cargo.lock | 39 + kitlang/Cargo.toml | 1 + kitlang/src/codegen/inference.rs | 4 +- kitlang/src/codegen/parser/binding_power.rs | 167 +++ kitlang/src/codegen/parser/diagnostics.rs | 198 +++ kitlang/src/codegen/parser/expr.rs | 429 ------- kitlang/src/codegen/parser/expr_pratt.rs | 1203 +++++++++++++++++++ kitlang/src/codegen/parser/mod.rs | 61 +- kitlang/src/lexer.rs | 381 ++++++ kitlang/src/lib.rs | 7 + 10 files changed, 2056 insertions(+), 434 deletions(-) create mode 100644 kitlang/src/codegen/parser/binding_power.rs create mode 100644 kitlang/src/codegen/parser/diagnostics.rs delete mode 100644 kitlang/src/codegen/parser/expr.rs create mode 100644 kitlang/src/codegen/parser/expr_pratt.rs create mode 100644 kitlang/src/lexer.rs diff --git a/Cargo.lock b/Cargo.lock index 3d87a0e..5617c4e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -227,6 +227,12 @@ dependencies = [ "num-traits", ] +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + [[package]] name = "generic-array" version = "0.14.7" @@ -292,6 +298,7 @@ name = "kitlang" version = "0.1.0" dependencies = [ "log", + "logos", "pest", "pest_derive", "strum", @@ -312,6 +319,38 @@ version = "0.4.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "113b30b4cd05f7c06868fdb2854f66a7b9fece9a48425351cd532e810d74024f" +[[package]] +name = "logos" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb2c55a318a87600ea870ff8c2012148b44bf18b74fad48d0f835c38c7d07c5f" +dependencies = [ + "logos-derive", +] + +[[package]] +name = "logos-codegen" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "58b3ffaa284e1350d017a57d04ada118c4583cf260c8fb01e0fe28a2e9cf8970" +dependencies = [ + "fnv", + "proc-macro2", + "quote", + "regex-automata", + "regex-syntax", + "syn", +] + +[[package]] +name = "logos-derive" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52d3a9855747c17eaf4383823f135220716ab49bea5fbea7dd42cc9a92f8aa31" +dependencies = [ + "logos-codegen", +] + [[package]] name = "memchr" version = "2.8.1" diff --git a/kitlang/Cargo.toml b/kitlang/Cargo.toml index 97d6b27..84eb3f7 100644 --- a/kitlang/Cargo.toml +++ b/kitlang/Cargo.toml @@ -5,6 +5,7 @@ edition = "2024" [dependencies] log = "0.4.27" +logos = "0.16.1" pest = "2.8.0" pest_derive = "2.8.0" strum = { version = "0.28.0", features = ["derive"] } diff --git a/kitlang/src/codegen/inference.rs b/kitlang/src/codegen/inference.rs index e02e380..ff60728 100644 --- a/kitlang/src/codegen/inference.rs +++ b/kitlang/src/codegen/inference.rs @@ -947,9 +947,7 @@ impl TypeInferencer { Type::CArray(elem_type, _) => self.store.new_known(*elem_type), Type::Ptr(inner) => self.store.new_known(*inner), _ => { - return Err(type_err!( - "Cannot index non-array type: {resolved:?}" - )); + return Err(type_err!("Cannot index non-array type: {resolved:?}")); } }; *ty = elem_ty; diff --git a/kitlang/src/codegen/parser/binding_power.rs b/kitlang/src/codegen/parser/binding_power.rs new file mode 100644 index 0000000..618580c --- /dev/null +++ b/kitlang/src/codegen/parser/binding_power.rs @@ -0,0 +1,167 @@ +//! Operator binding powers and token-to-operator conversions. + +use crate::codegen::types::{AssignmentOperator, BinaryOperator, UnaryOperator}; +use crate::lexer::Tok; + +/// Infix binding power: (left_bp, right_bp). None = not an infix operator. +/// lbp < rbp → right-associative (assignment); lbp == rbp → left-associative. +pub fn infix(tok: &Tok) -> Option<(u8, u8)> { + match tok { + Tok::Assign + | Tok::PlusEq + | Tok::MinusEq + | Tok::StarEq + | Tok::SlashEq + | Tok::PercentEq + | Tok::AmpEq + | Tok::PipeEq + | Tok::CaretEq + | Tok::ShlEq + | Tok::ShrEq => Some((0, 1)), + Tok::Ellipsis => Some((2, 2)), + Tok::OrOr => Some((3, 4)), + Tok::AndAnd => Some((5, 6)), + Tok::Pipe => Some((7, 8)), + Tok::Caret => Some((9, 10)), + Tok::Amp => Some((11, 12)), + Tok::EqEq | Tok::NotEq => Some((13, 14)), + Tok::Lt | Tok::Gt | Tok::LtEq | Tok::GtEq => Some((15, 16)), + Tok::Shl | Tok::Shr => Some((17, 18)), + Tok::Plus | Tok::Minus => Some((19, 20)), + Tok::Star | Tok::Slash | Tok::Percent => Some((21, 22)), + _ => None, + } +} + +/// Postfix binding power. All postfix ops use a single bp higher than any infix op. +pub fn postfix(tok: &Tok) -> Option { + match tok { + Tok::Dot | Tok::LBracket | Tok::LParen => Some(27), + _ => None, + } +} + +/// Prefix (unary) binding power. Returns None for non-prefix tokens. +pub fn prefix(tok: &Tok) -> Option { + match tok { + Tok::Bang + | Tok::Minus + | Tok::Star + | Tok::Amp + | Tok::Tilde + | Tok::PlusPlus + | Tok::MinusMinus => Some(25), + _ => None, + } +} + +/// True if the token is the range operator `...`. +pub fn is_range_op(tok: &Tok) -> bool { + matches!(tok, Tok::Ellipsis) +} + +/// Convert a `Tok` to a `BinaryOperator`. Returns None for non-binary operators. +pub fn tok_to_binary_op(tok: &Tok) -> Option { + Some(match tok { + Tok::Plus => BinaryOperator::Add, + Tok::Minus => BinaryOperator::Sub, + Tok::Star => BinaryOperator::Mul, + Tok::Slash => BinaryOperator::Div, + Tok::Percent => BinaryOperator::Mod, + Tok::EqEq => BinaryOperator::Eq, + Tok::NotEq => BinaryOperator::Ne, + Tok::Lt => BinaryOperator::Lt, + Tok::Gt => BinaryOperator::Gt, + Tok::LtEq => BinaryOperator::Le, + Tok::GtEq => BinaryOperator::Ge, + Tok::AndAnd => BinaryOperator::And, + Tok::OrOr => BinaryOperator::Or, + Tok::Amp => BinaryOperator::BitAnd, + Tok::Pipe => BinaryOperator::BitOr, + Tok::Caret => BinaryOperator::BitXor, + Tok::Shl => BinaryOperator::Shl, + Tok::Shr => BinaryOperator::Shr, + _ => return None, + }) +} + +/// Convert a `Tok` to an `AssignmentOperator`. +pub fn tok_to_assign_op(tok: &Tok) -> Option { + Some(match tok { + Tok::Assign => AssignmentOperator::Assign, + Tok::PlusEq => AssignmentOperator::AddAssign, + Tok::MinusEq => AssignmentOperator::SubAssign, + Tok::StarEq => AssignmentOperator::MulAssign, + Tok::SlashEq => AssignmentOperator::DivAssign, + Tok::PercentEq => AssignmentOperator::ModAssign, + Tok::AmpEq => AssignmentOperator::AndAssign, + Tok::PipeEq => AssignmentOperator::OrAssign, + Tok::CaretEq => AssignmentOperator::XorAssign, + Tok::ShlEq => AssignmentOperator::ShlAssign, + Tok::ShrEq => AssignmentOperator::ShrAssign, + _ => return None, + }) +} + +/// Convert a `Tok` to a `UnaryOperator`. +pub fn tok_to_unary_op(tok: &Tok) -> Option { + Some(match tok { + Tok::Bang => UnaryOperator::Not, + Tok::Minus => UnaryOperator::Neg, + Tok::Tilde => UnaryOperator::BitNot, + Tok::Amp => UnaryOperator::AddressOf, + Tok::Star => UnaryOperator::Dereference, + _ => return None, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::lexer::Tok; + + #[test] + fn infix_precedence_order() { + assert!(infix(&Tok::Assign).unwrap().0 < infix(&Tok::OrOr).unwrap().0); + assert!(infix(&Tok::OrOr).unwrap().0 < infix(&Tok::AndAnd).unwrap().0); + assert!(infix(&Tok::AndAnd).unwrap().0 < infix(&Tok::EqEq).unwrap().0); + assert!(infix(&Tok::EqEq).unwrap().0 < infix(&Tok::Plus).unwrap().0); + assert!(infix(&Tok::Plus).unwrap().0 < infix(&Tok::Star).unwrap().0); + } + + #[test] + fn assignment_is_right_associative() { + let (lbp, rbp) = infix(&Tok::Assign).unwrap(); + assert!(lbp < rbp); + } + + #[test] + fn additive_is_left_associative() { + // In Pratt parsers, left-associative operators have lbp < rbp + // (right operand binds tighter, so a+b+c = (a+b)+c). + let (lbp, rbp) = infix(&Tok::Plus).unwrap(); + assert!(lbp < rbp); + } + + #[test] + fn postfix_higher_than_infix() { + assert!(postfix(&Tok::Dot).unwrap() > infix(&Tok::Star).unwrap().1); + } + + #[test] + fn prefix_between_postfix_and_infix() { + assert!(prefix(&Tok::Minus).unwrap() > infix(&Tok::Star).unwrap().1); + assert!(prefix(&Tok::Minus).unwrap() < postfix(&Tok::Dot).unwrap()); + } + + #[test] + fn token_conversions() { + assert_eq!(tok_to_binary_op(&Tok::Plus), Some(BinaryOperator::Add)); + assert_eq!( + tok_to_assign_op(&Tok::PlusEq), + Some(AssignmentOperator::AddAssign) + ); + assert_eq!(tok_to_unary_op(&Tok::Minus), Some(UnaryOperator::Neg)); + assert_eq!(tok_to_binary_op(&Tok::Assign), None); + } +} diff --git a/kitlang/src/codegen/parser/diagnostics.rs b/kitlang/src/codegen/parser/diagnostics.rs new file mode 100644 index 0000000..83ce4d6 --- /dev/null +++ b/kitlang/src/codegen/parser/diagnostics.rs @@ -0,0 +1,198 @@ +//! Parser diagnostics: the internal error type used by the Pratt parser. +//! +//! This module exists to be the *single seam* through which future diagnostic +//! improvements (span attachment, "expected one of" rendering, error recovery) +//! can be added without touching the Pratt parser internals. +//! +//! Design rules: +//! 1. The Pratt parser produces `ExprParseError` values only. It never +//! formats messages, never prints, never holds source files. +//! 2. The conversion to the public `CompilationError` happens at exactly +//! one call site: the `PestExpr::parse` adapter in `parser/mod.rs`. +//! 3. Adding span support later is a per-variant field addition. The Pratt +//! parser's control flow does not change. +//! +//! Span-free in v1 by design. The variant shapes anticipate the addition +//! of a `span: Span` field; today, the conversion site synthesizes a +//! span-less error and a string message. + +use std::fmt; + +use crate::lexer::Tok; + +/// The internal error type produced by the Pratt parser. +/// +/// Every variant carries a `&'static [&'static str]` of "expected" token +/// names where applicable, so a future diagnostic system can render +/// "expected one of: ..." with no extra work. Today, [`to_human_message`] +/// joins these into a string for `CompilationError::ParseError`. +#[derive(Debug, Clone)] +#[allow(dead_code)] // variants will be constructed by diagnostic work in a follow-up +pub(crate) enum ExprParseError { + /// A token was found where it was not expected. + UnexpectedToken { + /// The token that was actually present. + found: Tok, + /// Human-readable names of what would have been acceptable. + expected: &'static [&'static str], + }, + /// The token stream ended before the expression was complete. + UnexpectedEof { + /// Human-readable names of what would have been acceptable. + expected: &'static [&'static str], + }, + /// A free-form parser-internal error. Used for cases that don't fit the "wrong token" / "ran + /// out" shape (e.g. type annotations that look like expressions but aren't). + Custom(String), +} + +impl ExprParseError { + /// Render this error as a human-readable string. + /// + /// This is the *only* place that turns structured error data into a + /// string. + /// + /// TODO: replace this with a structured rendering (e.g. a `miette::Report`) without changing + /// the parser. + pub(crate) fn to_human_message(&self) -> String { + match self { + Self::UnexpectedToken { found, expected } => { + let expected = expected.join(", "); + format!("unexpected token `{found:?}`, expected one of: {expected}") + } + Self::UnexpectedEof { expected } => { + format!( + "unexpected end of expression, expected one of: {}", + expected.join(", ") + ) + } + Self::Custom(msg) => msg.clone(), + } + } +} + +impl fmt::Display for ExprParseError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&self.to_human_message()) + } +} + +/// Map a `Tok` kind to a short human-readable name for error messages. +/// +/// Returns `'static` strings so `ExprParseError` can carry them as +/// `&'static [&'static str]` without allocation. Used by the binding-power +/// helpers in `expr_pratt.rs` to populate the `expected` list. +#[allow(dead_code)] // seam: will be called from diagnostic work in a follow-up +pub(crate) fn tok_name(kind: &Tok) -> &'static str { + match kind { + Tok::LParen => "`(`", + Tok::RParen => "`)`", + Tok::LBracket => "`[`", + Tok::RBracket => "`]`", + Tok::LBrace => "`{`", + Tok::RBrace => "`}`", + Tok::Comma => "`,`", + Tok::Semi => "`;`", + Tok::Dot => "`.`", + Tok::Colon => "`:`", + Tok::Ellipsis => "`...`", + Tok::Plus => "`+`", + Tok::Minus => "`-`", + Tok::Star => "`*`", + Tok::Slash => "`/`", + Tok::Percent => "`%`", + Tok::EqEq => "`==`", + Tok::NotEq => "`!=`", + Tok::LtEq => "`<=`", + Tok::GtEq => "`>=`", + Tok::Lt => "`<`", + Tok::Gt => "`>`", + Tok::AndAnd => "`&&`", + Tok::OrOr => "`||`", + Tok::Bang => "`!`", + Tok::Amp => "`&`", + Tok::Pipe => "`|`", + Tok::Caret => "`^`", + Tok::Tilde => "`~`", + Tok::Shl => "`<<`", + Tok::Shr => "`>>`", + Tok::PlusPlus => "`++`", + Tok::MinusMinus => "`--`", + Tok::ShlEq => "`<<=`", + Tok::ShrEq => "`>>=`", + Tok::PlusEq => "`+=`", + Tok::MinusEq => "`-=`", + Tok::StarEq => "`*=`", + Tok::SlashEq => "`/=`", + Tok::PercentEq => "`%=`", + Tok::AmpEq => "`&=`", + Tok::PipeEq => "`|=`", + Tok::CaretEq => "`^=`", + Tok::Assign => "`=`", + Tok::KwIf => "`if`", + Tok::KwThen => "`then`", + Tok::KwElse => "`else`", + Tok::KwTrue => "`true`", + Tok::KwFalse => "`false`", + Tok::KwNull => "`null`", + Tok::KwThis => "`this`", + Tok::KwSelf => "`Self`", + Tok::KwSizeof => "`sizeof`", + Tok::KwDefined => "`defined`", + Tok::KwUnsafe => "`unsafe`", + Tok::KwStatic => "`static`", + Tok::KwImplicit => "`implicit`", + Tok::KwEmpty => "`empty`", + Tok::KwStruct => "`struct`", + Tok::IntLit(_) => "integer literal", + Tok::FloatLit(_) => "float literal", + Tok::CharLit(_) => "char literal", + Tok::StringLit(_) => "string literal", + Tok::Ident(_) => "identifier", + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn unexpected_token_renders_expected_list() { + let err = ExprParseError::UnexpectedToken { + found: Tok::Plus, + expected: &["identifier", "integer literal", "string literal"], + }; + let msg = err.to_human_message(); + assert!(msg.contains("Plus"), "msg: {msg}"); + assert!(msg.contains("identifier"), "msg: {msg}"); + assert!(msg.contains("integer literal"), "msg: {msg}"); + assert!(msg.contains("string literal"), "msg: {msg}"); + } + + #[test] + fn unexpected_eof_renders_expected_list() { + let err = ExprParseError::UnexpectedEof { + expected: &[")", ",", "binary operator"], + }; + let msg = err.to_human_message(); + assert!(msg.contains("end of expression"), "msg: {msg}"); + assert!(msg.contains(")"), "msg: {msg}"); + assert!(msg.contains(","), "msg: {msg}"); + } + + #[test] + fn custom_message_passes_through() { + let err = ExprParseError::Custom("nope".to_string()); + assert_eq!(err.to_human_message(), "nope"); + } + + #[test] + fn tok_name_is_stable() { + // Sanity check: a few representative names. + assert_eq!(tok_name(&Tok::Plus), "`+`"); + assert_eq!(tok_name(&Tok::LParen), "`(`"); + assert_eq!(tok_name(&Tok::KwIf), "`if`"); + assert_eq!(tok_name(&Tok::IntLit(0)), "integer literal"); + assert_eq!(tok_name(&Tok::StringLit(String::new())), "string literal"); + } +} diff --git a/kitlang/src/codegen/parser/expr.rs b/kitlang/src/codegen/parser/expr.rs deleted file mode 100644 index ea1fd88..0000000 --- a/kitlang/src/codegen/parser/expr.rs +++ /dev/null @@ -1,429 +0,0 @@ -use std::str::FromStr; - -use pest::iterators::Pair; - -use crate::codegen::ast::{Expr, Literal}; -use crate::codegen::types::{AssignmentOperator, BinaryOperator, TypeId, UnaryOperator}; -use crate::error::{CompilationError, CompileResult}; -use crate::{Rule, parse_error}; - -use super::Parser; -use crate::codegen::type_ast::FieldInit; - -impl Parser { - pub(super) fn parse_expr(self, pair: Pair) -> CompileResult { - match pair.as_rule() { - Rule::expr => { - let inner = pair - .into_inner() - .next() - .ok_or_else(|| parse_error!("expr pair is empty"))?; - self.parse_expr(inner) - } - Rule::assign => self.parse_assign_expr(pair), - Rule::logical_or - | Rule::logical_and - | Rule::equality - | Rule::comparison - | Rule::additive - | Rule::multiplicative - | Rule::bitwise_or - | Rule::bitwise_xor - | Rule::bitwise_and - | Rule::shift => { - let mut inner = pair.into_inner(); - let mut left = self.parse_expr( - inner - .next() - .ok_or_else(|| parse_error!("binary op missing left operand"))?, - )?; - - while let Some(op_pair) = inner.next() { - let op = BinaryOperator::from_rule_pair(&op_pair)?; - let right = self.parse_expr( - inner - .next() - .ok_or_else(|| parse_error!("binary op missing right operand"))?, - )?; - left = Expr::BinaryOp { - op, - left: Box::new(left), - right: Box::new(right), - ty: TypeId::default(), - }; - } - Ok(left) - } - Rule::unary => { - let mut inner_pairs = pair.into_inner(); - let first_pair = inner_pairs - .next() - .ok_or_else(|| parse_error!("unary expression missing operand"))?; - match first_pair.as_rule() { - Rule::unary_op => { - let op_str = first_pair.as_str(); - let op = UnaryOperator::from_str(op_str) - .map_err(|_| parse_error!("invalid unary operation: {op_str}"))?; - let expr = self.parse_expr(inner_pairs.next().ok_or_else(|| { - parse_error!("unary expr missing operand after operator") - })?)?; - Ok(Expr::UnaryOp { - op, - expr: Box::new(expr), - ty: TypeId::default(), - }) - } - Rule::ADDRESS_OF_OP => { - let op = UnaryOperator::AddressOf; - let expr = self.parse_expr(inner_pairs.next().ok_or_else(|| { - parse_error!("unary expr missing operand after operator") - })?)?; - Ok(Expr::UnaryOp { - op, - expr: Box::new(expr), - ty: TypeId::default(), - }) - } - Rule::postfix | Rule::primary => self.parse_expr(first_pair), - other => Err(parse_error!("Unexpected rule in unary: {other:?}")), - } - } - Rule::identifier => Ok(Expr::Identifier { - name: Self::pair_text(pair), - ty: TypeId::default(), - }), - Rule::literal => { - let inner = pair - .into_inner() - .next() - .ok_or_else(|| parse_error!("literal is empty"))?; - - // Dispatch to the kind-specific literal parser - match inner.as_rule() { - Rule::number => { - let num_pair = inner - .into_inner() - .next() - .ok_or_else(|| parse_error!("number literal is empty"))?; - - // Dispatch to integer or float parser by number subtype - match num_pair.as_rule() { - Rule::integer => { - let s = num_pair.as_str(); - let i = s.parse::().map_err(|e| { - parse_error!("invalid integer literal '{s}': {:?}", e) - })?; - Ok(Expr::Literal { - value: Literal::Int(i), - ty: TypeId::default(), - }) - } - Rule::float => { - let s = num_pair.as_str(); - let f = s.parse::().map_err(|e| { - parse_error!("invalid float literal '{s}': {:?}", e) - })?; - Ok(Expr::Literal { - value: Literal::Float(f), - ty: TypeId::default(), - }) - } - _ => Err(parse_error!("Unexpected number type")), - } - } - Rule::boolean => Self::parse_bool_literal(inner.as_str()), - Rule::char_literal => { - let s = inner.as_str(); - let inner_content = &s[1..s.len() - 1]; - let c = Self::unescape(inner_content) - .and_then(|u| u.chars().next()) - .ok_or_else(|| parse_error!("invalid char literal: {s}"))?; - Ok(Expr::Literal { - value: Literal::Char(c), - ty: TypeId::default(), - }) - } - _ => Err(parse_error!( - "Unexpected literal type: {:?}", - inner.as_rule() - )), - } - } - Rule::string => { - let full = pair.as_str(); - let inner = &full[1..full.len() - 1]; - let unescaped = Self::unescape(inner).unwrap_or_else(|| inner.to_string()); - Ok(Expr::Literal { - value: Literal::String(unescaped), - ty: TypeId::default(), - }) - } - Rule::function_call_expr => { - let mut inner = pair.into_inner(); - let callee = Self::pair_text( - inner - .next() - .ok_or_else(|| parse_error!("function call missing callee"))?, - ); - let args = inner - .filter(|p: &Pair| p.as_rule() == Rule::expr) - .map(|p: Pair| self.parse_expr(p)) - .collect::, _>>()?; - Ok(Expr::Call { - callee, - args, - ty: TypeId::default(), - }) - } - Rule::if_expr => { - let mut inner = pair.into_inner(); - - let cond = self.parse_expr( - inner - .next() - .ok_or_else(|| parse_error!("if expr missing condition"))?, - )?; - - let then_branch = self.parse_expr( - inner - .next() - .ok_or_else(|| parse_error!("if expr missing then branch"))?, - )?; - - let else_branch = self.parse_expr( - inner - .next() - .ok_or_else(|| parse_error!("if expr missing else branch"))?, - )?; - - Ok(Expr::If { - cond: Box::new(cond), - then_branch: Box::new(then_branch), - else_branch: Box::new(else_branch), - ty: TypeId::default(), - }) - } - // Handle bare keyword primaries (null/true/false) and wrapped primaries that delegate inward - Rule::primary => { - let text = pair.as_str(); - let mut inner = pair.into_inner(); - if inner.peek().is_none() { - match text { - "null" => Ok(Expr::Literal { - value: Literal::Null, - ty: TypeId::default(), - }), - "true" | "false" => Self::parse_bool_literal(text), - other => Err(parse_error!("Unknown primary keyword: {}", other)), - } - } else { - let inner_pair = inner - .next() - .ok_or_else(|| parse_error!("primary expr is empty"))?; - match inner_pair.as_rule() { - Rule::identifier => Ok(Expr::Identifier { - name: Self::pair_text(inner_pair), - ty: TypeId::default(), - }), - Rule::literal - | Rule::function_call_expr - | Rule::array_literal - | Rule::struct_init - | Rule::union_init - | Rule::tuple_literal - | Rule::if_expr - | Rule::range_expr - | Rule::string - | Rule::expr - | Rule::unary => self.parse_expr(inner_pair), - _ => Err(parse_error!( - "Unexpected primary inner rule: {:?}", - inner_pair.as_rule() - )), - } - } - } - Rule::postfix => { - let mut inner = pair.into_inner(); - let mut expr = self.parse_expr( - inner - .next() - .ok_or_else(|| parse_error!("postfix expr missing base expression"))?, - )?; - - // Apply each chained member operation (field access or index) in order - for field_pair in inner { - // Skip non-postfix inner rules, only postfix_field carries a member operation - if field_pair.as_rule() == Rule::postfix_field { - let mut field_inner = field_pair.into_inner(); - let first = field_inner - .next() - .ok_or(parse_error!("Expected field or index in postfix"))?; - - // Branch on whether the operation is a member access or an index expression - match first.as_rule() { - Rule::identifier => { - let field_name = Self::pair_text(first); - expr = Expr::FieldAccess { - expr: Box::new(expr), - field_name, - ty: TypeId::default(), - }; - } - Rule::expr => { - let index = self.parse_expr(first)?; - expr = Expr::Index { - expr: Box::new(expr), - index: Box::new(index), - ty: TypeId::default(), - }; - } - other => { - return Err(parse_error!( - "Unexpected postfix_field inner rule: {other:?}" - )); - } - } - } - } - Ok(expr) - } - Rule::struct_init => self.parse_struct_init(pair), - Rule::range_expr => { - let mut inner = pair.into_inner(); - let start = self.parse_expr( - inner - .next() - .ok_or_else(|| parse_error!("range expr missing start"))?, - )?; - let end = self.parse_expr( - inner - .next() - .ok_or_else(|| parse_error!("range expr missing end"))?, - )?; - Ok(Expr::RangeLiteral { - start: Box::new(start), - end: Box::new(end), - }) - } - Rule::array_literal => { - let elements = pair - .into_inner() - .filter(|p: &Pair| p.as_rule() == Rule::expr) - .map(|p| self.parse_expr(p)) - .collect::, _>>()?; - Ok(Expr::ArrayLiteral { - elements, - ty: TypeId::default(), - }) - } - other => Err(CompilationError::ParseError(format!( - "Unexpected expr rule: {other:?}" - ))), - } - } - - fn parse_assign_expr(self, pair: Pair) -> CompileResult { - let mut inner = pair.into_inner(); - - let left_pair = inner - .next() - .ok_or_else(|| parse_error!("assign expr missing left operand"))?; - - let left = self.parse_expr(left_pair)?; - - // A plain lvalue (no operator) means the recursive call returned the inner expression unchanged - if let Some(assign_op_pair) = inner.next() { - let op = AssignmentOperator::from_rule_pair(&assign_op_pair)?; - - let right_assign_expr_pair = inner - .next() - .ok_or_else(|| parse_error!("assign expr missing right operand"))?; - - let right = self.parse_assign_expr(right_assign_expr_pair)?; - - Ok(Expr::Assign { - op, - left: Box::new(left), - right: Box::new(right), - ty: TypeId::default(), - }) - } else { - Ok(left) - } - } - - fn parse_struct_init(self, pair: Pair) -> CompileResult { - let mut inner = pair.into_inner(); - let type_pair = inner - .next() - .ok_or_else(|| parse_error!("struct init missing type name"))?; - - let struct_ty = self.parse_type(type_pair)?; - - let fields: Vec = inner - .filter(|p| p.as_rule() == Rule::field_init) // Drop separators, keep only field definitions - .map(|p| self.parse_field_init(p)) // Parse each definition into a FieldInit - .collect::>()?; // Short-circuit and propagate the first parse error - - Ok(Expr::StructInit { - ty: TypeId::default(), - struct_type: Some(struct_ty), - fields, - }) - } - - fn parse_field_init(self, pair: Pair) -> CompileResult { - let mut inner = pair.into_inner(); - let name = Self::pair_text( - inner - .next() - .ok_or_else(|| parse_error!("field init missing name"))?, - ); - let value = self.parse_expr( - inner - .next() - .ok_or_else(|| parse_error!("field init missing value"))?, - )?; - Ok(FieldInit { name, value }) - } - - fn parse_bool_literal(s: &str) -> CompileResult { - match s { - "true" => Ok(Expr::Literal { - value: Literal::Bool(true), - ty: TypeId::default(), - }), - "false" => Ok(Expr::Literal { - value: Literal::Bool(false), - ty: TypeId::default(), - }), - _ => Err(parse_error!("invalid boolean literal: {}", s)), - } - } - - /// Unescape common backslash sequences in a string slice. - /// - /// Returns `None` if the input ends with a stray backslash (no escape character follows it). - fn unescape(s: impl AsRef) -> Option { - let s = s.as_ref(); - let mut out = String::with_capacity(s.len()); - let mut chars = s.chars(); - while let Some(c) = chars.next() { - if c == '\\' { - match chars.next()? { - 'n' => out.push('\n'), - 'r' => out.push('\r'), - 't' => out.push('\t'), - '\\' => out.push('\\'), - '\'' => out.push('\''), - '"' => out.push('"'), - other => out.push(other), - } - } else { - out.push(c); - } - } - Some(out) - } -} diff --git a/kitlang/src/codegen/parser/expr_pratt.rs b/kitlang/src/codegen/parser/expr_pratt.rs new file mode 100644 index 0000000..fb64f26 --- /dev/null +++ b/kitlang/src/codegen/parser/expr_pratt.rs @@ -0,0 +1,1203 @@ +//! Hand-written Pratt parser for Kit expressions. +//! +//! This module takes over expression parsing from the pest-based grammar. +//! Pest still handles the program, declaration, statement, and type-annotation +//! grammars. For every `Pair<'_, Rule>` whose rule is an expression (i.e., +//! the 13 precedence levels `expr → assign → logical_or → ... → primary`), +//! the parser hands off to [`ExprParser::parse_expr`]. +//! +//! Pratt Parsing & Operator Precedence +//! ----------------------------------- +//! The pest grammar used 13 mutually recursive functions (one per precedence +//! level), which overflowed the default 1 MB stack on Windows. The Pratt +//! parser uses a single function with a binding-power loop, bounding stack +//! depth to `O(precedence levels) = O(13)` regardless of expression length. +//! Operator precedence is defined in [`binding_power::infix`], +//! [`binding_power::postfix`], [`binding_power::prefix`]. Each infix operator +//! has a (lbp, rbp) pair: `lbp < rbp` = right-associative (assignment), +//! `lbp == rbp` = left-associative (most binary ops). +//! +//! Errors & Source Spans +//! --------------------- +//! All parse errors are values of [`ExprParseError`] (in `diagnostics.rs`). +//! The parser never prints, never allocates strings for error messages, +//! and never holds source-file identity. Conversion to +//! `CompilationError::ParseError(String)` happens at `PestExpr::parse` +//! in `parser/mod.rs`. The token stream carries byte ranges; the parser +//! uses them internally but does not currently attach them to AST nodes. + +use crate::codegen::ast::{Expr, Literal}; +use crate::codegen::type_ast::FieldInit; +use crate::codegen::types::{Type, TypeId}; +use crate::lexer::{Span, SpannedTok, Tok, tokenize}; + +use super::binding_power::{ + infix, is_range_op, postfix, prefix, tok_to_assign_op, tok_to_binary_op, tok_to_unary_op, +}; +use super::diagnostics::ExprParseError; + +#[cfg(test)] +use crate::codegen::types::{AssignmentOperator, BinaryOperator, UnaryOperator}; + +// --------------------------------------------------------------------------- +// Parser +// --------------------------------------------------------------------------- + +/// A Pratt parser for Kit expressions. +/// +/// One instance is built per expression being parsed. The parser is +/// single-use: create a new one for each `parse_expr` call. +pub(crate) struct ExprParser<'a> { + tokens: &'a [SpannedTok], + pos: usize, +} + +impl<'a> ExprParser<'a> { + /// Build a parser over a token slice. The slice must be the result of + /// [`tokenize`] applied to the expression's source text. + pub(crate) fn new(tokens: &'a [SpannedTok]) -> Self { + Self { tokens, pos: 0 } + } + + /// Entry point. Parses one complete expression and returns it. + /// The expression may be followed by trailing tokens, which are left + /// in the stream (the caller can `pos()`-check or re-parse). + pub(crate) fn parse_expr(&mut self) -> Result { + self.parse_pratt(0) + } + + /// The current position in the token stream. Useful for tests and + /// for callers that want to know how many tokens were consumed. + #[allow(dead_code)] + pub(crate) fn pos(&self) -> usize { + self.pos + } + + // --- Token stream helpers (private) --- + + /// Peek the current token without consuming it. Returns a synthetic + /// "EOF" token when the stream is exhausted. + fn peek(&self) -> &SpannedTok { + // We never return None from peek; EOF is represented by a synthetic + // SpannedTok at span 0..0. This lets the Pratt loop compare with + // `==` against `Tok::...` cleanly. + static EOF: SpannedTok = SpannedTok { + kind: Tok::Semi, // any token that has no infix/postfix/prefix bp + span: 0..0, + }; + self.tokens.get(self.pos).unwrap_or(&EOF) + } + + /// Peek the next token (one past the current). Same EOF behavior. + #[allow(dead_code)] + fn peek_next(&self) -> &SpannedTok { + static EOF: SpannedTok = SpannedTok { + kind: Tok::Semi, + span: 0..0, + }; + self.tokens.get(self.pos + 1).unwrap_or(&EOF) + } + + /// Consume and return the current token. + fn advance(&mut self) -> &SpannedTok { + let tok = &self.tokens[self.pos]; + self.pos += 1; + tok + } + + /// True if the parser is at or past the end of the token stream. + #[allow(dead_code)] // seam: will be used for EOF diagnostics in a follow-up + fn at_eof(&self) -> bool { + self.pos >= self.tokens.len() + } + + // --- Pratt core --- + + /// Parse an expression with the given minimum binding power. + fn parse_pratt(&mut self, min_bp: u8) -> Result { + // Parse leading prefix operators (e.g. `-a`, `!!x`, `&arr[i]`). + // Prefix binds tighter than infix but looser than postfix, + // so `&arr[i]` = `&(arr[i])` (postfix on `arr` first). + let mut lhs = if let Some(pfx_bp) = prefix(&self.peek().kind) { + let op = tok_to_unary_op(&self.peek().kind).unwrap(); + self.advance(); + let rhs = self.parse_pratt(pfx_bp)?; + Expr::UnaryOp { + op, + expr: Box::new(rhs), + ty: TypeId::default(), + } + } else { + self.parse_primary()? + }; + + // Postfix chain: field access, index, call. + lhs = self.parse_postfix_chain(lhs)?; + + // Infix operators (binary ops, range). + loop { + let kind = self.peek().kind.clone(); + let Some((lbp, rbp)) = infix(&kind) else { + break; + }; + if lbp < min_bp { + break; + } + if tok_to_assign_op(&kind).is_some() { + break; + } + self.advance(); + if is_range_op(&kind) { + let rhs = self.parse_pratt(rbp)?; + lhs = Expr::RangeLiteral { + start: Box::new(lhs), + end: Box::new(rhs), + }; + continue; + } + let op = tok_to_binary_op(&kind).ok_or_else(|| { + ExprParseError::Custom(format!("internal: no binary op for {kind:?}")) + })?; + let rhs = self.parse_pratt(rbp)?; + lhs = Expr::BinaryOp { + op, + left: Box::new(lhs), + right: Box::new(rhs), + ty: TypeId::default(), + }; + } + + // Assignment (right-associative, lowest precedence). + loop { + let kind = self.peek().kind.clone(); + let Some(op) = tok_to_assign_op(&kind) else { + break; + }; + // Assignment has the lowest precedence (lbp=0, rbp=1 in the + // infix table, but we don't use that table here). Right- + // associativity: a = b = c means a = (b = c). Recurse with + // min_bp=0 so the rhs sees *all* operators including another + // assignment. + if 0 < min_bp { + break; + } + self.advance(); + let rhs = self.parse_pratt(0)?; + lhs = Expr::Assign { + op, + left: Box::new(lhs), + right: Box::new(rhs), + ty: TypeId::default(), + }; + } + + Ok(lhs) + } + + /// Iteratively apply postfix operators (call, index, field access) to + /// a base expression. Zero stack frames added per iteration. The + /// chain is bounded by the source's syntactic length, but the parser + /// is iterative, so the *call stack* depth is constant. + fn parse_postfix_chain(&mut self, mut base: Expr) -> Result { + loop { + let kind = self.peek().kind.clone(); + if postfix(&kind).is_none() { + break; + } + base = match kind { + Tok::Dot => self.parse_field_access(base)?, + Tok::LBracket => self.parse_index(base)?, + Tok::LParen => self.parse_call(base)?, + _ => unreachable!("postfix returned Some for {kind:?}"), + }; + } + Ok(base) + } + + /// Parse a primary expression: literals, identifiers, parenthesized + /// expressions, function calls, array literals, struct inits, and + /// the if-expression. Postfix operations (`.field`, `[i]`, `(args)`) + /// are handled in the outer Pratt loop, *not* here, so this function + /// only needs to produce the base expression. + fn parse_primary(&mut self) -> Result { + let tok = self.peek().kind.clone(); + // `span` is read for documentation; the parser doesn't currently + // attach it to AST nodes. Future PRs will use it. + let _span: Span = self.peek().span.clone(); + + match tok { + Tok::IntLit(n) => { + self.advance(); + Ok(Expr::Literal { + value: Literal::Int(n), + ty: TypeId::default(), + }) + } + Tok::FloatLit(f) => { + self.advance(); + Ok(Expr::Literal { + value: Literal::Float(f), + ty: TypeId::default(), + }) + } + Tok::CharLit(c) => { + self.advance(); + Ok(Expr::Literal { + value: Literal::Char(c), + ty: TypeId::default(), + }) + } + Tok::StringLit(s) => { + self.advance(); + Ok(Expr::Literal { + value: Literal::String(s), + ty: TypeId::default(), + }) + } + Tok::KwTrue => { + self.advance(); + Ok(Expr::Literal { + value: Literal::Bool(true), + ty: TypeId::default(), + }) + } + Tok::KwFalse => { + self.advance(); + Ok(Expr::Literal { + value: Literal::Bool(false), + ty: TypeId::default(), + }) + } + Tok::KwNull => { + self.advance(); + Ok(Expr::Literal { + value: Literal::Null, + ty: TypeId::default(), + }) + } + Tok::KwThis | Tok::KwSelf => { + // The pest parser treats these as `Identifier` with a + // fixed name; we follow suit. A future PR can introduce + // dedicated AST variants if needed. + let name = match tok { + Tok::KwThis => "this", + _ => "Self", + }; + self.advance(); + Ok(Expr::Identifier { + name: name.to_string(), + ty: TypeId::default(), + }) + } + Tok::Ident(name) => { + self.advance(); + Ok(Expr::Identifier { + name, + ty: TypeId::default(), + }) + } + Tok::LParen => { + // Either a parenthesized expression or the start of a + // tuple literal `(a, b, c)`. We parse the first + // expression; if the next token is `,` it's a tuple, else + // we expect a closing `)`. + self.advance(); // consume `(` + let first = self.parse_expr()?; + if self.peek().kind == Tok::Comma { + // Tuple literal: parse remaining comma-separated + // expressions, then expect `)`. The AST doesn't have + // a dedicated tuple variant, so we represent the + // tuple as a parenthesized expression with a + // synthetic structure. For now, produce a Custom + // error directing the caller to the pest path; this + // matches the existing parser's TODO for tuple + // literals. + return Err(ExprParseError::Custom( + "tuple literals are not yet supported by the Pratt parser".into(), + )); + } + self.expect(&Tok::RParen)?; + Ok(first) + } + Tok::LBracket => self.parse_array_literal(), + Tok::KwStruct => self.parse_struct_init(), + Tok::KwIf => self.parse_if_expr(), + Tok::KwEmpty => { + // The grammar's primary includes "empty" as a keyword. + // We don't have a dedicated AST variant; treat it as an + // identifier for now (semantics will be filled in by + // type inference downstream). + self.advance(); + Ok(Expr::Identifier { + name: "empty".to_string(), + ty: TypeId::default(), + }) + } + _ => Err(ExprParseError::UnexpectedToken { + found: tok, + expected: &[ + "integer literal", + "float literal", + "string literal", + "char literal", + "identifier", + "`(`", + "`[`", + "`if`", + "`null`", + "`true`", + "`false`", + ], + }), + } + } + + /// Parse a `.field` access postfix. + fn parse_field_access(&mut self, base: Expr) -> Result { + self.advance(); // consume `.` + let field_tok = self.peek().kind.clone(); + match field_tok { + Tok::Ident(name) => { + self.advance(); + Ok(Expr::FieldAccess { + expr: Box::new(base), + field_name: name, + ty: TypeId::default(), + }) + } + _ => Err(ExprParseError::UnexpectedToken { + found: field_tok, + expected: &["identifier"], + }), + } + } + + /// Parse a `[index]` postfix. + fn parse_index(&mut self, base: Expr) -> Result { + self.advance(); // consume `[` + let index = self.parse_expr()?; + self.expect(&Tok::RBracket)?; + Ok(Expr::Index { + expr: Box::new(base), + index: Box::new(index), + ty: TypeId::default(), + }) + } + + /// Parse a function call postfix: `(arg1, arg2, ...)`. + fn parse_call(&mut self, callee: Expr) -> Result { + // The callee expression's name lives in `Expr::Identifier.name` + // (possibly qualified with dots for things like + // `qualified_call.math.add`). The pest grammar's + // `function_call_expr` rule extracts a `path` which is a dot- + // separated identifier. We extract the name from the callee + // expression; for `Expr::FieldAccess` chains we concatenate with + // dots to recover the qualified name. This matches the pest + // parser's behavior in `parser/expr.rs:163-167` and the + // transpiler's qualified-name resolution in + // `transpile/mod.rs:493-500`. + let callee_name = expr_to_callee_name(&callee); + + self.advance(); // consume `(` + let args = self.parse_comma_list(Tok::RParen, |p| p.parse_expr())?; + Ok(Expr::Call { + callee: callee_name, + args, + ty: TypeId::default(), + }) + } + + /// Parse an array literal: `[expr, expr, ...]`. + fn parse_array_literal(&mut self) -> Result { + self.advance(); // consume `[` + let elements = self.parse_comma_list(Tok::RBracket, |p| p.parse_expr())?; + Ok(Expr::ArrayLiteral { + elements, + ty: TypeId::default(), + }) + } + + /// Parse a struct init: `struct Name { field: expr, ... }`. + fn parse_struct_init(&mut self) -> Result { + self.advance(); // consume `struct` + // The type annotation here is a single identifier (we don't + // handle complex type annotations in expressions). This matches + // the common case; the pest path still handles generics/pointers + // via the var_decl/init call site. + let type_tok = self.peek().kind.clone(); + let type_name = match type_tok { + Tok::Ident(name) => { + self.advance(); + name + } + _ => { + return Err(ExprParseError::UnexpectedToken { + found: type_tok, + expected: &["type name"], + }); + } + }; + self.expect(&Tok::LBrace)?; + let fields = self.parse_comma_list(Tok::RBrace, |p| { + let name = match &p.peek().kind { + Tok::Ident(n) => { + let n = n.clone(); + p.advance(); + n + } + _ => { + return Err(ExprParseError::UnexpectedToken { + found: p.peek().kind.clone(), + expected: &["field name"], + }); + } + }; + p.expect(&Tok::Colon)?; + let value = p.parse_expr()?; + Ok(FieldInit { name, value }) + })?; + Ok(Expr::StructInit { + ty: TypeId::default(), + struct_type: Some(Type::from_kit(&type_name)), + fields, + }) + } + + /// Parse an if-expression: `if cond then a else b`. + fn parse_if_expr(&mut self) -> Result { + self.advance(); // consume `if` + let cond = self.parse_expr()?; + self.expect(&Tok::KwThen)?; + let then_branch = self.parse_expr()?; + self.expect(&Tok::KwElse)?; + let else_branch = self.parse_expr()?; + Ok(Expr::If { + cond: Box::new(cond), + then_branch: Box::new(then_branch), + else_branch: Box::new(else_branch), + ty: TypeId::default(), + }) + } + + // --- Helpers --- + + /// Consume the current token if it matches `expected`, otherwise + /// return an `UnexpectedToken` error with `expected`'s name. + fn expect(&mut self, expected: &Tok) -> Result<(), ExprParseError> { + if &self.peek().kind == expected { + self.advance(); + Ok(()) + } else { + // We construct the `&'static [&'static str]` directly from + // a call to `tok_name`, which the compiler can verify is + // `'static`. Storing the result in a local first drops the + // `'static` lifetime, so we keep the call inline. + Err(ExprParseError::UnexpectedToken { + found: self.peek().kind.clone(), + expected: expected_name(expected), + }) + } + } + + /// Parse a comma-separated list of `T` terminated by `closer`. + /// Allows zero or more elements (an empty list is valid for fn + /// calls with no args, empty array literals, etc.). + fn parse_comma_list(&mut self, closer: Tok, mut f: F) -> Result, ExprParseError> + where + F: FnMut(&mut Self) -> Result, + { + let mut out = Vec::new(); + // Empty list case. + if self.peek().kind == closer { + self.advance(); + return Ok(out); + } + out.push(f(self)?); + while self.peek().kind == Tok::Comma { + self.advance(); + // Trailing comma is allowed (parses to empty trailing element). + if self.peek().kind == closer { + break; + } + out.push(f(self)?); + } + self.expect(&closer)?; + Ok(out) + } +} + +/// Extract a callee name from a function-call's leading expression. +/// +/// For `Expr::Identifier { name, .. }` this is just `name`. +/// For `Expr::FieldAccess { expr, field_name, .. }` chains like +/// `pkg.math.add`, this concatenates the path with `.`. +/// For other expressions (e.g. a parenthesized call), we fall back to +/// `Display`-formatting the expression, which the transpiler can still +/// route through name resolution. +fn expr_to_callee_name(expr: &Expr) -> String { + match expr { + Expr::Identifier { name, .. } => name.clone(), + Expr::FieldAccess { + expr: base, + field_name, + .. + } => { + let base_name = expr_to_callee_name(base); + format!("{base_name}.{field_name}") + } + other => format!("{other:?}"), + } +} + +/// Return a static slice of static strs containing the name of `expected`. +/// Used by `expect` to build an `UnexpectedToken` error variant. +fn expected_name(expected: &Tok) -> &'static [&'static str] { + match expected { + Tok::LParen => &["`(`"], + Tok::RParen => &["`)`"], + Tok::LBracket => &["`[`"], + Tok::RBracket => &["`]`"], + Tok::LBrace => &["`{`"], + Tok::RBrace => &["`}`"], + Tok::Comma => &["`,`"], + Tok::Semi => &["`;`"], + Tok::Dot => &["`.`"], + Tok::Colon => &["`:`"], + Tok::Ellipsis => &["`...`"], + Tok::Plus => &["`+`"], + Tok::Minus => &["`-`"], + Tok::Star => &["`*`"], + Tok::Slash => &["`/`"], + Tok::Percent => &["`%`"], + Tok::EqEq => &["`==`"], + Tok::NotEq => &["`!=`"], + Tok::LtEq => &["`<=`"], + Tok::GtEq => &["`>=`"], + Tok::Lt => &["`<`"], + Tok::Gt => &["`>`"], + Tok::AndAnd => &["`&&`"], + Tok::OrOr => &["`||`"], + Tok::Bang => &["`!`"], + Tok::Amp => &["`&`"], + Tok::Pipe => &["`|`"], + Tok::Caret => &["`^`"], + Tok::Tilde => &["`~`"], + Tok::Shl => &["`<<`"], + Tok::Shr => &["`>>`"], + Tok::PlusPlus => &["`++`"], + Tok::MinusMinus => &["`--`"], + Tok::ShlEq => &["`<<=`"], + Tok::ShrEq => &["`>>=`"], + Tok::PlusEq => &["`+=`"], + Tok::MinusEq => &["`-=`"], + Tok::StarEq => &["`*=`"], + Tok::SlashEq => &["`/=`"], + Tok::PercentEq => &["`%=`"], + Tok::AmpEq => &["`&=`"], + Tok::PipeEq => &["`|=`"], + Tok::CaretEq => &["`^=`"], + Tok::Assign => &["`=`"], + Tok::KwIf => &["`if`"], + Tok::KwThen => &["`then`"], + Tok::KwElse => &["`else`"], + Tok::KwTrue => &["`true`"], + Tok::KwFalse => &["`false`"], + Tok::KwNull => &["`null`"], + Tok::KwThis => &["`this`"], + Tok::KwSelf => &["`Self`"], + Tok::KwSizeof => &["`sizeof`"], + Tok::KwDefined => &["`defined`"], + Tok::KwUnsafe => &["`unsafe`"], + Tok::KwStatic => &["`static`"], + Tok::KwImplicit => &["`implicit`"], + Tok::KwEmpty => &["`empty`"], + Tok::KwStruct => &["`struct`"], + Tok::IntLit(_) => &["integer literal"], + Tok::FloatLit(_) => &["float literal"], + Tok::CharLit(_) => &["char literal"], + Tok::StringLit(_) => &["string literal"], + Tok::Ident(_) => &["identifier"], + } +} + +// --------------------------------------------------------------------------- +// Module surface: parse an expression from source text. +// --------------------------------------------------------------------------- + +/// Parse a Kit expression from source text. This is the public entry +/// point used by the pest-to-Pratt bridge (`PestExpr::parse`). +/// +/// The `text` should be the source text of the expression as a +/// `Pair::as_str()` slice. Tokenization, parsing, and conversion to an +/// `Expr` all happen here. +pub(crate) fn parse_kit_expr(text: &str) -> Result { + let tokens = tokenize(text); + let mut parser = ExprParser::new(&tokens); + parser.parse_expr() +} + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + + /// Convenience: parse an expression and unwrap. + fn p(text: &str) -> Expr { + parse_kit_expr(text).unwrap_or_else(|e| panic!("parse failed for `{text}`: {e}")) + } + + /// Convenience: parse and assert the error contains a substring. + fn p_err(text: &str, needle: &str) { + let err = parse_kit_expr(text) + .err() + .unwrap_or_else(|| panic!("expected error for `{text}`, got Ok")); + let msg = err.to_human_message(); + assert!( + msg.contains(needle), + "error `{msg}` does not contain `{needle}`" + ); + } + + // --- Literals --- + + #[test] + fn integer_literal() { + let e = p("42"); + assert!(matches!( + e, + Expr::Literal { + value: Literal::Int(42), + .. + } + )); + } + + #[test] + fn float_literal() { + let e = p("3.14"); + assert!( + matches!(e, Expr::Literal { value: Literal::Float(f), .. } if (f - 3.14).abs() < 1e-10) + ); + } + + #[test] + fn string_literal() { + let e = p(r#""hello""#); + assert!(matches!(e, Expr::Literal { value: Literal::String(s), .. } if s == "hello")); + } + + #[test] + fn bool_literals() { + assert!(matches!( + p("true"), + Expr::Literal { + value: Literal::Bool(true), + .. + } + )); + assert!(matches!( + p("false"), + Expr::Literal { + value: Literal::Bool(false), + .. + } + )); + } + + #[test] + fn null_literal() { + assert!(matches!( + p("null"), + Expr::Literal { + value: Literal::Null, + .. + } + )); + } + + // --- Identifiers --- + + #[test] + fn identifier() { + let e = p("foo"); + assert!(matches!(&e, Expr::Identifier { name, .. } if name == "foo")); + } + + #[test] + fn qualified_identifier_is_built_via_postfix_chain() { + // The lexer produces foo . bar . baz as three tokens. The postfix + // chain in the Pratt loop builds a FieldAccess tree. + let e = p("foo.bar.baz"); + // Walking the tree should give us three nested FieldAccess nodes + // with the leaf being `foo`. + let mut cur = &e; + let mut path = vec![]; + while let Expr::FieldAccess { + expr, field_name, .. + } = cur + { + path.push(field_name.clone()); + cur = expr; + } + if let Expr::Identifier { name, .. } = cur { + assert_eq!(name, "foo"); + } else { + panic!("expected leaf Identifier, got {cur:?}"); + } + assert_eq!(path, vec!["baz".to_string(), "bar".to_string()]); + } + + // --- Precedence --- + + #[test] + fn additive_vs_multiplicative() { + // 1 + 2 * 3 should be 1 + (2 * 3) + let e = p("1 + 2 * 3"); + // Top-level is BinaryOp::Add with right = BinaryOp::Mul + if let Expr::BinaryOp { op, right, .. } = &e { + assert_eq!(*op, BinaryOperator::Add); + if let Expr::BinaryOp { op: inner_op, .. } = right.as_ref() { + assert_eq!(*inner_op, BinaryOperator::Mul); + } else { + panic!("expected inner Mul, got {right:?}"); + } + } else { + panic!("expected top-level Add, got {e:?}"); + } + } + + #[test] + fn comparison_vs_equality() { + // a == b < c should be (a == b) < c, since == is lower precedence + // than comparison. Wait, looking at the table: equality (13/14) + // is LOWER precedence than comparison (15/16). So `a == b < c` + // means `a == (b < c)`. Hmm let me re-check the grammar. + // + // equality = { comparison ~ (eq_op ~ comparison)* } + // comparison = { bitwise_or ~ (comp_op ~ bitwise_or)* } + // + // This means: an equality expression is a comparison optionally + // followed by ==/!= and another comparison. So `a == b < c` would + // first match `a` as a comparison, then look for == or !=, find + // ==, parse `b < c` as a comparison (because comparison has + // higher precedence than equality), giving `a == (b < c)`. + // + // In our binding-power table: eq (13/14) < comparison (15/16), + // which is correct. So `a == b < c` is `a == (b < c)`. + let e = p("a == b < c"); + if let Expr::BinaryOp { + op, left, right, .. + } = &e + { + assert_eq!(*op, BinaryOperator::Eq); + // left should be `a`, right should be `b < c` + assert!(matches!(left.as_ref(), Expr::Identifier { name, .. } if name == "a")); + if let Expr::BinaryOp { op: inner_op, .. } = right.as_ref() { + assert_eq!(*inner_op, BinaryOperator::Lt); + } else { + panic!("expected inner Lt, got {right:?}"); + } + } else { + panic!("expected top-level Eq, got {e:?}"); + } + } + + #[test] + fn left_associative_addition() { + // 1 + 2 + 3 should be (1 + 2) + 3 + let e = p("1 + 2 + 3"); + if let Expr::BinaryOp { + op, left, right, .. + } = &e + { + assert_eq!(*op, BinaryOperator::Add); + assert!(matches!( + right.as_ref(), + Expr::Literal { + value: Literal::Int(3), + .. + } + )); + if let Expr::BinaryOp { op: inner_op, .. } = left.as_ref() { + assert_eq!(*inner_op, BinaryOperator::Add); + } else { + panic!("expected inner Add, got {left:?}"); + } + } else { + panic!("expected top-level Add, got {e:?}"); + } + } + + #[test] + fn right_associative_assignment() { + // a = b = c should be a = (b = c) -- right-associative + // The grammar's `assign = logical_or ~ ASSIGN_OP ~ assign | logical_or` + // recurses on the right, so this is the expected grouping. + // + // But: in our grammar, identifiers are not valid lvalues by + // themselves for assignment. The pest parser wraps the lhs in + // an `Expr::Identifier` and the rhs in another `Expr::Assign`. + // For now we test that the structure is right-associative; the + // type checker will reject `a = b = c` if identifiers aren't + // valid lvalues for that expression form. + // + // We use parenthesized variables to keep this test purely about + // precedence; type checking is downstream. + let e = p("a += b += c"); + if let Expr::Assign { + op, left, right, .. + } = &e + { + assert_eq!(*op, AssignmentOperator::AddAssign); + assert!(matches!(left.as_ref(), Expr::Identifier { name, .. } if name == "a")); + assert!(matches!(right.as_ref(), Expr::Assign { .. })); + } else { + panic!("expected top-level Assign, got {e:?}"); + } + } + + #[test] + fn unary_minus_binds_tighter_than_addition() { + // -a + b should be (-a) + b + let e = p("-a + b"); + if let Expr::BinaryOp { + op, left, right, .. + } = &e + { + assert_eq!(*op, BinaryOperator::Add); + assert!(matches!(right.as_ref(), Expr::Identifier { name, .. } if name == "b")); + assert!(matches!( + left.as_ref(), + Expr::UnaryOp { + op: UnaryOperator::Neg, + .. + } + )); + } else { + panic!("expected top-level Add, got {e:?}"); + } + } + + #[test] + fn unary_looser_than_postfix() { + // &arr[i] should be &(arr[i]) + let e = p("&arr[i]"); + if let Expr::UnaryOp { op, expr, .. } = &e { + assert_eq!(*op, UnaryOperator::AddressOf); + assert!(matches!(expr.as_ref(), Expr::Index { .. })); + } else { + panic!("expected top-level AddressOf, got {e:?}"); + } + } + + // --- Postfix chains --- + + #[test] + fn chained_field_access() { + // a.b.c.d.e - should produce 4 nested FieldAccess + let e = p("a.b.c.d.e"); + let mut depth = 0; + let mut cur = &e; + while let Expr::FieldAccess { expr, .. } = cur { + depth += 1; + cur = expr; + } + assert_eq!(depth, 4, "expected 4 field-access levels"); + assert!(matches!(cur, Expr::Identifier { name, .. } if name == "a")); + } + + #[test] + fn stress_deep_postfix_chain() { + // 100-deep chain. With the pest-based parser this would overflow + // the Windows 1MB stack; the Pratt parser handles it in one + // function call with a single loop. + let mut src = String::from("a"); + for i in 0..100 { + src.push('.'); + src.push_str(&format!("f{i}")); + } + let e = p(&src); + let mut depth = 0; + let mut cur = &e; + while let Expr::FieldAccess { expr, .. } = cur { + depth += 1; + cur = expr; + } + assert_eq!(depth, 100); + } + + #[test] + fn stress_deep_nested_parens() { + // 100-deep nesting. The pest-based parser would have a stack + // frame per `(` `)` pair; the Pratt parser recurses through + // `parse_pratt` once per pair but the binding-power loop bounds + // the recursion depth to the source's *syntactic* depth (which + // is the user's choice, not the grammar's). + let mut src = String::new(); + for _ in 0..100 { + src.push('('); + } + src.push('1'); + for _ in 0..100 { + src.push(')'); + } + let e = p(&src); + assert!(matches!( + e, + Expr::Literal { + value: Literal::Int(1), + .. + } + )); + } + + // --- Function calls --- + + #[test] + fn call_no_args() { + let e = p("f()"); + if let Expr::Call { callee, args, .. } = &e { + assert_eq!(callee, "f"); + assert!(args.is_empty()); + } else { + panic!("expected Call, got {e:?}"); + } + } + + #[test] + fn call_one_arg() { + let e = p("f(1)"); + if let Expr::Call { callee, args, .. } = &e { + assert_eq!(callee, "f"); + assert_eq!(args.len(), 1); + } else { + panic!("expected Call, got {e:?}"); + } + } + + #[test] + fn call_many_args() { + let e = p("f(1, 2, 3, 4, 5)"); + if let Expr::Call { args, .. } = &e { + assert_eq!(args.len(), 5); + } else { + panic!("expected Call, got {e:?}"); + } + } + + #[test] + fn call_qualified_name() { + let e = p("pkg.math.add(2, 3)"); + if let Expr::Call { callee, args, .. } = &e { + assert_eq!(callee, "pkg.math.add"); + assert_eq!(args.len(), 2); + } else { + panic!("expected Call, got {e:?}"); + } + } + + #[test] + fn call_with_nested_expressions_in_args() { + // The expression `f(g(1), h(2, 3))` exercises the recursive + // nature of the parser: each arg is itself a Pratt-parsed + // expression, and the call site itself is a postfix in the + // outer parse. + let e = p("f(g(1), h(2, 3))"); + if let Expr::Call { args, .. } = &e { + assert_eq!(args.len(), 2); + } else { + panic!("expected Call, got {e:?}"); + } + } + + // --- Indexing --- + + #[test] + fn index() { + let e = p("arr[0]"); + if let Expr::Index { expr, index, .. } = &e { + assert!(matches!(expr.as_ref(), Expr::Identifier { name, .. } if name == "arr")); + assert!(matches!( + index.as_ref(), + Expr::Literal { + value: Literal::Int(0), + .. + } + )); + } else { + panic!("expected Index, got {e:?}"); + } + } + + #[test] + fn chained_index() { + // a[i][j] - two index ops in sequence + let e = p("a[i][j]"); + let mut depth = 0; + let mut cur = &e; + while let Expr::Index { expr, .. } = cur { + depth += 1; + cur = expr; + } + assert_eq!(depth, 2); + } + + // --- Array literals --- + + #[test] + fn empty_array() { + let e = p("[]"); + if let Expr::ArrayLiteral { elements, .. } = &e { + assert!(elements.is_empty()); + } else { + panic!("expected ArrayLiteral, got {e:?}"); + } + } + + #[test] + fn array_with_elements() { + let e = p("[1, 2, 3]"); + if let Expr::ArrayLiteral { elements, .. } = &e { + assert_eq!(elements.len(), 3); + } else { + panic!("expected ArrayLiteral, got {e:?}"); + } + } + + // --- Struct init --- + + #[test] + fn struct_init() { + let e = p("struct Point { x: 10, y: 20 }"); + if let Expr::StructInit { fields, .. } = &e { + assert_eq!(fields.len(), 2); + assert_eq!(fields[0].name, "x"); + assert_eq!(fields[1].name, "y"); + } else { + panic!("expected StructInit, got {e:?}"); + } + } + + // --- If expressions --- + + #[test] + fn if_expr() { + let e = p("if a then b else c"); + if let Expr::If { + cond, + then_branch, + else_branch, + .. + } = &e + { + assert!(matches!(cond.as_ref(), Expr::Identifier { name, .. } if name == "a")); + assert!(matches!(then_branch.as_ref(), Expr::Identifier { name, .. } if name == "b")); + assert!(matches!(else_branch.as_ref(), Expr::Identifier { name, .. } if name == "c")); + } else { + panic!("expected If, got {e:?}"); + } + } + + // --- Logical operators --- + + #[test] + fn logical_and_vs_or() { + // a || b && c: && is higher precedence, so this is a || (b && c) + let e = p("a || b && c"); + if let Expr::BinaryOp { op, right, .. } = &e { + assert_eq!(*op, BinaryOperator::Or); + assert!(matches!( + right.as_ref(), + Expr::BinaryOp { + op: BinaryOperator::And, + .. + } + )); + } else { + panic!("expected top-level Or, got {e:?}"); + } + } + + // --- Errors --- + + #[test] + fn missing_rparen() { + p_err("(1 + 2", "`)`"); + } + + // --- Range literals --- + + #[test] + fn range_literal_simple() { + let e = p("1...5"); + if let Expr::RangeLiteral { start, end } = &e { + assert!(matches!( + start.as_ref(), + Expr::Literal { + value: Literal::Int(1), + .. + } + )); + assert!(matches!( + end.as_ref(), + Expr::Literal { + value: Literal::Int(5), + .. + } + )); + } else { + panic!("expected RangeLiteral, got {e:?}"); + } + } + + #[test] + fn range_literal_with_expressions() { + // a + 1...b - 1 + // Range binds tighter than assignment but looser than + // arithmetic, so this is (a + 1) ... (b - 1). + let e = p("a + 1...b - 1"); + if let Expr::RangeLiteral { start, end } = &e { + // start should be a + 1 + assert!(matches!( + start.as_ref(), + Expr::BinaryOp { + op: BinaryOperator::Add, + .. + } + )); + // end should be b - 1 + assert!(matches!( + end.as_ref(), + Expr::BinaryOp { + op: BinaryOperator::Sub, + .. + } + )); + } else { + panic!("expected RangeLiteral, got {e:?}"); + } + } + + #[test] + fn missing_rbracket() { + p_err("arr[0", "`]`"); + } + + #[test] + fn unexpected_token_at_start() { + p_err("+", "identifier"); + } + + #[test] + fn missing_field_name() { + p_err("foo.", "identifier"); + } + + #[test] + fn missing_else() { + p_err("if a then b", "`else`"); + } +} diff --git a/kitlang/src/codegen/parser/mod.rs b/kitlang/src/codegen/parser/mod.rs index 28e0a8e..ff3754c 100644 --- a/kitlang/src/codegen/parser/mod.rs +++ b/kitlang/src/codegen/parser/mod.rs @@ -1,11 +1,15 @@ -mod expr; +mod binding_power; +mod diagnostics; +mod expr_pratt; use pest::iterators::Pair; use crate::error::CompilationError; use crate::{Rule, parse_error}; -use super::ast::{Block, Function, GlobalDecl, Include, Literal, MetaArg, Metadata, Param, Stmt}; +use super::ast::{ + Block, Expr, Function, GlobalDecl, Include, Literal, MetaArg, Metadata, Param, Stmt, +}; use super::module::{ImportType, ModuleImport, ModulePath}; use super::type_ast::{ EnumDefinition, EnumVariant, Field, ImplDefinition, RuleDecl, RuleSet, StructDefinition, @@ -14,6 +18,40 @@ use super::type_ast::{ use super::types::{Type, TypeId}; use crate::error::CompileResult; +/// Bridge between pest and the Pratt parser. +/// +/// The pest-based parser walks the grammar tree and, when it encounters +/// an `expr` rule, hands the corresponding `Pair` off to the Pratt parser +/// via this adapter. The adapter: +/// +/// 1. Pulls the source text out of the `Pair` with `as_str()`. +/// 2. Tokenizes it with the Logos lexer. +/// 3. Parses the tokens with the Pratt parser. +/// 4. Converts the Pratt parser's `ExprParseError` to the public +/// `CompilationError`. +/// +/// This is the *only* conversion point between the two parsers; it's +/// also the only place the public error type is built from the +/// parser-internal one. Future diagnostic improvements (spans, severity, +/// pretty rendering) are added at this single seam. +pub(crate) struct PestExpr<'a> { + pair: Pair<'a, Rule>, +} + +impl<'a> PestExpr<'a> { + /// Wrap a pest `Pair` whose rule is an expression. + pub(crate) fn new(pair: Pair<'a, Rule>) -> Self { + Self { pair } + } + + /// Parse the wrapped pair as a Kit expression. + pub(crate) fn parse(self) -> CompileResult { + let text = self.pair.as_str(); + expr_pratt::parse_kit_expr(text) + .map_err(|e| CompilationError::ParseError(e.to_human_message())) + } +} + #[derive(Clone, Copy, Default, Debug)] pub struct Parser; @@ -144,6 +182,19 @@ impl Parser { } /// Parse a `function_decl` rule into a `Function`. + /// Parse an expression via the Pratt parser. This is the unified + /// entry point used by every pest-side call site that needs an + /// expression AST node. + /// + /// `Parser` is `Copy`, so we take `self` by value to match the + /// signature of the (now-deleted) pest-based `parse_expr`. The + /// caller doesn't need to mutate the parser; the Pratt parser + /// operates on its own `ExprParser` instance built from a token + /// slice derived from the `Pair`. + pub fn parse_expr(self, pair: Pair) -> CompileResult { + PestExpr::new(pair).parse() + } + pub fn parse_function(&self, pair: Pair) -> CompileResult { let mut inner = pair.into_inner(); @@ -748,4 +799,10 @@ impl Parser { )?; Ok(Stmt::For { var, iter, body }) } + + // The pest-based `parse_struct_init` and `parse_field_init` helpers + // were removed when the Pratt parser took over. Struct literals are + // now parsed from the token stream directly (see + // `ExprParser::parse_struct_init` in `expr_pratt.rs`), so the + // `Pair`-based versions in this `impl` are no longer reachable. } diff --git a/kitlang/src/lexer.rs b/kitlang/src/lexer.rs new file mode 100644 index 0000000..7e44d90 --- /dev/null +++ b/kitlang/src/lexer.rs @@ -0,0 +1,381 @@ +//! Tokenizer for Kit expressions. +//! +//! Tokenizes Kit expressions into a flat stream for the Pratt parser. +//! Pest handles program/declaration grammars; this covers only expressions. +//! Tokens carry byte spans for diagnostics. + +use logos::{Lexer, Logos}; +use std::ops::Range; + +/// Byte range in source. Promotable to a richer `Span` type later. +pub type Span = Range; + +/// Token with its source span. +#[derive(Debug, Clone, PartialEq)] +pub struct SpannedTok { + pub kind: Tok, + pub span: Span, +} + +/// Token kind for Kit expressions. Keywords absent by design (handled by pest). +/// Multi-char tokens declared first for explicit priority. +#[derive(Logos, Debug, Clone, PartialEq)] +#[logos(skip r"[ \t\r\n]+")] +#[logos(skip(r"//[^\r\n]*", allow_greedy = true))] +#[logos(skip(r"/\*([^*]|\*+[^*/])*\*/"))] +pub enum Tok { + // --- Punctuation --- + #[token("(")] + LParen, + #[token(")")] + RParen, + #[token("[")] + LBracket, + #[token("]")] + RBracket, + #[token("{")] + LBrace, + #[token("}")] + RBrace, + #[token(",")] + Comma, + #[token(";")] + Semi, + #[token(".")] + Dot, + #[token(":")] + Colon, + #[token("...")] + Ellipsis, + + // --- Arithmetic --- + #[token("+")] + Plus, + #[token("-")] + Minus, + #[token("*")] + Star, + #[token("/")] + Slash, + #[token("%")] + Percent, + + // --- Comparison / equality --- + // Longer forms first. + #[token("==")] + EqEq, + #[token("!=")] + NotEq, + #[token("<=")] + LtEq, + #[token(">=")] + GtEq, + #[token("<")] + Lt, + #[token(">")] + Gt, + + // --- Logical (longer first) --- + #[token("&&")] + AndAnd, + #[token("||")] + OrOr, + #[token("!")] + Bang, + + // --- Bitwise --- + // Note: `&&` and `||` are matched before `&` and `|` by virtue of being + // longer; Logos longest-match ensures we never see a stray single `&` + // that's actually the start of `&&`. Same for `<<` / `>>` below. + #[token("&")] + Amp, + #[token("|")] + Pipe, + #[token("^")] + Caret, + #[token("~")] + Tilde, + #[token("<<")] + Shl, + #[token(">>")] + Shr, + + // --- Increment / decrement --- + #[token("++")] + PlusPlus, + #[token("--")] + MinusMinus, + + // --- Assignment (longer forms first) --- + #[token("<<=")] + ShlEq, + #[token(">>=")] + ShrEq, + #[token("+=")] + PlusEq, + #[token("-=")] + MinusEq, + #[token("*=")] + StarEq, + #[token("/=")] + SlashEq, + #[token("%=")] + PercentEq, + #[token("&=")] + AmpEq, + #[token("|=")] + PipeEq, + #[token("^=")] + CaretEq, + #[token("=")] + Assign, + + // --- Keywords that appear in expressions --- + #[token("if")] + KwIf, + #[token("then")] + KwThen, + #[token("else")] + KwElse, + #[token("true")] + KwTrue, + #[token("false")] + KwFalse, + #[token("null")] + KwNull, + #[token("this")] + KwThis, + #[token("Self")] + KwSelf, + #[token("sizeof")] + KwSizeof, + #[token("defined")] + KwDefined, + #[token("unsafe")] + KwUnsafe, + #[token("static")] + KwStatic, + #[token("implicit")] + KwImplicit, + #[token("empty")] + KwEmpty, + #[token("struct")] + KwStruct, + + // --- Literals --- + // Integer: one or more ASCII digits. + #[regex(r"[0-9]+", priority = 4, callback = parse_int)] + IntLit(i64), + + // Float: digits.digits, optional exponent. + // Must come before IntLit conceptually (no `.` in IntLit) but Logos + // priority + longest-match handles this; we just give it a higher + // priority so a `1.0` source never matches as `IntLit(1)` + `.` + `0`. + #[regex(r"[0-9]+\.[0-9]+([eE][+-]?[0-9]+)?", priority = 5, callback = parse_float)] + FloatLit(f64), + + // Char literal: single character or escape between single quotes. + #[regex(r"'(\\.|[^'\\])'", priority = 6, callback = parse_char)] + CharLit(char), + + // String literal: any character or escape between double quotes. + #[regex(r#""(\\.|[^"\\])*""#, priority = 6, callback = parse_string)] + StringLit(String), + + // Identifier: must come last (lowest priority) and must NOT match the + // keyword forms. The grammar's `identifier` rule uses negative lookahead + // for the reserved words; we mirror that by giving keywords priority 2 + // and identifier priority 1. + #[regex(r"[a-zA-Z_][a-zA-Z0-9_]*", priority = 1, callback = parse_ident)] + Ident(String), +} + +fn parse_int(lex: &mut Lexer) -> Option { + lex.slice().parse::().ok() +} + +fn parse_float(lex: &mut Lexer) -> Option { + lex.slice().parse::().ok() +} + +fn parse_char(lex: &mut Lexer) -> Option { + // The slice includes the surrounding single quotes; strip them. + let s = lex.slice(); + let inner = &s[1..s.len() - 1]; + unescape_char(inner) +} + +fn parse_string(lex: &mut Lexer) -> Option { + let s = lex.slice(); + let inner = &s[1..s.len() - 1]; + Some(unescape_str(inner)) +} + +fn parse_ident(lex: &mut Lexer) -> Option { + Some(lex.slice().to_string()) +} + +fn unescape_char(s: &str) -> Option { + let mut chars = s.chars(); + let c = chars.next()?; + if c == '\\' { + let esc = chars.next()?; + Some(match esc { + 'n' => '\n', + 'r' => '\r', + 't' => '\t', + '\\' => '\\', + '\'' => '\'', + '"' => '"', + '0' => '\0', + other => other, + }) + } else { + Some(c) + } +} + +fn unescape_str(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + let mut chars = s.chars(); + while let Some(c) = chars.next() { + if c == '\\' { + if let Some(esc) = chars.next() { + match esc { + 'n' => out.push('\n'), + 'r' => out.push('\r'), + 't' => out.push('\t'), + '\\' => out.push('\\'), + '\'' => out.push('\''), + '"' => out.push('"'), + '0' => out.push('\0'), + other => out.push(other), + } + } + } else { + out.push(c); + } + } + out +} + +/// Tokenize source into `Vec`. Drops whitespace, comments, +/// and unrecognized characters. Returns tokens in source order; EOF is +/// end of vector. +pub fn tokenize(source: &str) -> Vec { + Tok::lexer(source) + .spanned() + .filter_map(|(res, span)| res.ok().map(|kind| SpannedTok { kind, span })) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + fn kinds(source: &str) -> Vec { + tokenize(source).into_iter().map(|t| t.kind).collect() + } + + #[test] + fn integer_literals() { + assert_eq!(kinds("0"), vec![Tok::IntLit(0)]); + assert_eq!(kinds("42"), vec![Tok::IntLit(42)]); + assert_eq!(kinds("1234567890"), vec![Tok::IntLit(1234567890)]); + } + + #[test] + fn float_literals() { + assert_eq!(kinds("3.14"), vec![Tok::FloatLit(3.14)]); + assert_eq!(kinds("1.0e10"), vec![Tok::FloatLit(1.0e10)]); + assert_eq!(kinds("2.5E-3"), vec![Tok::FloatLit(2.5E-3)]); + } + + #[test] + fn char_and_string_literals() { + assert_eq!(kinds("'a'"), vec![Tok::CharLit('a')]); + assert_eq!(kinds("'\\n'"), vec![Tok::CharLit('\n')]); + assert_eq!( + kinds(r#""hello""#), + vec![Tok::StringLit("hello".to_string())] + ); + assert_eq!(kinds(r#""a\nb""#), vec![Tok::StringLit("a\nb".to_string())]); + } + + #[test] + fn identifiers_and_keywords() { + assert_eq!(kinds("foo"), vec![Tok::Ident("foo".to_string())]); + assert_eq!( + kinds("foo_bar_123"), + vec![Tok::Ident("foo_bar_123".to_string())] + ); + // Keywords take priority over the identifier regex. + assert_eq!(kinds("if"), vec![Tok::KwIf]); + assert_eq!(kinds("Self"), vec![Tok::KwSelf]); + assert_eq!(kinds("null"), vec![Tok::KwNull]); + } + + #[test] + fn operators_longest_match() { + // `&&` must match as a single token, not two `&`s. + assert_eq!(kinds("&&"), vec![Tok::AndAnd]); + // `<<=` must match as a single token, not `<<` + `=`. + assert_eq!(kinds("<<="), vec![Tok::ShlEq]); + // `==` must match as a single token, not two `=`s. + assert_eq!(kinds("=="), vec![Tok::EqEq]); + } + + #[test] + fn arithmetic_operators() { + assert_eq!( + kinds("a + b * c"), + vec![ + Tok::Ident("a".to_string()), + Tok::Plus, + Tok::Ident("b".to_string()), + Tok::Star, + Tok::Ident("c".to_string()), + ] + ); + } + + #[test] + fn whitespace_is_skipped() { + assert_eq!(kinds(" a \t +\n b "), kinds("a+b")); + } + + #[test] + fn line_comments_are_skipped() { + assert_eq!(kinds("a // ignore me\n+ b"), kinds("a+b")); + } + + #[test] + fn block_comments_are_skipped() { + assert_eq!(kinds("a /* hi */ + b"), kinds("a+b")); + assert_eq!(kinds("a/*\nmulti\nline\n*/+b"), kinds("a+b")); + } + + #[test] + fn spans_are_byte_accurate() { + let toks = tokenize(" a + b "); + assert_eq!(toks[0].span, 2..3); // 'a' + assert_eq!(toks[1].span, 4..5); // '+' + assert_eq!(toks[2].span, 6..7); // 'b' + } + + #[test] + fn empty_source_produces_no_tokens() { + assert!(tokenize("").is_empty()); + } + + #[test] + fn unknown_characters_are_dropped() { + // `$` is not a Kit token; the lexer drops it. This mirrors how pest + // would surface the error at the grammar level, which is upstream + // of the Pratt parser and not its concern. + assert_eq!( + kinds("a $ b"), + vec![Tok::Ident("a".to_string()), Tok::Ident("b".to_string()),] + ); + } +} diff --git a/kitlang/src/lib.rs b/kitlang/src/lib.rs index 86abb75..1f0b3b7 100644 --- a/kitlang/src/lib.rs +++ b/kitlang/src/lib.rs @@ -11,5 +11,12 @@ pub use codegen::Toolchain; #[grammar = "grammar/kit.pest"] pub struct KitParser; +/// Tokenizer for expressions, used by the Pratt parser. +/// +/// Pest still handles the program/declaration/statement grammar; the Pratt +/// parser only takes over expression parsing, and it needs a token stream +/// to do so. The Logos-based lexer here is that token stream. +pub(crate) mod lexer; + /// Compilation error types. pub(crate) mod error; From fc1ce1e0efcf74f5333143b11c37bf8cafee8024 Mon Sep 17 00:00:00 2001 From: winlogon Date: Tue, 30 Jun 2026 06:38:15 +0200 Subject: [PATCH 2/4] refactor(parser): split expr_pratt into submodules The single 1200-line expr_pratt.rs mixed parser core, primary expression parsing, and tests, making it hard to navigate. The diagnostics module also duplicated its token-to-name mapping between tok_name and a local expected_name helper. Summary of changes: - Split expr_pratt.rs into mod.rs, primary.rs, and tests.rs via include! - Make diagnostics::expected_name the single canonical token-name match and derive tok_name from it instead of duplicating the match - Remove PlusPlus/MinusMinus from prefix binding power (not prefix ops) - Drop now-unneeded #[allow(dead_code)] attributes on used items --- kitlang/src/codegen/parser/binding_power.rs | 8 +- kitlang/src/codegen/parser/diagnostics.rs | 147 +- kitlang/src/codegen/parser/expr_pratt.rs | 1203 ----------------- kitlang/src/codegen/parser/expr_pratt/mod.rs | 282 ++++ .../src/codegen/parser/expr_pratt/primary.rs | 260 ++++ .../src/codegen/parser/expr_pratt/tests.rs | 502 +++++++ 6 files changed, 1121 insertions(+), 1281 deletions(-) delete mode 100644 kitlang/src/codegen/parser/expr_pratt.rs create mode 100644 kitlang/src/codegen/parser/expr_pratt/mod.rs create mode 100644 kitlang/src/codegen/parser/expr_pratt/primary.rs create mode 100644 kitlang/src/codegen/parser/expr_pratt/tests.rs diff --git a/kitlang/src/codegen/parser/binding_power.rs b/kitlang/src/codegen/parser/binding_power.rs index 618580c..84fd2d5 100644 --- a/kitlang/src/codegen/parser/binding_power.rs +++ b/kitlang/src/codegen/parser/binding_power.rs @@ -44,13 +44,7 @@ pub fn postfix(tok: &Tok) -> Option { /// Prefix (unary) binding power. Returns None for non-prefix tokens. pub fn prefix(tok: &Tok) -> Option { match tok { - Tok::Bang - | Tok::Minus - | Tok::Star - | Tok::Amp - | Tok::Tilde - | Tok::PlusPlus - | Tok::MinusMinus => Some(25), + Tok::Bang | Tok::Minus | Tok::Star | Tok::Amp | Tok::Tilde => Some(25), _ => None, } } diff --git a/kitlang/src/codegen/parser/diagnostics.rs b/kitlang/src/codegen/parser/diagnostics.rs index 83ce4d6..8bc8df6 100644 --- a/kitlang/src/codegen/parser/diagnostics.rs +++ b/kitlang/src/codegen/parser/diagnostics.rs @@ -27,7 +27,6 @@ use crate::lexer::Tok; /// "expected one of: ..." with no extra work. Today, [`to_human_message`] /// joins these into a string for `CompilationError::ParseError`. #[derive(Debug, Clone)] -#[allow(dead_code)] // variants will be constructed by diagnostic work in a follow-up pub(crate) enum ExprParseError { /// A token was found where it was not expected. UnexpectedToken { @@ -77,81 +76,87 @@ impl fmt::Display for ExprParseError { } } -/// Map a `Tok` kind to a short human-readable name for error messages. +/// Map a `Tok` kind to a short human-readable token name, as a static +/// slice of one element. Returns `&'static [&'static str]` for direct +/// use in `ExprParseError::UnexpectedToken.expected`. /// -/// Returns `'static` strings so `ExprParseError` can carry them as -/// `&'static [&'static str]` without allocation. Used by the binding-power -/// helpers in `expr_pratt.rs` to populate the `expected` list. -#[allow(dead_code)] // seam: will be called from diagnostic work in a follow-up -pub(crate) fn tok_name(kind: &Tok) -> &'static str { +/// This is the single canonical match for all token → name mappings. +/// [`tok_name`] is a zero-cost wrapper that extracts the first element. +pub(crate) fn expected_name(kind: &Tok) -> &'static [&'static str] { match kind { - Tok::LParen => "`(`", - Tok::RParen => "`)`", - Tok::LBracket => "`[`", - Tok::RBracket => "`]`", - Tok::LBrace => "`{`", - Tok::RBrace => "`}`", - Tok::Comma => "`,`", - Tok::Semi => "`;`", - Tok::Dot => "`.`", - Tok::Colon => "`:`", - Tok::Ellipsis => "`...`", - Tok::Plus => "`+`", - Tok::Minus => "`-`", - Tok::Star => "`*`", - Tok::Slash => "`/`", - Tok::Percent => "`%`", - Tok::EqEq => "`==`", - Tok::NotEq => "`!=`", - Tok::LtEq => "`<=`", - Tok::GtEq => "`>=`", - Tok::Lt => "`<`", - Tok::Gt => "`>`", - Tok::AndAnd => "`&&`", - Tok::OrOr => "`||`", - Tok::Bang => "`!`", - Tok::Amp => "`&`", - Tok::Pipe => "`|`", - Tok::Caret => "`^`", - Tok::Tilde => "`~`", - Tok::Shl => "`<<`", - Tok::Shr => "`>>`", - Tok::PlusPlus => "`++`", - Tok::MinusMinus => "`--`", - Tok::ShlEq => "`<<=`", - Tok::ShrEq => "`>>=`", - Tok::PlusEq => "`+=`", - Tok::MinusEq => "`-=`", - Tok::StarEq => "`*=`", - Tok::SlashEq => "`/=`", - Tok::PercentEq => "`%=`", - Tok::AmpEq => "`&=`", - Tok::PipeEq => "`|=`", - Tok::CaretEq => "`^=`", - Tok::Assign => "`=`", - Tok::KwIf => "`if`", - Tok::KwThen => "`then`", - Tok::KwElse => "`else`", - Tok::KwTrue => "`true`", - Tok::KwFalse => "`false`", - Tok::KwNull => "`null`", - Tok::KwThis => "`this`", - Tok::KwSelf => "`Self`", - Tok::KwSizeof => "`sizeof`", - Tok::KwDefined => "`defined`", - Tok::KwUnsafe => "`unsafe`", - Tok::KwStatic => "`static`", - Tok::KwImplicit => "`implicit`", - Tok::KwEmpty => "`empty`", - Tok::KwStruct => "`struct`", - Tok::IntLit(_) => "integer literal", - Tok::FloatLit(_) => "float literal", - Tok::CharLit(_) => "char literal", - Tok::StringLit(_) => "string literal", - Tok::Ident(_) => "identifier", + Tok::LParen => &["`(`"], + Tok::RParen => &["`)`"], + Tok::LBracket => &["`[`"], + Tok::RBracket => &["`]`"], + Tok::LBrace => &["`{`"], + Tok::RBrace => &["`}`"], + Tok::Comma => &["`,`"], + Tok::Semi => &["`;`"], + Tok::Dot => &["`.`"], + Tok::Colon => &["`:`"], + Tok::Ellipsis => &["`...`"], + Tok::Plus => &["`+`"], + Tok::Minus => &["`-`"], + Tok::Star => &["`*`"], + Tok::Slash => &["`/`"], + Tok::Percent => &["`%`"], + Tok::EqEq => &["`==`"], + Tok::NotEq => &["`!=`"], + Tok::LtEq => &["`<=`"], + Tok::GtEq => &["`>=`"], + Tok::Lt => &["`<`"], + Tok::Gt => &["`>`"], + Tok::AndAnd => &["`&&`"], + Tok::OrOr => &["`||`"], + Tok::Bang => &["`!`"], + Tok::Amp => &["`&`"], + Tok::Pipe => &["`|`"], + Tok::Caret => &["`^`"], + Tok::Tilde => &["`~`"], + Tok::Shl => &["`<<`"], + Tok::Shr => &["`>>`"], + Tok::PlusPlus => &["`++`"], + Tok::MinusMinus => &["`--`"], + Tok::ShlEq => &["`<<=`"], + Tok::ShrEq => &["`>>=`"], + Tok::PlusEq => &["`+=`"], + Tok::MinusEq => &["`-=`"], + Tok::StarEq => &["`*=`"], + Tok::SlashEq => &["`/=`"], + Tok::PercentEq => &["`%=`"], + Tok::AmpEq => &["`&=`"], + Tok::PipeEq => &["`|=`"], + Tok::CaretEq => &["`^=`"], + Tok::Assign => &["`=`"], + Tok::KwIf => &["`if`"], + Tok::KwThen => &["`then`"], + Tok::KwElse => &["`else`"], + Tok::KwTrue => &["`true`"], + Tok::KwFalse => &["`false`"], + Tok::KwNull => &["`null`"], + Tok::KwThis => &["`this`"], + Tok::KwSelf => &["`Self`"], + Tok::KwSizeof => &["`sizeof`"], + Tok::KwDefined => &["`defined`"], + Tok::KwUnsafe => &["`unsafe`"], + Tok::KwStatic => &["`static`"], + Tok::KwImplicit => &["`implicit`"], + Tok::KwEmpty => &["`empty`"], + Tok::KwStruct => &["`struct`"], + Tok::IntLit(_) => &["integer literal"], + Tok::FloatLit(_) => &["float literal"], + Tok::CharLit(_) => &["char literal"], + Tok::StringLit(_) => &["string literal"], + Tok::Ident(_) => &["identifier"], } } +/// Map a `Tok` kind to a short human-readable name for error messages. +/// Zero-cost wrapper around [`expected_name`]. +pub(crate) fn tok_name(kind: &Tok) -> &'static str { + expected_name(kind)[0] +} + #[cfg(test)] mod tests { use super::*; diff --git a/kitlang/src/codegen/parser/expr_pratt.rs b/kitlang/src/codegen/parser/expr_pratt.rs deleted file mode 100644 index fb64f26..0000000 --- a/kitlang/src/codegen/parser/expr_pratt.rs +++ /dev/null @@ -1,1203 +0,0 @@ -//! Hand-written Pratt parser for Kit expressions. -//! -//! This module takes over expression parsing from the pest-based grammar. -//! Pest still handles the program, declaration, statement, and type-annotation -//! grammars. For every `Pair<'_, Rule>` whose rule is an expression (i.e., -//! the 13 precedence levels `expr → assign → logical_or → ... → primary`), -//! the parser hands off to [`ExprParser::parse_expr`]. -//! -//! Pratt Parsing & Operator Precedence -//! ----------------------------------- -//! The pest grammar used 13 mutually recursive functions (one per precedence -//! level), which overflowed the default 1 MB stack on Windows. The Pratt -//! parser uses a single function with a binding-power loop, bounding stack -//! depth to `O(precedence levels) = O(13)` regardless of expression length. -//! Operator precedence is defined in [`binding_power::infix`], -//! [`binding_power::postfix`], [`binding_power::prefix`]. Each infix operator -//! has a (lbp, rbp) pair: `lbp < rbp` = right-associative (assignment), -//! `lbp == rbp` = left-associative (most binary ops). -//! -//! Errors & Source Spans -//! --------------------- -//! All parse errors are values of [`ExprParseError`] (in `diagnostics.rs`). -//! The parser never prints, never allocates strings for error messages, -//! and never holds source-file identity. Conversion to -//! `CompilationError::ParseError(String)` happens at `PestExpr::parse` -//! in `parser/mod.rs`. The token stream carries byte ranges; the parser -//! uses them internally but does not currently attach them to AST nodes. - -use crate::codegen::ast::{Expr, Literal}; -use crate::codegen::type_ast::FieldInit; -use crate::codegen::types::{Type, TypeId}; -use crate::lexer::{Span, SpannedTok, Tok, tokenize}; - -use super::binding_power::{ - infix, is_range_op, postfix, prefix, tok_to_assign_op, tok_to_binary_op, tok_to_unary_op, -}; -use super::diagnostics::ExprParseError; - -#[cfg(test)] -use crate::codegen::types::{AssignmentOperator, BinaryOperator, UnaryOperator}; - -// --------------------------------------------------------------------------- -// Parser -// --------------------------------------------------------------------------- - -/// A Pratt parser for Kit expressions. -/// -/// One instance is built per expression being parsed. The parser is -/// single-use: create a new one for each `parse_expr` call. -pub(crate) struct ExprParser<'a> { - tokens: &'a [SpannedTok], - pos: usize, -} - -impl<'a> ExprParser<'a> { - /// Build a parser over a token slice. The slice must be the result of - /// [`tokenize`] applied to the expression's source text. - pub(crate) fn new(tokens: &'a [SpannedTok]) -> Self { - Self { tokens, pos: 0 } - } - - /// Entry point. Parses one complete expression and returns it. - /// The expression may be followed by trailing tokens, which are left - /// in the stream (the caller can `pos()`-check or re-parse). - pub(crate) fn parse_expr(&mut self) -> Result { - self.parse_pratt(0) - } - - /// The current position in the token stream. Useful for tests and - /// for callers that want to know how many tokens were consumed. - #[allow(dead_code)] - pub(crate) fn pos(&self) -> usize { - self.pos - } - - // --- Token stream helpers (private) --- - - /// Peek the current token without consuming it. Returns a synthetic - /// "EOF" token when the stream is exhausted. - fn peek(&self) -> &SpannedTok { - // We never return None from peek; EOF is represented by a synthetic - // SpannedTok at span 0..0. This lets the Pratt loop compare with - // `==` against `Tok::...` cleanly. - static EOF: SpannedTok = SpannedTok { - kind: Tok::Semi, // any token that has no infix/postfix/prefix bp - span: 0..0, - }; - self.tokens.get(self.pos).unwrap_or(&EOF) - } - - /// Peek the next token (one past the current). Same EOF behavior. - #[allow(dead_code)] - fn peek_next(&self) -> &SpannedTok { - static EOF: SpannedTok = SpannedTok { - kind: Tok::Semi, - span: 0..0, - }; - self.tokens.get(self.pos + 1).unwrap_or(&EOF) - } - - /// Consume and return the current token. - fn advance(&mut self) -> &SpannedTok { - let tok = &self.tokens[self.pos]; - self.pos += 1; - tok - } - - /// True if the parser is at or past the end of the token stream. - #[allow(dead_code)] // seam: will be used for EOF diagnostics in a follow-up - fn at_eof(&self) -> bool { - self.pos >= self.tokens.len() - } - - // --- Pratt core --- - - /// Parse an expression with the given minimum binding power. - fn parse_pratt(&mut self, min_bp: u8) -> Result { - // Parse leading prefix operators (e.g. `-a`, `!!x`, `&arr[i]`). - // Prefix binds tighter than infix but looser than postfix, - // so `&arr[i]` = `&(arr[i])` (postfix on `arr` first). - let mut lhs = if let Some(pfx_bp) = prefix(&self.peek().kind) { - let op = tok_to_unary_op(&self.peek().kind).unwrap(); - self.advance(); - let rhs = self.parse_pratt(pfx_bp)?; - Expr::UnaryOp { - op, - expr: Box::new(rhs), - ty: TypeId::default(), - } - } else { - self.parse_primary()? - }; - - // Postfix chain: field access, index, call. - lhs = self.parse_postfix_chain(lhs)?; - - // Infix operators (binary ops, range). - loop { - let kind = self.peek().kind.clone(); - let Some((lbp, rbp)) = infix(&kind) else { - break; - }; - if lbp < min_bp { - break; - } - if tok_to_assign_op(&kind).is_some() { - break; - } - self.advance(); - if is_range_op(&kind) { - let rhs = self.parse_pratt(rbp)?; - lhs = Expr::RangeLiteral { - start: Box::new(lhs), - end: Box::new(rhs), - }; - continue; - } - let op = tok_to_binary_op(&kind).ok_or_else(|| { - ExprParseError::Custom(format!("internal: no binary op for {kind:?}")) - })?; - let rhs = self.parse_pratt(rbp)?; - lhs = Expr::BinaryOp { - op, - left: Box::new(lhs), - right: Box::new(rhs), - ty: TypeId::default(), - }; - } - - // Assignment (right-associative, lowest precedence). - loop { - let kind = self.peek().kind.clone(); - let Some(op) = tok_to_assign_op(&kind) else { - break; - }; - // Assignment has the lowest precedence (lbp=0, rbp=1 in the - // infix table, but we don't use that table here). Right- - // associativity: a = b = c means a = (b = c). Recurse with - // min_bp=0 so the rhs sees *all* operators including another - // assignment. - if 0 < min_bp { - break; - } - self.advance(); - let rhs = self.parse_pratt(0)?; - lhs = Expr::Assign { - op, - left: Box::new(lhs), - right: Box::new(rhs), - ty: TypeId::default(), - }; - } - - Ok(lhs) - } - - /// Iteratively apply postfix operators (call, index, field access) to - /// a base expression. Zero stack frames added per iteration. The - /// chain is bounded by the source's syntactic length, but the parser - /// is iterative, so the *call stack* depth is constant. - fn parse_postfix_chain(&mut self, mut base: Expr) -> Result { - loop { - let kind = self.peek().kind.clone(); - if postfix(&kind).is_none() { - break; - } - base = match kind { - Tok::Dot => self.parse_field_access(base)?, - Tok::LBracket => self.parse_index(base)?, - Tok::LParen => self.parse_call(base)?, - _ => unreachable!("postfix returned Some for {kind:?}"), - }; - } - Ok(base) - } - - /// Parse a primary expression: literals, identifiers, parenthesized - /// expressions, function calls, array literals, struct inits, and - /// the if-expression. Postfix operations (`.field`, `[i]`, `(args)`) - /// are handled in the outer Pratt loop, *not* here, so this function - /// only needs to produce the base expression. - fn parse_primary(&mut self) -> Result { - let tok = self.peek().kind.clone(); - // `span` is read for documentation; the parser doesn't currently - // attach it to AST nodes. Future PRs will use it. - let _span: Span = self.peek().span.clone(); - - match tok { - Tok::IntLit(n) => { - self.advance(); - Ok(Expr::Literal { - value: Literal::Int(n), - ty: TypeId::default(), - }) - } - Tok::FloatLit(f) => { - self.advance(); - Ok(Expr::Literal { - value: Literal::Float(f), - ty: TypeId::default(), - }) - } - Tok::CharLit(c) => { - self.advance(); - Ok(Expr::Literal { - value: Literal::Char(c), - ty: TypeId::default(), - }) - } - Tok::StringLit(s) => { - self.advance(); - Ok(Expr::Literal { - value: Literal::String(s), - ty: TypeId::default(), - }) - } - Tok::KwTrue => { - self.advance(); - Ok(Expr::Literal { - value: Literal::Bool(true), - ty: TypeId::default(), - }) - } - Tok::KwFalse => { - self.advance(); - Ok(Expr::Literal { - value: Literal::Bool(false), - ty: TypeId::default(), - }) - } - Tok::KwNull => { - self.advance(); - Ok(Expr::Literal { - value: Literal::Null, - ty: TypeId::default(), - }) - } - Tok::KwThis | Tok::KwSelf => { - // The pest parser treats these as `Identifier` with a - // fixed name; we follow suit. A future PR can introduce - // dedicated AST variants if needed. - let name = match tok { - Tok::KwThis => "this", - _ => "Self", - }; - self.advance(); - Ok(Expr::Identifier { - name: name.to_string(), - ty: TypeId::default(), - }) - } - Tok::Ident(name) => { - self.advance(); - Ok(Expr::Identifier { - name, - ty: TypeId::default(), - }) - } - Tok::LParen => { - // Either a parenthesized expression or the start of a - // tuple literal `(a, b, c)`. We parse the first - // expression; if the next token is `,` it's a tuple, else - // we expect a closing `)`. - self.advance(); // consume `(` - let first = self.parse_expr()?; - if self.peek().kind == Tok::Comma { - // Tuple literal: parse remaining comma-separated - // expressions, then expect `)`. The AST doesn't have - // a dedicated tuple variant, so we represent the - // tuple as a parenthesized expression with a - // synthetic structure. For now, produce a Custom - // error directing the caller to the pest path; this - // matches the existing parser's TODO for tuple - // literals. - return Err(ExprParseError::Custom( - "tuple literals are not yet supported by the Pratt parser".into(), - )); - } - self.expect(&Tok::RParen)?; - Ok(first) - } - Tok::LBracket => self.parse_array_literal(), - Tok::KwStruct => self.parse_struct_init(), - Tok::KwIf => self.parse_if_expr(), - Tok::KwEmpty => { - // The grammar's primary includes "empty" as a keyword. - // We don't have a dedicated AST variant; treat it as an - // identifier for now (semantics will be filled in by - // type inference downstream). - self.advance(); - Ok(Expr::Identifier { - name: "empty".to_string(), - ty: TypeId::default(), - }) - } - _ => Err(ExprParseError::UnexpectedToken { - found: tok, - expected: &[ - "integer literal", - "float literal", - "string literal", - "char literal", - "identifier", - "`(`", - "`[`", - "`if`", - "`null`", - "`true`", - "`false`", - ], - }), - } - } - - /// Parse a `.field` access postfix. - fn parse_field_access(&mut self, base: Expr) -> Result { - self.advance(); // consume `.` - let field_tok = self.peek().kind.clone(); - match field_tok { - Tok::Ident(name) => { - self.advance(); - Ok(Expr::FieldAccess { - expr: Box::new(base), - field_name: name, - ty: TypeId::default(), - }) - } - _ => Err(ExprParseError::UnexpectedToken { - found: field_tok, - expected: &["identifier"], - }), - } - } - - /// Parse a `[index]` postfix. - fn parse_index(&mut self, base: Expr) -> Result { - self.advance(); // consume `[` - let index = self.parse_expr()?; - self.expect(&Tok::RBracket)?; - Ok(Expr::Index { - expr: Box::new(base), - index: Box::new(index), - ty: TypeId::default(), - }) - } - - /// Parse a function call postfix: `(arg1, arg2, ...)`. - fn parse_call(&mut self, callee: Expr) -> Result { - // The callee expression's name lives in `Expr::Identifier.name` - // (possibly qualified with dots for things like - // `qualified_call.math.add`). The pest grammar's - // `function_call_expr` rule extracts a `path` which is a dot- - // separated identifier. We extract the name from the callee - // expression; for `Expr::FieldAccess` chains we concatenate with - // dots to recover the qualified name. This matches the pest - // parser's behavior in `parser/expr.rs:163-167` and the - // transpiler's qualified-name resolution in - // `transpile/mod.rs:493-500`. - let callee_name = expr_to_callee_name(&callee); - - self.advance(); // consume `(` - let args = self.parse_comma_list(Tok::RParen, |p| p.parse_expr())?; - Ok(Expr::Call { - callee: callee_name, - args, - ty: TypeId::default(), - }) - } - - /// Parse an array literal: `[expr, expr, ...]`. - fn parse_array_literal(&mut self) -> Result { - self.advance(); // consume `[` - let elements = self.parse_comma_list(Tok::RBracket, |p| p.parse_expr())?; - Ok(Expr::ArrayLiteral { - elements, - ty: TypeId::default(), - }) - } - - /// Parse a struct init: `struct Name { field: expr, ... }`. - fn parse_struct_init(&mut self) -> Result { - self.advance(); // consume `struct` - // The type annotation here is a single identifier (we don't - // handle complex type annotations in expressions). This matches - // the common case; the pest path still handles generics/pointers - // via the var_decl/init call site. - let type_tok = self.peek().kind.clone(); - let type_name = match type_tok { - Tok::Ident(name) => { - self.advance(); - name - } - _ => { - return Err(ExprParseError::UnexpectedToken { - found: type_tok, - expected: &["type name"], - }); - } - }; - self.expect(&Tok::LBrace)?; - let fields = self.parse_comma_list(Tok::RBrace, |p| { - let name = match &p.peek().kind { - Tok::Ident(n) => { - let n = n.clone(); - p.advance(); - n - } - _ => { - return Err(ExprParseError::UnexpectedToken { - found: p.peek().kind.clone(), - expected: &["field name"], - }); - } - }; - p.expect(&Tok::Colon)?; - let value = p.parse_expr()?; - Ok(FieldInit { name, value }) - })?; - Ok(Expr::StructInit { - ty: TypeId::default(), - struct_type: Some(Type::from_kit(&type_name)), - fields, - }) - } - - /// Parse an if-expression: `if cond then a else b`. - fn parse_if_expr(&mut self) -> Result { - self.advance(); // consume `if` - let cond = self.parse_expr()?; - self.expect(&Tok::KwThen)?; - let then_branch = self.parse_expr()?; - self.expect(&Tok::KwElse)?; - let else_branch = self.parse_expr()?; - Ok(Expr::If { - cond: Box::new(cond), - then_branch: Box::new(then_branch), - else_branch: Box::new(else_branch), - ty: TypeId::default(), - }) - } - - // --- Helpers --- - - /// Consume the current token if it matches `expected`, otherwise - /// return an `UnexpectedToken` error with `expected`'s name. - fn expect(&mut self, expected: &Tok) -> Result<(), ExprParseError> { - if &self.peek().kind == expected { - self.advance(); - Ok(()) - } else { - // We construct the `&'static [&'static str]` directly from - // a call to `tok_name`, which the compiler can verify is - // `'static`. Storing the result in a local first drops the - // `'static` lifetime, so we keep the call inline. - Err(ExprParseError::UnexpectedToken { - found: self.peek().kind.clone(), - expected: expected_name(expected), - }) - } - } - - /// Parse a comma-separated list of `T` terminated by `closer`. - /// Allows zero or more elements (an empty list is valid for fn - /// calls with no args, empty array literals, etc.). - fn parse_comma_list(&mut self, closer: Tok, mut f: F) -> Result, ExprParseError> - where - F: FnMut(&mut Self) -> Result, - { - let mut out = Vec::new(); - // Empty list case. - if self.peek().kind == closer { - self.advance(); - return Ok(out); - } - out.push(f(self)?); - while self.peek().kind == Tok::Comma { - self.advance(); - // Trailing comma is allowed (parses to empty trailing element). - if self.peek().kind == closer { - break; - } - out.push(f(self)?); - } - self.expect(&closer)?; - Ok(out) - } -} - -/// Extract a callee name from a function-call's leading expression. -/// -/// For `Expr::Identifier { name, .. }` this is just `name`. -/// For `Expr::FieldAccess { expr, field_name, .. }` chains like -/// `pkg.math.add`, this concatenates the path with `.`. -/// For other expressions (e.g. a parenthesized call), we fall back to -/// `Display`-formatting the expression, which the transpiler can still -/// route through name resolution. -fn expr_to_callee_name(expr: &Expr) -> String { - match expr { - Expr::Identifier { name, .. } => name.clone(), - Expr::FieldAccess { - expr: base, - field_name, - .. - } => { - let base_name = expr_to_callee_name(base); - format!("{base_name}.{field_name}") - } - other => format!("{other:?}"), - } -} - -/// Return a static slice of static strs containing the name of `expected`. -/// Used by `expect` to build an `UnexpectedToken` error variant. -fn expected_name(expected: &Tok) -> &'static [&'static str] { - match expected { - Tok::LParen => &["`(`"], - Tok::RParen => &["`)`"], - Tok::LBracket => &["`[`"], - Tok::RBracket => &["`]`"], - Tok::LBrace => &["`{`"], - Tok::RBrace => &["`}`"], - Tok::Comma => &["`,`"], - Tok::Semi => &["`;`"], - Tok::Dot => &["`.`"], - Tok::Colon => &["`:`"], - Tok::Ellipsis => &["`...`"], - Tok::Plus => &["`+`"], - Tok::Minus => &["`-`"], - Tok::Star => &["`*`"], - Tok::Slash => &["`/`"], - Tok::Percent => &["`%`"], - Tok::EqEq => &["`==`"], - Tok::NotEq => &["`!=`"], - Tok::LtEq => &["`<=`"], - Tok::GtEq => &["`>=`"], - Tok::Lt => &["`<`"], - Tok::Gt => &["`>`"], - Tok::AndAnd => &["`&&`"], - Tok::OrOr => &["`||`"], - Tok::Bang => &["`!`"], - Tok::Amp => &["`&`"], - Tok::Pipe => &["`|`"], - Tok::Caret => &["`^`"], - Tok::Tilde => &["`~`"], - Tok::Shl => &["`<<`"], - Tok::Shr => &["`>>`"], - Tok::PlusPlus => &["`++`"], - Tok::MinusMinus => &["`--`"], - Tok::ShlEq => &["`<<=`"], - Tok::ShrEq => &["`>>=`"], - Tok::PlusEq => &["`+=`"], - Tok::MinusEq => &["`-=`"], - Tok::StarEq => &["`*=`"], - Tok::SlashEq => &["`/=`"], - Tok::PercentEq => &["`%=`"], - Tok::AmpEq => &["`&=`"], - Tok::PipeEq => &["`|=`"], - Tok::CaretEq => &["`^=`"], - Tok::Assign => &["`=`"], - Tok::KwIf => &["`if`"], - Tok::KwThen => &["`then`"], - Tok::KwElse => &["`else`"], - Tok::KwTrue => &["`true`"], - Tok::KwFalse => &["`false`"], - Tok::KwNull => &["`null`"], - Tok::KwThis => &["`this`"], - Tok::KwSelf => &["`Self`"], - Tok::KwSizeof => &["`sizeof`"], - Tok::KwDefined => &["`defined`"], - Tok::KwUnsafe => &["`unsafe`"], - Tok::KwStatic => &["`static`"], - Tok::KwImplicit => &["`implicit`"], - Tok::KwEmpty => &["`empty`"], - Tok::KwStruct => &["`struct`"], - Tok::IntLit(_) => &["integer literal"], - Tok::FloatLit(_) => &["float literal"], - Tok::CharLit(_) => &["char literal"], - Tok::StringLit(_) => &["string literal"], - Tok::Ident(_) => &["identifier"], - } -} - -// --------------------------------------------------------------------------- -// Module surface: parse an expression from source text. -// --------------------------------------------------------------------------- - -/// Parse a Kit expression from source text. This is the public entry -/// point used by the pest-to-Pratt bridge (`PestExpr::parse`). -/// -/// The `text` should be the source text of the expression as a -/// `Pair::as_str()` slice. Tokenization, parsing, and conversion to an -/// `Expr` all happen here. -pub(crate) fn parse_kit_expr(text: &str) -> Result { - let tokens = tokenize(text); - let mut parser = ExprParser::new(&tokens); - parser.parse_expr() -} - -// --------------------------------------------------------------------------- -// Tests -// --------------------------------------------------------------------------- - -#[cfg(test)] -mod tests { - use super::*; - - /// Convenience: parse an expression and unwrap. - fn p(text: &str) -> Expr { - parse_kit_expr(text).unwrap_or_else(|e| panic!("parse failed for `{text}`: {e}")) - } - - /// Convenience: parse and assert the error contains a substring. - fn p_err(text: &str, needle: &str) { - let err = parse_kit_expr(text) - .err() - .unwrap_or_else(|| panic!("expected error for `{text}`, got Ok")); - let msg = err.to_human_message(); - assert!( - msg.contains(needle), - "error `{msg}` does not contain `{needle}`" - ); - } - - // --- Literals --- - - #[test] - fn integer_literal() { - let e = p("42"); - assert!(matches!( - e, - Expr::Literal { - value: Literal::Int(42), - .. - } - )); - } - - #[test] - fn float_literal() { - let e = p("3.14"); - assert!( - matches!(e, Expr::Literal { value: Literal::Float(f), .. } if (f - 3.14).abs() < 1e-10) - ); - } - - #[test] - fn string_literal() { - let e = p(r#""hello""#); - assert!(matches!(e, Expr::Literal { value: Literal::String(s), .. } if s == "hello")); - } - - #[test] - fn bool_literals() { - assert!(matches!( - p("true"), - Expr::Literal { - value: Literal::Bool(true), - .. - } - )); - assert!(matches!( - p("false"), - Expr::Literal { - value: Literal::Bool(false), - .. - } - )); - } - - #[test] - fn null_literal() { - assert!(matches!( - p("null"), - Expr::Literal { - value: Literal::Null, - .. - } - )); - } - - // --- Identifiers --- - - #[test] - fn identifier() { - let e = p("foo"); - assert!(matches!(&e, Expr::Identifier { name, .. } if name == "foo")); - } - - #[test] - fn qualified_identifier_is_built_via_postfix_chain() { - // The lexer produces foo . bar . baz as three tokens. The postfix - // chain in the Pratt loop builds a FieldAccess tree. - let e = p("foo.bar.baz"); - // Walking the tree should give us three nested FieldAccess nodes - // with the leaf being `foo`. - let mut cur = &e; - let mut path = vec![]; - while let Expr::FieldAccess { - expr, field_name, .. - } = cur - { - path.push(field_name.clone()); - cur = expr; - } - if let Expr::Identifier { name, .. } = cur { - assert_eq!(name, "foo"); - } else { - panic!("expected leaf Identifier, got {cur:?}"); - } - assert_eq!(path, vec!["baz".to_string(), "bar".to_string()]); - } - - // --- Precedence --- - - #[test] - fn additive_vs_multiplicative() { - // 1 + 2 * 3 should be 1 + (2 * 3) - let e = p("1 + 2 * 3"); - // Top-level is BinaryOp::Add with right = BinaryOp::Mul - if let Expr::BinaryOp { op, right, .. } = &e { - assert_eq!(*op, BinaryOperator::Add); - if let Expr::BinaryOp { op: inner_op, .. } = right.as_ref() { - assert_eq!(*inner_op, BinaryOperator::Mul); - } else { - panic!("expected inner Mul, got {right:?}"); - } - } else { - panic!("expected top-level Add, got {e:?}"); - } - } - - #[test] - fn comparison_vs_equality() { - // a == b < c should be (a == b) < c, since == is lower precedence - // than comparison. Wait, looking at the table: equality (13/14) - // is LOWER precedence than comparison (15/16). So `a == b < c` - // means `a == (b < c)`. Hmm let me re-check the grammar. - // - // equality = { comparison ~ (eq_op ~ comparison)* } - // comparison = { bitwise_or ~ (comp_op ~ bitwise_or)* } - // - // This means: an equality expression is a comparison optionally - // followed by ==/!= and another comparison. So `a == b < c` would - // first match `a` as a comparison, then look for == or !=, find - // ==, parse `b < c` as a comparison (because comparison has - // higher precedence than equality), giving `a == (b < c)`. - // - // In our binding-power table: eq (13/14) < comparison (15/16), - // which is correct. So `a == b < c` is `a == (b < c)`. - let e = p("a == b < c"); - if let Expr::BinaryOp { - op, left, right, .. - } = &e - { - assert_eq!(*op, BinaryOperator::Eq); - // left should be `a`, right should be `b < c` - assert!(matches!(left.as_ref(), Expr::Identifier { name, .. } if name == "a")); - if let Expr::BinaryOp { op: inner_op, .. } = right.as_ref() { - assert_eq!(*inner_op, BinaryOperator::Lt); - } else { - panic!("expected inner Lt, got {right:?}"); - } - } else { - panic!("expected top-level Eq, got {e:?}"); - } - } - - #[test] - fn left_associative_addition() { - // 1 + 2 + 3 should be (1 + 2) + 3 - let e = p("1 + 2 + 3"); - if let Expr::BinaryOp { - op, left, right, .. - } = &e - { - assert_eq!(*op, BinaryOperator::Add); - assert!(matches!( - right.as_ref(), - Expr::Literal { - value: Literal::Int(3), - .. - } - )); - if let Expr::BinaryOp { op: inner_op, .. } = left.as_ref() { - assert_eq!(*inner_op, BinaryOperator::Add); - } else { - panic!("expected inner Add, got {left:?}"); - } - } else { - panic!("expected top-level Add, got {e:?}"); - } - } - - #[test] - fn right_associative_assignment() { - // a = b = c should be a = (b = c) -- right-associative - // The grammar's `assign = logical_or ~ ASSIGN_OP ~ assign | logical_or` - // recurses on the right, so this is the expected grouping. - // - // But: in our grammar, identifiers are not valid lvalues by - // themselves for assignment. The pest parser wraps the lhs in - // an `Expr::Identifier` and the rhs in another `Expr::Assign`. - // For now we test that the structure is right-associative; the - // type checker will reject `a = b = c` if identifiers aren't - // valid lvalues for that expression form. - // - // We use parenthesized variables to keep this test purely about - // precedence; type checking is downstream. - let e = p("a += b += c"); - if let Expr::Assign { - op, left, right, .. - } = &e - { - assert_eq!(*op, AssignmentOperator::AddAssign); - assert!(matches!(left.as_ref(), Expr::Identifier { name, .. } if name == "a")); - assert!(matches!(right.as_ref(), Expr::Assign { .. })); - } else { - panic!("expected top-level Assign, got {e:?}"); - } - } - - #[test] - fn unary_minus_binds_tighter_than_addition() { - // -a + b should be (-a) + b - let e = p("-a + b"); - if let Expr::BinaryOp { - op, left, right, .. - } = &e - { - assert_eq!(*op, BinaryOperator::Add); - assert!(matches!(right.as_ref(), Expr::Identifier { name, .. } if name == "b")); - assert!(matches!( - left.as_ref(), - Expr::UnaryOp { - op: UnaryOperator::Neg, - .. - } - )); - } else { - panic!("expected top-level Add, got {e:?}"); - } - } - - #[test] - fn unary_looser_than_postfix() { - // &arr[i] should be &(arr[i]) - let e = p("&arr[i]"); - if let Expr::UnaryOp { op, expr, .. } = &e { - assert_eq!(*op, UnaryOperator::AddressOf); - assert!(matches!(expr.as_ref(), Expr::Index { .. })); - } else { - panic!("expected top-level AddressOf, got {e:?}"); - } - } - - // --- Postfix chains --- - - #[test] - fn chained_field_access() { - // a.b.c.d.e - should produce 4 nested FieldAccess - let e = p("a.b.c.d.e"); - let mut depth = 0; - let mut cur = &e; - while let Expr::FieldAccess { expr, .. } = cur { - depth += 1; - cur = expr; - } - assert_eq!(depth, 4, "expected 4 field-access levels"); - assert!(matches!(cur, Expr::Identifier { name, .. } if name == "a")); - } - - #[test] - fn stress_deep_postfix_chain() { - // 100-deep chain. With the pest-based parser this would overflow - // the Windows 1MB stack; the Pratt parser handles it in one - // function call with a single loop. - let mut src = String::from("a"); - for i in 0..100 { - src.push('.'); - src.push_str(&format!("f{i}")); - } - let e = p(&src); - let mut depth = 0; - let mut cur = &e; - while let Expr::FieldAccess { expr, .. } = cur { - depth += 1; - cur = expr; - } - assert_eq!(depth, 100); - } - - #[test] - fn stress_deep_nested_parens() { - // 100-deep nesting. The pest-based parser would have a stack - // frame per `(` `)` pair; the Pratt parser recurses through - // `parse_pratt` once per pair but the binding-power loop bounds - // the recursion depth to the source's *syntactic* depth (which - // is the user's choice, not the grammar's). - let mut src = String::new(); - for _ in 0..100 { - src.push('('); - } - src.push('1'); - for _ in 0..100 { - src.push(')'); - } - let e = p(&src); - assert!(matches!( - e, - Expr::Literal { - value: Literal::Int(1), - .. - } - )); - } - - // --- Function calls --- - - #[test] - fn call_no_args() { - let e = p("f()"); - if let Expr::Call { callee, args, .. } = &e { - assert_eq!(callee, "f"); - assert!(args.is_empty()); - } else { - panic!("expected Call, got {e:?}"); - } - } - - #[test] - fn call_one_arg() { - let e = p("f(1)"); - if let Expr::Call { callee, args, .. } = &e { - assert_eq!(callee, "f"); - assert_eq!(args.len(), 1); - } else { - panic!("expected Call, got {e:?}"); - } - } - - #[test] - fn call_many_args() { - let e = p("f(1, 2, 3, 4, 5)"); - if let Expr::Call { args, .. } = &e { - assert_eq!(args.len(), 5); - } else { - panic!("expected Call, got {e:?}"); - } - } - - #[test] - fn call_qualified_name() { - let e = p("pkg.math.add(2, 3)"); - if let Expr::Call { callee, args, .. } = &e { - assert_eq!(callee, "pkg.math.add"); - assert_eq!(args.len(), 2); - } else { - panic!("expected Call, got {e:?}"); - } - } - - #[test] - fn call_with_nested_expressions_in_args() { - // The expression `f(g(1), h(2, 3))` exercises the recursive - // nature of the parser: each arg is itself a Pratt-parsed - // expression, and the call site itself is a postfix in the - // outer parse. - let e = p("f(g(1), h(2, 3))"); - if let Expr::Call { args, .. } = &e { - assert_eq!(args.len(), 2); - } else { - panic!("expected Call, got {e:?}"); - } - } - - // --- Indexing --- - - #[test] - fn index() { - let e = p("arr[0]"); - if let Expr::Index { expr, index, .. } = &e { - assert!(matches!(expr.as_ref(), Expr::Identifier { name, .. } if name == "arr")); - assert!(matches!( - index.as_ref(), - Expr::Literal { - value: Literal::Int(0), - .. - } - )); - } else { - panic!("expected Index, got {e:?}"); - } - } - - #[test] - fn chained_index() { - // a[i][j] - two index ops in sequence - let e = p("a[i][j]"); - let mut depth = 0; - let mut cur = &e; - while let Expr::Index { expr, .. } = cur { - depth += 1; - cur = expr; - } - assert_eq!(depth, 2); - } - - // --- Array literals --- - - #[test] - fn empty_array() { - let e = p("[]"); - if let Expr::ArrayLiteral { elements, .. } = &e { - assert!(elements.is_empty()); - } else { - panic!("expected ArrayLiteral, got {e:?}"); - } - } - - #[test] - fn array_with_elements() { - let e = p("[1, 2, 3]"); - if let Expr::ArrayLiteral { elements, .. } = &e { - assert_eq!(elements.len(), 3); - } else { - panic!("expected ArrayLiteral, got {e:?}"); - } - } - - // --- Struct init --- - - #[test] - fn struct_init() { - let e = p("struct Point { x: 10, y: 20 }"); - if let Expr::StructInit { fields, .. } = &e { - assert_eq!(fields.len(), 2); - assert_eq!(fields[0].name, "x"); - assert_eq!(fields[1].name, "y"); - } else { - panic!("expected StructInit, got {e:?}"); - } - } - - // --- If expressions --- - - #[test] - fn if_expr() { - let e = p("if a then b else c"); - if let Expr::If { - cond, - then_branch, - else_branch, - .. - } = &e - { - assert!(matches!(cond.as_ref(), Expr::Identifier { name, .. } if name == "a")); - assert!(matches!(then_branch.as_ref(), Expr::Identifier { name, .. } if name == "b")); - assert!(matches!(else_branch.as_ref(), Expr::Identifier { name, .. } if name == "c")); - } else { - panic!("expected If, got {e:?}"); - } - } - - // --- Logical operators --- - - #[test] - fn logical_and_vs_or() { - // a || b && c: && is higher precedence, so this is a || (b && c) - let e = p("a || b && c"); - if let Expr::BinaryOp { op, right, .. } = &e { - assert_eq!(*op, BinaryOperator::Or); - assert!(matches!( - right.as_ref(), - Expr::BinaryOp { - op: BinaryOperator::And, - .. - } - )); - } else { - panic!("expected top-level Or, got {e:?}"); - } - } - - // --- Errors --- - - #[test] - fn missing_rparen() { - p_err("(1 + 2", "`)`"); - } - - // --- Range literals --- - - #[test] - fn range_literal_simple() { - let e = p("1...5"); - if let Expr::RangeLiteral { start, end } = &e { - assert!(matches!( - start.as_ref(), - Expr::Literal { - value: Literal::Int(1), - .. - } - )); - assert!(matches!( - end.as_ref(), - Expr::Literal { - value: Literal::Int(5), - .. - } - )); - } else { - panic!("expected RangeLiteral, got {e:?}"); - } - } - - #[test] - fn range_literal_with_expressions() { - // a + 1...b - 1 - // Range binds tighter than assignment but looser than - // arithmetic, so this is (a + 1) ... (b - 1). - let e = p("a + 1...b - 1"); - if let Expr::RangeLiteral { start, end } = &e { - // start should be a + 1 - assert!(matches!( - start.as_ref(), - Expr::BinaryOp { - op: BinaryOperator::Add, - .. - } - )); - // end should be b - 1 - assert!(matches!( - end.as_ref(), - Expr::BinaryOp { - op: BinaryOperator::Sub, - .. - } - )); - } else { - panic!("expected RangeLiteral, got {e:?}"); - } - } - - #[test] - fn missing_rbracket() { - p_err("arr[0", "`]`"); - } - - #[test] - fn unexpected_token_at_start() { - p_err("+", "identifier"); - } - - #[test] - fn missing_field_name() { - p_err("foo.", "identifier"); - } - - #[test] - fn missing_else() { - p_err("if a then b", "`else`"); - } -} diff --git a/kitlang/src/codegen/parser/expr_pratt/mod.rs b/kitlang/src/codegen/parser/expr_pratt/mod.rs new file mode 100644 index 0000000..a797a7a --- /dev/null +++ b/kitlang/src/codegen/parser/expr_pratt/mod.rs @@ -0,0 +1,282 @@ +//! Hand-written Pratt parser for Kit expressions. +//! +//! This module takes over expression parsing from the pest-based grammar. +//! Pest still handles the program, declaration, statement, and type-annotation +//! grammars. For every `Pair<'_, Rule>` whose rule is an expression (i.e., +//! the 13 precedence levels `expr → assign → logical_or → ... → primary`), +//! the parser hands off to [`ExprParser::parse_expr`]. +//! +//! Pratt Parsing & Operator Precedence +//! ----------------------------------- +//! The pest grammar used 13 mutually recursive functions (one per precedence +//! level), which overflowed the default 1 MB stack on Windows. The Pratt +//! parser uses a single function with a binding-power loop, bounding stack +//! depth to `O(precedence levels) = O(13)` regardless of expression length. +//! Operator precedence is defined in [`binding_power::infix`], +//! [`binding_power::postfix`], [`binding_power::prefix`]. Each infix operator +//! has a (lbp, rbp) pair: `lbp < rbp` = right-associative (assignment), +//! `lbp == rbp` = left-associative (most binary ops). +//! +//! Errors & Source Spans +//! --------------------- +//! All parse errors are values of [`ExprParseError`] (in `diagnostics.rs`). +//! The parser never prints, never allocates strings for error messages, +//! and never holds source-file identity. Conversion to +//! `CompilationError::ParseError(String)` happens at `PestExpr::parse` +//! in `parser/mod.rs`. The token stream carries byte ranges; the parser +//! uses them internally but does not currently attach them to AST nodes. + +use crate::codegen::ast::{Expr, Literal}; +use crate::codegen::type_ast::FieldInit; +use crate::codegen::types::{Type, TypeId}; +use crate::lexer::{Span, SpannedTok, Tok, tokenize}; + +use super::binding_power::{ + infix, is_range_op, postfix, prefix, tok_to_assign_op, tok_to_binary_op, tok_to_unary_op, +}; +use super::diagnostics::{ExprParseError, expected_name}; + +// --------------------------------------------------------------------------- +// Parser +// --------------------------------------------------------------------------- + +/// A Pratt parser for Kit expressions. +/// +/// One instance is built per expression being parsed. The parser is +/// single-use: create a new one for each `parse_expr` call. +pub(crate) struct ExprParser<'a> { + tokens: &'a [SpannedTok], + pos: usize, +} + +impl<'a> ExprParser<'a> { + /// Build a parser over a token slice. The slice must be the result of + /// [`tokenize`] applied to the expression's source text. + pub(crate) fn new(tokens: &'a [SpannedTok]) -> Self { + Self { tokens, pos: 0 } + } + + /// Entry point. Parses one complete expression and returns it. + /// The expression may be followed by trailing tokens, which are left + /// in the stream (the caller can `pos()`-check or re-parse). + pub(crate) fn parse_expr(&mut self) -> Result { + self.parse_pratt(0) + } + + /// The current position in the token stream. Useful for tests and + /// for callers that want to know how many tokens were consumed. + pub(crate) fn pos(&self) -> usize { + self.pos + } + + // --- Token stream helpers (private) --- + + /// Peek the current token without consuming it. Returns a synthetic + /// "EOF" token when the stream is exhausted. + fn peek(&self) -> &SpannedTok { + // We never return None from peek; EOF is represented by a synthetic + // SpannedTok at span 0..0. This lets the Pratt loop compare with + // `==` against `Tok::...` cleanly. + static EOF: SpannedTok = SpannedTok { + kind: Tok::Semi, // any token that has no infix/postfix/prefix bp + span: 0..0, + }; + self.tokens.get(self.pos).unwrap_or(&EOF) + } + + /// Peek the next token (one past the current). Same EOF behavior. + fn peek_next(&self) -> &SpannedTok { + static EOF: SpannedTok = SpannedTok { + kind: Tok::Semi, + span: 0..0, + }; + self.tokens.get(self.pos + 1).unwrap_or(&EOF) + } + + /// Consume and return the current token. + fn advance(&mut self) -> &SpannedTok { + let tok = &self.tokens[self.pos]; + self.pos += 1; + tok + } + + /// True if the parser is at or past the end of the token stream. + fn at_eof(&self) -> bool { + self.pos >= self.tokens.len() + } + + // --- Pratt core --- + + /// Parse an expression with the given minimum binding power. + fn parse_pratt(&mut self, min_bp: u8) -> Result { + // Parse leading prefix operators (e.g. `-a`, `!!x`, `&arr[i]`). + // Prefix binds tighter than infix but looser than postfix, + // so `&arr[i]` = `&(arr[i])` (postfix on `arr` first). + let mut lhs = if let Some(pfx_bp) = prefix(&self.peek().kind) { + let op = tok_to_unary_op(&self.peek().kind).unwrap(); + self.advance(); + let rhs = self.parse_pratt(pfx_bp)?; + Expr::UnaryOp { + op, + expr: Box::new(rhs), + ty: TypeId::default(), + } + } else { + self.parse_primary()? + }; + + // Postfix chain: field access, index, call. + lhs = self.parse_postfix_chain(lhs)?; + + // Infix operators (binary ops, range). + loop { + let kind = self.peek().kind.clone(); + let Some((lbp, rbp)) = infix(&kind) else { + break; + }; + if lbp < min_bp { + break; + } + if tok_to_assign_op(&kind).is_some() { + break; + } + self.advance(); + if is_range_op(&kind) { + let rhs = self.parse_pratt(rbp)?; + lhs = Expr::RangeLiteral { + start: Box::new(lhs), + end: Box::new(rhs), + }; + continue; + } + let op = tok_to_binary_op(&kind).ok_or_else(|| { + ExprParseError::Custom(format!("internal: no binary op for {kind:?}")) + })?; + let rhs = self.parse_pratt(rbp)?; + lhs = Expr::BinaryOp { + op, + left: Box::new(lhs), + right: Box::new(rhs), + ty: TypeId::default(), + }; + } + + // Assignment (right-associative, lowest precedence). + loop { + let kind = self.peek().kind.clone(); + let Some(op) = tok_to_assign_op(&kind) else { + break; + }; + // Assignment has the lowest precedence (lbp=0, rbp=1 in the + // infix table, but we don't use that table here). Right- + // associativity: a = b = c means a = (b = c). Recurse with + // min_bp=0 so the rhs sees *all* operators including another + // assignment. + if 0 < min_bp { + break; + } + self.advance(); + let rhs = self.parse_pratt(0)?; + lhs = Expr::Assign { + op, + left: Box::new(lhs), + right: Box::new(rhs), + ty: TypeId::default(), + }; + } + + Ok(lhs) + } + + // --- Helpers --- + + /// Consume the current token if it matches `expected`, otherwise + /// return an `UnexpectedToken` error with `expected`'s name. + fn expect(&mut self, expected: &Tok) -> Result<(), ExprParseError> { + if &self.peek().kind == expected { + self.advance(); + Ok(()) + } else { + Err(ExprParseError::UnexpectedToken { + found: self.peek().kind.clone(), + expected: expected_name(expected), + }) + } + } + + /// Parse a comma-separated list of `T` terminated by `closer`. + /// Allows zero or more elements (an empty list is valid for fn + /// calls with no args, empty array literals, etc.). + fn parse_comma_list(&mut self, closer: Tok, mut f: F) -> Result, ExprParseError> + where + F: FnMut(&mut Self) -> Result, + { + let mut out = Vec::new(); + // Empty list case. + if self.peek().kind == closer { + self.advance(); + return Ok(out); + } + out.push(f(self)?); + while self.peek().kind == Tok::Comma { + self.advance(); + // Trailing comma is allowed (parses to empty trailing element). + if self.peek().kind == closer { + break; + } + out.push(f(self)?); + } + self.expect(&closer)?; + Ok(out) + } +} + +/// Extract a callee name from a function-call's leading expression. +/// +/// For `Expr::Identifier { name, .. }` this is just `name`. +/// For `Expr::FieldAccess { expr, field_name, .. }` chains like +/// `pkg.math.add`, this concatenates the path with `.`. +/// For other expressions (e.g. a parenthesized call), we fall back to +/// `Display`-formatting the expression, which the transpiler can still +/// route through name resolution. +fn expr_to_callee_name(expr: &Expr) -> String { + match expr { + Expr::Identifier { name, .. } => name.clone(), + Expr::FieldAccess { + expr: base, + field_name, + .. + } => { + let base_name = expr_to_callee_name(base); + format!("{base_name}.{field_name}") + } + other => format!("{other:?}"), + } +} + +// --------------------------------------------------------------------------- +// Module surface: parse an expression from source text. +// --------------------------------------------------------------------------- + +/// Parse a Kit expression from source text. This is the public entry +/// point used by the pest-to-Pratt bridge (`PestExpr::parse`). +/// +/// The `text` should be the source text of the expression as a +/// `Pair::as_str()` slice. Tokenization, parsing, and conversion to an +/// `Expr` all happen here. +pub(crate) fn parse_kit_expr(text: &str) -> Result { + let tokens = tokenize(text); + let mut parser = ExprParser::new(&tokens); + parser.parse_expr() +} + +// --------------------------------------------------------------------------- +// Primary expression parsers (include'd into this module scope). +// --------------------------------------------------------------------------- +include!("primary.rs"); + +// --------------------------------------------------------------------------- +// Tests (include'd into this module scope, test builds only). +// --------------------------------------------------------------------------- +#[cfg(test)] +include!("tests.rs"); diff --git a/kitlang/src/codegen/parser/expr_pratt/primary.rs b/kitlang/src/codegen/parser/expr_pratt/primary.rs new file mode 100644 index 0000000..a2029f9 --- /dev/null +++ b/kitlang/src/codegen/parser/expr_pratt/primary.rs @@ -0,0 +1,260 @@ +// Primary expression parsers. Included into `mod.rs` via `include!`. +// This file is in the same module scope as `mod.rs`, so imports are +// inherited and all `ExprParser` private methods are accessible. + +impl<'a> ExprParser<'a> { + /// Iteratively apply postfix operators (call, index, field access) to + /// a base expression. Zero stack frames added per iteration. The + /// chain is bounded by the source's syntactic length, but the parser + /// is iterative, so the *call stack* depth is constant. + pub(crate) fn parse_postfix_chain( + &mut self, + mut base: Expr, + ) -> Result { + loop { + let kind = self.peek().kind.clone(); + if postfix(&kind).is_none() { + break; + } + base = match kind { + Tok::Dot => self.parse_field_access(base)?, + Tok::LBracket => self.parse_index(base)?, + Tok::LParen => self.parse_call(base)?, + _ => unreachable!("postfix returned Some for {kind:?}"), + }; + } + Ok(base) + } + + /// Parse a primary expression: literals, identifiers, parenthesized + /// expressions, function calls, array literals, struct inits, and + /// the if-expression. Postfix operations (`.field`, `[i]`, `(args)`) + /// are handled in the outer Pratt loop, *not* here, so this function + /// only needs to produce the base expression. + pub(crate) fn parse_primary(&mut self) -> Result { + let tok = self.peek().kind.clone(); + // `span` is read for documentation; the parser doesn't currently + // attach it to AST nodes. Future PRs will use it. + let _span: Span = self.peek().span.clone(); + + match tok { + Tok::IntLit(n) => { + self.advance(); + Ok(Expr::Literal { + value: Literal::Int(n), + ty: TypeId::default(), + }) + } + Tok::FloatLit(f) => { + self.advance(); + Ok(Expr::Literal { + value: Literal::Float(f), + ty: TypeId::default(), + }) + } + Tok::CharLit(c) => { + self.advance(); + Ok(Expr::Literal { + value: Literal::Char(c), + ty: TypeId::default(), + }) + } + Tok::StringLit(s) => { + self.advance(); + Ok(Expr::Literal { + value: Literal::String(s), + ty: TypeId::default(), + }) + } + Tok::KwTrue => { + self.advance(); + Ok(Expr::Literal { + value: Literal::Bool(true), + ty: TypeId::default(), + }) + } + Tok::KwFalse => { + self.advance(); + Ok(Expr::Literal { + value: Literal::Bool(false), + ty: TypeId::default(), + }) + } + Tok::KwNull => { + self.advance(); + Ok(Expr::Literal { + value: Literal::Null, + ty: TypeId::default(), + }) + } + Tok::KwThis | Tok::KwSelf => { + let name = match tok { + Tok::KwThis => "this", + _ => "Self", + }; + self.advance(); + Ok(Expr::Identifier { + name: name.to_string(), + ty: TypeId::default(), + }) + } + Tok::Ident(name) => { + self.advance(); + Ok(Expr::Identifier { + name, + ty: TypeId::default(), + }) + } + Tok::LParen => { + self.advance(); // consume `(` + let first = self.parse_expr()?; + if self.peek().kind == Tok::Comma { + return Err(ExprParseError::Custom( + "tuple literals are not yet supported by the Pratt parser".into(), + )); + } + self.expect(&Tok::RParen)?; + Ok(first) + } + Tok::LBracket => self.parse_array_literal(), + Tok::KwStruct => self.parse_struct_init(), + Tok::KwIf => self.parse_if_expr(), + Tok::KwEmpty => { + self.advance(); + Ok(Expr::Identifier { + name: "empty".to_string(), + ty: TypeId::default(), + }) + } + _ => Err(ExprParseError::UnexpectedToken { + found: tok, + expected: &[ + "integer literal", + "float literal", + "string literal", + "char literal", + "identifier", + "`(`", + "`[`", + "`if`", + "`null`", + "`true`", + "`false`", + ], + }), + } + } + + /// Parse a `.field` access postfix. + fn parse_field_access(&mut self, base: Expr) -> Result { + self.advance(); // consume `.` + let field_tok = self.peek().kind.clone(); + match field_tok { + Tok::Ident(name) => { + self.advance(); + Ok(Expr::FieldAccess { + expr: Box::new(base), + field_name: name, + ty: TypeId::default(), + }) + } + _ => Err(ExprParseError::UnexpectedToken { + found: field_tok, + expected: &["identifier"], + }), + } + } + + /// Parse a `[index]` postfix. + fn parse_index(&mut self, base: Expr) -> Result { + self.advance(); // consume `[` + let index = self.parse_expr()?; + self.expect(&Tok::RBracket)?; + Ok(Expr::Index { + expr: Box::new(base), + index: Box::new(index), + ty: TypeId::default(), + }) + } + + /// Parse a function call postfix: `(arg1, arg2, ...)`. + fn parse_call(&mut self, callee: Expr) -> Result { + let callee_name = expr_to_callee_name(&callee); + + self.advance(); // consume `(` + let args = self.parse_comma_list(Tok::RParen, |p| p.parse_expr())?; + Ok(Expr::Call { + callee: callee_name, + args, + ty: TypeId::default(), + }) + } + + /// Parse an array literal: `[expr, expr, ...]`. + fn parse_array_literal(&mut self) -> Result { + self.advance(); // consume `[` + let elements = self.parse_comma_list(Tok::RBracket, |p| p.parse_expr())?; + Ok(Expr::ArrayLiteral { + elements, + ty: TypeId::default(), + }) + } + + /// Parse a struct init: `struct Name { field: expr, ... }`. + fn parse_struct_init(&mut self) -> Result { + self.advance(); // consume `struct` + let type_tok = self.peek().kind.clone(); + let type_name = match type_tok { + Tok::Ident(name) => { + self.advance(); + name + } + _ => { + return Err(ExprParseError::UnexpectedToken { + found: type_tok, + expected: &["type name"], + }); + } + }; + self.expect(&Tok::LBrace)?; + let fields = self.parse_comma_list(Tok::RBrace, |p| { + let name = match &p.peek().kind { + Tok::Ident(n) => { + let n = n.clone(); + p.advance(); + n + } + _ => { + return Err(ExprParseError::UnexpectedToken { + found: p.peek().kind.clone(), + expected: &["field name"], + }); + } + }; + p.expect(&Tok::Colon)?; + let value = p.parse_expr()?; + Ok(FieldInit { name, value }) + })?; + Ok(Expr::StructInit { + ty: TypeId::default(), + struct_type: Some(Type::from_kit(&type_name)), + fields, + }) + } + + /// Parse an if-expression: `if cond then a else b`. + fn parse_if_expr(&mut self) -> Result { + self.advance(); // consume `if` + let cond = self.parse_expr()?; + self.expect(&Tok::KwThen)?; + let then_branch = self.parse_expr()?; + self.expect(&Tok::KwElse)?; + let else_branch = self.parse_expr()?; + Ok(Expr::If { + cond: Box::new(cond), + then_branch: Box::new(then_branch), + else_branch: Box::new(else_branch), + ty: TypeId::default(), + }) + } +} diff --git a/kitlang/src/codegen/parser/expr_pratt/tests.rs b/kitlang/src/codegen/parser/expr_pratt/tests.rs new file mode 100644 index 0000000..fce4b29 --- /dev/null +++ b/kitlang/src/codegen/parser/expr_pratt/tests.rs @@ -0,0 +1,502 @@ +// Unit tests for the Pratt parser. Included into `mod.rs` via `include!`, +// so this file is in the same module scope. Only compiled in test builds. + +use crate::codegen::types::{AssignmentOperator, BinaryOperator, UnaryOperator}; + +/// Convenience: parse an expression and unwrap. +fn p(text: &str) -> Expr { + parse_kit_expr(text).unwrap_or_else(|e| panic!("parse failed for `{text}`: {e}")) +} + +/// Convenience: parse and assert the error contains a substring. +fn p_err(text: &str, needle: &str) { + let err = parse_kit_expr(text) + .err() + .unwrap_or_else(|| panic!("expected error for `{text}`, got Ok")); + let msg = err.to_human_message(); + assert!( + msg.contains(needle), + "error `{msg}` does not contain `{needle}`" + ); +} + +// --- Literals --- + +#[test] +fn integer_literal() { + let e = p("42"); + assert!(matches!( + e, + Expr::Literal { + value: Literal::Int(42), + .. + } + )); +} + +#[test] +fn float_literal() { + let e = p("3.14"); + assert!( + matches!(e, Expr::Literal { value: Literal::Float(f), .. } if (f - 3.14).abs() < 1e-10) + ); +} + +#[test] +fn string_literal() { + let e = p(r#""hello""#); + assert!(matches!(e, Expr::Literal { value: Literal::String(s), .. } if s == "hello")); +} + +#[test] +fn bool_literals() { + assert!(matches!( + p("true"), + Expr::Literal { + value: Literal::Bool(true), + .. + } + )); + assert!(matches!( + p("false"), + Expr::Literal { + value: Literal::Bool(false), + .. + } + )); +} + +#[test] +fn null_literal() { + assert!(matches!( + p("null"), + Expr::Literal { + value: Literal::Null, + .. + } + )); +} + +// --- Identifiers --- + +#[test] +fn identifier() { + let e = p("foo"); + assert!(matches!(&e, Expr::Identifier { name, .. } if name == "foo")); +} + +#[test] +fn qualified_identifier_is_built_via_postfix_chain() { + let e = p("foo.bar.baz"); + let mut cur = &e; + let mut path = vec![]; + while let Expr::FieldAccess { + expr, field_name, .. + } = cur + { + path.push(field_name.clone()); + cur = expr; + } + if let Expr::Identifier { name, .. } = cur { + assert_eq!(name, "foo"); + } else { + panic!("expected leaf Identifier, got {cur:?}"); + } + assert_eq!(path, vec!["baz".to_string(), "bar".to_string()]); +} + +// --- Precedence --- + +#[test] +fn additive_vs_multiplicative() { + let e = p("1 + 2 * 3"); + if let Expr::BinaryOp { op, right, .. } = &e { + assert_eq!(*op, BinaryOperator::Add); + if let Expr::BinaryOp { op: inner_op, .. } = right.as_ref() { + assert_eq!(*inner_op, BinaryOperator::Mul); + } else { + panic!("expected inner Mul, got {right:?}"); + } + } else { + panic!("expected top-level Add, got {e:?}"); + } +} + +#[test] +fn comparison_vs_equality() { + let e = p("a == b < c"); + if let Expr::BinaryOp { + op, left, right, .. + } = &e + { + assert_eq!(*op, BinaryOperator::Eq); + assert!(matches!(left.as_ref(), Expr::Identifier { name, .. } if name == "a")); + if let Expr::BinaryOp { op: inner_op, .. } = right.as_ref() { + assert_eq!(*inner_op, BinaryOperator::Lt); + } else { + panic!("expected inner Lt, got {right:?}"); + } + } else { + panic!("expected top-level Eq, got {e:?}"); + } +} + +#[test] +fn left_associative_addition() { + let e = p("1 + 2 + 3"); + if let Expr::BinaryOp { + op, left, right, .. + } = &e + { + assert_eq!(*op, BinaryOperator::Add); + assert!(matches!( + right.as_ref(), + Expr::Literal { + value: Literal::Int(3), + .. + } + )); + if let Expr::BinaryOp { op: inner_op, .. } = left.as_ref() { + assert_eq!(*inner_op, BinaryOperator::Add); + } else { + panic!("expected inner Add, got {left:?}"); + } + } else { + panic!("expected top-level Add, got {e:?}"); + } +} + +#[test] +fn right_associative_assignment() { + let e = p("a += b += c"); + if let Expr::Assign { + op, left, right, .. + } = &e + { + assert_eq!(*op, AssignmentOperator::AddAssign); + assert!(matches!(left.as_ref(), Expr::Identifier { name, .. } if name == "a")); + assert!(matches!(right.as_ref(), Expr::Assign { .. })); + } else { + panic!("expected top-level Assign, got {e:?}"); + } +} + +#[test] +fn unary_minus_binds_tighter_than_addition() { + let e = p("-a + b"); + if let Expr::BinaryOp { + op, left, right, .. + } = &e + { + assert_eq!(*op, BinaryOperator::Add); + assert!(matches!(right.as_ref(), Expr::Identifier { name, .. } if name == "b")); + assert!(matches!( + left.as_ref(), + Expr::UnaryOp { + op: UnaryOperator::Neg, + .. + } + )); + } else { + panic!("expected top-level Add, got {e:?}"); + } +} + +#[test] +fn unary_looser_than_postfix() { + let e = p("&arr[i]"); + if let Expr::UnaryOp { op, expr, .. } = &e { + assert_eq!(*op, UnaryOperator::AddressOf); + assert!(matches!(expr.as_ref(), Expr::Index { .. })); + } else { + panic!("expected top-level AddressOf, got {e:?}"); + } +} + +// --- Postfix chains --- + +#[test] +fn chained_field_access() { + let e = p("a.b.c.d.e"); + let mut depth = 0; + let mut cur = &e; + while let Expr::FieldAccess { expr, .. } = cur { + depth += 1; + cur = expr; + } + assert_eq!(depth, 4, "expected 4 field-access levels"); + assert!(matches!(cur, Expr::Identifier { name, .. } if name == "a")); +} + +#[test] +fn stress_deep_postfix_chain() { + let mut src = String::from("a"); + for i in 0..100 { + src.push('.'); + src.push_str(&format!("f{i}")); + } + let e = p(&src); + let mut depth = 0; + let mut cur = &e; + while let Expr::FieldAccess { expr, .. } = cur { + depth += 1; + cur = expr; + } + assert_eq!(depth, 100); +} + +#[test] +fn stress_deep_nested_parens() { + let mut src = String::new(); + for _ in 0..100 { + src.push('('); + } + src.push('1'); + for _ in 0..100 { + src.push(')'); + } + let e = p(&src); + assert!(matches!( + e, + Expr::Literal { + value: Literal::Int(1), + .. + } + )); +} + +// --- Function calls --- + +#[test] +fn call_no_args() { + let e = p("f()"); + if let Expr::Call { callee, args, .. } = &e { + assert_eq!(callee, "f"); + assert!(args.is_empty()); + } else { + panic!("expected Call, got {e:?}"); + } +} + +#[test] +fn call_one_arg() { + let e = p("f(1)"); + if let Expr::Call { callee, args, .. } = &e { + assert_eq!(callee, "f"); + assert_eq!(args.len(), 1); + } else { + panic!("expected Call, got {e:?}"); + } +} + +#[test] +fn call_many_args() { + let e = p("f(1, 2, 3, 4, 5)"); + if let Expr::Call { args, .. } = &e { + assert_eq!(args.len(), 5); + } else { + panic!("expected Call, got {e:?}"); + } +} + +#[test] +fn call_qualified_name() { + let e = p("pkg.math.add(2, 3)"); + if let Expr::Call { callee, args, .. } = &e { + assert_eq!(callee, "pkg.math.add"); + assert_eq!(args.len(), 2); + } else { + panic!("expected Call, got {e:?}"); + } +} + +#[test] +fn call_with_nested_expressions_in_args() { + let e = p("f(g(1), h(2, 3))"); + if let Expr::Call { args, .. } = &e { + assert_eq!(args.len(), 2); + } else { + panic!("expected Call, got {e:?}"); + } +} + +// --- Indexing --- + +#[test] +fn index() { + let e = p("arr[0]"); + if let Expr::Index { expr, index, .. } = &e { + assert!(matches!(expr.as_ref(), Expr::Identifier { name, .. } if name == "arr")); + assert!(matches!( + index.as_ref(), + Expr::Literal { + value: Literal::Int(0), + .. + } + )); + } else { + panic!("expected Index, got {e:?}"); + } +} + +#[test] +fn chained_index() { + let e = p("a[i][j]"); + let mut depth = 0; + let mut cur = &e; + while let Expr::Index { expr, .. } = cur { + depth += 1; + cur = expr; + } + assert_eq!(depth, 2); +} + +// --- Array literals --- + +#[test] +fn empty_array() { + let e = p("[]"); + if let Expr::ArrayLiteral { elements, .. } = &e { + assert!(elements.is_empty()); + } else { + panic!("expected ArrayLiteral, got {e:?}"); + } +} + +#[test] +fn array_with_elements() { + let e = p("[1, 2, 3]"); + if let Expr::ArrayLiteral { elements, .. } = &e { + assert_eq!(elements.len(), 3); + } else { + panic!("expected ArrayLiteral, got {e:?}"); + } +} + +// --- Struct init --- + +#[test] +fn struct_init() { + let e = p("struct Point { x: 10, y: 20 }"); + if let Expr::StructInit { fields, .. } = &e { + assert_eq!(fields.len(), 2); + assert_eq!(fields[0].name, "x"); + assert_eq!(fields[1].name, "y"); + } else { + panic!("expected StructInit, got {e:?}"); + } +} + +// --- If expressions --- + +#[test] +fn if_expr() { + let e = p("if a then b else c"); + if let Expr::If { + cond, + then_branch, + else_branch, + .. + } = &e + { + assert!(matches!(cond.as_ref(), Expr::Identifier { name, .. } if name == "a")); + assert!(matches!(then_branch.as_ref(), Expr::Identifier { name, .. } if name == "b")); + assert!(matches!(else_branch.as_ref(), Expr::Identifier { name, .. } if name == "c")); + } else { + panic!("expected If, got {e:?}"); + } +} + +// --- Logical operators --- + +#[test] +fn logical_and_vs_or() { + let e = p("a || b && c"); + if let Expr::BinaryOp { op, right, .. } = &e { + assert_eq!(*op, BinaryOperator::Or); + assert!(matches!( + right.as_ref(), + Expr::BinaryOp { + op: BinaryOperator::And, + .. + } + )); + } else { + panic!("expected top-level Or, got {e:?}"); + } +} + +// --- Errors --- + +#[test] +fn missing_rparen() { + p_err("(1 + 2", "`)`"); +} + +// --- Range literals --- + +#[test] +fn range_literal_simple() { + let e = p("1...5"); + if let Expr::RangeLiteral { start, end } = &e { + assert!(matches!( + start.as_ref(), + Expr::Literal { + value: Literal::Int(1), + .. + } + )); + assert!(matches!( + end.as_ref(), + Expr::Literal { + value: Literal::Int(5), + .. + } + )); + } else { + panic!("expected RangeLiteral, got {e:?}"); + } +} + +#[test] +fn range_literal_with_expressions() { + let e = p("a + 1...b - 1"); + if let Expr::RangeLiteral { start, end } = &e { + assert!(matches!( + start.as_ref(), + Expr::BinaryOp { + op: BinaryOperator::Add, + .. + } + )); + assert!(matches!( + end.as_ref(), + Expr::BinaryOp { + op: BinaryOperator::Sub, + .. + } + )); + } else { + panic!("expected RangeLiteral, got {e:?}"); + } +} + +#[test] +fn missing_rbracket() { + p_err("arr[0", "`]`"); +} + +#[test] +fn unexpected_token_at_start() { + p_err("+", "identifier"); +} + +#[test] +fn missing_field_name() { + p_err("foo.", "identifier"); +} + +#[test] +fn missing_else() { + p_err("if a then b", "`else`"); +} From b6a733311cc6c88173ed1a8339b1f30661670895 Mon Sep 17 00:00:00 2001 From: winlogon Date: Fri, 10 Jul 2026 05:29:57 +0200 Subject: [PATCH 3/4] fix(parser): improve error reporting and validation The lexer now properly reports unrecognized characters and integer overflow instead of silently dropping them. The parser validates no trailing tokens remain and provides better EOF error messages. Unsupported constructs like indirect calls are now explicitly rejected with clear diagnostics. Summary of changes: - Change tokenize() to return Result and report lexical errors - Add LexicalError enum for UnexpectedCharacter and IntegerOverflow - Validate trailing tokens in parse_kit_expr() - Distinguish UnexpectedEof from UnexpectedToken in errors - Reject indirect function calls with clear diagnostic message - Fix binding power checks in assignment operator handling - Update expect() to return UnexpectedEof when at EOF - Improve documentation accuracy and clarity - Add comprehensive test coverage for all error cases --- kitlang/src/codegen/parser/binding_power.rs | 2 +- kitlang/src/codegen/parser/diagnostics.rs | 2 +- kitlang/src/codegen/parser/expr_pratt/mod.rs | 85 ++++++++++---- .../src/codegen/parser/expr_pratt/primary.rs | 54 ++++++--- .../src/codegen/parser/expr_pratt/tests.rs | 108 ++++++++++++++++++ kitlang/src/lexer.rs | 94 ++++++++++++--- 6 files changed, 284 insertions(+), 61 deletions(-) diff --git a/kitlang/src/codegen/parser/binding_power.rs b/kitlang/src/codegen/parser/binding_power.rs index 84fd2d5..6df73a0 100644 --- a/kitlang/src/codegen/parser/binding_power.rs +++ b/kitlang/src/codegen/parser/binding_power.rs @@ -4,7 +4,7 @@ use crate::codegen::types::{AssignmentOperator, BinaryOperator, UnaryOperator}; use crate::lexer::Tok; /// Infix binding power: (left_bp, right_bp). None = not an infix operator. -/// lbp < rbp → right-associative (assignment); lbp == rbp → left-associative. +/// lbp < rbp = right-associative (assignment); lbp == rbp = left-associative. pub fn infix(tok: &Tok) -> Option<(u8, u8)> { match tok { Tok::Assign diff --git a/kitlang/src/codegen/parser/diagnostics.rs b/kitlang/src/codegen/parser/diagnostics.rs index 8bc8df6..badd1b6 100644 --- a/kitlang/src/codegen/parser/diagnostics.rs +++ b/kitlang/src/codegen/parser/diagnostics.rs @@ -80,7 +80,7 @@ impl fmt::Display for ExprParseError { /// slice of one element. Returns `&'static [&'static str]` for direct /// use in `ExprParseError::UnexpectedToken.expected`. /// -/// This is the single canonical match for all token → name mappings. +/// This is the single canonical match for all token-to-name mappings. /// [`tok_name`] is a zero-cost wrapper that extracts the first element. pub(crate) fn expected_name(kind: &Tok) -> &'static [&'static str] { match kind { diff --git a/kitlang/src/codegen/parser/expr_pratt/mod.rs b/kitlang/src/codegen/parser/expr_pratt/mod.rs index a797a7a..86218d5 100644 --- a/kitlang/src/codegen/parser/expr_pratt/mod.rs +++ b/kitlang/src/codegen/parser/expr_pratt/mod.rs @@ -3,15 +3,17 @@ //! This module takes over expression parsing from the pest-based grammar. //! Pest still handles the program, declaration, statement, and type-annotation //! grammars. For every `Pair<'_, Rule>` whose rule is an expression (i.e., -//! the 13 precedence levels `expr → assign → logical_or → ... → primary`), +//! the 13 precedence levels `expr -> assign -> logical_or -> ... -> primary`), //! the parser hands off to [`ExprParser::parse_expr`]. //! //! Pratt Parsing & Operator Precedence //! ----------------------------------- //! The pest grammar used 13 mutually recursive functions (one per precedence //! level), which overflowed the default 1 MB stack on Windows. The Pratt -//! parser uses a single function with a binding-power loop, bounding stack -//! depth to `O(precedence levels) = O(13)` regardless of expression length. +//! parser uses a single function with a binding-power loop, dramatically +//! reducing recursive depth compared to the old 13-level grammar chain. +//! Parenthesized sub-expressions still recurse through `parse_primary` +//! and `parse_expr`, so stack depth is `O(parenthetical nesting)`. //! Operator precedence is defined in [`binding_power::infix`], //! [`binding_power::postfix`], [`binding_power::prefix`]. Each infix operator //! has a (lbp, rbp) pair: `lbp < rbp` = right-associative (assignment), @@ -29,7 +31,7 @@ use crate::codegen::ast::{Expr, Literal}; use crate::codegen::type_ast::FieldInit; use crate::codegen::types::{Type, TypeId}; -use crate::lexer::{Span, SpannedTok, Tok, tokenize}; +use crate::lexer::{LexicalError, Span, SpannedTok, Tok, tokenize}; use super::binding_power::{ infix, is_range_op, postfix, prefix, tok_to_assign_op, tok_to_binary_op, tok_to_unary_op, @@ -57,8 +59,8 @@ impl<'a> ExprParser<'a> { } /// Entry point. Parses one complete expression and returns it. - /// The expression may be followed by trailing tokens, which are left - /// in the stream (the caller can `pos()`-check or re-parse). + /// Callers must check `pos()` against the token length to ensure + /// no trailing tokens were left unparsed. pub(crate) fn parse_expr(&mut self) -> Result { self.parse_pratt(0) } @@ -162,21 +164,23 @@ impl<'a> ExprParser<'a> { } // Assignment (right-associative, lowest precedence). + // Right-associativity requires the RHS to be parsed at the same + // binding power as the operator's lbp, so `a = b = c` becomes + // `a = (b = c)`. We use `lbp` (not `rbp` from the table) since + // assignment needs `rbp <= lbp` for the RHS to see the operator. loop { let kind = self.peek().kind.clone(); let Some(op) = tok_to_assign_op(&kind) else { break; }; - // Assignment has the lowest precedence (lbp=0, rbp=1 in the - // infix table, but we don't use that table here). Right- - // associativity: a = b = c means a = (b = c). Recurse with - // min_bp=0 so the rhs sees *all* operators including another - // assignment. - if 0 < min_bp { + let Some((lbp, _rbp)) = infix(&kind) else { + break; + }; + if lbp < min_bp { break; } self.advance(); - let rhs = self.parse_pratt(0)?; + let rhs = self.parse_pratt(lbp)?; lhs = Expr::Assign { op, left: Box::new(lhs), @@ -191,11 +195,16 @@ impl<'a> ExprParser<'a> { // --- Helpers --- /// Consume the current token if it matches `expected`, otherwise - /// return an `UnexpectedToken` error with `expected`'s name. + /// return an `UnexpectedToken` (or `UnexpectedEof` if at end) error + /// with `expected`'s name. fn expect(&mut self, expected: &Tok) -> Result<(), ExprParseError> { if &self.peek().kind == expected { self.advance(); Ok(()) + } else if self.at_eof() { + Err(ExprParseError::UnexpectedEof { + expected: expected_name(expected), + }) } else { Err(ExprParseError::UnexpectedToken { found: self.peek().kind.clone(), @@ -236,21 +245,24 @@ impl<'a> ExprParser<'a> { /// For `Expr::Identifier { name, .. }` this is just `name`. /// For `Expr::FieldAccess { expr, field_name, .. }` chains like /// `pkg.math.add`, this concatenates the path with `.`. -/// For other expressions (e.g. a parenthesized call), we fall back to -/// `Display`-formatting the expression, which the transpiler can still -/// route through name resolution. -fn expr_to_callee_name(expr: &Expr) -> String { +/// Rejects non-trivial call targets (e.g. calling the result of another +/// call) with an error. +fn expr_to_callee_name(expr: &Expr) -> Result { match expr { - Expr::Identifier { name, .. } => name.clone(), + Expr::Identifier { name, .. } => Ok(name.clone()), Expr::FieldAccess { expr: base, field_name, .. } => { - let base_name = expr_to_callee_name(base); - format!("{base_name}.{field_name}") + let base_name = expr_to_callee_name(base)?; + Ok(format!("{base_name}.{field_name}")) } - other => format!("{other:?}"), + _ => Err(ExprParseError::Custom( + "indirect calls (calling the result of another expression) \ + are not supported by the Kit compiler" + .into(), + )), } } @@ -264,10 +276,35 @@ fn expr_to_callee_name(expr: &Expr) -> String { /// The `text` should be the source text of the expression as a /// `Pair::as_str()` slice. Tokenization, parsing, and conversion to an /// `Expr` all happen here. +/// +/// Errors are returned for: +/// - Unrecognized characters in the source +/// - Integer literals that overflow `i64` +/// - Trailing tokens after the expression +/// - Any parse error from the Pratt loop pub(crate) fn parse_kit_expr(text: &str) -> Result { - let tokens = tokenize(text); + let tokens = tokenize(text).map_err(|e| match e { + LexicalError::UnexpectedCharacter { offset } => { + ExprParseError::Custom(format!("unexpected character at byte offset {}", offset)) + } + LexicalError::IntegerOverflow { text, .. } => { + ExprParseError::Custom(format!("integer literal `{text}` is out of range for i64")) + } + })?; + let mut parser = ExprParser::new(&tokens); - parser.parse_expr() + let expr = parser.parse_expr()?; + + // Reject leftover tokens (e.g. from a stray character that was dropped + // or from genuinely malformed input like `a b`). + if parser.pos() < tokens.len() { + return Err(ExprParseError::UnexpectedToken { + found: tokens[parser.pos()].kind.clone(), + expected: &["end of expression"], + }); + } + + Ok(expr) } // --------------------------------------------------------------------------- diff --git a/kitlang/src/codegen/parser/expr_pratt/primary.rs b/kitlang/src/codegen/parser/expr_pratt/primary.rs index a2029f9..584401e 100644 --- a/kitlang/src/codegen/parser/expr_pratt/primary.rs +++ b/kitlang/src/codegen/parser/expr_pratt/primary.rs @@ -126,22 +126,42 @@ impl<'a> ExprParser<'a> { ty: TypeId::default(), }) } - _ => Err(ExprParseError::UnexpectedToken { - found: tok, - expected: &[ - "integer literal", - "float literal", - "string literal", - "char literal", - "identifier", - "`(`", - "`[`", - "`if`", - "`null`", - "`true`", - "`false`", - ], - }), + _ => { + if self.at_eof() { + Err(ExprParseError::UnexpectedEof { + expected: &[ + "integer literal", + "float literal", + "string literal", + "char literal", + "identifier", + "`(`", + "`[`", + "`if`", + "`null`", + "`true`", + "`false`", + ], + }) + } else { + Err(ExprParseError::UnexpectedToken { + found: tok, + expected: &[ + "integer literal", + "float literal", + "string literal", + "char literal", + "identifier", + "`(`", + "`[`", + "`if`", + "`null`", + "`true`", + "`false`", + ], + }) + } + } } } @@ -179,7 +199,7 @@ impl<'a> ExprParser<'a> { /// Parse a function call postfix: `(arg1, arg2, ...)`. fn parse_call(&mut self, callee: Expr) -> Result { - let callee_name = expr_to_callee_name(&callee); + let callee_name = expr_to_callee_name(&callee)?; self.advance(); // consume `(` let args = self.parse_comma_list(Tok::RParen, |p| p.parse_expr())?; diff --git a/kitlang/src/codegen/parser/expr_pratt/tests.rs b/kitlang/src/codegen/parser/expr_pratt/tests.rs index fce4b29..7c12021 100644 --- a/kitlang/src/codegen/parser/expr_pratt/tests.rs +++ b/kitlang/src/codegen/parser/expr_pratt/tests.rs @@ -500,3 +500,111 @@ fn missing_field_name() { fn missing_else() { p_err("if a then b", "`else`"); } + +#[test] +fn trailing_tokens_produce_error() { + p_err("a b", "end of expression"); +} + +#[test] +fn unrecognized_characters_produce_error() { + let err = parse_kit_expr("a $ b").err().expect("should error on $"); + let msg = err.to_human_message(); + assert!(msg.contains("unexpected character"), "msg: {msg}"); +} + +#[test] +fn integer_overflow_is_detected() { + let err = parse_kit_expr("99999999999999999999") + .err() + .expect("should error on overflow literal"); + let msg = err.to_human_message(); + assert!( + msg.contains("out of range"), + "expected 'out of range', got: {msg}" + ); +} + +#[test] +fn integer_overflow_as_left_operand_is_detected() { + let err = parse_kit_expr("99999999999999999999 + 1") + .err() + .expect("should error on overflow literal"); + let msg = err.to_human_message(); + assert!( + msg.contains("out of range"), + "expected 'out of range', got: {msg}" + ); +} + +#[test] +fn integer_overflow_as_right_operand_is_detected() { + let err = parse_kit_expr("1 + 99999999999999999999") + .err() + .expect("should error on overflow literal"); + let msg = err.to_human_message(); + assert!( + msg.contains("out of range"), + "expected 'out of range', got: {msg}" + ); +} + +#[test] +fn eof_uses_unexpected_eof_not_semi() { + let err = parse_kit_expr("(1 + 2") + .err() + .expect("should error on missing )"); + let msg = err.to_human_message(); + assert!( + msg.contains("end of expression"), + "expected 'end of expression', got: {msg}" + ); + // Also verify the existing test still works: + let missing_rparen = parse_kit_expr("(1 + 2").err().unwrap(); + assert!(missing_rparen.to_human_message().contains("`)`")); +} + +#[test] +fn indirect_call_errors_cleanly() { + let err = parse_kit_expr("f()()") + .err() + .expect("should error on indirect call"); + let msg = err.to_human_message(); + assert!( + msg.contains("indirect calls"), + "expected 'indirect calls' error, got: {msg}" + ); +} + +#[test] +fn postfix_increment_is_now_an_error() { + let err = parse_kit_expr("x++") + .err() + .expect("should error on trailing ++"); + let msg = err.to_human_message(); + assert!( + msg.contains("PlusPlus") || msg.contains("end of expression"), + "msg: {msg}" + ); +} + +#[test] +fn postfix_decrement_is_now_an_error() { + let err = parse_kit_expr("x--") + .err() + .expect("should error on trailing --"); + let msg = err.to_human_message(); + assert!( + msg.contains("MinusMinus") || msg.contains("end of expression"), + "msg: {msg}" + ); +} + +#[test] +fn sizeof_is_not_supported() { + let err = parse_kit_expr("sizeof(i32)") + .err() + .expect("sizeof should error"); + let msg = err.to_human_message(); + assert!(msg.contains("Sizeof"), "msg: {msg}"); +} diff --git a/kitlang/src/lexer.rs b/kitlang/src/lexer.rs index 7e44d90..be320b0 100644 --- a/kitlang/src/lexer.rs +++ b/kitlang/src/lexer.rs @@ -259,14 +259,43 @@ fn unescape_str(s: &str) -> String { out } -/// Tokenize source into `Vec`. Drops whitespace, comments, -/// and unrecognized characters. Returns tokens in source order; EOF is -/// end of vector. -pub fn tokenize(source: &str) -> Vec { - Tok::lexer(source) - .spanned() - .filter_map(|(res, span)| res.ok().map(|kind| SpannedTok { kind, span })) - .collect() +/// A lexical error encountered during tokenization. +#[derive(Debug, Clone)] +pub enum LexicalError { + /// An unrecognized character that matches no token pattern. + UnexpectedCharacter { offset: usize }, + /// An integer literal that overflows `i64`. + IntegerOverflow { + text: String, + #[allow(dead_code)] + offset: usize, + }, +} + +/// Tokenize source into `Vec`. Returns an error on the first +/// unrecognized character or overflowed literal. Whitespace and comments +/// are skipped silently. +pub fn tokenize(source: &str) -> Result, LexicalError> { + let mut tokens = Vec::new(); + for (res, span) in Tok::lexer(source).spanned() { + match res { + Ok(kind) => tokens.push(SpannedTok { kind, span }), + Err(()) => { + // Logos produces Err when either no regex matches OR a callback + // returns None (e.g. parse_int on overflow). Distinguish by + // checking if the rejected span is all digits. + let rejected = &source[span.clone()]; + if rejected.chars().all(|c| c.is_ascii_digit()) { + return Err(LexicalError::IntegerOverflow { + text: rejected.to_string(), + offset: span.start, + }); + } + return Err(LexicalError::UnexpectedCharacter { offset: span.start }); + } + } + } + Ok(tokens) } #[cfg(test)] @@ -274,7 +303,11 @@ mod tests { use super::*; fn kinds(source: &str) -> Vec { - tokenize(source).into_iter().map(|t| t.kind).collect() + tokenize(source) + .unwrap_or_else(|e| panic!("tokenize failed for `{source}`: {e:?}")) + .into_iter() + .map(|t| t.kind) + .collect() } #[test] @@ -357,7 +390,7 @@ mod tests { #[test] fn spans_are_byte_accurate() { - let toks = tokenize(" a + b "); + let toks = tokenize(" a + b ").unwrap(); assert_eq!(toks[0].span, 2..3); // 'a' assert_eq!(toks[1].span, 4..5); // '+' assert_eq!(toks[2].span, 6..7); // 'b' @@ -365,17 +398,42 @@ mod tests { #[test] fn empty_source_produces_no_tokens() { - assert!(tokenize("").is_empty()); + assert!(tokenize("").unwrap().is_empty()); } #[test] - fn unknown_characters_are_dropped() { - // `$` is not a Kit token; the lexer drops it. This mirrors how pest - // would surface the error at the grammar level, which is upstream - // of the Pratt parser and not its concern. - assert_eq!( - kinds("a $ b"), - vec![Tok::Ident("a".to_string()), Tok::Ident("b".to_string()),] + fn unknown_characters_are_errors() { + // `$` is not a Kit token; the lexer now surfaces an error instead of + // silently dropping it, so the Pratt parser can produce a diagnostic. + let err = tokenize("a $ b").unwrap_err(); + assert!(matches!( + err, + LexicalError::UnexpectedCharacter { offset: 2 } + )); + } + + #[test] + fn integer_overflow_produces_error() { + // Logos produces Err when parse_int returns None for overflow. + // Our tokenize catches this as IntegerOverflow. + let err = tokenize("99999999999999999999").unwrap_err(); + assert!( + matches!(&err, LexicalError::IntegerOverflow { text, .. } if text == "99999999999999999999") ); } + + #[test] + fn mixed_overflow_and_valid_tokens_produces_error() { + // Overflow literal causes a tokenize error before valid tokens are reached + let err = tokenize("99999999999999999999 + 1").unwrap_err(); + assert!(matches!(&err, LexicalError::IntegerOverflow { .. })); + } + + #[test] + fn overflow_does_not_affect_valid_expression_with_small_int() { + // "1 + 99999999999999999999" -- overflow is NOT the first token, + // so Logos produces Err at the overflow position after the `1` and `+` + let err = tokenize("1 + 99999999999999999999").unwrap_err(); + assert!(matches!(&err, LexicalError::IntegerOverflow { .. })); + } } From 617e89aee01215dc09bb0fe58ff37c7407a2cf28 Mon Sep 17 00:00:00 2001 From: winlogon Date: Sat, 11 Jul 2026 12:17:27 +0200 Subject: [PATCH 4/4] feat: add support for higher-order functions kitc now supports passing functions as first-class values, enabling higher-order programming patterns. Functions can be passed as arguments, returned from functions, and stored in variables. The parser, type system, and code generator were updated to handle function types throughout the compilation pipeline. Summary of changes: - Change Call expression callee from String to Box for indirect calls - Add Function type variant to support function-type values - Update Pratt parser to allow and parse indirect call expressions - Implement type inference for indirect calls via three paths: named, indirect, C interop - Register functions as first-class values in symbol table with function type - Add type unification logic for function types with parameter matching - Add function pointer parameter formatting with correct C declarator syntax - Support transpiling both named and indirect function calls - Remove parse-time rejection of indirect calls; validate at type-check time - Add test example demonstrating indirect function calls - Update parser tests to reflect indirect call support - Remove unused expr_to_callee_name function and replace with callee_name --- examples/indirect_call.kit | 13 + examples/indirect_call.kit.expected | 1 + kitc/tests/examples.rs | 5 + kitlang/src/codegen/ast.rs | 4 +- kitlang/src/codegen/inference.rs | 147 +++++++-- kitlang/src/codegen/parser/diagnostics.rs | 17 - kitlang/src/codegen/parser/expr_pratt/mod.rs | 95 ++---- .../src/codegen/parser/expr_pratt/primary.rs | 5 +- .../src/codegen/parser/expr_pratt/tests.rs | 118 ++++--- kitlang/src/codegen/parser/mod.rs | 40 ++- kitlang/src/codegen/transpile/enum_gen.rs | 13 +- kitlang/src/codegen/transpile/mod.rs | 312 +++++++++++------- kitlang/src/codegen/types.rs | 65 +++- 13 files changed, 542 insertions(+), 293 deletions(-) create mode 100644 examples/indirect_call.kit create mode 100644 examples/indirect_call.kit.expected diff --git a/examples/indirect_call.kit b/examples/indirect_call.kit new file mode 100644 index 0000000..a902425 --- /dev/null +++ b/examples/indirect_call.kit @@ -0,0 +1,13 @@ +include "stdio.h"; + +function f(i: Int): Float { + return i; +} + +function g(func: function (Int) -> Float, i: Int): Float { + return func(i); +} + +function main() { + printf("%.1f\n", g(f, 4)); +} diff --git a/examples/indirect_call.kit.expected b/examples/indirect_call.kit.expected new file mode 100644 index 0000000..5186d07 --- /dev/null +++ b/examples/indirect_call.kit.expected @@ -0,0 +1 @@ +4.0 diff --git a/kitc/tests/examples.rs b/kitc/tests/examples.rs index e834b47..8d7a4c8 100644 --- a/kitc/tests/examples.rs +++ b/kitc/tests/examples.rs @@ -384,3 +384,8 @@ fn test_typedef() -> Result<(), Box> { fn test_array_literal() -> Result<(), Box> { run_example_test("array_literal", None) } + +#[test] +fn test_indirect_call() -> Result<(), Box> { + run_example_test("indirect_call", None) +} diff --git a/kitlang/src/codegen/ast.rs b/kitlang/src/codegen/ast.rs index efc1fc3..6f2e3d5 100644 --- a/kitlang/src/codegen/ast.rs +++ b/kitlang/src/codegen/ast.rs @@ -197,8 +197,8 @@ pub enum Expr { Literal { value: Literal, ty: TypeId }, /// Function call. Call { - /// Name of the callee function. - callee: String, + /// Callee expression (any expression that evaluates to a callable type). + callee: Box, /// Arguments passed to the function. args: Vec, /// Inferred return type. diff --git a/kitlang/src/codegen/inference.rs b/kitlang/src/codegen/inference.rs index ff60728..f6610ee 100644 --- a/kitlang/src/codegen/inference.rs +++ b/kitlang/src/codegen/inference.rs @@ -5,6 +5,7 @@ use super::ast::{Block, Expr, Function, GlobalDecl, Literal, Program, Stmt}; use super::symbols::{EnumVariantInfo, SymbolTable}; use super::type_ast::{EnumDefinition, FieldInit, StructDefinition}; use super::types::{BinaryOperator, Type, TypeId, TypeStore, UnaryOperator}; +use crate::codegen::parser::expr_pratt::callee_name; use crate::error::{CompilationError, CompileResult}; use crate::type_err; @@ -21,10 +22,10 @@ fn set_expr_type(expr: &mut Expr, ty: TypeId) -> &mut Expr { | Expr::StructInit { ty: t, .. } | Expr::FieldAccess { ty: t, .. } | Expr::EnumVariant { ty: t, .. } - | Expr::EnumInit { ty: t, .. } => *t = ty, + | Expr::EnumInit { ty: t, .. } + | Expr::ArrayLiteral { ty: t, .. } + | Expr::Index { ty: t, .. } => *t = ty, Expr::RangeLiteral { .. } => {} - Expr::ArrayLiteral { ty: t, .. } => *t = ty, - Expr::Index { ty: t, .. } => *t = ty, } expr } @@ -193,9 +194,34 @@ impl TypeInferencer { self.symbols.pop_scope(); // Register function signature in symbol table - if let Some(ret_ty) = func.inferred_return { - let param_tys: Vec = func.params.iter().map(|p| p.ty).collect(); - self.symbols.define_function(&func.name, param_tys, ret_ty); + if let Some(ret_ty_id) = func.inferred_return { + let param_ids: Vec = func.params.iter().map(|p| p.ty).collect(); + self.symbols + .define_function(&func.name, param_ids.clone(), ret_ty_id); + + // Register function as a value (for higher-order calls like `g(f)`). + // Resolve TypeIds back to Type values since Type::Function stores by value. + let param_tys: Vec = param_ids + .iter() + .filter_map(|id| self.store.resolve(*id).ok()) + .collect(); + let ret_ty = self.store.resolve(ret_ty_id).ok(); + if param_tys.len() == param_ids.len() + && let Some(ret_ty) = ret_ty + { + let fn_ty = Type::Function { + param_tys, + ret_ty: Box::new(ret_ty), + }; + let fn_ty_id = self.store.new_known(fn_ty); + self.symbols.define_global(&func.name, fn_ty_id); + } else { + eprintln!( + "Warning: function '{}' is not usable as a first-class value \ + because a parameter or return type could not be resolved", + func.name + ); + } } Ok(()) @@ -350,10 +376,11 @@ impl TypeInferencer { fn is_call_enum_constructor(&self, expr: &Expr) -> bool { match expr { - Expr::Call { callee, .. } => self - .symbols - .lookup_enum_variant_by_simple_name(callee) - .is_some(), + Expr::Call { callee, .. } => callee_name(callee).is_some_and(|name| { + self.symbols + .lookup_enum_variant_by_simple_name(&name) + .is_some() + }), _ => false, } } @@ -436,9 +463,10 @@ impl TypeInferencer { let Expr::Call { callee, args, ty } = expr else { unreachable!("infer_enum_constructor_call called on non-Call"); }; + let callee_str = callee_name(callee).expect("guard ensures this is valid"); let variant_info = self .symbols - .lookup_enum_variant_by_simple_name(callee) + .lookup_enum_variant_by_simple_name(&callee_str) .expect("guard ensures this exists"); let args_clone = args.clone(); let enum_def = self.symbols.lookup_enum(&variant_info.enum_name).cloned(); @@ -475,17 +503,98 @@ impl TypeInferencer { let Expr::Call { callee, args, ty } = expr else { unreachable!("infer_function_call called on non-Call"); }; - let (param_tys, ret_ty) = if let Some(sig) = self.symbols.lookup_function(callee) { - sig - } else { + + // Named function: lookup by string name in symbol table. + if let Some(name) = callee_name(callee) + && let Some((param_tys, ret_ty)) = self.symbols.lookup_function(&name) + { + return self.infer_call_with_sig(&name, param_tys, ret_ty, args, ty); + } + + // Indirect call: infer callee type and check callability. + let mut infer_failed_on_name = false; + match self.infer_expr(callee) { + Ok(callee_ty_id) => { + if let Ok(callee_ty) = self.store.resolve(callee_ty_id) { + let sig = match &callee_ty { + Type::Function { param_tys, ret_ty } => Some((param_tys, ret_ty.as_ref())), + Type::Ptr(inner) => { + if let Type::Function { param_tys, ret_ty } = inner.as_ref() { + Some((param_tys, ret_ty.as_ref())) + } else { + None + } + } + _ => None, + }; + if let Some((param_tys, ret_ty)) = sig { + if args.len() != param_tys.len() { + return Err(type_err!( + "Function expects {} arguments, got {}", + param_tys.len(), + args.len() + )); + } + for (arg, param_ty) in args.iter_mut().zip(param_tys.iter()) { + let arg_ty = self.infer_expr(arg)?; + let param_ty_id = self.store.new_known(param_ty.clone()); + self.unify(arg_ty, param_ty_id)?; + } + let ret_ty_id = self.store.new_known((*ret_ty).clone()); + *ty = ret_ty_id; + return Ok(ret_ty_id); + } + // Resolved to a non-callable type. + return Err(type_err!("Cannot call a value of type {callee_ty:?}")); + } + } + Err(e) => { + // infer_expr failed. If the callee is a pure name (identifier or + // field-access chain), it may be an external symbol. Fall through to + // the C interop path. Otherwise propagate the error. + if callee_name(callee).is_none() { + return Err(e); + } + infer_failed_on_name = true; + } + } + + // C interop: only for names absent from the symbol table. + if let Some(name) = callee_name(callee) + && self.symbols.lookup_function(&name).is_none() + && self.symbols.lookup_global(&name).is_none() + { let void_ty = self.store.new_known(Type::Void); - (vec![], void_ty) + for arg in args.iter_mut() { + self.infer_expr(arg)?; + } + *ty = void_ty; + return Ok(void_ty); + } + + let msg = if infer_failed_on_name { + format!( + "Cannot call '{}': not a known function, global, or external symbol", + callee_name(callee).as_deref().unwrap_or("?") + ) + } else { + "Expression is not callable".to_string() }; + Err(type_err!("{msg}")) + } + /// Infer a call to a known function from the symbol table. + fn infer_call_with_sig( + &mut self, + name: &str, + param_tys: Vec, + ret_ty: TypeId, + args: &mut [Expr], + call_ty: &mut TypeId, + ) -> Result { if !param_tys.is_empty() && args.len() != param_tys.len() { return Err(type_err!( - "Function '{}' expects {} arguments, got {}", - callee, + "Function '{name}' expects {} arguments, got {}", param_tys.len(), args.len() )); @@ -503,7 +612,7 @@ impl TypeInferencer { } } - *ty = ret_ty; + *call_ty = ret_ty; Ok(ret_ty) } @@ -706,8 +815,6 @@ impl TypeInferencer { .map(|f| (f.name.clone(), f.annotation.clone(), f.default.clone())) .collect(); - let _ = struct_def; - // inject default values for missing optional fields for field_info in &field_infos { let field_name = &field_info.0; diff --git a/kitlang/src/codegen/parser/diagnostics.rs b/kitlang/src/codegen/parser/diagnostics.rs index badd1b6..d3dfb1c 100644 --- a/kitlang/src/codegen/parser/diagnostics.rs +++ b/kitlang/src/codegen/parser/diagnostics.rs @@ -81,7 +81,6 @@ impl fmt::Display for ExprParseError { /// use in `ExprParseError::UnexpectedToken.expected`. /// /// This is the single canonical match for all token-to-name mappings. -/// [`tok_name`] is a zero-cost wrapper that extracts the first element. pub(crate) fn expected_name(kind: &Tok) -> &'static [&'static str] { match kind { Tok::LParen => &["`(`"], @@ -151,12 +150,6 @@ pub(crate) fn expected_name(kind: &Tok) -> &'static [&'static str] { } } -/// Map a `Tok` kind to a short human-readable name for error messages. -/// Zero-cost wrapper around [`expected_name`]. -pub(crate) fn tok_name(kind: &Tok) -> &'static str { - expected_name(kind)[0] -} - #[cfg(test)] mod tests { use super::*; @@ -190,14 +183,4 @@ mod tests { let err = ExprParseError::Custom("nope".to_string()); assert_eq!(err.to_human_message(), "nope"); } - - #[test] - fn tok_name_is_stable() { - // Sanity check: a few representative names. - assert_eq!(tok_name(&Tok::Plus), "`+`"); - assert_eq!(tok_name(&Tok::LParen), "`(`"); - assert_eq!(tok_name(&Tok::KwIf), "`if`"); - assert_eq!(tok_name(&Tok::IntLit(0)), "integer literal"); - assert_eq!(tok_name(&Tok::StringLit(String::new())), "string literal"); - } } diff --git a/kitlang/src/codegen/parser/expr_pratt/mod.rs b/kitlang/src/codegen/parser/expr_pratt/mod.rs index 86218d5..abcaf86 100644 --- a/kitlang/src/codegen/parser/expr_pratt/mod.rs +++ b/kitlang/src/codegen/parser/expr_pratt/mod.rs @@ -1,32 +1,36 @@ //! Hand-written Pratt parser for Kit expressions. //! //! This module takes over expression parsing from the pest-based grammar. -//! Pest still handles the program, declaration, statement, and type-annotation -//! grammars. For every `Pair<'_, Rule>` whose rule is an expression (i.e., -//! the 13 precedence levels `expr -> assign -> logical_or -> ... -> primary`), -//! the parser hands off to [`ExprParser::parse_expr`]. +//! +//! Pest still handles the program, declaration, statement, and type-annotation grammars. For every +//! `Pair<'_, Rule>` whose rule is an expression (i.e. the 13 precedence levels `expr -> assign -> +//! logical_or -> ... -> primary`), the parser hands off to [`ExprParser::parse_expr`]. //! //! Pratt Parsing & Operator Precedence //! ----------------------------------- -//! The pest grammar used 13 mutually recursive functions (one per precedence -//! level), which overflowed the default 1 MB stack on Windows. The Pratt -//! parser uses a single function with a binding-power loop, dramatically -//! reducing recursive depth compared to the old 13-level grammar chain. -//! Parenthesized sub-expressions still recurse through `parse_primary` -//! and `parse_expr`, so stack depth is `O(parenthetical nesting)`. -//! Operator precedence is defined in [`binding_power::infix`], -//! [`binding_power::postfix`], [`binding_power::prefix`]. Each infix operator -//! has a (lbp, rbp) pair: `lbp < rbp` = right-associative (assignment), +//! The pest grammar used 13 mutually recursive functions (one per precedence level, which +//! overflowed the default 1 MB stack on Windows. +//! +//! The Pratt parser uses a single function with a binding-power loop, dramatically reducing +//! recursive depth compared to the old 13-level grammar chain. +//! +//! Parenthesized sub-expressions still recurse through `parse_primary` and `parse_expr`, so stack +//! depth is `O(parenthetical nesting)`. Operator precedence is defined in +//! [`binding_power::infix`], [`binding_power::postfix`], [`binding_power::prefix`]. +//! +//! Each infix operator has a (lbp, rbp) pair: `lbp < rbp` = right-associative (assignment), //! `lbp == rbp` = left-associative (most binary ops). //! //! Errors & Source Spans //! --------------------- //! All parse errors are values of [`ExprParseError`] (in `diagnostics.rs`). -//! The parser never prints, never allocates strings for error messages, -//! and never holds source-file identity. Conversion to -//! `CompilationError::ParseError(String)` happens at `PestExpr::parse` -//! in `parser/mod.rs`. The token stream carries byte ranges; the parser -//! uses them internally but does not currently attach them to AST nodes. +//! +//! The parser never prints, never allocates strings for error messages, and never holds +//! source-file identity. Conversion to `Compilation::ParseError(String)` happens at +//! `PestExpr::parse` in `parser/mod.rs`. +//! +//! The token stream carries byte ranges; the parser uses them internally but does not currently +//! attach them to AST nodes. use crate::codegen::ast::{Expr, Literal}; use crate::codegen::type_ast::FieldInit; @@ -38,10 +42,6 @@ use super::binding_power::{ }; use super::diagnostics::{ExprParseError, expected_name}; -// --------------------------------------------------------------------------- -// Parser -// --------------------------------------------------------------------------- - /// A Pratt parser for Kit expressions. /// /// One instance is built per expression being parsed. The parser is @@ -86,15 +86,6 @@ impl<'a> ExprParser<'a> { self.tokens.get(self.pos).unwrap_or(&EOF) } - /// Peek the next token (one past the current). Same EOF behavior. - fn peek_next(&self) -> &SpannedTok { - static EOF: SpannedTok = SpannedTok { - kind: Tok::Semi, - span: 0..0, - }; - self.tokens.get(self.pos + 1).unwrap_or(&EOF) - } - /// Consume and return the current token. fn advance(&mut self) -> &SpannedTok { let tok = &self.tokens[self.pos]; @@ -130,7 +121,7 @@ impl<'a> ExprParser<'a> { // Postfix chain: field access, index, call. lhs = self.parse_postfix_chain(lhs)?; - // Infix operators (binary ops, range). + // Infix operators (binary ops, range) loop { let kind = self.peek().kind.clone(); let Some((lbp, rbp)) = infix(&kind) else { @@ -240,29 +231,18 @@ impl<'a> ExprParser<'a> { } } -/// Extract a callee name from a function-call's leading expression. -/// -/// For `Expr::Identifier { name, .. }` this is just `name`. -/// For `Expr::FieldAccess { expr, field_name, .. }` chains like -/// `pkg.math.add`, this concatenates the path with `.`. -/// Rejects non-trivial call targets (e.g. calling the result of another -/// call) with an error. -fn expr_to_callee_name(expr: &Expr) -> Result { +/// Extract a string callee name from an expression for name-mangling +/// and symbol-table lookup. Returns `None` for indirect calls that +/// must be resolved by the callee's inferred type. +pub(crate) fn callee_name(expr: &Expr) -> Option { match expr { - Expr::Identifier { name, .. } => Ok(name.clone()), + Expr::Identifier { name, .. } => Some(name.clone()), Expr::FieldAccess { expr: base, field_name, .. - } => { - let base_name = expr_to_callee_name(base)?; - Ok(format!("{base_name}.{field_name}")) - } - _ => Err(ExprParseError::Custom( - "indirect calls (calling the result of another expression) \ - are not supported by the Kit compiler" - .into(), - )), + } => Some(format!("{}.{}", callee_name(base)?, field_name)), + _ => None, } } @@ -270,12 +250,13 @@ fn expr_to_callee_name(expr: &Expr) -> Result { // Module surface: parse an expression from source text. // --------------------------------------------------------------------------- -/// Parse a Kit expression from source text. This is the public entry -/// point used by the pest-to-Pratt bridge (`PestExpr::parse`). +/// Parse a Kit expression from source text. This is the public entry point used by the +/// pest-to-Pratt bridge (`PestExpr::parse`). /// -/// The `text` should be the source text of the expression as a -/// `Pair::as_str()` slice. Tokenization, parsing, and conversion to an -/// `Expr` all happen here. +/// The `text` should be the source text of the expression as a `Pair::as_str()` slice. +/// Tokenization, parsing, and conversion to an `Expr` all happen here. +/// +/// # Errors /// /// Errors are returned for: /// - Unrecognized characters in the source @@ -307,13 +288,9 @@ pub(crate) fn parse_kit_expr(text: &str) -> Result { Ok(expr) } -// --------------------------------------------------------------------------- // Primary expression parsers (include'd into this module scope). -// --------------------------------------------------------------------------- include!("primary.rs"); -// --------------------------------------------------------------------------- // Tests (include'd into this module scope, test builds only). -// --------------------------------------------------------------------------- #[cfg(test)] include!("tests.rs"); diff --git a/kitlang/src/codegen/parser/expr_pratt/primary.rs b/kitlang/src/codegen/parser/expr_pratt/primary.rs index 584401e..06f92f8 100644 --- a/kitlang/src/codegen/parser/expr_pratt/primary.rs +++ b/kitlang/src/codegen/parser/expr_pratt/primary.rs @@ -198,13 +198,12 @@ impl<'a> ExprParser<'a> { } /// Parse a function call postfix: `(arg1, arg2, ...)`. + /// The callee is any expression; no rejection of indirect calls. fn parse_call(&mut self, callee: Expr) -> Result { - let callee_name = expr_to_callee_name(&callee)?; - self.advance(); // consume `(` let args = self.parse_comma_list(Tok::RParen, |p| p.parse_expr())?; Ok(Expr::Call { - callee: callee_name, + callee: Box::new(callee), args, ty: TypeId::default(), }) diff --git a/kitlang/src/codegen/parser/expr_pratt/tests.rs b/kitlang/src/codegen/parser/expr_pratt/tests.rs index 7c12021..d60bc25 100644 --- a/kitlang/src/codegen/parser/expr_pratt/tests.rs +++ b/kitlang/src/codegen/parser/expr_pratt/tests.rs @@ -3,12 +3,12 @@ use crate::codegen::types::{AssignmentOperator, BinaryOperator, UnaryOperator}; -/// Convenience: parse an expression and unwrap. +/// Parse an expression and unwrap fn p(text: &str) -> Expr { parse_kit_expr(text).unwrap_or_else(|e| panic!("parse failed for `{text}`: {e}")) } -/// Convenience: parse and assert the error contains a substring. +/// Parse and assert the error contains a substring. fn p_err(text: &str, needle: &str) { let err = parse_kit_expr(text) .err() @@ -22,6 +22,7 @@ fn p_err(text: &str, needle: &str) { // --- Literals --- +/// Integer literal `42` is parsed as `Literal::Int(42)`. #[test] fn integer_literal() { let e = p("42"); @@ -34,6 +35,7 @@ fn integer_literal() { )); } +/// Float literal `3.14` is parsed with correct precision. #[test] fn float_literal() { let e = p("3.14"); @@ -42,12 +44,14 @@ fn float_literal() { ); } +/// Double-quoted string `"hello"` is parsed as `Literal::String`. #[test] fn string_literal() { let e = p(r#""hello""#); assert!(matches!(e, Expr::Literal { value: Literal::String(s), .. } if s == "hello")); } +/// `true` and `false` both parse as `Literal::Bool`. #[test] fn bool_literals() { assert!(matches!( @@ -66,6 +70,7 @@ fn bool_literals() { )); } +/// `null` is parsed as `Literal::Null`. #[test] fn null_literal() { assert!(matches!( @@ -79,6 +84,7 @@ fn null_literal() { // --- Identifiers --- +/// Bare identifier `foo` is parsed as `Expr::Identifier`. #[test] fn identifier() { let e = p("foo"); @@ -107,6 +113,7 @@ fn qualified_identifier_is_built_via_postfix_chain() { // --- Precedence --- +/// `1 + 2 * 3` - `*` binds tighter than `+`. #[test] fn additive_vs_multiplicative() { let e = p("1 + 2 * 3"); @@ -122,6 +129,7 @@ fn additive_vs_multiplicative() { } } +/// `a == b < c` - `==` is looser than `<`, parsed as `(a == (b < c))`. #[test] fn comparison_vs_equality() { let e = p("a == b < c"); @@ -141,6 +149,7 @@ fn comparison_vs_equality() { } } +/// `1 + 2 + 3` - `+` is left-associative: `((1 + 2) + 3)`. #[test] fn left_associative_addition() { let e = p("1 + 2 + 3"); @@ -166,6 +175,7 @@ fn left_associative_addition() { } } +/// `a += b += c` - `+=` is right-associative: `(a += (b += c))`. #[test] fn right_associative_assignment() { let e = p("a += b += c"); @@ -181,6 +191,7 @@ fn right_associative_assignment() { } } +/// `-a + b` - prefix `-` binds tighter than `+`. #[test] fn unary_minus_binds_tighter_than_addition() { let e = p("-a + b"); @@ -202,6 +213,7 @@ fn unary_minus_binds_tighter_than_addition() { } } +/// `&arr[i]` - prefix `&` is looser than postfix `[]`, parsed as `&(arr[i])`. #[test] fn unary_looser_than_postfix() { let e = p("&arr[i]"); @@ -215,6 +227,7 @@ fn unary_looser_than_postfix() { // --- Postfix chains --- +/// `a.b.c.d.e` - field access chains to at least 4 levels. #[test] fn chained_field_access() { let e = p("a.b.c.d.e"); @@ -228,6 +241,7 @@ fn chained_field_access() { assert!(matches!(cur, Expr::Identifier { name, .. } if name == "a")); } +/// 100-level field-access chain `a.f0.f1...f99` does not overflow the stack. #[test] fn stress_deep_postfix_chain() { let mut src = String::from("a"); @@ -245,6 +259,7 @@ fn stress_deep_postfix_chain() { assert_eq!(depth, 100); } +/// 100 levels of nested parentheses are parsed correctly. #[test] fn stress_deep_nested_parens() { let mut src = String::new(); @@ -267,28 +282,31 @@ fn stress_deep_nested_parens() { // --- Function calls --- +/// `f()` - function call with zero arguments. #[test] fn call_no_args() { let e = p("f()"); if let Expr::Call { callee, args, .. } = &e { - assert_eq!(callee, "f"); + assert_eq!(callee_name(callee), Some("f".to_string())); assert!(args.is_empty()); } else { panic!("expected Call, got {e:?}"); } } +/// `f(1)` - function call with one argument. #[test] fn call_one_arg() { let e = p("f(1)"); if let Expr::Call { callee, args, .. } = &e { - assert_eq!(callee, "f"); + assert_eq!(callee_name(callee), Some("f".to_string())); assert_eq!(args.len(), 1); } else { panic!("expected Call, got {e:?}"); } } +/// `f(1, 2, 3, 4, 5)` - function call with five arguments. #[test] fn call_many_args() { let e = p("f(1, 2, 3, 4, 5)"); @@ -299,17 +317,19 @@ fn call_many_args() { } } +/// `pkg.math.add(2, 3)` - qualified name via field-access chain is the callee. #[test] fn call_qualified_name() { let e = p("pkg.math.add(2, 3)"); if let Expr::Call { callee, args, .. } = &e { - assert_eq!(callee, "pkg.math.add"); + assert_eq!(callee_name(callee), Some("pkg.math.add".to_string())); assert_eq!(args.len(), 2); } else { panic!("expected Call, got {e:?}"); } } +/// `f(g(1), h(2, 3))` - nested calls as arguments. #[test] fn call_with_nested_expressions_in_args() { let e = p("f(g(1), h(2, 3))"); @@ -322,6 +342,7 @@ fn call_with_nested_expressions_in_args() { // --- Indexing --- +/// `arr[0]` - index expression with identifier base and integer index. #[test] fn index() { let e = p("arr[0]"); @@ -339,6 +360,7 @@ fn index() { } } +/// `a[i][j]` - chained index produces two `Index` nodes. #[test] fn chained_index() { let e = p("a[i][j]"); @@ -353,6 +375,7 @@ fn chained_index() { // --- Array literals --- +/// `[]` - empty array literal. #[test] fn empty_array() { let e = p("[]"); @@ -363,6 +386,7 @@ fn empty_array() { } } +/// `[1, 2, 3]` - array literal with three elements. #[test] fn array_with_elements() { let e = p("[1, 2, 3]"); @@ -375,6 +399,7 @@ fn array_with_elements() { // --- Struct init --- +/// `struct Point { x: 10, y: 20 }` - struct init with two named fields. #[test] fn struct_init() { let e = p("struct Point { x: 10, y: 20 }"); @@ -389,6 +414,7 @@ fn struct_init() { // --- If expressions --- +/// `if a then b else c` - if expression with all three branches. #[test] fn if_expr() { let e = p("if a then b else c"); @@ -409,6 +435,7 @@ fn if_expr() { // --- Logical operators --- +/// `a || b && c` - `&&` binds tighter than `||`, parsed as `(a || (b && c))`. #[test] fn logical_and_vs_or() { let e = p("a || b && c"); @@ -428,6 +455,7 @@ fn logical_and_vs_or() { // --- Errors --- +/// `(1 + 2` - missing `)` produces error mentioning `)`. #[test] fn missing_rparen() { p_err("(1 + 2", "`)`"); @@ -435,6 +463,7 @@ fn missing_rparen() { // --- Range literals --- +/// `1...5` - range literal with integer start and end. #[test] fn range_literal_simple() { let e = p("1...5"); @@ -458,6 +487,7 @@ fn range_literal_simple() { } } +/// `a + 1...b - 1` - range bounds can be arbitrary expressions. #[test] fn range_literal_with_expressions() { let e = p("a + 1...b - 1"); @@ -481,43 +511,48 @@ fn range_literal_with_expressions() { } } +/// `arr[0` - missing `]` produces error mentioning `]`. #[test] fn missing_rbracket() { p_err("arr[0", "`]`"); } +/// `+` at start - parser rejects leading binary operator. #[test] fn unexpected_token_at_start() { p_err("+", "identifier"); } +/// `foo.` - trailing dot is rejected with expected identifier. #[test] fn missing_field_name() { p_err("foo.", "identifier"); } +/// `if a then b` - missing `else` produces error mentioning `else`. #[test] fn missing_else() { p_err("if a then b", "`else`"); } +/// `a b` - two adjacent expressions produce end-of-expression error. #[test] fn trailing_tokens_produce_error() { p_err("a b", "end of expression"); } +/// `a $ b` - `$` is an unrecognized character error. #[test] fn unrecognized_characters_produce_error() { - let err = parse_kit_expr("a $ b").err().expect("should error on $"); + let err = parse_kit_expr("a $ b").expect_err("should error on $"); let msg = err.to_human_message(); assert!(msg.contains("unexpected character"), "msg: {msg}"); } +/// Literal exceeding i64 range produces overflow error. #[test] fn integer_overflow_is_detected() { - let err = parse_kit_expr("99999999999999999999") - .err() - .expect("should error on overflow literal"); + let err = parse_kit_expr("99999999999999999999").expect_err("should error on overflow literal"); let msg = err.to_human_message(); assert!( msg.contains("out of range"), @@ -525,11 +560,11 @@ fn integer_overflow_is_detected() { ); } +/// Overflow in left operand of binary expression. #[test] fn integer_overflow_as_left_operand_is_detected() { - let err = parse_kit_expr("99999999999999999999 + 1") - .err() - .expect("should error on overflow literal"); + let err = + parse_kit_expr("99999999999999999999 + 1").expect_err("should error on overflow literal"); let msg = err.to_human_message(); assert!( msg.contains("out of range"), @@ -537,11 +572,11 @@ fn integer_overflow_as_left_operand_is_detected() { ); } +/// Overflow in right operand of binary expression. #[test] fn integer_overflow_as_right_operand_is_detected() { - let err = parse_kit_expr("1 + 99999999999999999999") - .err() - .expect("should error on overflow literal"); + let err = + parse_kit_expr("1 + 99999999999999999999").expect_err("should error on overflow literal"); let msg = err.to_human_message(); assert!( msg.contains("out of range"), @@ -549,38 +584,47 @@ fn integer_overflow_as_right_operand_is_detected() { ); } +/// Missing `)` uses `ExpectedEof` variant, not the `Semi` sentinel. #[test] fn eof_uses_unexpected_eof_not_semi() { - let err = parse_kit_expr("(1 + 2") - .err() - .expect("should error on missing )"); + let err = parse_kit_expr("(1 + 2").expect_err("should error on missing )"); let msg = err.to_human_message(); assert!( msg.contains("end of expression"), "expected 'end of expression', got: {msg}" ); // Also verify the existing test still works: - let missing_rparen = parse_kit_expr("(1 + 2").err().unwrap(); + let missing_rparen = parse_kit_expr("(1 + 2").expect_err("should error on missing )"); assert!(missing_rparen.to_human_message().contains("`)`")); } -#[test] -fn indirect_call_errors_cleanly() { - let err = parse_kit_expr("f()()") - .err() - .expect("should error on indirect call"); - let msg = err.to_human_message(); - assert!( - msg.contains("indirect calls"), - "expected 'indirect calls' error, got: {msg}" - ); +/// `f()()` - nested calls: result of `f()` is called with no args. +#[test] +fn indirect_call_is_parsed() { + let e = p("f()()"); + match &e { + Expr::Call { callee, args, .. } => { + assert!(args.is_empty(), "outer call should have no args"); + match callee.as_ref() { + Expr::Call { + callee: inner, + args: inner_args, + .. + } => { + assert_eq!(callee_name(inner), Some("f".to_string())); + assert!(inner_args.is_empty()); + } + _ => panic!("expected inner Call, got {callee:?}"), + } + } + _ => panic!("expected outer Call, got {e:?}"), + } } +/// `x++` - postfix `++` is rejected (not supported in Kit). #[test] fn postfix_increment_is_now_an_error() { - let err = parse_kit_expr("x++") - .err() - .expect("should error on trailing ++"); + let err = parse_kit_expr("x++").expect_err("should error on trailing ++"); let msg = err.to_human_message(); assert!( msg.contains("PlusPlus") || msg.contains("end of expression"), @@ -588,11 +632,10 @@ fn postfix_increment_is_now_an_error() { ); } +/// `x--` - postfix `--` is rejected (not supported in Kit). #[test] fn postfix_decrement_is_now_an_error() { - let err = parse_kit_expr("x--") - .err() - .expect("should error on trailing --"); + let err = parse_kit_expr("x--").expect_err("should error on trailing --"); let msg = err.to_human_message(); assert!( msg.contains("MinusMinus") || msg.contains("end of expression"), @@ -600,11 +643,10 @@ fn postfix_decrement_is_now_an_error() { ); } +/// `sizeof(i32)` - `sizeof` keyword is rejected (not supported in Kit). #[test] fn sizeof_is_not_supported() { - let err = parse_kit_expr("sizeof(i32)") - .err() - .expect("sizeof should error"); + let err = parse_kit_expr("sizeof(i32)").expect_err("sizeof should error"); let msg = err.to_human_message(); assert!(msg.contains("Sizeof"), "msg: {msg}"); } diff --git a/kitlang/src/codegen/parser/mod.rs b/kitlang/src/codegen/parser/mod.rs index ff3754c..e9c41d1 100644 --- a/kitlang/src/codegen/parser/mod.rs +++ b/kitlang/src/codegen/parser/mod.rs @@ -1,6 +1,6 @@ mod binding_power; mod diagnostics; -mod expr_pratt; +pub(crate) mod expr_pratt; use pest::iterators::Pair; @@ -168,14 +168,13 @@ impl Parser { args.push(MetaArg::Literal(Literal::String(val.to_string()))); } else if let Ok(n) = text.parse::() { args.push(MetaArg::Literal(Literal::Int(n))); - } else if text == "true" { - args.push(MetaArg::Literal(Literal::Bool(true))); - } else if text == "false" { - args.push(MetaArg::Literal(Literal::Bool(false))); - } else if text == "null" { - args.push(MetaArg::Literal(Literal::Null)); } else { - args.push(MetaArg::Identifier(text)); + match text.as_str() { + "true" => args.push(MetaArg::Literal(Literal::Bool(true))), + "false" => args.push(MetaArg::Literal(Literal::Bool(false))), + "null" => args.push(MetaArg::Literal(Literal::Null)), + _ => args.push(MetaArg::Identifier(text)), + } } } Ok(Metadata { name, args }) @@ -425,7 +424,7 @@ impl Parser { .next() .ok_or(parse_error!("trait impl missing 'for' type"))?, )?; - // For now, return a simple placeholder + // TODO: For now, return a simple placeholder Ok(ImplDefinition { name: String::new(), trait_type, @@ -694,7 +693,22 @@ impl Parser { let inner_ty = self.parse_type(inner_ptr_type)?; Ok(Type::Ptr(Box::new(inner_ty))) } - // TODO: Handle other type_annotation rules like function_type, tuple_type + Rule::function_type => { + let inner = inner_rule.into_inner(); + // All type_annotation pairs from the params (zero or more), + // followed by the return type as the last pair. + let mut type_pairs: Vec> = inner.collect(); + let ret_pair = type_pairs + .pop() + .ok_or_else(|| parse_error!("function_type missing return type"))?; + let ret_ty = self.parse_type(ret_pair)?; + let param_tys: Result, CompilationError> = + type_pairs.into_iter().map(|p| self.parse_type(p)).collect(); + Ok(Type::Function { + param_tys: param_tys?, + ret_ty: Box::new(ret_ty), + }) + } _ => Err(CompilationError::ParseError(format!( "Unexpected rule in type_annotation: {:?}", inner_rule.as_rule() @@ -799,10 +813,4 @@ impl Parser { )?; Ok(Stmt::For { var, iter, body }) } - - // The pest-based `parse_struct_init` and `parse_field_init` helpers - // were removed when the Pratt parser took over. Struct literals are - // now parsed from the token stream directly (see - // `ExprParser::parse_struct_init` in `expr_pratt.rs`), so the - // `Pair`-based versions in this `impl` are no longer reachable. } diff --git a/kitlang/src/codegen/transpile/enum_gen.rs b/kitlang/src/codegen/transpile/enum_gen.rs index dc60b57..8e8c4d0 100644 --- a/kitlang/src/codegen/transpile/enum_gen.rs +++ b/kitlang/src/codegen/transpile/enum_gen.rs @@ -4,7 +4,7 @@ use crate::codegen::ast::Attributed; use crate::codegen::module::ModulePath; use crate::codegen::name_mangling::{mangle_enum_variant, mangle_name}; use crate::codegen::type_ast::{EnumDefinition, EnumVariant, Field, StructDefinition}; -use crate::codegen::types::{ToCRepr, Type}; +use crate::codegen::types::Type; use super::CodegenCtx; @@ -34,9 +34,9 @@ impl CodegenCtx<'_> { .iter() .map(|field| { let ty = self.resolve_field_type(field); + let formatted = self.format_type_with_name(&ty, &field.name, &self.current_module); let prefix = if field.is_const { "const " } else { "" }; - let cname = self.type_to_c_name(&ty); - format!(" {}{} {};", prefix, cname, field.name) + format!(" {}{};", prefix, formatted) }) .collect(); @@ -127,7 +127,10 @@ impl CodegenCtx<'_> { .iter() .map(|arg| { let ty = self.resolve_field_type(arg); - format!(" {} {};", ty.to_c_repr().name, arg.name) + format!( + " {};", + self.format_type_with_name(&ty, &arg.name, &self.current_module) + ) }) .collect(); let _ = write!( @@ -178,7 +181,7 @@ impl CodegenCtx<'_> { .iter() .map(|arg| { let ty = self.resolve_field_type(arg); - format!("{} {}", ty.to_c_repr().name, arg.name) + self.format_type_with_name(&ty, &arg.name, &self.current_module) }) .collect(); let arg_names: Vec = v.args.iter().map(|arg| arg.name.clone()).collect(); diff --git a/kitlang/src/codegen/transpile/mod.rs b/kitlang/src/codegen/transpile/mod.rs index aef5027..607f206 100644 --- a/kitlang/src/codegen/transpile/mod.rs +++ b/kitlang/src/codegen/transpile/mod.rs @@ -9,6 +9,8 @@ use std::path::PathBuf; use crate::codegen::ast::{Attributed, Block, Expr, Function, GlobalDecl, Program, Stmt}; use crate::codegen::module::{ModulePath, ModuleRegistry}; use crate::codegen::name_mangling::{mangle_enum_variant, mangle_name}; +use crate::codegen::parser::expr_pratt::callee_name; +use crate::codegen::type_ast::FieldInit; use crate::codegen::types::{ToCRepr, Type, TypeId}; use super::ast::Param; @@ -125,9 +127,9 @@ impl CodegenCtx<'_> { | Expr::StructInit { ty, .. } | Expr::FieldAccess { ty, .. } | Expr::EnumVariant { ty, .. } - | Expr::EnumInit { ty, .. } => *ty, - Expr::ArrayLiteral { ty, .. } => *ty, - Expr::Index { ty, .. } => *ty, + | Expr::EnumInit { ty, .. } + | Expr::ArrayLiteral { ty, .. } + | Expr::Index { ty, .. } => *ty, Expr::RangeLiteral { .. } => TypeId::default(), } } @@ -403,6 +405,25 @@ impl CodegenCtx<'_> { self.format_function_params_with_module(params, &self.current_module) } + /// Format a type with a variable name, handling function-pointer declarator syntax. + /// Function/ptr-to-function types need `ret (*name)(params)` instead of `ret(*)(params) name`. + fn format_type_with_name(&self, ty: &Type, name: &str, module: &ModulePath) -> String { + if let Some(fp) = self.format_fn_ptr_param(ty, name, module) { + fp + } else { + // Resolve typedef aliases so the variable uses the underlying C type name. + let resolved = self + .inferencer + .store + .resolve_typedef_type(ty) + .unwrap_or_else(|| ty.clone()); + format!( + "{} {name}", + self.type_to_c_name_with_module(&resolved, module) + ) + } + } + /// Format a variable declaration with proper C syntax. /// /// For CArray types (e.g., `CArray(Int, 3)`), this produces `int name[3]` instead of the @@ -414,6 +435,7 @@ impl CodegenCtx<'_> { let elem_c_name = self.type_to_c_name(&elem_type); format!("{elem_c_name} {name}[{size}]") } + Ok(ref ty) => self.format_type_with_name(ty, name, &self.current_module), _ => { let ty_str = self.resolve_type_to_c_name(type_id, "int"); format!("{ty_str} {name}") @@ -425,6 +447,13 @@ impl CodegenCtx<'_> { params .iter() .map(|p| { + // C requires function pointer parameters to embed the name in + // the declarator: `ret (*name)(params)` instead of `ret(*)(params) name`. + if let Ok(ty) = self.inferencer.store.resolve(p.ty) + && let Some(fp) = self.format_fn_ptr_param(&ty, &p.name, module) + { + return fp; + } format!( "{} {}", self.format_function_param_type_with_module(p, module), @@ -435,6 +464,30 @@ impl CodegenCtx<'_> { .join(", ") } + /// If `ty` is a function type or pointer-to-function type, format it as a C + /// function pointer parameter declaration (`ret (*name)(params)`). Returns + /// `None` for non-function types. + fn format_fn_ptr_param(&self, ty: &Type, name: &str, module: &ModulePath) -> Option { + let (ret_ty, param_tys) = match ty { + Type::Function { param_tys, ret_ty } => (ret_ty.as_ref(), param_tys.as_slice()), + Type::Ptr(inner) if matches!(inner.as_ref(), Type::Function { .. }) => { + if let Type::Function { param_tys, ret_ty } = inner.as_ref() { + (ret_ty.as_ref(), param_tys.as_slice()) + } else { + return None; + } + } + _ => return None, + }; + let ret_c = self.type_to_c_name_with_module(ret_ty, module); + let params_c = param_tys + .iter() + .map(|t| self.type_to_c_name_with_module(t, module)) + .collect::>() + .join(", "); + Some(format!("{ret_c} (*{name})({params_c})")) + } + fn mangled_enum_variant(&self, enum_name: &str, variant_name: &str) -> String { let is_simple = self .inferencer @@ -453,15 +506,139 @@ impl CodegenCtx<'_> { } } + fn transpile_call(&self, callee: &Expr, args: &[Expr]) -> String { + if let Some(name) = callee_name(callee) { + self.transpile_named_call(&name, args) + } else { + let callee_c = self.transpile_expr(callee); + let a = args + .iter() + .map(|a| self.transpile_expr(a)) + .collect::>() + .join(", "); + format!("({})({})", callee_c, a) + } + } + + fn transpile_named_call(&self, name: &str, args: &[Expr]) -> String { + if let Some(info) = self + .inferencer + .symbols() + .lookup_enum_variant_by_simple_name(name) + { + let a = args + .iter() + .map(|a| self.transpile_expr(a)) + .collect::>() + .join(", "); + let ctor = + mangle_enum_variant(&self.current_module, &info.enum_name, &info.variant_name); + return format!("{}_new({})", ctor, a); + } + let (mod_path, base_name) = if let Some((mp, bn)) = self.resolve_function_name(name) { + (Some(mp), bn) + } else { + let last = name.rsplit('.').next().unwrap_or(name); + (None, last.to_string()) + }; + let mangled = if name == "main" { + name.to_string() + } else if let Some(mp) = &mod_path { + if has_no_mangle_in_module!(self.registry, mp, base_name.as_str(), functions) { + base_name + } else { + mangle_name(mp, &base_name) + } + } else if self.inferencer.symbols().lookup_function(name).is_some() + && !self.current_module.is_empty() + { + mangle_name(&self.current_module, name) + } else { + name.to_string() + }; + let a = args + .iter() + .map(|a| self.transpile_expr(a)) + .collect::>() + .join(", "); + format!("{mangled}({a})") + } + + fn transpile_field_access(&self, expr: &Expr, field_name: &str) -> String { + let container = self.transpile_expr(expr); + let container_ty = Self::expr_type_id(expr); + + if let Ok(Type::Named(type_name)) = self.inferencer.store.resolve(container_ty) + && let Some(enum_def) = self.inferencer.symbols().lookup_enum(&type_name) + && let Some(variant) = enum_def + .variants + .iter() + .find(|v| !v.args.is_empty() && v.args.iter().any(|a| a.name == *field_name)) + { + return format!( + "{}._variant.{}.{}", + container, + variant.name.to_lowercase(), + field_name + ); + } + format!("{}.{}", container, field_name) + } + + fn transpile_array_literal(&self, ty: TypeId, elements: &[Expr]) -> String { + let array_c_name = self + .inferencer + .store + .resolve(ty) + .ok() + .map(|t| self.type_to_c_name(&t)) + .unwrap_or_else(|| "int[]".to_string()); + let elems = elements + .iter() + .map(|e| self.transpile_expr(e)) + .collect::>() + .join(", "); + format!("({array_c_name}){{{elems}}}") + } + + fn transpile_struct_init(&self, ty: TypeId, fields: &[FieldInit]) -> String { + let name = match self.inferencer.store.resolve(ty) { + Ok(Type::Struct { name, .. } | Type::Named(name)) => name, + Ok(_) => "UNKNOWN_STRUCT".to_string(), + Err(e) => { + eprintln!("Warning: Failed to resolve struct type: {e}"); + "UNKNOWN_STRUCT".to_string() + } + }; + let mangled = mangle_name(&self.current_module, &name); + let inits = fields + .iter() + .map(|f| format!(".{} = {}", f.name, self.transpile_expr(&f.value))) + .collect::>() + .join(", "); + format!("(struct {mangled}){{{inits}}}") + } + fn transpile_expr(&self, expr: &Expr) -> String { match expr { Expr::Identifier { name, .. } => { if let Some(mod_path) = self.find_global_module(name) { + // Global variable reference. if has_no_mangle_in_module!(self.registry, &mod_path, name.as_str(), globals) { name.clone() } else { mangle_name(&mod_path, name) } + } else if let Some((mp, bn)) = self.resolve_function_name(name) + && self.inferencer.symbols().lookup_function(&bn).is_some() + { + // Function reference used as a value (e.g. `g(f)`). + // Reuse the call-path mangling for cross-module correctness + no_mangle. + if has_no_mangle_in_module!(self.registry, &mp, bn.as_str(), functions) { + bn + } else { + mangle_name(&mp, &bn) + } } else { name.clone() } @@ -472,66 +649,7 @@ impl CodegenCtx<'_> { }); lit.to_c_with_float(is_c_float) } - Expr::Call { callee, args, .. } => { - if let Some(info) = self - .inferencer - .symbols() - .lookup_enum_variant_by_simple_name(callee) - { - let a = args - .iter() - .map(|a| self.transpile_expr(a)) - .collect::>() - .join(", "); - let ctor = mangle_enum_variant( - &self.current_module, - &info.enum_name, - &info.variant_name, - ); - format!("{}_new({})", ctor, a) - } else { - // XXX: name resolution cascade - qualified name -> module-scoped -> bare (C interop) - let (mod_path, base_name) = - if let Some((mp, bn)) = self.resolve_function_name(callee) { - (Some(mp), bn) - } else { - let last = callee.rsplit('.').next().unwrap_or(callee); - (None, last.to_string()) - }; - - // XXX: 5-condition mangling ladder: - // 1. main is never mangled - // 2. extern/expose items skip mangling - // 3. known functions in non-empty module get module prefix - // 4. everything else passes through as-is (C interop) - let mangled = if callee == "main" { - callee.clone() - } else if let Some(mp) = mod_path { - if has_no_mangle_in_module!( - self.registry, - &mp, - base_name.as_str(), - functions - ) { - base_name.clone() - } else { - mangle_name(&mp, &base_name) - } - } else if self.inferencer.symbols().lookup_function(callee).is_some() - && !self.current_module.is_empty() - { - mangle_name(&self.current_module, callee) - } else { - callee.clone() - }; - let a = args - .iter() - .map(|a| self.transpile_expr(a)) - .collect::>() - .join(", "); - format!("{mangled}({a})") - } - } + Expr::Call { callee, args, .. } => self.transpile_call(callee, args), Expr::UnaryOp { op, expr, .. } => { format!("{}({})", op.to_c_str(), self.transpile_expr(expr)) } @@ -561,55 +679,10 @@ impl CodegenCtx<'_> { format!("({c} ? {t} : {e})") } Expr::RangeLiteral { .. } => "/* range literal */ 0".to_string(), - Expr::StructInit { - ty, - struct_type: _, - fields, - } => { - let name = match self.inferencer.store.resolve(*ty) { - Ok(Type::Struct { name, .. } | Type::Named(name)) => name, - Ok(_) => "UNKNOWN_STRUCT".to_string(), - Err(e) => { - eprintln!("Warning: Failed to resolve struct type: {}", e); - "UNKNOWN_STRUCT".to_string() - } - }; - let mangled = mangle_name(&self.current_module, &name); - let inits = fields - .iter() - .map(|f| format!(".{} = {}", f.name, self.transpile_expr(&f.value))) - .collect::>() - .join(", "); - format!("(struct {}){{{}}}", mangled, inits) - } + Expr::StructInit { ty, fields, .. } => self.transpile_struct_init(*ty, fields), Expr::FieldAccess { expr, field_name, .. - } => { - let container = self.transpile_expr(expr); - let container_ty = Self::expr_type_id(expr); - - // Try to resolve the inferred type of the container expression - if let Ok(Type::Named(type_name)) = self.inferencer.store.resolve(container_ty) - // We only care about named types (structs/enums), not primitives or generics - && let Some(enum_def) = self.inferencer.symbols().lookup_enum(&type_name) - // Ensure the named type is actually an enum in our symbol table - // and retrieve its definition - && let Some(variant) = enum_def.variants.iter().find(|v| { - // Look for a variant that has at least one field/argument - // and where any of those fields match the requested field name - !v.args.is_empty() && v.args.iter().any(|a| a.name == *field_name) - }) { - // If we found a matching enum variant + field, build a fully qualified access path: - // container -> variant (lowercased) -> field - return format!( - "{}._variant.{}.{}", - container, - variant.name.to_lowercase(), - field_name - ); - } - format!("{}.{}", container, field_name) - } + } => self.transpile_field_access(expr, field_name), Expr::Index { expr, index, .. } => { let container = self.transpile_expr(expr); let idx = self.transpile_expr(index); @@ -626,24 +699,7 @@ impl CodegenCtx<'_> { variant_name, .. } => self.mangled_enum_variant(enum_name, variant_name), - Expr::ArrayLiteral { elements, ty } => { - // Resolve the array type to get the element type name for the compound literal - let array_c_name = self - .inferencer - .store - .resolve(*ty) - .ok() - .map(|t| self.type_to_c_name(&t)) - .unwrap_or_else(|| "int[]".to_string()); - - // Construct the compound literal - let elems = elements - .iter() - .map(|e| self.transpile_expr(e)) - .collect::>() - .join(", "); - format!("({array_c_name}){{{elems}}}") - } + Expr::ArrayLiteral { elements, ty } => self.transpile_array_literal(*ty, elements), Expr::EnumInit { enum_name, variant_name, diff --git a/kitlang/src/codegen/types.rs b/kitlang/src/codegen/types.rs index bd42f16..debb9ee 100644 --- a/kitlang/src/codegen/types.rs +++ b/kitlang/src/codegen/types.rs @@ -224,7 +224,11 @@ impl TypeStore { if a == b && !matches!( a, - Type::Ptr(_) | Type::Tuple(_) | Type::CArray(..) | Type::Named(_) + Type::Ptr(_) + | Type::Tuple(_) + | Type::CArray(..) + | Type::Named(_) + | Type::Function { .. } ) { return Ok(()); @@ -238,7 +242,11 @@ impl TypeStore { if a_ty == b_ty && !matches!( a_ty, - Type::Ptr(_) | Type::Tuple(_) | Type::CArray(..) | Type::Named(_) + Type::Ptr(_) + | Type::Tuple(_) + | Type::CArray(..) + | Type::Named(_) + | Type::Function { .. } ) { return Ok(()); @@ -282,6 +290,30 @@ impl TypeStore { } } + // Function types: unify element-wise + ( + Type::Function { + param_tys: p1, + ret_ty: r1, + }, + Type::Function { + param_tys: p2, + ret_ty: r2, + }, + ) => { + if p1.len() != p2.len() { + return Err(format!( + "Cannot unify functions with different parameter counts: {} vs {}", + p1.len(), + p2.len() + )); + } + for (t1, t2) in p1.iter().zip(p2.iter()) { + self.unify_type_ids(t1.clone(), t2.clone())?; + } + self.unify_type_ids((**r1).clone(), (**r2).clone()) + } + // Numeric type promotion: allow mixed-width integer/float types to unify _ if Self::is_numeric(&a_ty) && Self::is_numeric(&b_ty) => Ok(()), @@ -320,9 +352,7 @@ impl TypeStore { } } -/// Represents a type in the Kit language. -/// -/// TODO: further description +/// A type in the Kit language: primitives, composites (struct/enum/tuple), references (pointers/named aliases), and function types. #[derive(Clone, Debug, PartialEq, Hash)] pub enum Type { /// User-defined named type (fallback for types not covered by other variants). @@ -374,6 +404,14 @@ pub enum Type { /// Field definitions for the struct. fields: Vec<(String, TypeId)>, }, + /// Function type (e.g., `function (Int) -> Float`). + /// Parameter and return types are stored by value. When needed for + /// unification, they are converted to TypeId via [`TypeStore::new_known`] + /// (see `unify_types` for the same pattern used by `Tuple`). + Function { + param_tys: Vec, + ret_ty: Box, + }, } impl Type { @@ -456,6 +494,23 @@ impl ToCRepr for Type { headers: elem_repr.headers, } } + Type::Function { param_tys, ret_ty } => { + let ret_repr = ret_ty.to_c_repr(); + let mut all_headers = ret_repr.headers.clone(); + let params: Vec = param_tys + .iter() + .map(|t| { + let r = t.to_c_repr(); + all_headers.extend(r.headers); + r.name + }) + .collect(); + CRepr { + name: format!("{}(*)({})", ret_repr.name, params.join(", ")), + declaration: None, + headers: all_headers, + } + } Type::Named(name) => simple_c_type(name, &[]), Type::Struct { name, fields: _ } => CRepr { name: format!("struct {}", name),