diff --git a/Cargo.lock b/Cargo.lock index 3d87a0e..5617c4e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -227,6 +227,12 @@ dependencies = [ "num-traits", ] +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + [[package]] name = "generic-array" version = "0.14.7" @@ -292,6 +298,7 @@ name = "kitlang" version = "0.1.0" dependencies = [ "log", + "logos", "pest", "pest_derive", "strum", @@ -312,6 +319,38 @@ version = "0.4.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "113b30b4cd05f7c06868fdb2854f66a7b9fece9a48425351cd532e810d74024f" +[[package]] +name = "logos" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb2c55a318a87600ea870ff8c2012148b44bf18b74fad48d0f835c38c7d07c5f" +dependencies = [ + "logos-derive", +] + +[[package]] +name = "logos-codegen" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "58b3ffaa284e1350d017a57d04ada118c4583cf260c8fb01e0fe28a2e9cf8970" +dependencies = [ + "fnv", + "proc-macro2", + "quote", + "regex-automata", + "regex-syntax", + "syn", +] + +[[package]] +name = "logos-derive" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52d3a9855747c17eaf4383823f135220716ab49bea5fbea7dd42cc9a92f8aa31" +dependencies = [ + "logos-codegen", +] + [[package]] name = "memchr" version = "2.8.1" diff --git a/examples/indirect_call.kit b/examples/indirect_call.kit new file mode 100644 index 0000000..a902425 --- /dev/null +++ b/examples/indirect_call.kit @@ -0,0 +1,13 @@ +include "stdio.h"; + +function f(i: Int): Float { + return i; +} + +function g(func: function (Int) -> Float, i: Int): Float { + return func(i); +} + +function main() { + printf("%.1f\n", g(f, 4)); +} diff --git a/examples/indirect_call.kit.expected b/examples/indirect_call.kit.expected new file mode 100644 index 0000000..5186d07 --- /dev/null +++ b/examples/indirect_call.kit.expected @@ -0,0 +1 @@ +4.0 diff --git a/kitc/tests/examples.rs b/kitc/tests/examples.rs index e834b47..8d7a4c8 100644 --- a/kitc/tests/examples.rs +++ b/kitc/tests/examples.rs @@ -384,3 +384,8 @@ fn test_typedef() -> Result<(), Box> { fn test_array_literal() -> Result<(), Box> { run_example_test("array_literal", None) } + +#[test] +fn test_indirect_call() -> Result<(), Box> { + run_example_test("indirect_call", None) +} diff --git a/kitlang/Cargo.toml b/kitlang/Cargo.toml index 97d6b27..84eb3f7 100644 --- a/kitlang/Cargo.toml +++ b/kitlang/Cargo.toml @@ -5,6 +5,7 @@ edition = "2024" [dependencies] log = "0.4.27" +logos = "0.16.1" pest = "2.8.0" pest_derive = "2.8.0" strum = { version = "0.28.0", features = ["derive"] } diff --git a/kitlang/src/codegen/ast.rs b/kitlang/src/codegen/ast.rs index efc1fc3..6f2e3d5 100644 --- a/kitlang/src/codegen/ast.rs +++ b/kitlang/src/codegen/ast.rs @@ -197,8 +197,8 @@ pub enum Expr { Literal { value: Literal, ty: TypeId }, /// Function call. Call { - /// Name of the callee function. - callee: String, + /// Callee expression (any expression that evaluates to a callable type). + callee: Box, /// Arguments passed to the function. args: Vec, /// Inferred return type. diff --git a/kitlang/src/codegen/inference.rs b/kitlang/src/codegen/inference.rs index e02e380..f6610ee 100644 --- a/kitlang/src/codegen/inference.rs +++ b/kitlang/src/codegen/inference.rs @@ -5,6 +5,7 @@ use super::ast::{Block, Expr, Function, GlobalDecl, Literal, Program, Stmt}; use super::symbols::{EnumVariantInfo, SymbolTable}; use super::type_ast::{EnumDefinition, FieldInit, StructDefinition}; use super::types::{BinaryOperator, Type, TypeId, TypeStore, UnaryOperator}; +use crate::codegen::parser::expr_pratt::callee_name; use crate::error::{CompilationError, CompileResult}; use crate::type_err; @@ -21,10 +22,10 @@ fn set_expr_type(expr: &mut Expr, ty: TypeId) -> &mut Expr { | Expr::StructInit { ty: t, .. } | Expr::FieldAccess { ty: t, .. } | Expr::EnumVariant { ty: t, .. } - | Expr::EnumInit { ty: t, .. } => *t = ty, + | Expr::EnumInit { ty: t, .. } + | Expr::ArrayLiteral { ty: t, .. } + | Expr::Index { ty: t, .. } => *t = ty, Expr::RangeLiteral { .. } => {} - Expr::ArrayLiteral { ty: t, .. } => *t = ty, - Expr::Index { ty: t, .. } => *t = ty, } expr } @@ -193,9 +194,34 @@ impl TypeInferencer { self.symbols.pop_scope(); // Register function signature in symbol table - if let Some(ret_ty) = func.inferred_return { - let param_tys: Vec = func.params.iter().map(|p| p.ty).collect(); - self.symbols.define_function(&func.name, param_tys, ret_ty); + if let Some(ret_ty_id) = func.inferred_return { + let param_ids: Vec = func.params.iter().map(|p| p.ty).collect(); + self.symbols + .define_function(&func.name, param_ids.clone(), ret_ty_id); + + // Register function as a value (for higher-order calls like `g(f)`). + // Resolve TypeIds back to Type values since Type::Function stores by value. + let param_tys: Vec = param_ids + .iter() + .filter_map(|id| self.store.resolve(*id).ok()) + .collect(); + let ret_ty = self.store.resolve(ret_ty_id).ok(); + if param_tys.len() == param_ids.len() + && let Some(ret_ty) = ret_ty + { + let fn_ty = Type::Function { + param_tys, + ret_ty: Box::new(ret_ty), + }; + let fn_ty_id = self.store.new_known(fn_ty); + self.symbols.define_global(&func.name, fn_ty_id); + } else { + eprintln!( + "Warning: function '{}' is not usable as a first-class value \ + because a parameter or return type could not be resolved", + func.name + ); + } } Ok(()) @@ -350,10 +376,11 @@ impl TypeInferencer { fn is_call_enum_constructor(&self, expr: &Expr) -> bool { match expr { - Expr::Call { callee, .. } => self - .symbols - .lookup_enum_variant_by_simple_name(callee) - .is_some(), + Expr::Call { callee, .. } => callee_name(callee).is_some_and(|name| { + self.symbols + .lookup_enum_variant_by_simple_name(&name) + .is_some() + }), _ => false, } } @@ -436,9 +463,10 @@ impl TypeInferencer { let Expr::Call { callee, args, ty } = expr else { unreachable!("infer_enum_constructor_call called on non-Call"); }; + let callee_str = callee_name(callee).expect("guard ensures this is valid"); let variant_info = self .symbols - .lookup_enum_variant_by_simple_name(callee) + .lookup_enum_variant_by_simple_name(&callee_str) .expect("guard ensures this exists"); let args_clone = args.clone(); let enum_def = self.symbols.lookup_enum(&variant_info.enum_name).cloned(); @@ -475,17 +503,98 @@ impl TypeInferencer { let Expr::Call { callee, args, ty } = expr else { unreachable!("infer_function_call called on non-Call"); }; - let (param_tys, ret_ty) = if let Some(sig) = self.symbols.lookup_function(callee) { - sig - } else { + + // Named function: lookup by string name in symbol table. + if let Some(name) = callee_name(callee) + && let Some((param_tys, ret_ty)) = self.symbols.lookup_function(&name) + { + return self.infer_call_with_sig(&name, param_tys, ret_ty, args, ty); + } + + // Indirect call: infer callee type and check callability. + let mut infer_failed_on_name = false; + match self.infer_expr(callee) { + Ok(callee_ty_id) => { + if let Ok(callee_ty) = self.store.resolve(callee_ty_id) { + let sig = match &callee_ty { + Type::Function { param_tys, ret_ty } => Some((param_tys, ret_ty.as_ref())), + Type::Ptr(inner) => { + if let Type::Function { param_tys, ret_ty } = inner.as_ref() { + Some((param_tys, ret_ty.as_ref())) + } else { + None + } + } + _ => None, + }; + if let Some((param_tys, ret_ty)) = sig { + if args.len() != param_tys.len() { + return Err(type_err!( + "Function expects {} arguments, got {}", + param_tys.len(), + args.len() + )); + } + for (arg, param_ty) in args.iter_mut().zip(param_tys.iter()) { + let arg_ty = self.infer_expr(arg)?; + let param_ty_id = self.store.new_known(param_ty.clone()); + self.unify(arg_ty, param_ty_id)?; + } + let ret_ty_id = self.store.new_known((*ret_ty).clone()); + *ty = ret_ty_id; + return Ok(ret_ty_id); + } + // Resolved to a non-callable type. + return Err(type_err!("Cannot call a value of type {callee_ty:?}")); + } + } + Err(e) => { + // infer_expr failed. If the callee is a pure name (identifier or + // field-access chain), it may be an external symbol. Fall through to + // the C interop path. Otherwise propagate the error. + if callee_name(callee).is_none() { + return Err(e); + } + infer_failed_on_name = true; + } + } + + // C interop: only for names absent from the symbol table. + if let Some(name) = callee_name(callee) + && self.symbols.lookup_function(&name).is_none() + && self.symbols.lookup_global(&name).is_none() + { let void_ty = self.store.new_known(Type::Void); - (vec![], void_ty) + for arg in args.iter_mut() { + self.infer_expr(arg)?; + } + *ty = void_ty; + return Ok(void_ty); + } + + let msg = if infer_failed_on_name { + format!( + "Cannot call '{}': not a known function, global, or external symbol", + callee_name(callee).as_deref().unwrap_or("?") + ) + } else { + "Expression is not callable".to_string() }; + Err(type_err!("{msg}")) + } + /// Infer a call to a known function from the symbol table. + fn infer_call_with_sig( + &mut self, + name: &str, + param_tys: Vec, + ret_ty: TypeId, + args: &mut [Expr], + call_ty: &mut TypeId, + ) -> Result { if !param_tys.is_empty() && args.len() != param_tys.len() { return Err(type_err!( - "Function '{}' expects {} arguments, got {}", - callee, + "Function '{name}' expects {} arguments, got {}", param_tys.len(), args.len() )); @@ -503,7 +612,7 @@ impl TypeInferencer { } } - *ty = ret_ty; + *call_ty = ret_ty; Ok(ret_ty) } @@ -706,8 +815,6 @@ impl TypeInferencer { .map(|f| (f.name.clone(), f.annotation.clone(), f.default.clone())) .collect(); - let _ = struct_def; - // inject default values for missing optional fields for field_info in &field_infos { let field_name = &field_info.0; @@ -947,9 +1054,7 @@ impl TypeInferencer { Type::CArray(elem_type, _) => self.store.new_known(*elem_type), Type::Ptr(inner) => self.store.new_known(*inner), _ => { - return Err(type_err!( - "Cannot index non-array type: {resolved:?}" - )); + return Err(type_err!("Cannot index non-array type: {resolved:?}")); } }; *ty = elem_ty; diff --git a/kitlang/src/codegen/parser/binding_power.rs b/kitlang/src/codegen/parser/binding_power.rs new file mode 100644 index 0000000..6df73a0 --- /dev/null +++ b/kitlang/src/codegen/parser/binding_power.rs @@ -0,0 +1,161 @@ +//! Operator binding powers and token-to-operator conversions. + +use crate::codegen::types::{AssignmentOperator, BinaryOperator, UnaryOperator}; +use crate::lexer::Tok; + +/// Infix binding power: (left_bp, right_bp). None = not an infix operator. +/// lbp < rbp = right-associative (assignment); lbp == rbp = left-associative. +pub fn infix(tok: &Tok) -> Option<(u8, u8)> { + match tok { + Tok::Assign + | Tok::PlusEq + | Tok::MinusEq + | Tok::StarEq + | Tok::SlashEq + | Tok::PercentEq + | Tok::AmpEq + | Tok::PipeEq + | Tok::CaretEq + | Tok::ShlEq + | Tok::ShrEq => Some((0, 1)), + Tok::Ellipsis => Some((2, 2)), + Tok::OrOr => Some((3, 4)), + Tok::AndAnd => Some((5, 6)), + Tok::Pipe => Some((7, 8)), + Tok::Caret => Some((9, 10)), + Tok::Amp => Some((11, 12)), + Tok::EqEq | Tok::NotEq => Some((13, 14)), + Tok::Lt | Tok::Gt | Tok::LtEq | Tok::GtEq => Some((15, 16)), + Tok::Shl | Tok::Shr => Some((17, 18)), + Tok::Plus | Tok::Minus => Some((19, 20)), + Tok::Star | Tok::Slash | Tok::Percent => Some((21, 22)), + _ => None, + } +} + +/// Postfix binding power. All postfix ops use a single bp higher than any infix op. +pub fn postfix(tok: &Tok) -> Option { + match tok { + Tok::Dot | Tok::LBracket | Tok::LParen => Some(27), + _ => None, + } +} + +/// Prefix (unary) binding power. Returns None for non-prefix tokens. +pub fn prefix(tok: &Tok) -> Option { + match tok { + Tok::Bang | Tok::Minus | Tok::Star | Tok::Amp | Tok::Tilde => Some(25), + _ => None, + } +} + +/// True if the token is the range operator `...`. +pub fn is_range_op(tok: &Tok) -> bool { + matches!(tok, Tok::Ellipsis) +} + +/// Convert a `Tok` to a `BinaryOperator`. Returns None for non-binary operators. +pub fn tok_to_binary_op(tok: &Tok) -> Option { + Some(match tok { + Tok::Plus => BinaryOperator::Add, + Tok::Minus => BinaryOperator::Sub, + Tok::Star => BinaryOperator::Mul, + Tok::Slash => BinaryOperator::Div, + Tok::Percent => BinaryOperator::Mod, + Tok::EqEq => BinaryOperator::Eq, + Tok::NotEq => BinaryOperator::Ne, + Tok::Lt => BinaryOperator::Lt, + Tok::Gt => BinaryOperator::Gt, + Tok::LtEq => BinaryOperator::Le, + Tok::GtEq => BinaryOperator::Ge, + Tok::AndAnd => BinaryOperator::And, + Tok::OrOr => BinaryOperator::Or, + Tok::Amp => BinaryOperator::BitAnd, + Tok::Pipe => BinaryOperator::BitOr, + Tok::Caret => BinaryOperator::BitXor, + Tok::Shl => BinaryOperator::Shl, + Tok::Shr => BinaryOperator::Shr, + _ => return None, + }) +} + +/// Convert a `Tok` to an `AssignmentOperator`. +pub fn tok_to_assign_op(tok: &Tok) -> Option { + Some(match tok { + Tok::Assign => AssignmentOperator::Assign, + Tok::PlusEq => AssignmentOperator::AddAssign, + Tok::MinusEq => AssignmentOperator::SubAssign, + Tok::StarEq => AssignmentOperator::MulAssign, + Tok::SlashEq => AssignmentOperator::DivAssign, + Tok::PercentEq => AssignmentOperator::ModAssign, + Tok::AmpEq => AssignmentOperator::AndAssign, + Tok::PipeEq => AssignmentOperator::OrAssign, + Tok::CaretEq => AssignmentOperator::XorAssign, + Tok::ShlEq => AssignmentOperator::ShlAssign, + Tok::ShrEq => AssignmentOperator::ShrAssign, + _ => return None, + }) +} + +/// Convert a `Tok` to a `UnaryOperator`. +pub fn tok_to_unary_op(tok: &Tok) -> Option { + Some(match tok { + Tok::Bang => UnaryOperator::Not, + Tok::Minus => UnaryOperator::Neg, + Tok::Tilde => UnaryOperator::BitNot, + Tok::Amp => UnaryOperator::AddressOf, + Tok::Star => UnaryOperator::Dereference, + _ => return None, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::lexer::Tok; + + #[test] + fn infix_precedence_order() { + assert!(infix(&Tok::Assign).unwrap().0 < infix(&Tok::OrOr).unwrap().0); + assert!(infix(&Tok::OrOr).unwrap().0 < infix(&Tok::AndAnd).unwrap().0); + assert!(infix(&Tok::AndAnd).unwrap().0 < infix(&Tok::EqEq).unwrap().0); + assert!(infix(&Tok::EqEq).unwrap().0 < infix(&Tok::Plus).unwrap().0); + assert!(infix(&Tok::Plus).unwrap().0 < infix(&Tok::Star).unwrap().0); + } + + #[test] + fn assignment_is_right_associative() { + let (lbp, rbp) = infix(&Tok::Assign).unwrap(); + assert!(lbp < rbp); + } + + #[test] + fn additive_is_left_associative() { + // In Pratt parsers, left-associative operators have lbp < rbp + // (right operand binds tighter, so a+b+c = (a+b)+c). + let (lbp, rbp) = infix(&Tok::Plus).unwrap(); + assert!(lbp < rbp); + } + + #[test] + fn postfix_higher_than_infix() { + assert!(postfix(&Tok::Dot).unwrap() > infix(&Tok::Star).unwrap().1); + } + + #[test] + fn prefix_between_postfix_and_infix() { + assert!(prefix(&Tok::Minus).unwrap() > infix(&Tok::Star).unwrap().1); + assert!(prefix(&Tok::Minus).unwrap() < postfix(&Tok::Dot).unwrap()); + } + + #[test] + fn token_conversions() { + assert_eq!(tok_to_binary_op(&Tok::Plus), Some(BinaryOperator::Add)); + assert_eq!( + tok_to_assign_op(&Tok::PlusEq), + Some(AssignmentOperator::AddAssign) + ); + assert_eq!(tok_to_unary_op(&Tok::Minus), Some(UnaryOperator::Neg)); + assert_eq!(tok_to_binary_op(&Tok::Assign), None); + } +} diff --git a/kitlang/src/codegen/parser/diagnostics.rs b/kitlang/src/codegen/parser/diagnostics.rs new file mode 100644 index 0000000..d3dfb1c --- /dev/null +++ b/kitlang/src/codegen/parser/diagnostics.rs @@ -0,0 +1,186 @@ +//! Parser diagnostics: the internal error type used by the Pratt parser. +//! +//! This module exists to be the *single seam* through which future diagnostic +//! improvements (span attachment, "expected one of" rendering, error recovery) +//! can be added without touching the Pratt parser internals. +//! +//! Design rules: +//! 1. The Pratt parser produces `ExprParseError` values only. It never +//! formats messages, never prints, never holds source files. +//! 2. The conversion to the public `CompilationError` happens at exactly +//! one call site: the `PestExpr::parse` adapter in `parser/mod.rs`. +//! 3. Adding span support later is a per-variant field addition. The Pratt +//! parser's control flow does not change. +//! +//! Span-free in v1 by design. The variant shapes anticipate the addition +//! of a `span: Span` field; today, the conversion site synthesizes a +//! span-less error and a string message. + +use std::fmt; + +use crate::lexer::Tok; + +/// The internal error type produced by the Pratt parser. +/// +/// Every variant carries a `&'static [&'static str]` of "expected" token +/// names where applicable, so a future diagnostic system can render +/// "expected one of: ..." with no extra work. Today, [`to_human_message`] +/// joins these into a string for `CompilationError::ParseError`. +#[derive(Debug, Clone)] +pub(crate) enum ExprParseError { + /// A token was found where it was not expected. + UnexpectedToken { + /// The token that was actually present. + found: Tok, + /// Human-readable names of what would have been acceptable. + expected: &'static [&'static str], + }, + /// The token stream ended before the expression was complete. + UnexpectedEof { + /// Human-readable names of what would have been acceptable. + expected: &'static [&'static str], + }, + /// A free-form parser-internal error. Used for cases that don't fit the "wrong token" / "ran + /// out" shape (e.g. type annotations that look like expressions but aren't). + Custom(String), +} + +impl ExprParseError { + /// Render this error as a human-readable string. + /// + /// This is the *only* place that turns structured error data into a + /// string. + /// + /// TODO: replace this with a structured rendering (e.g. a `miette::Report`) without changing + /// the parser. + pub(crate) fn to_human_message(&self) -> String { + match self { + Self::UnexpectedToken { found, expected } => { + let expected = expected.join(", "); + format!("unexpected token `{found:?}`, expected one of: {expected}") + } + Self::UnexpectedEof { expected } => { + format!( + "unexpected end of expression, expected one of: {}", + expected.join(", ") + ) + } + Self::Custom(msg) => msg.clone(), + } + } +} + +impl fmt::Display for ExprParseError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&self.to_human_message()) + } +} + +/// Map a `Tok` kind to a short human-readable token name, as a static +/// slice of one element. Returns `&'static [&'static str]` for direct +/// use in `ExprParseError::UnexpectedToken.expected`. +/// +/// This is the single canonical match for all token-to-name mappings. +pub(crate) fn expected_name(kind: &Tok) -> &'static [&'static str] { + match kind { + Tok::LParen => &["`(`"], + Tok::RParen => &["`)`"], + Tok::LBracket => &["`[`"], + Tok::RBracket => &["`]`"], + Tok::LBrace => &["`{`"], + Tok::RBrace => &["`}`"], + Tok::Comma => &["`,`"], + Tok::Semi => &["`;`"], + Tok::Dot => &["`.`"], + Tok::Colon => &["`:`"], + Tok::Ellipsis => &["`...`"], + Tok::Plus => &["`+`"], + Tok::Minus => &["`-`"], + Tok::Star => &["`*`"], + Tok::Slash => &["`/`"], + Tok::Percent => &["`%`"], + Tok::EqEq => &["`==`"], + Tok::NotEq => &["`!=`"], + Tok::LtEq => &["`<=`"], + Tok::GtEq => &["`>=`"], + Tok::Lt => &["`<`"], + Tok::Gt => &["`>`"], + Tok::AndAnd => &["`&&`"], + Tok::OrOr => &["`||`"], + Tok::Bang => &["`!`"], + Tok::Amp => &["`&`"], + Tok::Pipe => &["`|`"], + Tok::Caret => &["`^`"], + Tok::Tilde => &["`~`"], + Tok::Shl => &["`<<`"], + Tok::Shr => &["`>>`"], + Tok::PlusPlus => &["`++`"], + Tok::MinusMinus => &["`--`"], + Tok::ShlEq => &["`<<=`"], + Tok::ShrEq => &["`>>=`"], + Tok::PlusEq => &["`+=`"], + Tok::MinusEq => &["`-=`"], + Tok::StarEq => &["`*=`"], + Tok::SlashEq => &["`/=`"], + Tok::PercentEq => &["`%=`"], + Tok::AmpEq => &["`&=`"], + Tok::PipeEq => &["`|=`"], + Tok::CaretEq => &["`^=`"], + Tok::Assign => &["`=`"], + Tok::KwIf => &["`if`"], + Tok::KwThen => &["`then`"], + Tok::KwElse => &["`else`"], + Tok::KwTrue => &["`true`"], + Tok::KwFalse => &["`false`"], + Tok::KwNull => &["`null`"], + Tok::KwThis => &["`this`"], + Tok::KwSelf => &["`Self`"], + Tok::KwSizeof => &["`sizeof`"], + Tok::KwDefined => &["`defined`"], + Tok::KwUnsafe => &["`unsafe`"], + Tok::KwStatic => &["`static`"], + Tok::KwImplicit => &["`implicit`"], + Tok::KwEmpty => &["`empty`"], + Tok::KwStruct => &["`struct`"], + Tok::IntLit(_) => &["integer literal"], + Tok::FloatLit(_) => &["float literal"], + Tok::CharLit(_) => &["char literal"], + Tok::StringLit(_) => &["string literal"], + Tok::Ident(_) => &["identifier"], + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn unexpected_token_renders_expected_list() { + let err = ExprParseError::UnexpectedToken { + found: Tok::Plus, + expected: &["identifier", "integer literal", "string literal"], + }; + let msg = err.to_human_message(); + assert!(msg.contains("Plus"), "msg: {msg}"); + assert!(msg.contains("identifier"), "msg: {msg}"); + assert!(msg.contains("integer literal"), "msg: {msg}"); + assert!(msg.contains("string literal"), "msg: {msg}"); + } + + #[test] + fn unexpected_eof_renders_expected_list() { + let err = ExprParseError::UnexpectedEof { + expected: &[")", ",", "binary operator"], + }; + let msg = err.to_human_message(); + assert!(msg.contains("end of expression"), "msg: {msg}"); + assert!(msg.contains(")"), "msg: {msg}"); + assert!(msg.contains(","), "msg: {msg}"); + } + + #[test] + fn custom_message_passes_through() { + let err = ExprParseError::Custom("nope".to_string()); + assert_eq!(err.to_human_message(), "nope"); + } +} diff --git a/kitlang/src/codegen/parser/expr.rs b/kitlang/src/codegen/parser/expr.rs deleted file mode 100644 index ea1fd88..0000000 --- a/kitlang/src/codegen/parser/expr.rs +++ /dev/null @@ -1,429 +0,0 @@ -use std::str::FromStr; - -use pest::iterators::Pair; - -use crate::codegen::ast::{Expr, Literal}; -use crate::codegen::types::{AssignmentOperator, BinaryOperator, TypeId, UnaryOperator}; -use crate::error::{CompilationError, CompileResult}; -use crate::{Rule, parse_error}; - -use super::Parser; -use crate::codegen::type_ast::FieldInit; - -impl Parser { - pub(super) fn parse_expr(self, pair: Pair) -> CompileResult { - match pair.as_rule() { - Rule::expr => { - let inner = pair - .into_inner() - .next() - .ok_or_else(|| parse_error!("expr pair is empty"))?; - self.parse_expr(inner) - } - Rule::assign => self.parse_assign_expr(pair), - Rule::logical_or - | Rule::logical_and - | Rule::equality - | Rule::comparison - | Rule::additive - | Rule::multiplicative - | Rule::bitwise_or - | Rule::bitwise_xor - | Rule::bitwise_and - | Rule::shift => { - let mut inner = pair.into_inner(); - let mut left = self.parse_expr( - inner - .next() - .ok_or_else(|| parse_error!("binary op missing left operand"))?, - )?; - - while let Some(op_pair) = inner.next() { - let op = BinaryOperator::from_rule_pair(&op_pair)?; - let right = self.parse_expr( - inner - .next() - .ok_or_else(|| parse_error!("binary op missing right operand"))?, - )?; - left = Expr::BinaryOp { - op, - left: Box::new(left), - right: Box::new(right), - ty: TypeId::default(), - }; - } - Ok(left) - } - Rule::unary => { - let mut inner_pairs = pair.into_inner(); - let first_pair = inner_pairs - .next() - .ok_or_else(|| parse_error!("unary expression missing operand"))?; - match first_pair.as_rule() { - Rule::unary_op => { - let op_str = first_pair.as_str(); - let op = UnaryOperator::from_str(op_str) - .map_err(|_| parse_error!("invalid unary operation: {op_str}"))?; - let expr = self.parse_expr(inner_pairs.next().ok_or_else(|| { - parse_error!("unary expr missing operand after operator") - })?)?; - Ok(Expr::UnaryOp { - op, - expr: Box::new(expr), - ty: TypeId::default(), - }) - } - Rule::ADDRESS_OF_OP => { - let op = UnaryOperator::AddressOf; - let expr = self.parse_expr(inner_pairs.next().ok_or_else(|| { - parse_error!("unary expr missing operand after operator") - })?)?; - Ok(Expr::UnaryOp { - op, - expr: Box::new(expr), - ty: TypeId::default(), - }) - } - Rule::postfix | Rule::primary => self.parse_expr(first_pair), - other => Err(parse_error!("Unexpected rule in unary: {other:?}")), - } - } - Rule::identifier => Ok(Expr::Identifier { - name: Self::pair_text(pair), - ty: TypeId::default(), - }), - Rule::literal => { - let inner = pair - .into_inner() - .next() - .ok_or_else(|| parse_error!("literal is empty"))?; - - // Dispatch to the kind-specific literal parser - match inner.as_rule() { - Rule::number => { - let num_pair = inner - .into_inner() - .next() - .ok_or_else(|| parse_error!("number literal is empty"))?; - - // Dispatch to integer or float parser by number subtype - match num_pair.as_rule() { - Rule::integer => { - let s = num_pair.as_str(); - let i = s.parse::().map_err(|e| { - parse_error!("invalid integer literal '{s}': {:?}", e) - })?; - Ok(Expr::Literal { - value: Literal::Int(i), - ty: TypeId::default(), - }) - } - Rule::float => { - let s = num_pair.as_str(); - let f = s.parse::().map_err(|e| { - parse_error!("invalid float literal '{s}': {:?}", e) - })?; - Ok(Expr::Literal { - value: Literal::Float(f), - ty: TypeId::default(), - }) - } - _ => Err(parse_error!("Unexpected number type")), - } - } - Rule::boolean => Self::parse_bool_literal(inner.as_str()), - Rule::char_literal => { - let s = inner.as_str(); - let inner_content = &s[1..s.len() - 1]; - let c = Self::unescape(inner_content) - .and_then(|u| u.chars().next()) - .ok_or_else(|| parse_error!("invalid char literal: {s}"))?; - Ok(Expr::Literal { - value: Literal::Char(c), - ty: TypeId::default(), - }) - } - _ => Err(parse_error!( - "Unexpected literal type: {:?}", - inner.as_rule() - )), - } - } - Rule::string => { - let full = pair.as_str(); - let inner = &full[1..full.len() - 1]; - let unescaped = Self::unescape(inner).unwrap_or_else(|| inner.to_string()); - Ok(Expr::Literal { - value: Literal::String(unescaped), - ty: TypeId::default(), - }) - } - Rule::function_call_expr => { - let mut inner = pair.into_inner(); - let callee = Self::pair_text( - inner - .next() - .ok_or_else(|| parse_error!("function call missing callee"))?, - ); - let args = inner - .filter(|p: &Pair| p.as_rule() == Rule::expr) - .map(|p: Pair| self.parse_expr(p)) - .collect::, _>>()?; - Ok(Expr::Call { - callee, - args, - ty: TypeId::default(), - }) - } - Rule::if_expr => { - let mut inner = pair.into_inner(); - - let cond = self.parse_expr( - inner - .next() - .ok_or_else(|| parse_error!("if expr missing condition"))?, - )?; - - let then_branch = self.parse_expr( - inner - .next() - .ok_or_else(|| parse_error!("if expr missing then branch"))?, - )?; - - let else_branch = self.parse_expr( - inner - .next() - .ok_or_else(|| parse_error!("if expr missing else branch"))?, - )?; - - Ok(Expr::If { - cond: Box::new(cond), - then_branch: Box::new(then_branch), - else_branch: Box::new(else_branch), - ty: TypeId::default(), - }) - } - // Handle bare keyword primaries (null/true/false) and wrapped primaries that delegate inward - Rule::primary => { - let text = pair.as_str(); - let mut inner = pair.into_inner(); - if inner.peek().is_none() { - match text { - "null" => Ok(Expr::Literal { - value: Literal::Null, - ty: TypeId::default(), - }), - "true" | "false" => Self::parse_bool_literal(text), - other => Err(parse_error!("Unknown primary keyword: {}", other)), - } - } else { - let inner_pair = inner - .next() - .ok_or_else(|| parse_error!("primary expr is empty"))?; - match inner_pair.as_rule() { - Rule::identifier => Ok(Expr::Identifier { - name: Self::pair_text(inner_pair), - ty: TypeId::default(), - }), - Rule::literal - | Rule::function_call_expr - | Rule::array_literal - | Rule::struct_init - | Rule::union_init - | Rule::tuple_literal - | Rule::if_expr - | Rule::range_expr - | Rule::string - | Rule::expr - | Rule::unary => self.parse_expr(inner_pair), - _ => Err(parse_error!( - "Unexpected primary inner rule: {:?}", - inner_pair.as_rule() - )), - } - } - } - Rule::postfix => { - let mut inner = pair.into_inner(); - let mut expr = self.parse_expr( - inner - .next() - .ok_or_else(|| parse_error!("postfix expr missing base expression"))?, - )?; - - // Apply each chained member operation (field access or index) in order - for field_pair in inner { - // Skip non-postfix inner rules, only postfix_field carries a member operation - if field_pair.as_rule() == Rule::postfix_field { - let mut field_inner = field_pair.into_inner(); - let first = field_inner - .next() - .ok_or(parse_error!("Expected field or index in postfix"))?; - - // Branch on whether the operation is a member access or an index expression - match first.as_rule() { - Rule::identifier => { - let field_name = Self::pair_text(first); - expr = Expr::FieldAccess { - expr: Box::new(expr), - field_name, - ty: TypeId::default(), - }; - } - Rule::expr => { - let index = self.parse_expr(first)?; - expr = Expr::Index { - expr: Box::new(expr), - index: Box::new(index), - ty: TypeId::default(), - }; - } - other => { - return Err(parse_error!( - "Unexpected postfix_field inner rule: {other:?}" - )); - } - } - } - } - Ok(expr) - } - Rule::struct_init => self.parse_struct_init(pair), - Rule::range_expr => { - let mut inner = pair.into_inner(); - let start = self.parse_expr( - inner - .next() - .ok_or_else(|| parse_error!("range expr missing start"))?, - )?; - let end = self.parse_expr( - inner - .next() - .ok_or_else(|| parse_error!("range expr missing end"))?, - )?; - Ok(Expr::RangeLiteral { - start: Box::new(start), - end: Box::new(end), - }) - } - Rule::array_literal => { - let elements = pair - .into_inner() - .filter(|p: &Pair| p.as_rule() == Rule::expr) - .map(|p| self.parse_expr(p)) - .collect::, _>>()?; - Ok(Expr::ArrayLiteral { - elements, - ty: TypeId::default(), - }) - } - other => Err(CompilationError::ParseError(format!( - "Unexpected expr rule: {other:?}" - ))), - } - } - - fn parse_assign_expr(self, pair: Pair) -> CompileResult { - let mut inner = pair.into_inner(); - - let left_pair = inner - .next() - .ok_or_else(|| parse_error!("assign expr missing left operand"))?; - - let left = self.parse_expr(left_pair)?; - - // A plain lvalue (no operator) means the recursive call returned the inner expression unchanged - if let Some(assign_op_pair) = inner.next() { - let op = AssignmentOperator::from_rule_pair(&assign_op_pair)?; - - let right_assign_expr_pair = inner - .next() - .ok_or_else(|| parse_error!("assign expr missing right operand"))?; - - let right = self.parse_assign_expr(right_assign_expr_pair)?; - - Ok(Expr::Assign { - op, - left: Box::new(left), - right: Box::new(right), - ty: TypeId::default(), - }) - } else { - Ok(left) - } - } - - fn parse_struct_init(self, pair: Pair) -> CompileResult { - let mut inner = pair.into_inner(); - let type_pair = inner - .next() - .ok_or_else(|| parse_error!("struct init missing type name"))?; - - let struct_ty = self.parse_type(type_pair)?; - - let fields: Vec = inner - .filter(|p| p.as_rule() == Rule::field_init) // Drop separators, keep only field definitions - .map(|p| self.parse_field_init(p)) // Parse each definition into a FieldInit - .collect::>()?; // Short-circuit and propagate the first parse error - - Ok(Expr::StructInit { - ty: TypeId::default(), - struct_type: Some(struct_ty), - fields, - }) - } - - fn parse_field_init(self, pair: Pair) -> CompileResult { - let mut inner = pair.into_inner(); - let name = Self::pair_text( - inner - .next() - .ok_or_else(|| parse_error!("field init missing name"))?, - ); - let value = self.parse_expr( - inner - .next() - .ok_or_else(|| parse_error!("field init missing value"))?, - )?; - Ok(FieldInit { name, value }) - } - - fn parse_bool_literal(s: &str) -> CompileResult { - match s { - "true" => Ok(Expr::Literal { - value: Literal::Bool(true), - ty: TypeId::default(), - }), - "false" => Ok(Expr::Literal { - value: Literal::Bool(false), - ty: TypeId::default(), - }), - _ => Err(parse_error!("invalid boolean literal: {}", s)), - } - } - - /// Unescape common backslash sequences in a string slice. - /// - /// Returns `None` if the input ends with a stray backslash (no escape character follows it). - fn unescape(s: impl AsRef) -> Option { - let s = s.as_ref(); - let mut out = String::with_capacity(s.len()); - let mut chars = s.chars(); - while let Some(c) = chars.next() { - if c == '\\' { - match chars.next()? { - 'n' => out.push('\n'), - 'r' => out.push('\r'), - 't' => out.push('\t'), - '\\' => out.push('\\'), - '\'' => out.push('\''), - '"' => out.push('"'), - other => out.push(other), - } - } else { - out.push(c); - } - } - Some(out) - } -} diff --git a/kitlang/src/codegen/parser/expr_pratt/mod.rs b/kitlang/src/codegen/parser/expr_pratt/mod.rs new file mode 100644 index 0000000..abcaf86 --- /dev/null +++ b/kitlang/src/codegen/parser/expr_pratt/mod.rs @@ -0,0 +1,296 @@ +//! Hand-written Pratt parser for Kit expressions. +//! +//! This module takes over expression parsing from the pest-based grammar. +//! +//! Pest still handles the program, declaration, statement, and type-annotation grammars. For every +//! `Pair<'_, Rule>` whose rule is an expression (i.e. the 13 precedence levels `expr -> assign -> +//! logical_or -> ... -> primary`), the parser hands off to [`ExprParser::parse_expr`]. +//! +//! Pratt Parsing & Operator Precedence +//! ----------------------------------- +//! The pest grammar used 13 mutually recursive functions (one per precedence level, which +//! overflowed the default 1 MB stack on Windows. +//! +//! The Pratt parser uses a single function with a binding-power loop, dramatically reducing +//! recursive depth compared to the old 13-level grammar chain. +//! +//! Parenthesized sub-expressions still recurse through `parse_primary` and `parse_expr`, so stack +//! depth is `O(parenthetical nesting)`. Operator precedence is defined in +//! [`binding_power::infix`], [`binding_power::postfix`], [`binding_power::prefix`]. +//! +//! Each infix operator has a (lbp, rbp) pair: `lbp < rbp` = right-associative (assignment), +//! `lbp == rbp` = left-associative (most binary ops). +//! +//! Errors & Source Spans +//! --------------------- +//! All parse errors are values of [`ExprParseError`] (in `diagnostics.rs`). +//! +//! The parser never prints, never allocates strings for error messages, and never holds +//! source-file identity. Conversion to `Compilation::ParseError(String)` happens at +//! `PestExpr::parse` in `parser/mod.rs`. +//! +//! The token stream carries byte ranges; the parser uses them internally but does not currently +//! attach them to AST nodes. + +use crate::codegen::ast::{Expr, Literal}; +use crate::codegen::type_ast::FieldInit; +use crate::codegen::types::{Type, TypeId}; +use crate::lexer::{LexicalError, Span, SpannedTok, Tok, tokenize}; + +use super::binding_power::{ + infix, is_range_op, postfix, prefix, tok_to_assign_op, tok_to_binary_op, tok_to_unary_op, +}; +use super::diagnostics::{ExprParseError, expected_name}; + +/// A Pratt parser for Kit expressions. +/// +/// One instance is built per expression being parsed. The parser is +/// single-use: create a new one for each `parse_expr` call. +pub(crate) struct ExprParser<'a> { + tokens: &'a [SpannedTok], + pos: usize, +} + +impl<'a> ExprParser<'a> { + /// Build a parser over a token slice. The slice must be the result of + /// [`tokenize`] applied to the expression's source text. + pub(crate) fn new(tokens: &'a [SpannedTok]) -> Self { + Self { tokens, pos: 0 } + } + + /// Entry point. Parses one complete expression and returns it. + /// Callers must check `pos()` against the token length to ensure + /// no trailing tokens were left unparsed. + pub(crate) fn parse_expr(&mut self) -> Result { + self.parse_pratt(0) + } + + /// The current position in the token stream. Useful for tests and + /// for callers that want to know how many tokens were consumed. + pub(crate) fn pos(&self) -> usize { + self.pos + } + + // --- Token stream helpers (private) --- + + /// Peek the current token without consuming it. Returns a synthetic + /// "EOF" token when the stream is exhausted. + fn peek(&self) -> &SpannedTok { + // We never return None from peek; EOF is represented by a synthetic + // SpannedTok at span 0..0. This lets the Pratt loop compare with + // `==` against `Tok::...` cleanly. + static EOF: SpannedTok = SpannedTok { + kind: Tok::Semi, // any token that has no infix/postfix/prefix bp + span: 0..0, + }; + self.tokens.get(self.pos).unwrap_or(&EOF) + } + + /// Consume and return the current token. + fn advance(&mut self) -> &SpannedTok { + let tok = &self.tokens[self.pos]; + self.pos += 1; + tok + } + + /// True if the parser is at or past the end of the token stream. + fn at_eof(&self) -> bool { + self.pos >= self.tokens.len() + } + + // --- Pratt core --- + + /// Parse an expression with the given minimum binding power. + fn parse_pratt(&mut self, min_bp: u8) -> Result { + // Parse leading prefix operators (e.g. `-a`, `!!x`, `&arr[i]`). + // Prefix binds tighter than infix but looser than postfix, + // so `&arr[i]` = `&(arr[i])` (postfix on `arr` first). + let mut lhs = if let Some(pfx_bp) = prefix(&self.peek().kind) { + let op = tok_to_unary_op(&self.peek().kind).unwrap(); + self.advance(); + let rhs = self.parse_pratt(pfx_bp)?; + Expr::UnaryOp { + op, + expr: Box::new(rhs), + ty: TypeId::default(), + } + } else { + self.parse_primary()? + }; + + // Postfix chain: field access, index, call. + lhs = self.parse_postfix_chain(lhs)?; + + // Infix operators (binary ops, range) + loop { + let kind = self.peek().kind.clone(); + let Some((lbp, rbp)) = infix(&kind) else { + break; + }; + if lbp < min_bp { + break; + } + if tok_to_assign_op(&kind).is_some() { + break; + } + self.advance(); + if is_range_op(&kind) { + let rhs = self.parse_pratt(rbp)?; + lhs = Expr::RangeLiteral { + start: Box::new(lhs), + end: Box::new(rhs), + }; + continue; + } + let op = tok_to_binary_op(&kind).ok_or_else(|| { + ExprParseError::Custom(format!("internal: no binary op for {kind:?}")) + })?; + let rhs = self.parse_pratt(rbp)?; + lhs = Expr::BinaryOp { + op, + left: Box::new(lhs), + right: Box::new(rhs), + ty: TypeId::default(), + }; + } + + // Assignment (right-associative, lowest precedence). + // Right-associativity requires the RHS to be parsed at the same + // binding power as the operator's lbp, so `a = b = c` becomes + // `a = (b = c)`. We use `lbp` (not `rbp` from the table) since + // assignment needs `rbp <= lbp` for the RHS to see the operator. + loop { + let kind = self.peek().kind.clone(); + let Some(op) = tok_to_assign_op(&kind) else { + break; + }; + let Some((lbp, _rbp)) = infix(&kind) else { + break; + }; + if lbp < min_bp { + break; + } + self.advance(); + let rhs = self.parse_pratt(lbp)?; + lhs = Expr::Assign { + op, + left: Box::new(lhs), + right: Box::new(rhs), + ty: TypeId::default(), + }; + } + + Ok(lhs) + } + + // --- Helpers --- + + /// Consume the current token if it matches `expected`, otherwise + /// return an `UnexpectedToken` (or `UnexpectedEof` if at end) error + /// with `expected`'s name. + fn expect(&mut self, expected: &Tok) -> Result<(), ExprParseError> { + if &self.peek().kind == expected { + self.advance(); + Ok(()) + } else if self.at_eof() { + Err(ExprParseError::UnexpectedEof { + expected: expected_name(expected), + }) + } else { + Err(ExprParseError::UnexpectedToken { + found: self.peek().kind.clone(), + expected: expected_name(expected), + }) + } + } + + /// Parse a comma-separated list of `T` terminated by `closer`. + /// Allows zero or more elements (an empty list is valid for fn + /// calls with no args, empty array literals, etc.). + fn parse_comma_list(&mut self, closer: Tok, mut f: F) -> Result, ExprParseError> + where + F: FnMut(&mut Self) -> Result, + { + let mut out = Vec::new(); + // Empty list case. + if self.peek().kind == closer { + self.advance(); + return Ok(out); + } + out.push(f(self)?); + while self.peek().kind == Tok::Comma { + self.advance(); + // Trailing comma is allowed (parses to empty trailing element). + if self.peek().kind == closer { + break; + } + out.push(f(self)?); + } + self.expect(&closer)?; + Ok(out) + } +} + +/// Extract a string callee name from an expression for name-mangling +/// and symbol-table lookup. Returns `None` for indirect calls that +/// must be resolved by the callee's inferred type. +pub(crate) fn callee_name(expr: &Expr) -> Option { + match expr { + Expr::Identifier { name, .. } => Some(name.clone()), + Expr::FieldAccess { + expr: base, + field_name, + .. + } => Some(format!("{}.{}", callee_name(base)?, field_name)), + _ => None, + } +} + +// --------------------------------------------------------------------------- +// Module surface: parse an expression from source text. +// --------------------------------------------------------------------------- + +/// Parse a Kit expression from source text. This is the public entry point used by the +/// pest-to-Pratt bridge (`PestExpr::parse`). +/// +/// The `text` should be the source text of the expression as a `Pair::as_str()` slice. +/// Tokenization, parsing, and conversion to an `Expr` all happen here. +/// +/// # Errors +/// +/// Errors are returned for: +/// - Unrecognized characters in the source +/// - Integer literals that overflow `i64` +/// - Trailing tokens after the expression +/// - Any parse error from the Pratt loop +pub(crate) fn parse_kit_expr(text: &str) -> Result { + let tokens = tokenize(text).map_err(|e| match e { + LexicalError::UnexpectedCharacter { offset } => { + ExprParseError::Custom(format!("unexpected character at byte offset {}", offset)) + } + LexicalError::IntegerOverflow { text, .. } => { + ExprParseError::Custom(format!("integer literal `{text}` is out of range for i64")) + } + })?; + + let mut parser = ExprParser::new(&tokens); + let expr = parser.parse_expr()?; + + // Reject leftover tokens (e.g. from a stray character that was dropped + // or from genuinely malformed input like `a b`). + if parser.pos() < tokens.len() { + return Err(ExprParseError::UnexpectedToken { + found: tokens[parser.pos()].kind.clone(), + expected: &["end of expression"], + }); + } + + Ok(expr) +} + +// Primary expression parsers (include'd into this module scope). +include!("primary.rs"); + +// Tests (include'd into this module scope, test builds only). +#[cfg(test)] +include!("tests.rs"); diff --git a/kitlang/src/codegen/parser/expr_pratt/primary.rs b/kitlang/src/codegen/parser/expr_pratt/primary.rs new file mode 100644 index 0000000..06f92f8 --- /dev/null +++ b/kitlang/src/codegen/parser/expr_pratt/primary.rs @@ -0,0 +1,279 @@ +// Primary expression parsers. Included into `mod.rs` via `include!`. +// This file is in the same module scope as `mod.rs`, so imports are +// inherited and all `ExprParser` private methods are accessible. + +impl<'a> ExprParser<'a> { + /// Iteratively apply postfix operators (call, index, field access) to + /// a base expression. Zero stack frames added per iteration. The + /// chain is bounded by the source's syntactic length, but the parser + /// is iterative, so the *call stack* depth is constant. + pub(crate) fn parse_postfix_chain( + &mut self, + mut base: Expr, + ) -> Result { + loop { + let kind = self.peek().kind.clone(); + if postfix(&kind).is_none() { + break; + } + base = match kind { + Tok::Dot => self.parse_field_access(base)?, + Tok::LBracket => self.parse_index(base)?, + Tok::LParen => self.parse_call(base)?, + _ => unreachable!("postfix returned Some for {kind:?}"), + }; + } + Ok(base) + } + + /// Parse a primary expression: literals, identifiers, parenthesized + /// expressions, function calls, array literals, struct inits, and + /// the if-expression. Postfix operations (`.field`, `[i]`, `(args)`) + /// are handled in the outer Pratt loop, *not* here, so this function + /// only needs to produce the base expression. + pub(crate) fn parse_primary(&mut self) -> Result { + let tok = self.peek().kind.clone(); + // `span` is read for documentation; the parser doesn't currently + // attach it to AST nodes. Future PRs will use it. + let _span: Span = self.peek().span.clone(); + + match tok { + Tok::IntLit(n) => { + self.advance(); + Ok(Expr::Literal { + value: Literal::Int(n), + ty: TypeId::default(), + }) + } + Tok::FloatLit(f) => { + self.advance(); + Ok(Expr::Literal { + value: Literal::Float(f), + ty: TypeId::default(), + }) + } + Tok::CharLit(c) => { + self.advance(); + Ok(Expr::Literal { + value: Literal::Char(c), + ty: TypeId::default(), + }) + } + Tok::StringLit(s) => { + self.advance(); + Ok(Expr::Literal { + value: Literal::String(s), + ty: TypeId::default(), + }) + } + Tok::KwTrue => { + self.advance(); + Ok(Expr::Literal { + value: Literal::Bool(true), + ty: TypeId::default(), + }) + } + Tok::KwFalse => { + self.advance(); + Ok(Expr::Literal { + value: Literal::Bool(false), + ty: TypeId::default(), + }) + } + Tok::KwNull => { + self.advance(); + Ok(Expr::Literal { + value: Literal::Null, + ty: TypeId::default(), + }) + } + Tok::KwThis | Tok::KwSelf => { + let name = match tok { + Tok::KwThis => "this", + _ => "Self", + }; + self.advance(); + Ok(Expr::Identifier { + name: name.to_string(), + ty: TypeId::default(), + }) + } + Tok::Ident(name) => { + self.advance(); + Ok(Expr::Identifier { + name, + ty: TypeId::default(), + }) + } + Tok::LParen => { + self.advance(); // consume `(` + let first = self.parse_expr()?; + if self.peek().kind == Tok::Comma { + return Err(ExprParseError::Custom( + "tuple literals are not yet supported by the Pratt parser".into(), + )); + } + self.expect(&Tok::RParen)?; + Ok(first) + } + Tok::LBracket => self.parse_array_literal(), + Tok::KwStruct => self.parse_struct_init(), + Tok::KwIf => self.parse_if_expr(), + Tok::KwEmpty => { + self.advance(); + Ok(Expr::Identifier { + name: "empty".to_string(), + ty: TypeId::default(), + }) + } + _ => { + if self.at_eof() { + Err(ExprParseError::UnexpectedEof { + expected: &[ + "integer literal", + "float literal", + "string literal", + "char literal", + "identifier", + "`(`", + "`[`", + "`if`", + "`null`", + "`true`", + "`false`", + ], + }) + } else { + Err(ExprParseError::UnexpectedToken { + found: tok, + expected: &[ + "integer literal", + "float literal", + "string literal", + "char literal", + "identifier", + "`(`", + "`[`", + "`if`", + "`null`", + "`true`", + "`false`", + ], + }) + } + } + } + } + + /// Parse a `.field` access postfix. + fn parse_field_access(&mut self, base: Expr) -> Result { + self.advance(); // consume `.` + let field_tok = self.peek().kind.clone(); + match field_tok { + Tok::Ident(name) => { + self.advance(); + Ok(Expr::FieldAccess { + expr: Box::new(base), + field_name: name, + ty: TypeId::default(), + }) + } + _ => Err(ExprParseError::UnexpectedToken { + found: field_tok, + expected: &["identifier"], + }), + } + } + + /// Parse a `[index]` postfix. + fn parse_index(&mut self, base: Expr) -> Result { + self.advance(); // consume `[` + let index = self.parse_expr()?; + self.expect(&Tok::RBracket)?; + Ok(Expr::Index { + expr: Box::new(base), + index: Box::new(index), + ty: TypeId::default(), + }) + } + + /// Parse a function call postfix: `(arg1, arg2, ...)`. + /// The callee is any expression; no rejection of indirect calls. + fn parse_call(&mut self, callee: Expr) -> Result { + self.advance(); // consume `(` + let args = self.parse_comma_list(Tok::RParen, |p| p.parse_expr())?; + Ok(Expr::Call { + callee: Box::new(callee), + args, + ty: TypeId::default(), + }) + } + + /// Parse an array literal: `[expr, expr, ...]`. + fn parse_array_literal(&mut self) -> Result { + self.advance(); // consume `[` + let elements = self.parse_comma_list(Tok::RBracket, |p| p.parse_expr())?; + Ok(Expr::ArrayLiteral { + elements, + ty: TypeId::default(), + }) + } + + /// Parse a struct init: `struct Name { field: expr, ... }`. + fn parse_struct_init(&mut self) -> Result { + self.advance(); // consume `struct` + let type_tok = self.peek().kind.clone(); + let type_name = match type_tok { + Tok::Ident(name) => { + self.advance(); + name + } + _ => { + return Err(ExprParseError::UnexpectedToken { + found: type_tok, + expected: &["type name"], + }); + } + }; + self.expect(&Tok::LBrace)?; + let fields = self.parse_comma_list(Tok::RBrace, |p| { + let name = match &p.peek().kind { + Tok::Ident(n) => { + let n = n.clone(); + p.advance(); + n + } + _ => { + return Err(ExprParseError::UnexpectedToken { + found: p.peek().kind.clone(), + expected: &["field name"], + }); + } + }; + p.expect(&Tok::Colon)?; + let value = p.parse_expr()?; + Ok(FieldInit { name, value }) + })?; + Ok(Expr::StructInit { + ty: TypeId::default(), + struct_type: Some(Type::from_kit(&type_name)), + fields, + }) + } + + /// Parse an if-expression: `if cond then a else b`. + fn parse_if_expr(&mut self) -> Result { + self.advance(); // consume `if` + let cond = self.parse_expr()?; + self.expect(&Tok::KwThen)?; + let then_branch = self.parse_expr()?; + self.expect(&Tok::KwElse)?; + let else_branch = self.parse_expr()?; + Ok(Expr::If { + cond: Box::new(cond), + then_branch: Box::new(then_branch), + else_branch: Box::new(else_branch), + ty: TypeId::default(), + }) + } +} diff --git a/kitlang/src/codegen/parser/expr_pratt/tests.rs b/kitlang/src/codegen/parser/expr_pratt/tests.rs new file mode 100644 index 0000000..d60bc25 --- /dev/null +++ b/kitlang/src/codegen/parser/expr_pratt/tests.rs @@ -0,0 +1,652 @@ +// Unit tests for the Pratt parser. Included into `mod.rs` via `include!`, +// so this file is in the same module scope. Only compiled in test builds. + +use crate::codegen::types::{AssignmentOperator, BinaryOperator, UnaryOperator}; + +/// Parse an expression and unwrap +fn p(text: &str) -> Expr { + parse_kit_expr(text).unwrap_or_else(|e| panic!("parse failed for `{text}`: {e}")) +} + +/// Parse and assert the error contains a substring. +fn p_err(text: &str, needle: &str) { + let err = parse_kit_expr(text) + .err() + .unwrap_or_else(|| panic!("expected error for `{text}`, got Ok")); + let msg = err.to_human_message(); + assert!( + msg.contains(needle), + "error `{msg}` does not contain `{needle}`" + ); +} + +// --- Literals --- + +/// Integer literal `42` is parsed as `Literal::Int(42)`. +#[test] +fn integer_literal() { + let e = p("42"); + assert!(matches!( + e, + Expr::Literal { + value: Literal::Int(42), + .. + } + )); +} + +/// Float literal `3.14` is parsed with correct precision. +#[test] +fn float_literal() { + let e = p("3.14"); + assert!( + matches!(e, Expr::Literal { value: Literal::Float(f), .. } if (f - 3.14).abs() < 1e-10) + ); +} + +/// Double-quoted string `"hello"` is parsed as `Literal::String`. +#[test] +fn string_literal() { + let e = p(r#""hello""#); + assert!(matches!(e, Expr::Literal { value: Literal::String(s), .. } if s == "hello")); +} + +/// `true` and `false` both parse as `Literal::Bool`. +#[test] +fn bool_literals() { + assert!(matches!( + p("true"), + Expr::Literal { + value: Literal::Bool(true), + .. + } + )); + assert!(matches!( + p("false"), + Expr::Literal { + value: Literal::Bool(false), + .. + } + )); +} + +/// `null` is parsed as `Literal::Null`. +#[test] +fn null_literal() { + assert!(matches!( + p("null"), + Expr::Literal { + value: Literal::Null, + .. + } + )); +} + +// --- Identifiers --- + +/// Bare identifier `foo` is parsed as `Expr::Identifier`. +#[test] +fn identifier() { + let e = p("foo"); + assert!(matches!(&e, Expr::Identifier { name, .. } if name == "foo")); +} + +#[test] +fn qualified_identifier_is_built_via_postfix_chain() { + let e = p("foo.bar.baz"); + let mut cur = &e; + let mut path = vec![]; + while let Expr::FieldAccess { + expr, field_name, .. + } = cur + { + path.push(field_name.clone()); + cur = expr; + } + if let Expr::Identifier { name, .. } = cur { + assert_eq!(name, "foo"); + } else { + panic!("expected leaf Identifier, got {cur:?}"); + } + assert_eq!(path, vec!["baz".to_string(), "bar".to_string()]); +} + +// --- Precedence --- + +/// `1 + 2 * 3` - `*` binds tighter than `+`. +#[test] +fn additive_vs_multiplicative() { + let e = p("1 + 2 * 3"); + if let Expr::BinaryOp { op, right, .. } = &e { + assert_eq!(*op, BinaryOperator::Add); + if let Expr::BinaryOp { op: inner_op, .. } = right.as_ref() { + assert_eq!(*inner_op, BinaryOperator::Mul); + } else { + panic!("expected inner Mul, got {right:?}"); + } + } else { + panic!("expected top-level Add, got {e:?}"); + } +} + +/// `a == b < c` - `==` is looser than `<`, parsed as `(a == (b < c))`. +#[test] +fn comparison_vs_equality() { + let e = p("a == b < c"); + if let Expr::BinaryOp { + op, left, right, .. + } = &e + { + assert_eq!(*op, BinaryOperator::Eq); + assert!(matches!(left.as_ref(), Expr::Identifier { name, .. } if name == "a")); + if let Expr::BinaryOp { op: inner_op, .. } = right.as_ref() { + assert_eq!(*inner_op, BinaryOperator::Lt); + } else { + panic!("expected inner Lt, got {right:?}"); + } + } else { + panic!("expected top-level Eq, got {e:?}"); + } +} + +/// `1 + 2 + 3` - `+` is left-associative: `((1 + 2) + 3)`. +#[test] +fn left_associative_addition() { + let e = p("1 + 2 + 3"); + if let Expr::BinaryOp { + op, left, right, .. + } = &e + { + assert_eq!(*op, BinaryOperator::Add); + assert!(matches!( + right.as_ref(), + Expr::Literal { + value: Literal::Int(3), + .. + } + )); + if let Expr::BinaryOp { op: inner_op, .. } = left.as_ref() { + assert_eq!(*inner_op, BinaryOperator::Add); + } else { + panic!("expected inner Add, got {left:?}"); + } + } else { + panic!("expected top-level Add, got {e:?}"); + } +} + +/// `a += b += c` - `+=` is right-associative: `(a += (b += c))`. +#[test] +fn right_associative_assignment() { + let e = p("a += b += c"); + if let Expr::Assign { + op, left, right, .. + } = &e + { + assert_eq!(*op, AssignmentOperator::AddAssign); + assert!(matches!(left.as_ref(), Expr::Identifier { name, .. } if name == "a")); + assert!(matches!(right.as_ref(), Expr::Assign { .. })); + } else { + panic!("expected top-level Assign, got {e:?}"); + } +} + +/// `-a + b` - prefix `-` binds tighter than `+`. +#[test] +fn unary_minus_binds_tighter_than_addition() { + let e = p("-a + b"); + if let Expr::BinaryOp { + op, left, right, .. + } = &e + { + assert_eq!(*op, BinaryOperator::Add); + assert!(matches!(right.as_ref(), Expr::Identifier { name, .. } if name == "b")); + assert!(matches!( + left.as_ref(), + Expr::UnaryOp { + op: UnaryOperator::Neg, + .. + } + )); + } else { + panic!("expected top-level Add, got {e:?}"); + } +} + +/// `&arr[i]` - prefix `&` is looser than postfix `[]`, parsed as `&(arr[i])`. +#[test] +fn unary_looser_than_postfix() { + let e = p("&arr[i]"); + if let Expr::UnaryOp { op, expr, .. } = &e { + assert_eq!(*op, UnaryOperator::AddressOf); + assert!(matches!(expr.as_ref(), Expr::Index { .. })); + } else { + panic!("expected top-level AddressOf, got {e:?}"); + } +} + +// --- Postfix chains --- + +/// `a.b.c.d.e` - field access chains to at least 4 levels. +#[test] +fn chained_field_access() { + let e = p("a.b.c.d.e"); + let mut depth = 0; + let mut cur = &e; + while let Expr::FieldAccess { expr, .. } = cur { + depth += 1; + cur = expr; + } + assert_eq!(depth, 4, "expected 4 field-access levels"); + assert!(matches!(cur, Expr::Identifier { name, .. } if name == "a")); +} + +/// 100-level field-access chain `a.f0.f1...f99` does not overflow the stack. +#[test] +fn stress_deep_postfix_chain() { + let mut src = String::from("a"); + for i in 0..100 { + src.push('.'); + src.push_str(&format!("f{i}")); + } + let e = p(&src); + let mut depth = 0; + let mut cur = &e; + while let Expr::FieldAccess { expr, .. } = cur { + depth += 1; + cur = expr; + } + assert_eq!(depth, 100); +} + +/// 100 levels of nested parentheses are parsed correctly. +#[test] +fn stress_deep_nested_parens() { + let mut src = String::new(); + for _ in 0..100 { + src.push('('); + } + src.push('1'); + for _ in 0..100 { + src.push(')'); + } + let e = p(&src); + assert!(matches!( + e, + Expr::Literal { + value: Literal::Int(1), + .. + } + )); +} + +// --- Function calls --- + +/// `f()` - function call with zero arguments. +#[test] +fn call_no_args() { + let e = p("f()"); + if let Expr::Call { callee, args, .. } = &e { + assert_eq!(callee_name(callee), Some("f".to_string())); + assert!(args.is_empty()); + } else { + panic!("expected Call, got {e:?}"); + } +} + +/// `f(1)` - function call with one argument. +#[test] +fn call_one_arg() { + let e = p("f(1)"); + if let Expr::Call { callee, args, .. } = &e { + assert_eq!(callee_name(callee), Some("f".to_string())); + assert_eq!(args.len(), 1); + } else { + panic!("expected Call, got {e:?}"); + } +} + +/// `f(1, 2, 3, 4, 5)` - function call with five arguments. +#[test] +fn call_many_args() { + let e = p("f(1, 2, 3, 4, 5)"); + if let Expr::Call { args, .. } = &e { + assert_eq!(args.len(), 5); + } else { + panic!("expected Call, got {e:?}"); + } +} + +/// `pkg.math.add(2, 3)` - qualified name via field-access chain is the callee. +#[test] +fn call_qualified_name() { + let e = p("pkg.math.add(2, 3)"); + if let Expr::Call { callee, args, .. } = &e { + assert_eq!(callee_name(callee), Some("pkg.math.add".to_string())); + assert_eq!(args.len(), 2); + } else { + panic!("expected Call, got {e:?}"); + } +} + +/// `f(g(1), h(2, 3))` - nested calls as arguments. +#[test] +fn call_with_nested_expressions_in_args() { + let e = p("f(g(1), h(2, 3))"); + if let Expr::Call { args, .. } = &e { + assert_eq!(args.len(), 2); + } else { + panic!("expected Call, got {e:?}"); + } +} + +// --- Indexing --- + +/// `arr[0]` - index expression with identifier base and integer index. +#[test] +fn index() { + let e = p("arr[0]"); + if let Expr::Index { expr, index, .. } = &e { + assert!(matches!(expr.as_ref(), Expr::Identifier { name, .. } if name == "arr")); + assert!(matches!( + index.as_ref(), + Expr::Literal { + value: Literal::Int(0), + .. + } + )); + } else { + panic!("expected Index, got {e:?}"); + } +} + +/// `a[i][j]` - chained index produces two `Index` nodes. +#[test] +fn chained_index() { + let e = p("a[i][j]"); + let mut depth = 0; + let mut cur = &e; + while let Expr::Index { expr, .. } = cur { + depth += 1; + cur = expr; + } + assert_eq!(depth, 2); +} + +// --- Array literals --- + +/// `[]` - empty array literal. +#[test] +fn empty_array() { + let e = p("[]"); + if let Expr::ArrayLiteral { elements, .. } = &e { + assert!(elements.is_empty()); + } else { + panic!("expected ArrayLiteral, got {e:?}"); + } +} + +/// `[1, 2, 3]` - array literal with three elements. +#[test] +fn array_with_elements() { + let e = p("[1, 2, 3]"); + if let Expr::ArrayLiteral { elements, .. } = &e { + assert_eq!(elements.len(), 3); + } else { + panic!("expected ArrayLiteral, got {e:?}"); + } +} + +// --- Struct init --- + +/// `struct Point { x: 10, y: 20 }` - struct init with two named fields. +#[test] +fn struct_init() { + let e = p("struct Point { x: 10, y: 20 }"); + if let Expr::StructInit { fields, .. } = &e { + assert_eq!(fields.len(), 2); + assert_eq!(fields[0].name, "x"); + assert_eq!(fields[1].name, "y"); + } else { + panic!("expected StructInit, got {e:?}"); + } +} + +// --- If expressions --- + +/// `if a then b else c` - if expression with all three branches. +#[test] +fn if_expr() { + let e = p("if a then b else c"); + if let Expr::If { + cond, + then_branch, + else_branch, + .. + } = &e + { + assert!(matches!(cond.as_ref(), Expr::Identifier { name, .. } if name == "a")); + assert!(matches!(then_branch.as_ref(), Expr::Identifier { name, .. } if name == "b")); + assert!(matches!(else_branch.as_ref(), Expr::Identifier { name, .. } if name == "c")); + } else { + panic!("expected If, got {e:?}"); + } +} + +// --- Logical operators --- + +/// `a || b && c` - `&&` binds tighter than `||`, parsed as `(a || (b && c))`. +#[test] +fn logical_and_vs_or() { + let e = p("a || b && c"); + if let Expr::BinaryOp { op, right, .. } = &e { + assert_eq!(*op, BinaryOperator::Or); + assert!(matches!( + right.as_ref(), + Expr::BinaryOp { + op: BinaryOperator::And, + .. + } + )); + } else { + panic!("expected top-level Or, got {e:?}"); + } +} + +// --- Errors --- + +/// `(1 + 2` - missing `)` produces error mentioning `)`. +#[test] +fn missing_rparen() { + p_err("(1 + 2", "`)`"); +} + +// --- Range literals --- + +/// `1...5` - range literal with integer start and end. +#[test] +fn range_literal_simple() { + let e = p("1...5"); + if let Expr::RangeLiteral { start, end } = &e { + assert!(matches!( + start.as_ref(), + Expr::Literal { + value: Literal::Int(1), + .. + } + )); + assert!(matches!( + end.as_ref(), + Expr::Literal { + value: Literal::Int(5), + .. + } + )); + } else { + panic!("expected RangeLiteral, got {e:?}"); + } +} + +/// `a + 1...b - 1` - range bounds can be arbitrary expressions. +#[test] +fn range_literal_with_expressions() { + let e = p("a + 1...b - 1"); + if let Expr::RangeLiteral { start, end } = &e { + assert!(matches!( + start.as_ref(), + Expr::BinaryOp { + op: BinaryOperator::Add, + .. + } + )); + assert!(matches!( + end.as_ref(), + Expr::BinaryOp { + op: BinaryOperator::Sub, + .. + } + )); + } else { + panic!("expected RangeLiteral, got {e:?}"); + } +} + +/// `arr[0` - missing `]` produces error mentioning `]`. +#[test] +fn missing_rbracket() { + p_err("arr[0", "`]`"); +} + +/// `+` at start - parser rejects leading binary operator. +#[test] +fn unexpected_token_at_start() { + p_err("+", "identifier"); +} + +/// `foo.` - trailing dot is rejected with expected identifier. +#[test] +fn missing_field_name() { + p_err("foo.", "identifier"); +} + +/// `if a then b` - missing `else` produces error mentioning `else`. +#[test] +fn missing_else() { + p_err("if a then b", "`else`"); +} + +/// `a b` - two adjacent expressions produce end-of-expression error. +#[test] +fn trailing_tokens_produce_error() { + p_err("a b", "end of expression"); +} + +/// `a $ b` - `$` is an unrecognized character error. +#[test] +fn unrecognized_characters_produce_error() { + let err = parse_kit_expr("a $ b").expect_err("should error on $"); + let msg = err.to_human_message(); + assert!(msg.contains("unexpected character"), "msg: {msg}"); +} + +/// Literal exceeding i64 range produces overflow error. +#[test] +fn integer_overflow_is_detected() { + let err = parse_kit_expr("99999999999999999999").expect_err("should error on overflow literal"); + let msg = err.to_human_message(); + assert!( + msg.contains("out of range"), + "expected 'out of range', got: {msg}" + ); +} + +/// Overflow in left operand of binary expression. +#[test] +fn integer_overflow_as_left_operand_is_detected() { + let err = + parse_kit_expr("99999999999999999999 + 1").expect_err("should error on overflow literal"); + let msg = err.to_human_message(); + assert!( + msg.contains("out of range"), + "expected 'out of range', got: {msg}" + ); +} + +/// Overflow in right operand of binary expression. +#[test] +fn integer_overflow_as_right_operand_is_detected() { + let err = + parse_kit_expr("1 + 99999999999999999999").expect_err("should error on overflow literal"); + let msg = err.to_human_message(); + assert!( + msg.contains("out of range"), + "expected 'out of range', got: {msg}" + ); +} + +/// Missing `)` uses `ExpectedEof` variant, not the `Semi` sentinel. +#[test] +fn eof_uses_unexpected_eof_not_semi() { + let err = parse_kit_expr("(1 + 2").expect_err("should error on missing )"); + let msg = err.to_human_message(); + assert!( + msg.contains("end of expression"), + "expected 'end of expression', got: {msg}" + ); + // Also verify the existing test still works: + let missing_rparen = parse_kit_expr("(1 + 2").expect_err("should error on missing )"); + assert!(missing_rparen.to_human_message().contains("`)`")); +} + +/// `f()()` - nested calls: result of `f()` is called with no args. +#[test] +fn indirect_call_is_parsed() { + let e = p("f()()"); + match &e { + Expr::Call { callee, args, .. } => { + assert!(args.is_empty(), "outer call should have no args"); + match callee.as_ref() { + Expr::Call { + callee: inner, + args: inner_args, + .. + } => { + assert_eq!(callee_name(inner), Some("f".to_string())); + assert!(inner_args.is_empty()); + } + _ => panic!("expected inner Call, got {callee:?}"), + } + } + _ => panic!("expected outer Call, got {e:?}"), + } +} + +/// `x++` - postfix `++` is rejected (not supported in Kit). +#[test] +fn postfix_increment_is_now_an_error() { + let err = parse_kit_expr("x++").expect_err("should error on trailing ++"); + let msg = err.to_human_message(); + assert!( + msg.contains("PlusPlus") || msg.contains("end of expression"), + "msg: {msg}" + ); +} + +/// `x--` - postfix `--` is rejected (not supported in Kit). +#[test] +fn postfix_decrement_is_now_an_error() { + let err = parse_kit_expr("x--").expect_err("should error on trailing --"); + let msg = err.to_human_message(); + assert!( + msg.contains("MinusMinus") || msg.contains("end of expression"), + "msg: {msg}" + ); +} + +/// `sizeof(i32)` - `sizeof` keyword is rejected (not supported in Kit). +#[test] +fn sizeof_is_not_supported() { + let err = parse_kit_expr("sizeof(i32)").expect_err("sizeof should error"); + let msg = err.to_human_message(); + assert!(msg.contains("Sizeof"), "msg: {msg}"); +} diff --git a/kitlang/src/codegen/parser/mod.rs b/kitlang/src/codegen/parser/mod.rs index 28e0a8e..e9c41d1 100644 --- a/kitlang/src/codegen/parser/mod.rs +++ b/kitlang/src/codegen/parser/mod.rs @@ -1,11 +1,15 @@ -mod expr; +mod binding_power; +mod diagnostics; +pub(crate) mod expr_pratt; use pest::iterators::Pair; use crate::error::CompilationError; use crate::{Rule, parse_error}; -use super::ast::{Block, Function, GlobalDecl, Include, Literal, MetaArg, Metadata, Param, Stmt}; +use super::ast::{ + Block, Expr, Function, GlobalDecl, Include, Literal, MetaArg, Metadata, Param, Stmt, +}; use super::module::{ImportType, ModuleImport, ModulePath}; use super::type_ast::{ EnumDefinition, EnumVariant, Field, ImplDefinition, RuleDecl, RuleSet, StructDefinition, @@ -14,6 +18,40 @@ use super::type_ast::{ use super::types::{Type, TypeId}; use crate::error::CompileResult; +/// Bridge between pest and the Pratt parser. +/// +/// The pest-based parser walks the grammar tree and, when it encounters +/// an `expr` rule, hands the corresponding `Pair` off to the Pratt parser +/// via this adapter. The adapter: +/// +/// 1. Pulls the source text out of the `Pair` with `as_str()`. +/// 2. Tokenizes it with the Logos lexer. +/// 3. Parses the tokens with the Pratt parser. +/// 4. Converts the Pratt parser's `ExprParseError` to the public +/// `CompilationError`. +/// +/// This is the *only* conversion point between the two parsers; it's +/// also the only place the public error type is built from the +/// parser-internal one. Future diagnostic improvements (spans, severity, +/// pretty rendering) are added at this single seam. +pub(crate) struct PestExpr<'a> { + pair: Pair<'a, Rule>, +} + +impl<'a> PestExpr<'a> { + /// Wrap a pest `Pair` whose rule is an expression. + pub(crate) fn new(pair: Pair<'a, Rule>) -> Self { + Self { pair } + } + + /// Parse the wrapped pair as a Kit expression. + pub(crate) fn parse(self) -> CompileResult { + let text = self.pair.as_str(); + expr_pratt::parse_kit_expr(text) + .map_err(|e| CompilationError::ParseError(e.to_human_message())) + } +} + #[derive(Clone, Copy, Default, Debug)] pub struct Parser; @@ -130,20 +168,32 @@ impl Parser { args.push(MetaArg::Literal(Literal::String(val.to_string()))); } else if let Ok(n) = text.parse::() { args.push(MetaArg::Literal(Literal::Int(n))); - } else if text == "true" { - args.push(MetaArg::Literal(Literal::Bool(true))); - } else if text == "false" { - args.push(MetaArg::Literal(Literal::Bool(false))); - } else if text == "null" { - args.push(MetaArg::Literal(Literal::Null)); } else { - args.push(MetaArg::Identifier(text)); + match text.as_str() { + "true" => args.push(MetaArg::Literal(Literal::Bool(true))), + "false" => args.push(MetaArg::Literal(Literal::Bool(false))), + "null" => args.push(MetaArg::Literal(Literal::Null)), + _ => args.push(MetaArg::Identifier(text)), + } } } Ok(Metadata { name, args }) } /// Parse a `function_decl` rule into a `Function`. + /// Parse an expression via the Pratt parser. This is the unified + /// entry point used by every pest-side call site that needs an + /// expression AST node. + /// + /// `Parser` is `Copy`, so we take `self` by value to match the + /// signature of the (now-deleted) pest-based `parse_expr`. The + /// caller doesn't need to mutate the parser; the Pratt parser + /// operates on its own `ExprParser` instance built from a token + /// slice derived from the `Pair`. + pub fn parse_expr(self, pair: Pair) -> CompileResult { + PestExpr::new(pair).parse() + } + pub fn parse_function(&self, pair: Pair) -> CompileResult { let mut inner = pair.into_inner(); @@ -374,7 +424,7 @@ impl Parser { .next() .ok_or(parse_error!("trait impl missing 'for' type"))?, )?; - // For now, return a simple placeholder + // TODO: For now, return a simple placeholder Ok(ImplDefinition { name: String::new(), trait_type, @@ -643,7 +693,22 @@ impl Parser { let inner_ty = self.parse_type(inner_ptr_type)?; Ok(Type::Ptr(Box::new(inner_ty))) } - // TODO: Handle other type_annotation rules like function_type, tuple_type + Rule::function_type => { + let inner = inner_rule.into_inner(); + // All type_annotation pairs from the params (zero or more), + // followed by the return type as the last pair. + let mut type_pairs: Vec> = inner.collect(); + let ret_pair = type_pairs + .pop() + .ok_or_else(|| parse_error!("function_type missing return type"))?; + let ret_ty = self.parse_type(ret_pair)?; + let param_tys: Result, CompilationError> = + type_pairs.into_iter().map(|p| self.parse_type(p)).collect(); + Ok(Type::Function { + param_tys: param_tys?, + ret_ty: Box::new(ret_ty), + }) + } _ => Err(CompilationError::ParseError(format!( "Unexpected rule in type_annotation: {:?}", inner_rule.as_rule() diff --git a/kitlang/src/codegen/transpile/enum_gen.rs b/kitlang/src/codegen/transpile/enum_gen.rs index dc60b57..8e8c4d0 100644 --- a/kitlang/src/codegen/transpile/enum_gen.rs +++ b/kitlang/src/codegen/transpile/enum_gen.rs @@ -4,7 +4,7 @@ use crate::codegen::ast::Attributed; use crate::codegen::module::ModulePath; use crate::codegen::name_mangling::{mangle_enum_variant, mangle_name}; use crate::codegen::type_ast::{EnumDefinition, EnumVariant, Field, StructDefinition}; -use crate::codegen::types::{ToCRepr, Type}; +use crate::codegen::types::Type; use super::CodegenCtx; @@ -34,9 +34,9 @@ impl CodegenCtx<'_> { .iter() .map(|field| { let ty = self.resolve_field_type(field); + let formatted = self.format_type_with_name(&ty, &field.name, &self.current_module); let prefix = if field.is_const { "const " } else { "" }; - let cname = self.type_to_c_name(&ty); - format!(" {}{} {};", prefix, cname, field.name) + format!(" {}{};", prefix, formatted) }) .collect(); @@ -127,7 +127,10 @@ impl CodegenCtx<'_> { .iter() .map(|arg| { let ty = self.resolve_field_type(arg); - format!(" {} {};", ty.to_c_repr().name, arg.name) + format!( + " {};", + self.format_type_with_name(&ty, &arg.name, &self.current_module) + ) }) .collect(); let _ = write!( @@ -178,7 +181,7 @@ impl CodegenCtx<'_> { .iter() .map(|arg| { let ty = self.resolve_field_type(arg); - format!("{} {}", ty.to_c_repr().name, arg.name) + self.format_type_with_name(&ty, &arg.name, &self.current_module) }) .collect(); let arg_names: Vec = v.args.iter().map(|arg| arg.name.clone()).collect(); diff --git a/kitlang/src/codegen/transpile/mod.rs b/kitlang/src/codegen/transpile/mod.rs index aef5027..607f206 100644 --- a/kitlang/src/codegen/transpile/mod.rs +++ b/kitlang/src/codegen/transpile/mod.rs @@ -9,6 +9,8 @@ use std::path::PathBuf; use crate::codegen::ast::{Attributed, Block, Expr, Function, GlobalDecl, Program, Stmt}; use crate::codegen::module::{ModulePath, ModuleRegistry}; use crate::codegen::name_mangling::{mangle_enum_variant, mangle_name}; +use crate::codegen::parser::expr_pratt::callee_name; +use crate::codegen::type_ast::FieldInit; use crate::codegen::types::{ToCRepr, Type, TypeId}; use super::ast::Param; @@ -125,9 +127,9 @@ impl CodegenCtx<'_> { | Expr::StructInit { ty, .. } | Expr::FieldAccess { ty, .. } | Expr::EnumVariant { ty, .. } - | Expr::EnumInit { ty, .. } => *ty, - Expr::ArrayLiteral { ty, .. } => *ty, - Expr::Index { ty, .. } => *ty, + | Expr::EnumInit { ty, .. } + | Expr::ArrayLiteral { ty, .. } + | Expr::Index { ty, .. } => *ty, Expr::RangeLiteral { .. } => TypeId::default(), } } @@ -403,6 +405,25 @@ impl CodegenCtx<'_> { self.format_function_params_with_module(params, &self.current_module) } + /// Format a type with a variable name, handling function-pointer declarator syntax. + /// Function/ptr-to-function types need `ret (*name)(params)` instead of `ret(*)(params) name`. + fn format_type_with_name(&self, ty: &Type, name: &str, module: &ModulePath) -> String { + if let Some(fp) = self.format_fn_ptr_param(ty, name, module) { + fp + } else { + // Resolve typedef aliases so the variable uses the underlying C type name. + let resolved = self + .inferencer + .store + .resolve_typedef_type(ty) + .unwrap_or_else(|| ty.clone()); + format!( + "{} {name}", + self.type_to_c_name_with_module(&resolved, module) + ) + } + } + /// Format a variable declaration with proper C syntax. /// /// For CArray types (e.g., `CArray(Int, 3)`), this produces `int name[3]` instead of the @@ -414,6 +435,7 @@ impl CodegenCtx<'_> { let elem_c_name = self.type_to_c_name(&elem_type); format!("{elem_c_name} {name}[{size}]") } + Ok(ref ty) => self.format_type_with_name(ty, name, &self.current_module), _ => { let ty_str = self.resolve_type_to_c_name(type_id, "int"); format!("{ty_str} {name}") @@ -425,6 +447,13 @@ impl CodegenCtx<'_> { params .iter() .map(|p| { + // C requires function pointer parameters to embed the name in + // the declarator: `ret (*name)(params)` instead of `ret(*)(params) name`. + if let Ok(ty) = self.inferencer.store.resolve(p.ty) + && let Some(fp) = self.format_fn_ptr_param(&ty, &p.name, module) + { + return fp; + } format!( "{} {}", self.format_function_param_type_with_module(p, module), @@ -435,6 +464,30 @@ impl CodegenCtx<'_> { .join(", ") } + /// If `ty` is a function type or pointer-to-function type, format it as a C + /// function pointer parameter declaration (`ret (*name)(params)`). Returns + /// `None` for non-function types. + fn format_fn_ptr_param(&self, ty: &Type, name: &str, module: &ModulePath) -> Option { + let (ret_ty, param_tys) = match ty { + Type::Function { param_tys, ret_ty } => (ret_ty.as_ref(), param_tys.as_slice()), + Type::Ptr(inner) if matches!(inner.as_ref(), Type::Function { .. }) => { + if let Type::Function { param_tys, ret_ty } = inner.as_ref() { + (ret_ty.as_ref(), param_tys.as_slice()) + } else { + return None; + } + } + _ => return None, + }; + let ret_c = self.type_to_c_name_with_module(ret_ty, module); + let params_c = param_tys + .iter() + .map(|t| self.type_to_c_name_with_module(t, module)) + .collect::>() + .join(", "); + Some(format!("{ret_c} (*{name})({params_c})")) + } + fn mangled_enum_variant(&self, enum_name: &str, variant_name: &str) -> String { let is_simple = self .inferencer @@ -453,15 +506,139 @@ impl CodegenCtx<'_> { } } + fn transpile_call(&self, callee: &Expr, args: &[Expr]) -> String { + if let Some(name) = callee_name(callee) { + self.transpile_named_call(&name, args) + } else { + let callee_c = self.transpile_expr(callee); + let a = args + .iter() + .map(|a| self.transpile_expr(a)) + .collect::>() + .join(", "); + format!("({})({})", callee_c, a) + } + } + + fn transpile_named_call(&self, name: &str, args: &[Expr]) -> String { + if let Some(info) = self + .inferencer + .symbols() + .lookup_enum_variant_by_simple_name(name) + { + let a = args + .iter() + .map(|a| self.transpile_expr(a)) + .collect::>() + .join(", "); + let ctor = + mangle_enum_variant(&self.current_module, &info.enum_name, &info.variant_name); + return format!("{}_new({})", ctor, a); + } + let (mod_path, base_name) = if let Some((mp, bn)) = self.resolve_function_name(name) { + (Some(mp), bn) + } else { + let last = name.rsplit('.').next().unwrap_or(name); + (None, last.to_string()) + }; + let mangled = if name == "main" { + name.to_string() + } else if let Some(mp) = &mod_path { + if has_no_mangle_in_module!(self.registry, mp, base_name.as_str(), functions) { + base_name + } else { + mangle_name(mp, &base_name) + } + } else if self.inferencer.symbols().lookup_function(name).is_some() + && !self.current_module.is_empty() + { + mangle_name(&self.current_module, name) + } else { + name.to_string() + }; + let a = args + .iter() + .map(|a| self.transpile_expr(a)) + .collect::>() + .join(", "); + format!("{mangled}({a})") + } + + fn transpile_field_access(&self, expr: &Expr, field_name: &str) -> String { + let container = self.transpile_expr(expr); + let container_ty = Self::expr_type_id(expr); + + if let Ok(Type::Named(type_name)) = self.inferencer.store.resolve(container_ty) + && let Some(enum_def) = self.inferencer.symbols().lookup_enum(&type_name) + && let Some(variant) = enum_def + .variants + .iter() + .find(|v| !v.args.is_empty() && v.args.iter().any(|a| a.name == *field_name)) + { + return format!( + "{}._variant.{}.{}", + container, + variant.name.to_lowercase(), + field_name + ); + } + format!("{}.{}", container, field_name) + } + + fn transpile_array_literal(&self, ty: TypeId, elements: &[Expr]) -> String { + let array_c_name = self + .inferencer + .store + .resolve(ty) + .ok() + .map(|t| self.type_to_c_name(&t)) + .unwrap_or_else(|| "int[]".to_string()); + let elems = elements + .iter() + .map(|e| self.transpile_expr(e)) + .collect::>() + .join(", "); + format!("({array_c_name}){{{elems}}}") + } + + fn transpile_struct_init(&self, ty: TypeId, fields: &[FieldInit]) -> String { + let name = match self.inferencer.store.resolve(ty) { + Ok(Type::Struct { name, .. } | Type::Named(name)) => name, + Ok(_) => "UNKNOWN_STRUCT".to_string(), + Err(e) => { + eprintln!("Warning: Failed to resolve struct type: {e}"); + "UNKNOWN_STRUCT".to_string() + } + }; + let mangled = mangle_name(&self.current_module, &name); + let inits = fields + .iter() + .map(|f| format!(".{} = {}", f.name, self.transpile_expr(&f.value))) + .collect::>() + .join(", "); + format!("(struct {mangled}){{{inits}}}") + } + fn transpile_expr(&self, expr: &Expr) -> String { match expr { Expr::Identifier { name, .. } => { if let Some(mod_path) = self.find_global_module(name) { + // Global variable reference. if has_no_mangle_in_module!(self.registry, &mod_path, name.as_str(), globals) { name.clone() } else { mangle_name(&mod_path, name) } + } else if let Some((mp, bn)) = self.resolve_function_name(name) + && self.inferencer.symbols().lookup_function(&bn).is_some() + { + // Function reference used as a value (e.g. `g(f)`). + // Reuse the call-path mangling for cross-module correctness + no_mangle. + if has_no_mangle_in_module!(self.registry, &mp, bn.as_str(), functions) { + bn + } else { + mangle_name(&mp, &bn) + } } else { name.clone() } @@ -472,66 +649,7 @@ impl CodegenCtx<'_> { }); lit.to_c_with_float(is_c_float) } - Expr::Call { callee, args, .. } => { - if let Some(info) = self - .inferencer - .symbols() - .lookup_enum_variant_by_simple_name(callee) - { - let a = args - .iter() - .map(|a| self.transpile_expr(a)) - .collect::>() - .join(", "); - let ctor = mangle_enum_variant( - &self.current_module, - &info.enum_name, - &info.variant_name, - ); - format!("{}_new({})", ctor, a) - } else { - // XXX: name resolution cascade - qualified name -> module-scoped -> bare (C interop) - let (mod_path, base_name) = - if let Some((mp, bn)) = self.resolve_function_name(callee) { - (Some(mp), bn) - } else { - let last = callee.rsplit('.').next().unwrap_or(callee); - (None, last.to_string()) - }; - - // XXX: 5-condition mangling ladder: - // 1. main is never mangled - // 2. extern/expose items skip mangling - // 3. known functions in non-empty module get module prefix - // 4. everything else passes through as-is (C interop) - let mangled = if callee == "main" { - callee.clone() - } else if let Some(mp) = mod_path { - if has_no_mangle_in_module!( - self.registry, - &mp, - base_name.as_str(), - functions - ) { - base_name.clone() - } else { - mangle_name(&mp, &base_name) - } - } else if self.inferencer.symbols().lookup_function(callee).is_some() - && !self.current_module.is_empty() - { - mangle_name(&self.current_module, callee) - } else { - callee.clone() - }; - let a = args - .iter() - .map(|a| self.transpile_expr(a)) - .collect::>() - .join(", "); - format!("{mangled}({a})") - } - } + Expr::Call { callee, args, .. } => self.transpile_call(callee, args), Expr::UnaryOp { op, expr, .. } => { format!("{}({})", op.to_c_str(), self.transpile_expr(expr)) } @@ -561,55 +679,10 @@ impl CodegenCtx<'_> { format!("({c} ? {t} : {e})") } Expr::RangeLiteral { .. } => "/* range literal */ 0".to_string(), - Expr::StructInit { - ty, - struct_type: _, - fields, - } => { - let name = match self.inferencer.store.resolve(*ty) { - Ok(Type::Struct { name, .. } | Type::Named(name)) => name, - Ok(_) => "UNKNOWN_STRUCT".to_string(), - Err(e) => { - eprintln!("Warning: Failed to resolve struct type: {}", e); - "UNKNOWN_STRUCT".to_string() - } - }; - let mangled = mangle_name(&self.current_module, &name); - let inits = fields - .iter() - .map(|f| format!(".{} = {}", f.name, self.transpile_expr(&f.value))) - .collect::>() - .join(", "); - format!("(struct {}){{{}}}", mangled, inits) - } + Expr::StructInit { ty, fields, .. } => self.transpile_struct_init(*ty, fields), Expr::FieldAccess { expr, field_name, .. - } => { - let container = self.transpile_expr(expr); - let container_ty = Self::expr_type_id(expr); - - // Try to resolve the inferred type of the container expression - if let Ok(Type::Named(type_name)) = self.inferencer.store.resolve(container_ty) - // We only care about named types (structs/enums), not primitives or generics - && let Some(enum_def) = self.inferencer.symbols().lookup_enum(&type_name) - // Ensure the named type is actually an enum in our symbol table - // and retrieve its definition - && let Some(variant) = enum_def.variants.iter().find(|v| { - // Look for a variant that has at least one field/argument - // and where any of those fields match the requested field name - !v.args.is_empty() && v.args.iter().any(|a| a.name == *field_name) - }) { - // If we found a matching enum variant + field, build a fully qualified access path: - // container -> variant (lowercased) -> field - return format!( - "{}._variant.{}.{}", - container, - variant.name.to_lowercase(), - field_name - ); - } - format!("{}.{}", container, field_name) - } + } => self.transpile_field_access(expr, field_name), Expr::Index { expr, index, .. } => { let container = self.transpile_expr(expr); let idx = self.transpile_expr(index); @@ -626,24 +699,7 @@ impl CodegenCtx<'_> { variant_name, .. } => self.mangled_enum_variant(enum_name, variant_name), - Expr::ArrayLiteral { elements, ty } => { - // Resolve the array type to get the element type name for the compound literal - let array_c_name = self - .inferencer - .store - .resolve(*ty) - .ok() - .map(|t| self.type_to_c_name(&t)) - .unwrap_or_else(|| "int[]".to_string()); - - // Construct the compound literal - let elems = elements - .iter() - .map(|e| self.transpile_expr(e)) - .collect::>() - .join(", "); - format!("({array_c_name}){{{elems}}}") - } + Expr::ArrayLiteral { elements, ty } => self.transpile_array_literal(*ty, elements), Expr::EnumInit { enum_name, variant_name, diff --git a/kitlang/src/codegen/types.rs b/kitlang/src/codegen/types.rs index bd42f16..debb9ee 100644 --- a/kitlang/src/codegen/types.rs +++ b/kitlang/src/codegen/types.rs @@ -224,7 +224,11 @@ impl TypeStore { if a == b && !matches!( a, - Type::Ptr(_) | Type::Tuple(_) | Type::CArray(..) | Type::Named(_) + Type::Ptr(_) + | Type::Tuple(_) + | Type::CArray(..) + | Type::Named(_) + | Type::Function { .. } ) { return Ok(()); @@ -238,7 +242,11 @@ impl TypeStore { if a_ty == b_ty && !matches!( a_ty, - Type::Ptr(_) | Type::Tuple(_) | Type::CArray(..) | Type::Named(_) + Type::Ptr(_) + | Type::Tuple(_) + | Type::CArray(..) + | Type::Named(_) + | Type::Function { .. } ) { return Ok(()); @@ -282,6 +290,30 @@ impl TypeStore { } } + // Function types: unify element-wise + ( + Type::Function { + param_tys: p1, + ret_ty: r1, + }, + Type::Function { + param_tys: p2, + ret_ty: r2, + }, + ) => { + if p1.len() != p2.len() { + return Err(format!( + "Cannot unify functions with different parameter counts: {} vs {}", + p1.len(), + p2.len() + )); + } + for (t1, t2) in p1.iter().zip(p2.iter()) { + self.unify_type_ids(t1.clone(), t2.clone())?; + } + self.unify_type_ids((**r1).clone(), (**r2).clone()) + } + // Numeric type promotion: allow mixed-width integer/float types to unify _ if Self::is_numeric(&a_ty) && Self::is_numeric(&b_ty) => Ok(()), @@ -320,9 +352,7 @@ impl TypeStore { } } -/// Represents a type in the Kit language. -/// -/// TODO: further description +/// A type in the Kit language: primitives, composites (struct/enum/tuple), references (pointers/named aliases), and function types. #[derive(Clone, Debug, PartialEq, Hash)] pub enum Type { /// User-defined named type (fallback for types not covered by other variants). @@ -374,6 +404,14 @@ pub enum Type { /// Field definitions for the struct. fields: Vec<(String, TypeId)>, }, + /// Function type (e.g., `function (Int) -> Float`). + /// Parameter and return types are stored by value. When needed for + /// unification, they are converted to TypeId via [`TypeStore::new_known`] + /// (see `unify_types` for the same pattern used by `Tuple`). + Function { + param_tys: Vec, + ret_ty: Box, + }, } impl Type { @@ -456,6 +494,23 @@ impl ToCRepr for Type { headers: elem_repr.headers, } } + Type::Function { param_tys, ret_ty } => { + let ret_repr = ret_ty.to_c_repr(); + let mut all_headers = ret_repr.headers.clone(); + let params: Vec = param_tys + .iter() + .map(|t| { + let r = t.to_c_repr(); + all_headers.extend(r.headers); + r.name + }) + .collect(); + CRepr { + name: format!("{}(*)({})", ret_repr.name, params.join(", ")), + declaration: None, + headers: all_headers, + } + } Type::Named(name) => simple_c_type(name, &[]), Type::Struct { name, fields: _ } => CRepr { name: format!("struct {}", name), diff --git a/kitlang/src/lexer.rs b/kitlang/src/lexer.rs new file mode 100644 index 0000000..be320b0 --- /dev/null +++ b/kitlang/src/lexer.rs @@ -0,0 +1,439 @@ +//! Tokenizer for Kit expressions. +//! +//! Tokenizes Kit expressions into a flat stream for the Pratt parser. +//! Pest handles program/declaration grammars; this covers only expressions. +//! Tokens carry byte spans for diagnostics. + +use logos::{Lexer, Logos}; +use std::ops::Range; + +/// Byte range in source. Promotable to a richer `Span` type later. +pub type Span = Range; + +/// Token with its source span. +#[derive(Debug, Clone, PartialEq)] +pub struct SpannedTok { + pub kind: Tok, + pub span: Span, +} + +/// Token kind for Kit expressions. Keywords absent by design (handled by pest). +/// Multi-char tokens declared first for explicit priority. +#[derive(Logos, Debug, Clone, PartialEq)] +#[logos(skip r"[ \t\r\n]+")] +#[logos(skip(r"//[^\r\n]*", allow_greedy = true))] +#[logos(skip(r"/\*([^*]|\*+[^*/])*\*/"))] +pub enum Tok { + // --- Punctuation --- + #[token("(")] + LParen, + #[token(")")] + RParen, + #[token("[")] + LBracket, + #[token("]")] + RBracket, + #[token("{")] + LBrace, + #[token("}")] + RBrace, + #[token(",")] + Comma, + #[token(";")] + Semi, + #[token(".")] + Dot, + #[token(":")] + Colon, + #[token("...")] + Ellipsis, + + // --- Arithmetic --- + #[token("+")] + Plus, + #[token("-")] + Minus, + #[token("*")] + Star, + #[token("/")] + Slash, + #[token("%")] + Percent, + + // --- Comparison / equality --- + // Longer forms first. + #[token("==")] + EqEq, + #[token("!=")] + NotEq, + #[token("<=")] + LtEq, + #[token(">=")] + GtEq, + #[token("<")] + Lt, + #[token(">")] + Gt, + + // --- Logical (longer first) --- + #[token("&&")] + AndAnd, + #[token("||")] + OrOr, + #[token("!")] + Bang, + + // --- Bitwise --- + // Note: `&&` and `||` are matched before `&` and `|` by virtue of being + // longer; Logos longest-match ensures we never see a stray single `&` + // that's actually the start of `&&`. Same for `<<` / `>>` below. + #[token("&")] + Amp, + #[token("|")] + Pipe, + #[token("^")] + Caret, + #[token("~")] + Tilde, + #[token("<<")] + Shl, + #[token(">>")] + Shr, + + // --- Increment / decrement --- + #[token("++")] + PlusPlus, + #[token("--")] + MinusMinus, + + // --- Assignment (longer forms first) --- + #[token("<<=")] + ShlEq, + #[token(">>=")] + ShrEq, + #[token("+=")] + PlusEq, + #[token("-=")] + MinusEq, + #[token("*=")] + StarEq, + #[token("/=")] + SlashEq, + #[token("%=")] + PercentEq, + #[token("&=")] + AmpEq, + #[token("|=")] + PipeEq, + #[token("^=")] + CaretEq, + #[token("=")] + Assign, + + // --- Keywords that appear in expressions --- + #[token("if")] + KwIf, + #[token("then")] + KwThen, + #[token("else")] + KwElse, + #[token("true")] + KwTrue, + #[token("false")] + KwFalse, + #[token("null")] + KwNull, + #[token("this")] + KwThis, + #[token("Self")] + KwSelf, + #[token("sizeof")] + KwSizeof, + #[token("defined")] + KwDefined, + #[token("unsafe")] + KwUnsafe, + #[token("static")] + KwStatic, + #[token("implicit")] + KwImplicit, + #[token("empty")] + KwEmpty, + #[token("struct")] + KwStruct, + + // --- Literals --- + // Integer: one or more ASCII digits. + #[regex(r"[0-9]+", priority = 4, callback = parse_int)] + IntLit(i64), + + // Float: digits.digits, optional exponent. + // Must come before IntLit conceptually (no `.` in IntLit) but Logos + // priority + longest-match handles this; we just give it a higher + // priority so a `1.0` source never matches as `IntLit(1)` + `.` + `0`. + #[regex(r"[0-9]+\.[0-9]+([eE][+-]?[0-9]+)?", priority = 5, callback = parse_float)] + FloatLit(f64), + + // Char literal: single character or escape between single quotes. + #[regex(r"'(\\.|[^'\\])'", priority = 6, callback = parse_char)] + CharLit(char), + + // String literal: any character or escape between double quotes. + #[regex(r#""(\\.|[^"\\])*""#, priority = 6, callback = parse_string)] + StringLit(String), + + // Identifier: must come last (lowest priority) and must NOT match the + // keyword forms. The grammar's `identifier` rule uses negative lookahead + // for the reserved words; we mirror that by giving keywords priority 2 + // and identifier priority 1. + #[regex(r"[a-zA-Z_][a-zA-Z0-9_]*", priority = 1, callback = parse_ident)] + Ident(String), +} + +fn parse_int(lex: &mut Lexer) -> Option { + lex.slice().parse::().ok() +} + +fn parse_float(lex: &mut Lexer) -> Option { + lex.slice().parse::().ok() +} + +fn parse_char(lex: &mut Lexer) -> Option { + // The slice includes the surrounding single quotes; strip them. + let s = lex.slice(); + let inner = &s[1..s.len() - 1]; + unescape_char(inner) +} + +fn parse_string(lex: &mut Lexer) -> Option { + let s = lex.slice(); + let inner = &s[1..s.len() - 1]; + Some(unescape_str(inner)) +} + +fn parse_ident(lex: &mut Lexer) -> Option { + Some(lex.slice().to_string()) +} + +fn unescape_char(s: &str) -> Option { + let mut chars = s.chars(); + let c = chars.next()?; + if c == '\\' { + let esc = chars.next()?; + Some(match esc { + 'n' => '\n', + 'r' => '\r', + 't' => '\t', + '\\' => '\\', + '\'' => '\'', + '"' => '"', + '0' => '\0', + other => other, + }) + } else { + Some(c) + } +} + +fn unescape_str(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + let mut chars = s.chars(); + while let Some(c) = chars.next() { + if c == '\\' { + if let Some(esc) = chars.next() { + match esc { + 'n' => out.push('\n'), + 'r' => out.push('\r'), + 't' => out.push('\t'), + '\\' => out.push('\\'), + '\'' => out.push('\''), + '"' => out.push('"'), + '0' => out.push('\0'), + other => out.push(other), + } + } + } else { + out.push(c); + } + } + out +} + +/// A lexical error encountered during tokenization. +#[derive(Debug, Clone)] +pub enum LexicalError { + /// An unrecognized character that matches no token pattern. + UnexpectedCharacter { offset: usize }, + /// An integer literal that overflows `i64`. + IntegerOverflow { + text: String, + #[allow(dead_code)] + offset: usize, + }, +} + +/// Tokenize source into `Vec`. Returns an error on the first +/// unrecognized character or overflowed literal. Whitespace and comments +/// are skipped silently. +pub fn tokenize(source: &str) -> Result, LexicalError> { + let mut tokens = Vec::new(); + for (res, span) in Tok::lexer(source).spanned() { + match res { + Ok(kind) => tokens.push(SpannedTok { kind, span }), + Err(()) => { + // Logos produces Err when either no regex matches OR a callback + // returns None (e.g. parse_int on overflow). Distinguish by + // checking if the rejected span is all digits. + let rejected = &source[span.clone()]; + if rejected.chars().all(|c| c.is_ascii_digit()) { + return Err(LexicalError::IntegerOverflow { + text: rejected.to_string(), + offset: span.start, + }); + } + return Err(LexicalError::UnexpectedCharacter { offset: span.start }); + } + } + } + Ok(tokens) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn kinds(source: &str) -> Vec { + tokenize(source) + .unwrap_or_else(|e| panic!("tokenize failed for `{source}`: {e:?}")) + .into_iter() + .map(|t| t.kind) + .collect() + } + + #[test] + fn integer_literals() { + assert_eq!(kinds("0"), vec![Tok::IntLit(0)]); + assert_eq!(kinds("42"), vec![Tok::IntLit(42)]); + assert_eq!(kinds("1234567890"), vec![Tok::IntLit(1234567890)]); + } + + #[test] + fn float_literals() { + assert_eq!(kinds("3.14"), vec![Tok::FloatLit(3.14)]); + assert_eq!(kinds("1.0e10"), vec![Tok::FloatLit(1.0e10)]); + assert_eq!(kinds("2.5E-3"), vec![Tok::FloatLit(2.5E-3)]); + } + + #[test] + fn char_and_string_literals() { + assert_eq!(kinds("'a'"), vec![Tok::CharLit('a')]); + assert_eq!(kinds("'\\n'"), vec![Tok::CharLit('\n')]); + assert_eq!( + kinds(r#""hello""#), + vec![Tok::StringLit("hello".to_string())] + ); + assert_eq!(kinds(r#""a\nb""#), vec![Tok::StringLit("a\nb".to_string())]); + } + + #[test] + fn identifiers_and_keywords() { + assert_eq!(kinds("foo"), vec![Tok::Ident("foo".to_string())]); + assert_eq!( + kinds("foo_bar_123"), + vec![Tok::Ident("foo_bar_123".to_string())] + ); + // Keywords take priority over the identifier regex. + assert_eq!(kinds("if"), vec![Tok::KwIf]); + assert_eq!(kinds("Self"), vec![Tok::KwSelf]); + assert_eq!(kinds("null"), vec![Tok::KwNull]); + } + + #[test] + fn operators_longest_match() { + // `&&` must match as a single token, not two `&`s. + assert_eq!(kinds("&&"), vec![Tok::AndAnd]); + // `<<=` must match as a single token, not `<<` + `=`. + assert_eq!(kinds("<<="), vec![Tok::ShlEq]); + // `==` must match as a single token, not two `=`s. + assert_eq!(kinds("=="), vec![Tok::EqEq]); + } + + #[test] + fn arithmetic_operators() { + assert_eq!( + kinds("a + b * c"), + vec![ + Tok::Ident("a".to_string()), + Tok::Plus, + Tok::Ident("b".to_string()), + Tok::Star, + Tok::Ident("c".to_string()), + ] + ); + } + + #[test] + fn whitespace_is_skipped() { + assert_eq!(kinds(" a \t +\n b "), kinds("a+b")); + } + + #[test] + fn line_comments_are_skipped() { + assert_eq!(kinds("a // ignore me\n+ b"), kinds("a+b")); + } + + #[test] + fn block_comments_are_skipped() { + assert_eq!(kinds("a /* hi */ + b"), kinds("a+b")); + assert_eq!(kinds("a/*\nmulti\nline\n*/+b"), kinds("a+b")); + } + + #[test] + fn spans_are_byte_accurate() { + let toks = tokenize(" a + b ").unwrap(); + assert_eq!(toks[0].span, 2..3); // 'a' + assert_eq!(toks[1].span, 4..5); // '+' + assert_eq!(toks[2].span, 6..7); // 'b' + } + + #[test] + fn empty_source_produces_no_tokens() { + assert!(tokenize("").unwrap().is_empty()); + } + + #[test] + fn unknown_characters_are_errors() { + // `$` is not a Kit token; the lexer now surfaces an error instead of + // silently dropping it, so the Pratt parser can produce a diagnostic. + let err = tokenize("a $ b").unwrap_err(); + assert!(matches!( + err, + LexicalError::UnexpectedCharacter { offset: 2 } + )); + } + + #[test] + fn integer_overflow_produces_error() { + // Logos produces Err when parse_int returns None for overflow. + // Our tokenize catches this as IntegerOverflow. + let err = tokenize("99999999999999999999").unwrap_err(); + assert!( + matches!(&err, LexicalError::IntegerOverflow { text, .. } if text == "99999999999999999999") + ); + } + + #[test] + fn mixed_overflow_and_valid_tokens_produces_error() { + // Overflow literal causes a tokenize error before valid tokens are reached + let err = tokenize("99999999999999999999 + 1").unwrap_err(); + assert!(matches!(&err, LexicalError::IntegerOverflow { .. })); + } + + #[test] + fn overflow_does_not_affect_valid_expression_with_small_int() { + // "1 + 99999999999999999999" -- overflow is NOT the first token, + // so Logos produces Err at the overflow position after the `1` and `+` + let err = tokenize("1 + 99999999999999999999").unwrap_err(); + assert!(matches!(&err, LexicalError::IntegerOverflow { .. })); + } +} diff --git a/kitlang/src/lib.rs b/kitlang/src/lib.rs index 86abb75..1f0b3b7 100644 --- a/kitlang/src/lib.rs +++ b/kitlang/src/lib.rs @@ -11,5 +11,12 @@ pub use codegen::Toolchain; #[grammar = "grammar/kit.pest"] pub struct KitParser; +/// Tokenizer for expressions, used by the Pratt parser. +/// +/// Pest still handles the program/declaration/statement grammar; the Pratt +/// parser only takes over expression parsing, and it needs a token stream +/// to do so. The Logos-based lexer here is that token stream. +pub(crate) mod lexer; + /// Compilation error types. pub(crate) mod error;