From 939ee7606fbfe3afa6b6a008fb617120a6dd8114 Mon Sep 17 00:00:00 2001 From: Mathias Magnusson Date: Mon, 2 Jun 2025 15:01:58 +0200 Subject: make Lexer peekable without a wrapper --- src/Lexer.zig | 38 ++++++++++++++++++++++++++------------ src/main.zig | 32 +++++++++++++------------------- src/parse.zig | 20 ++++++++------------ src/peek.zig | 51 --------------------------------------------------- 4 files changed, 47 insertions(+), 94 deletions(-) delete mode 100644 src/peek.zig diff --git a/src/Lexer.zig b/src/Lexer.zig index 62b2352..8c23b26 100644 --- a/src/Lexer.zig +++ b/src/Lexer.zig @@ -44,13 +44,27 @@ pub const Location = struct { } }; +pub fn peek(self: *Self) Token { + if (self.peeked == null) { + self.peeked = self.getNext(); + } + return self.peeked.?; +} + +pub fn next(self: *Self) Token { + const token = self.peek(); + self.peeked = null; + return token; +} + source: []const u8, start: usize = 0, pos: usize = 0, +peeked: ?Token = null, -pub fn next(self: *Self) ?Token { +fn getNext(self: *Self) Token { self.start = self.pos; - return s: switch (self.eat() orelse return self.create(.eof)) { + return s: switch (self.eatChar() orelse return self.create(.eof)) { '(' => self.create(.left_paren), ')' => self.create(.right_paren), '0', '1', '2', '3', '4', '5', '6', '7', '8', '9' => self.integerLiteral(), @@ -58,7 +72,7 @@ pub fn next(self: *Self) ?Token { '-' => self.create(.minus), ' ' => { self.start = self.pos; - continue :s (self.eat() orelse return self.create(.eof)); + continue :s (self.eatChar() orelse return self.create(.eof)); }, else => |c| if ('a' <= c and c <= 'z' or 'A' <= c and c <= 'Z') self.identifierOrKeyword() @@ -69,7 +83,7 @@ pub fn next(self: *Self) ?Token { fn integerLiteral(self: *Self) Token { var value: ?u64 = self.source[self.start] - '0'; - while (digitValue(self.peek())) |v| { + while (digitValue(self.peekChar())) |v| { var nxt: ?u64 = null; if (value) |val| if (std.math.mul(u64, val, 10) catch null) |p| @@ -78,7 +92,7 @@ fn integerLiteral(self: *Self) Token { }; value = nxt; - _ = self.eat(); + _ = self.eatChar(); } return if (value != null) self.create(.integer_literal) @@ -88,9 +102,9 @@ fn integerLiteral(self: *Self) Token { fn identifierOrKeyword(self: *Self) Token { while (true) { - const c = self.peek() orelse 0; + const c = self.peekChar() orelse 0; if ('a' <= c and c <= 'z' or 'A' <= c and c <= 'Z' or c == '_') { - _ = self.eat(); + _ = self.eatChar(); continue; } const value = self.source[self.start..self.pos]; @@ -101,18 +115,18 @@ fn identifierOrKeyword(self: *Self) Token { } } -fn create(self: *Self, tajp: Token.Type) Token { +fn create(self: *Self, ty: Token.Type) Token { const start = self.start; - return .{ .loc = .{ .start = start, .end = self.pos }, .type = tajp }; + return .{ .loc = .{ .start = start, .end = self.pos }, .type = ty }; } -fn eat(self: *Self) ?u8 { - const token = self.peek(); +fn eatChar(self: *Self) ?u8 { + const token = self.peekChar(); if (token != null) self.pos += 1; return token; } -fn peek(self: *Self) ?u8 { +fn peekChar(self: *Self) ?u8 { return if (self.pos < self.source.len) self.source[self.pos] else null; } diff --git a/src/main.zig b/src/main.zig index 3c21f19..87dc493 100644 --- a/src/main.zig +++ b/src/main.zig @@ -1,10 +1,7 @@ const std = @import("std"); const parse = @import("./parse.zig"); -const peek = @import("./peek.zig"); const codegen = @import("./codegen.zig"); -pub const peekable = peek.peekable; -pub const Peekable = peek.Peekable; pub const Expr = parse.Expr; pub const Lexer = @import("./Lexer.zig"); pub const compile = @import("./compile.zig"); @@ -34,22 +31,23 @@ pub fn main() !void { // const source = "17216961135462248174 + 4095 - 4294967295 + 2147483647"; const source = "print 18446744073709551615"; - // var lexer = Lexer{ .source = source }; - // while (true) { - // const token = lexer.next().?; - // std.debug.print("{}\n", .{token}); - // if (token.type == .Eof) break; - // } - // std.debug.print("\n", .{}); - var lexer = peekable(Lexer{ .source = source }); + var lexer: Lexer = .{ .source = source }; + std.debug.print("Tokens:\n", .{}); + while (true) { + const token = lexer.next(); + std.debug.print(" {}\n", .{token}); + if (token.type == .eof) break; + } + lexer = .{ .source = source }; const expr = try parse.expression(allocator, &lexer); std.debug.print("{}\n", .{expr}); - if (lexer.next()) |token| if (token.type != .eof) { - std.debug.print("Unexpected token {}, expected end of file\n", .{token}); - }; const block = try compile.compile(allocator, source, expr); + if (lexer.peek().type != .eof) { + std.debug.print("Unexpected token {}, expected end of file\n", .{lexer.next()}); + } + std.debug.print("Bytecode instructions:\n", .{}); for (block.instrs) |instr| { - std.debug.print("{}\n", .{instr}); + std.debug.print(" {}\n", .{instr}); } const elf = try codegen.create_elf(allocator, block); try output.writeAll(elf); @@ -81,7 +79,3 @@ fn HashMapFormatter(HashMap: type) type { pub fn fmtHashMap(hash_map: anytype) HashMapFormatter(@TypeOf(hash_map)) { return .{ .data = hash_map }; } - -test { - _ = peek; -} diff --git a/src/parse.zig b/src/parse.zig index d5d3e9b..901ce0d 100644 --- a/src/parse.zig +++ b/src/parse.zig @@ -4,8 +4,6 @@ const Allocator = std.mem.Allocator; const root = @import("root"); const Lexer = root.Lexer; const Token = root.Lexer.Token; -const Peekable = root.Peekable; -const peekable = root.peekable; pub const Expr = struct { loc: Lexer.Location, @@ -57,20 +55,18 @@ pub const Expr = struct { } }; -pub fn expression(allocator: Allocator, lexer: *Peekable(Lexer)) error{OutOfMemory}!*Expr { +pub fn expression(allocator: Allocator, lexer: *Lexer) error{OutOfMemory}!*Expr { return addExpr(allocator, lexer); } -pub fn addExpr(allocator: Allocator, lexer: *Peekable(Lexer)) !*Expr { +pub fn addExpr(allocator: Allocator, lexer: *Lexer) !*Expr { var lhs = try callExpr(allocator, lexer); while (true) { - const token: Lexer.Token = if (lexer.peek()) |t| t else break; - const op: Expr.Type.BinOp.Op = switch (token.type) { + const op: Expr.Type.BinOp.Op = switch (lexer.peek().type) { .plus => .plus, .minus => .minus, else => break, }; - _ = lexer.next(); const rhs = try callExpr(allocator, lexer); @@ -82,10 +78,10 @@ pub fn addExpr(allocator: Allocator, lexer: *Peekable(Lexer)) !*Expr { return lhs; } -pub fn callExpr(allocator: Allocator, lexer: *Peekable(Lexer)) !*Expr { +pub fn callExpr(allocator: Allocator, lexer: *Lexer) !*Expr { var proc = try primaryExpr(allocator, lexer); while (true) { - switch (lexer.peek().?.type) { + switch (lexer.peek().type) { .left_paren, .integer_literal => {}, else => break, } @@ -98,13 +94,13 @@ pub fn callExpr(allocator: Allocator, lexer: *Peekable(Lexer)) !*Expr { return proc; } -pub fn primaryExpr(allocator: Allocator, lexer: *Peekable(Lexer)) !*Expr { - const token = lexer.next().?; +pub fn primaryExpr(allocator: Allocator, lexer: *Lexer) !*Expr { + const token = lexer.next(); // std.debug.print("term {}\n", .{token}); return allocate(allocator, switch (token.type) { .left_paren => { const res = expression(allocator, lexer); - const right_paren = lexer.next().?; + const right_paren = lexer.next(); if (right_paren.type != .right_paren) return allocate(allocator, .{ .loc = right_paren.loc, diff --git a/src/peek.zig b/src/peek.zig deleted file mode 100644 index 58911e9..0000000 --- a/src/peek.zig +++ /dev/null @@ -1,51 +0,0 @@ -pub fn peekable(iterator: anytype) Peekable(@TypeOf(iterator)) { - return .{ .iterator = iterator }; -} - -pub fn Peekable(Iterator: type) type { - const ActualIterator = switch (@typeInfo(Iterator)) { - .pointer => |p| p.child, - else => Iterator, - }; - const Next = @typeInfo(@TypeOf(ActualIterator.next)).@"fn"; - const Item = switch (@typeInfo(Next.return_type.?)) { - .optional => |o| o.child, - else => |i| i, - }; - return struct { - iterator: Iterator, - peeked: ?Item = null, - - pub fn peek(self: *Self) ?Item { - if (self.peeked) |peeked| return peeked; - const item = self.iterator.next(); - self.peeked = item; - return item; - } - - pub fn next(self: *Self) ?Item { - const item = if (self.peeked) |peeked| peeked else self.iterator.next(); - self.peeked = null; - return item; - } - - const Self = @This(); - }; -} - -test peekable { - const expect = std.testing.expect; - // std.meta.de - - var it = std.mem.window(u8, &[_]u8{ 1, 2, 3 }, 1, 1); - var peek = peekable(&it); - try expect(peek.next().?[0] == 1); - try expect(peek.peek().?[0] == 2); - try expect(peek.peek().?[0] == 2); - try expect(peek.next().?[0] == 2); - try expect(peek.next().?[0] == 3); - try expect(peek.peek() == null); - try expect(peek.next() == null); -} - -const std = @import("std"); -- cgit v1.2.3