diff --git a/logos-codegen/src/generator/context.rs b/logos-codegen/src/generator/context.rs index 515785ee..82f76f39 100644 --- a/logos-codegen/src/generator/context.rs +++ b/logos-codegen/src/generator/context.rs @@ -59,21 +59,12 @@ impl Context { self.available.saturating_sub(self.at) } - pub fn read_unchecked(&mut self, len: usize) -> TokenStream { + pub fn read_byte_unchecked(&mut self) -> TokenStream { let at = self.at; - match len { - 0 => { - self.advance(1); + self.advance(1); - quote!(lex.read_unchecked::(#at)) - } - l => { - self.advance(l); - - quote!(lex.read_unchecked::<&[u8; #l]>(#at)) - } - } + quote!(lex.read_byte_unchecked(#at)) } pub fn read(&mut self, len: usize) -> TokenStream { diff --git a/logos-codegen/src/generator/fork.rs b/logos-codegen/src/generator/fork.rs index 0a96056b..a0631f93 100644 --- a/logos-codegen/src/generator/fork.rs +++ b/logos-codegen/src/generator/fork.rs @@ -127,7 +127,7 @@ impl<'a> Generator<'a> { let min_read = self.meta[this].min_read; if ctx.remainder() >= max(min_read, 1) { - let read = ctx.read_unchecked(0); + let read = ctx.read_byte_unchecked(); return (quote!(byte), quote!(let byte = unsafe { #read };)); } diff --git a/logos/src/internal.rs b/logos/src/internal.rs index 125eab1c..2957fede 100644 --- a/logos/src/internal.rs +++ b/logos/src/internal.rs @@ -16,8 +16,8 @@ pub trait LexerInternal<'source> { /// Read a chunk at current position, offset by `n`. fn read_at>(&self, n: usize) -> Option; - /// Unchecked read a chunk at current position, offset by `n`. - unsafe fn read_unchecked>(&self, n: usize) -> T; + /// Unchecked read a byte at current position, offset by `n`. + unsafe fn read_byte_unchecked(&self, n: usize) -> u8; /// Test a chunk at current position with a closure. fn test, F: FnOnce(T) -> bool>(&self, test: F) -> bool; diff --git a/logos/src/lexer.rs b/logos/src/lexer.rs index 2340b750..f5b12b2d 100644 --- a/logos/src/lexer.rs +++ b/logos/src/lexer.rs @@ -302,11 +302,8 @@ where } #[inline] - unsafe fn read_unchecked(&self, n: usize) -> Chunk - where - Chunk: source::Chunk<'source>, - { - self.source.read_unchecked(self.token_end + n) + unsafe fn read_byte_unchecked(&self, n: usize) -> u8 { + self.source.read_byte_unchecked(self.token_end + n) } /// Test a chunk at current position with a closure. diff --git a/logos/src/source.rs b/logos/src/source.rs index 4cd09e08..a796274f 100644 --- a/logos/src/source.rs +++ b/logos/src/source.rs @@ -43,14 +43,12 @@ pub trait Source { where Chunk: self::Chunk<'a>; - /// Read a chunk of bytes into an array without doing bounds checks. + /// Read a byte without doing bounds checks. /// /// # Safety /// /// Offset should not exceed bounds. - unsafe fn read_unchecked<'a, Chunk>(&'a self, offset: usize) -> Chunk - where - Chunk: self::Chunk<'a>; + unsafe fn read_byte_unchecked(&self, offset: usize) -> u8; /// Get a slice of the source at given range. This is analogous to /// `slice::get(range)`. @@ -119,10 +117,7 @@ impl Source for str { } #[inline] - unsafe fn read_unchecked<'a, Chunk>(&'a self, offset: usize) -> Chunk - where - Chunk: self::Chunk<'a>, - { + unsafe fn read_byte_unchecked(&self, offset: usize) -> u8 { Chunk::from_ptr(self.as_ptr().add(offset)) } @@ -179,10 +174,7 @@ impl Source for [u8] { } #[inline] - unsafe fn read_unchecked<'a, Chunk>(&'a self, offset: usize) -> Chunk - where - Chunk: self::Chunk<'a>, - { + unsafe fn read_byte_unchecked(&self, offset: usize) -> u8 { Chunk::from_ptr(self.as_ptr().add(offset)) } diff --git a/tests/tests/source.rs b/tests/tests/source.rs index a0024a53..c747a354 100644 --- a/tests/tests/source.rs +++ b/tests/tests/source.rs @@ -19,11 +19,8 @@ impl<'s, S: ?Sized + Source> Source for RefSource<'s, S> { self.0.read(offset) } - unsafe fn read_unchecked<'a, Chunk>(&'a self, offset: usize) -> Chunk - where - Chunk: logos::source::Chunk<'a>, - { - self.0.read_unchecked(offset) + unsafe fn read_byte_unchecked(&self, offset: usize) -> u8 { + self.0.read_byte_unchecked(offset) } fn slice(&self, range: Range) -> Option> {