logos/
lexer.rs

1use super::internal::LexerInternal;
2use super::Logos;
3use crate::source::{self, Source};
4
5use core::fmt::{self, Debug};
6use core::mem::ManuallyDrop;
7
8/// Byte range in the source.
9pub type Span = core::ops::Range<usize>;
10
11/// `Lexer` is the main struct of the crate that allows you to read through a
12/// `Source` and produce tokens for enums implementing the `Logos` trait.
13pub struct Lexer<'source, Token: Logos<'source>> {
14    source: &'source Token::Source,
15    token: ManuallyDrop<Option<Token>>,
16    token_start: usize,
17    token_end: usize,
18
19    /// Extras associated with the `Token`.
20    pub extras: Token::Extras,
21}
22
23impl<'source, Token> Debug for Lexer<'source, Token>
24where
25    Token: Logos<'source>,
26    Token::Source: Debug,
27    Token::Extras: Debug,
28{
29    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
30        fmt.debug_map()
31            .entry(&"source", &self.source)
32            .entry(&"extras", &self.extras)
33            .finish()
34    }
35}
36
37impl<'source, Token: Logos<'source>> Lexer<'source, Token> {
38    /// Create a new `Lexer`.
39    ///
40    /// Due to type inference, it might be more ergonomic to construct
41    /// it by calling [`Token::lexer`](./trait.Logos.html#method.lexer) on any `Token` with derived `Logos`.
42    pub fn new(source: &'source Token::Source) -> Self
43    where
44        Token::Extras: Default,
45    {
46        Self::with_extras(source, Default::default())
47    }
48
49    /// Create a new `Lexer` with the provided `Extras`.
50    ///
51    /// Due to type inference, it might be more ergonomic to construct
52    /// it by calling [`Token::lexer_with_extras`](./trait.Logos.html#method.lexer_with_extras) on any `Token` with derived `Logos`.
53    pub fn with_extras(source: &'source Token::Source, extras: Token::Extras) -> Self {
54        Lexer {
55            source,
56            token: ManuallyDrop::new(None),
57            extras,
58            token_start: 0,
59            token_end: 0,
60        }
61    }
62
63    /// Source from which this Lexer is reading tokens.
64    #[inline]
65    pub fn source(&self) -> &'source Token::Source {
66        self.source
67    }
68
69    /// Wrap the `Lexer` in an [`Iterator`](https://doc.rust-lang.org/std/iter/trait.Iterator.html)
70    /// that produces tuples of `(Token, `[`Span`](./type.Span.html)`)`.
71    ///
72    /// # Example
73    ///
74    /// ```
75    /// use logos::Logos;
76    ///
77    /// #[derive(Logos, Debug, PartialEq)]
78    /// enum Example {
79    ///     #[regex(r"[ \n\t\f]+", logos::skip)]
80    ///     #[error]
81    ///     Error,
82    ///
83    ///     #[regex("-?[0-9]+", |lex| lex.slice().parse())]
84    ///     Integer(i64),
85    ///
86    ///     #[regex("-?[0-9]+\\.[0-9]+", |lex| lex.slice().parse())]
87    ///     Float(f64),
88    /// }
89    ///
90    /// let tokens: Vec<_> = Example::lexer("42 3.14 -5 f").spanned().collect();
91    ///
92    /// assert_eq!(
93    ///     tokens,
94    ///     &[
95    ///         (Example::Integer(42), 0..2),
96    ///         (Example::Float(3.14), 3..7),
97    ///         (Example::Integer(-5), 8..10),
98    ///         (Example::Error, 11..12), // 'f' is not a recognized token
99    ///     ],
100    /// );
101    /// ```
102    #[inline]
103    pub fn spanned(self) -> SpannedIter<'source, Token> {
104        SpannedIter { lexer: self }
105    }
106
107    #[inline]
108    #[doc(hidden)]
109    #[deprecated(since = "0.11.0", note = "please use `span` instead")]
110    pub fn range(&self) -> Span {
111        self.span()
112    }
113
114    /// Get the range for the current token in `Source`.
115    #[inline]
116    pub fn span(&self) -> Span {
117        self.token_start..self.token_end
118    }
119
120    /// Get a string slice of the current token.
121    #[inline]
122    pub fn slice(&self) -> &'source <Token::Source as Source>::Slice {
123        unsafe { self.source.slice_unchecked(self.span()) }
124    }
125
126    /// Get a slice of remaining source, starting at the end of current token.
127    #[inline]
128    pub fn remainder(&self) -> &'source <Token::Source as Source>::Slice {
129        unsafe {
130            self.source
131                .slice_unchecked(self.token_end..self.source.len())
132        }
133    }
134
135    /// Turn this lexer into a lexer for a new token type.
136    ///
137    /// The new lexer continues to point at the same span as the current lexer,
138    /// and the current token becomes the error token of the new token type.
139    pub fn morph<Token2>(self) -> Lexer<'source, Token2>
140    where
141        Token2: Logos<'source, Source = Token::Source>,
142        Token::Extras: Into<Token2::Extras>,
143    {
144        Lexer {
145            source: self.source,
146            token: ManuallyDrop::new(None),
147            extras: self.extras.into(),
148            token_start: self.token_start,
149            token_end: self.token_end,
150        }
151    }
152
153    /// Bumps the end of currently lexed token by `n` bytes.
154    ///
155    /// # Panics
156    ///
157    /// Panics if adding `n` to current offset would place the `Lexer` beyond the last byte,
158    /// or in the middle of an UTF-8 code point (does not apply when lexing raw `&[u8]`).
159    pub fn bump(&mut self, n: usize) {
160        self.token_end += n;
161
162        assert!(
163            self.source.is_boundary(self.token_end),
164            "Invalid Lexer bump",
165        )
166    }
167}
168
169impl<'source, Token> Clone for Lexer<'source, Token>
170where
171    Token: Logos<'source> + Clone,
172    Token::Extras: Clone,
173{
174    fn clone(&self) -> Self {
175        Lexer {
176            extras: self.extras.clone(),
177            token: self.token.clone(),
178            ..*self
179        }
180    }
181}
182
183impl<'source, Token> Iterator for Lexer<'source, Token>
184where
185    Token: Logos<'source>,
186{
187    type Item = Token;
188
189    #[inline]
190    fn next(&mut self) -> Option<Token> {
191        self.token_start = self.token_end;
192
193        Token::lex(self);
194
195        // This basically treats self.token as a temporary field.
196        // Since we always immediately return a newly set token here,
197        // we don't have to replace it with `None` or manually drop
198        // it later.
199        unsafe { ManuallyDrop::take(&mut self.token) }
200    }
201}
202
203/// Iterator that pairs tokens with their position in the source.
204///
205/// Look at [`Lexer::spanned`](./struct.Lexer.html#method.spanned) for documentation.
206pub struct SpannedIter<'source, Token: Logos<'source>> {
207    lexer: Lexer<'source, Token>,
208}
209
210impl<'source, Token> Iterator for SpannedIter<'source, Token>
211where
212    Token: Logos<'source>,
213{
214    type Item = (Token, Span);
215
216    fn next(&mut self) -> Option<Self::Item> {
217        self.lexer.next().map(|token| (token, self.lexer.span()))
218    }
219}
220
221#[doc(hidden)]
222/// # WARNING!
223///
224/// **This trait, and it's methods, are not meant to be used outside of the
225/// code produced by `#[derive(Logos)]` macro.**
226impl<'source, Token> LexerInternal<'source> for Lexer<'source, Token>
227where
228    Token: Logos<'source>,
229{
230    type Token = Token;
231
232    /// Read a `Chunk` at current position of the `Lexer`. If end
233    /// of the `Source` has been reached, this will return `0`.
234    #[inline]
235    fn read<Chunk>(&self) -> Option<Chunk>
236    where
237        Chunk: source::Chunk<'source>,
238    {
239        self.source.read(self.token_end)
240    }
241
242    /// Read a `Chunk` at a position offset by `n`.
243    #[inline]
244    fn read_at<Chunk>(&self, n: usize) -> Option<Chunk>
245    where
246        Chunk: source::Chunk<'source>,
247    {
248        self.source.read(self.token_end + n)
249    }
250
251    #[inline]
252    unsafe fn read_unchecked<Chunk>(&self, n: usize) -> Chunk
253    where
254        Chunk: source::Chunk<'source>,
255    {
256        self.source.read_unchecked(self.token_end + n)
257    }
258
259    /// Test a chunk at current position with a closure.
260    #[inline]
261    fn test<T, F>(&self, test: F) -> bool
262    where
263        T: source::Chunk<'source>,
264        F: FnOnce(T) -> bool,
265    {
266        match self.source.read::<T>(self.token_end) {
267            Some(chunk) => test(chunk),
268            None => false,
269        }
270    }
271
272    /// Test a chunk at current position offset by `n` with a closure.
273    #[inline]
274    fn test_at<T, F>(&self, n: usize, test: F) -> bool
275    where
276        T: source::Chunk<'source>,
277        F: FnOnce(T) -> bool,
278    {
279        match self.source.read::<T>(self.token_end + n) {
280            Some(chunk) => test(chunk),
281            None => false,
282        }
283    }
284
285    /// Bump the position `Lexer` is reading from by `size`.
286    #[inline]
287    fn bump_unchecked(&mut self, size: usize) {
288        debug_assert!(
289            self.token_end + size <= self.source.len(),
290            "Bumping out of bounds!"
291        );
292
293        self.token_end += size;
294    }
295
296    /// Reset `token_start` to `token_end`.
297    #[inline]
298    fn trivia(&mut self) {
299        self.token_start = self.token_end;
300    }
301
302    /// Set the current token to appropriate `#[error]` variant.
303    /// Guarantee that `token_end` is at char boundary for `&str`.
304    #[inline]
305    fn error(&mut self) {
306        self.token_end = self.source.find_boundary(self.token_end);
307        self.token = ManuallyDrop::new(Some(Token::ERROR));
308    }
309
310    #[inline]
311    fn end(&mut self) {
312        self.token = ManuallyDrop::new(None);
313    }
314
315    #[inline]
316    fn set(&mut self, token: Token) {
317        self.token = ManuallyDrop::new(Some(token));
318    }
319}
logos/lexer.rs

logos/
lexer.rs
