logos/lib.rs
1//! <img src="https://raw.githubusercontent.com/maciejhirsz/logos/master/logos.svg?sanitize=true" alt="Logos logo" width="250" align="right">
2//!
3//! # Logos
4//!
5//! _Create ridiculously fast Lexers._
6//!
7//! **Logos** has two goals:
8//!
9//! + To make it easy to create a Lexer, so you can focus on more complex problems.
10//! + To make the generated Lexer faster than anything you'd write by hand.
11//!
12//! To achieve those, **Logos**:
13//!
14//! + Combines all token definitions into a single [deterministic state machine](https://en.wikipedia.org/wiki/Deterministic_finite_automaton).
15//! + Optimizes branches into [lookup tables](https://en.wikipedia.org/wiki/Lookup_table) or [jump tables](https://en.wikipedia.org/wiki/Branch_table).
16//! + Prevents [backtracking](https://en.wikipedia.org/wiki/ReDoS) inside token definitions.
17//! + [Unwinds loops](https://en.wikipedia.org/wiki/Loop_unrolling), and batches reads to minimize bounds checking.
18//! + Does all of that heavy lifting at compile time.
19//!
20//! ## Example
21//!
22//! ```rust
23//! use logos::Logos;
24//!
25//! #[derive(Logos, Debug, PartialEq)]
26//! enum Token {
27//! // Tokens can be literal strings, of any length.
28//! #[token("fast")]
29//! Fast,
30//!
31//! #[token(".")]
32//! Period,
33//!
34//! // Or regular expressions.
35//! #[regex("[a-zA-Z]+")]
36//! Text,
37//!
38//! // Logos requires one token variant to handle errors,
39//! // it can be named anything you wish.
40//! #[error]
41//! // We can also use this variant to define whitespace,
42//! // or any other matches we wish to skip.
43//! #[regex(r"[ \t\n\f]+", logos::skip)]
44//! Error,
45//! }
46//!
47//! fn main() {
48//! let mut lex = Token::lexer("Create ridiculously fast Lexers.");
49//!
50//! assert_eq!(lex.next(), Some(Token::Text));
51//! assert_eq!(lex.span(), 0..6);
52//! assert_eq!(lex.slice(), "Create");
53//!
54//! assert_eq!(lex.next(), Some(Token::Text));
55//! assert_eq!(lex.span(), 7..19);
56//! assert_eq!(lex.slice(), "ridiculously");
57//!
58//! assert_eq!(lex.next(), Some(Token::Fast));
59//! assert_eq!(lex.span(), 20..24);
60//! assert_eq!(lex.slice(), "fast");
61//!
62//! assert_eq!(lex.next(), Some(Token::Text));
63//! assert_eq!(lex.slice(), "Lexers");
64//! assert_eq!(lex.span(), 25..31);
65//!
66//! assert_eq!(lex.next(), Some(Token::Period));
67//! assert_eq!(lex.span(), 31..32);
68//! assert_eq!(lex.slice(), ".");
69//!
70//! assert_eq!(lex.next(), None);
71//! }
72//! ```
73//!
74//! ### Callbacks
75//!
76//! **Logos** can also call arbitrary functions whenever a pattern is matched,
77//! which can be used to put data into a variant:
78//!
79//! ```rust
80//! use logos::{Logos, Lexer};
81//!
82//! // Note: callbacks can return `Option` or `Result`
83//! fn kilo(lex: &mut Lexer<Token>) -> Option<u64> {
84//! let slice = lex.slice();
85//! let n: u64 = slice[..slice.len() - 1].parse().ok()?; // skip 'k'
86//! Some(n * 1_000)
87//! }
88//!
89//! fn mega(lex: &mut Lexer<Token>) -> Option<u64> {
90//! let slice = lex.slice();
91//! let n: u64 = slice[..slice.len() - 1].parse().ok()?; // skip 'm'
92//! Some(n * 1_000_000)
93//! }
94//!
95//! #[derive(Logos, Debug, PartialEq)]
96//! enum Token {
97//! #[regex(r"[ \t\n\f]+", logos::skip)]
98//! #[error]
99//! Error,
100//!
101//! // Callbacks can use closure syntax, or refer
102//! // to a function defined elsewhere.
103//! //
104//! // Each pattern can have it's own callback.
105//! #[regex("[0-9]+", |lex| lex.slice().parse())]
106//! #[regex("[0-9]+k", kilo)]
107//! #[regex("[0-9]+m", mega)]
108//! Number(u64),
109//! }
110//!
111//! fn main() {
112//! let mut lex = Token::lexer("5 42k 75m");
113//!
114//! assert_eq!(lex.next(), Some(Token::Number(5)));
115//! assert_eq!(lex.slice(), "5");
116//!
117//! assert_eq!(lex.next(), Some(Token::Number(42_000)));
118//! assert_eq!(lex.slice(), "42k");
119//!
120//! assert_eq!(lex.next(), Some(Token::Number(75_000_000)));
121//! assert_eq!(lex.slice(), "75m");
122//!
123//! assert_eq!(lex.next(), None);
124//! }
125//! ```
126//!
127//! Logos can handle callbacks with following return types:
128//!
129//! | Return type | Produces |
130//! |-----------------------------------|----------------------------------------------------|
131//! | `()` | `Token::Unit` |
132//! | `bool` | `Token::Unit` **or** `<Token as Logos>::ERROR` |
133//! | `Result<(), _>` | `Token::Unit` **or** `<Token as Logos>::ERROR` |
134//! | `T` | `Token::Value(T)` |
135//! | `Option<T>` | `Token::Value(T)` **or** `<Token as Logos>::ERROR` |
136//! | `Result<T, _>` | `Token::Value(T)` **or** `<Token as Logos>::ERROR` |
137//! | [`Skip`](./struct.Skip.html) | _skips matched input_ |
138//! | [`Filter<T>`](./enum.Filter.html) | `Token::Value(T)` **or** _skips matched input_ |
139//!
140//! Callbacks can be also used to do perform more specialized lexing in place
141//! where regular expressions are too limiting. For specifics look at
142//! [`Lexer::remainder`](./struct.Lexer.html#method.remainder) and
143//! [`Lexer::bump`](./struct.Lexer.html#method.bump).
144//!
145//! ## Token disambiguation
146//!
147//! Rule of thumb is:
148//!
149//! + Longer beats shorter.
150//! + Specific beats generic.
151//!
152//! If any two definitions could match the same input, like `fast` and `[a-zA-Z]+`
153//! in the example above, it's the longer and more specific definition of `Token::Fast`
154//! that will be the result.
155//!
156//! This is done by comparing numeric priority attached to each definition. Every consecutive,
157//! non-repeating single byte adds 2 to the priority, while every range or regex class adds 1.
158//! Loops or optional blocks are ignored, while alternations count the shortest alternative:
159//!
160//! + `[a-zA-Z]+` has a priority of 1 (lowest possible), because at minimum it can match a single byte to a class.
161//! + `foobar` has a priority of 12.
162//! + `(foo|hello)(bar)?` has a priority of 6, `foo` being it's shortest possible match.
163
164#![cfg_attr(not(feature = "std"), no_std)]
165#![warn(missing_docs)]
166#![doc(html_logo_url = "https://maciej.codes/kosz/logos.png")]
167
168#[cfg(not(feature = "std"))]
169extern crate core as std;
170
171#[cfg(feature = "export_derive")]
172pub use logos_derive::Logos;
173
174mod lexer;
175pub mod source;
176
177#[doc(hidden)]
178pub mod internal;
179
180pub use crate::lexer::{Lexer, Span, SpannedIter};
181pub use crate::source::Source;
182
183/// Trait implemented for an enum representing all tokens. You should never have
184/// to implement it manually, use the `#[derive(Logos)]` attribute on your enum.
185pub trait Logos<'source>: Sized {
186 /// Associated type `Extras` for the particular lexer. This can be set using
187 /// `#[logos(extras = MyExtras)]` and accessed inside callbacks.
188 type Extras;
189
190 /// Source type this token can be lexed from. This will default to `str`,
191 /// unless one of the defined patterns explicitly uses non-unicode byte values
192 /// or byte slices, in which case that implementation will use `[u8]`.
193 type Source: Source + ?Sized + 'source;
194
195 /// Helper `const` of the variant marked as `#[error]`.
196 const ERROR: Self;
197
198 /// The heart of Logos. Called by the `Lexer`. The implementation for this function
199 /// is generated by the `logos-derive` crate.
200 fn lex(lexer: &mut Lexer<'source, Self>);
201
202 /// Create a new instance of a `Lexer` that will produce tokens implementing
203 /// this `Logos`.
204 fn lexer(source: &'source Self::Source) -> Lexer<'source, Self>
205 where
206 Self::Extras: Default,
207 {
208 Lexer::new(source)
209 }
210
211 /// Create a new instance of a `Lexer` with the provided `Extras` that will
212 /// produce tokens implementing this `Logos`.
213 fn lexer_with_extras(
214 source: &'source Self::Source,
215 extras: Self::Extras,
216 ) -> Lexer<'source, Self> {
217 Lexer::with_extras(source, extras)
218 }
219}
220
221/// Type that can be returned from a callback, informing the `Lexer`, to skip
222/// current token match. See also [`logos::skip`](./fn.skip.html).
223///
224/// # Example
225///
226/// ```rust
227/// use logos::{Logos, Skip};
228///
229/// #[derive(Logos, Debug, PartialEq)]
230/// enum Token<'a> {
231/// // We will treat "abc" as if it was whitespace.
232/// // This is identical to using `logos::skip`.
233/// #[regex(" |abc", |_| Skip)]
234/// #[error]
235/// Error,
236///
237/// #[regex("[a-zA-Z]+")]
238/// Text(&'a str),
239/// }
240///
241/// let tokens: Vec<_> = Token::lexer("Hello abc world").collect();
242///
243/// assert_eq!(
244/// tokens,
245/// &[
246/// Token::Text("Hello"),
247/// Token::Text("world"),
248/// ],
249/// );
250/// ```
251pub struct Skip;
252
253/// Type that can be returned from a callback, either producing a field
254/// for a token, or skipping it.
255///
256/// # Example
257///
258/// ```rust
259/// use logos::{Logos, Filter};
260///
261/// #[derive(Logos, Debug, PartialEq)]
262/// enum Token {
263/// #[regex(r"[ \n\f\t]+", logos::skip)]
264/// #[error]
265/// Error,
266///
267/// #[regex("[0-9]+", |lex| {
268/// let n: u64 = lex.slice().parse().unwrap();
269///
270/// // Only emit a token if `n` is an even number
271/// match n % 2 {
272/// 0 => Filter::Emit(n),
273/// _ => Filter::Skip,
274/// }
275/// })]
276/// EvenNumber(u64)
277/// }
278///
279/// let tokens: Vec<_> = Token::lexer("20 11 42 23 100 8002").collect();
280///
281/// assert_eq!(
282/// tokens,
283/// &[
284/// Token::EvenNumber(20),
285/// // skipping 11
286/// Token::EvenNumber(42),
287/// // skipping 23
288/// Token::EvenNumber(100),
289/// Token::EvenNumber(8002),
290/// ]
291/// );
292/// ```
293pub enum Filter<T> {
294 /// Emit a token with a given value `T`. Use `()` for unit variants without fields.
295 Emit(T),
296 /// Skip current match, analog to [`Skip`](./struct.Skip.html).
297 Skip,
298}
299
300/// Type that can be returned from a callback, either producing a field
301/// for a token, skipping it, or emitting an error.
302///
303/// # Example
304///
305/// ```rust
306/// use logos::{Logos, FilterResult};
307///
308/// #[derive(Logos, Debug, PartialEq)]
309/// enum Token {
310/// #[regex(r"[ \n\f\t]+", logos::skip)]
311/// #[error]
312/// Error,
313///
314/// #[regex("[0-9]+", |lex| {
315/// let n: u64 = lex.slice().parse().unwrap();
316///
317/// // Only emit a token if `n` is an even number.
318/// if n % 2 == 0 {
319/// // Emit an error if `n` is 10.
320/// if n == 10 {
321/// FilterResult::Error
322/// } else {
323/// FilterResult::Emit(n)
324/// }
325/// } else {
326/// FilterResult::Skip
327/// }
328/// })]
329/// NiceEvenNumber(u64)
330/// }
331///
332/// let tokens: Vec<_> = Token::lexer("20 11 42 23 100 10").collect();
333///
334/// assert_eq!(
335/// tokens,
336/// &[
337/// Token::NiceEvenNumber(20),
338/// // skipping 11
339/// Token::NiceEvenNumber(42),
340/// // skipping 23
341/// Token::NiceEvenNumber(100),
342/// // error at 10
343/// Token::Error,
344/// ]
345/// );
346/// ```
347pub enum FilterResult<T> {
348 /// Emit a token with a given value `T`. Use `()` for unit variants without fields.
349 Emit(T),
350 /// Skip current match, analog to [`Skip`](./struct.Skip.html).
351 Skip,
352 /// Emit a `<Token as Logos>::ERROR` token.
353 Error,
354}
355
356/// Predefined callback that will inform the `Lexer` to skip a definition.
357///
358/// # Example
359///
360/// ```rust
361/// use logos::Logos;
362///
363/// #[derive(Logos, Debug, PartialEq)]
364/// enum Token<'a> {
365/// // We will treat "abc" as if it was whitespace
366/// #[regex(" |abc", logos::skip)]
367/// #[error]
368/// Error,
369///
370/// #[regex("[a-zA-Z]+")]
371/// Text(&'a str),
372/// }
373///
374/// let tokens: Vec<_> = Token::lexer("Hello abc world").collect();
375///
376/// assert_eq!(
377/// tokens,
378/// &[
379/// Token::Text("Hello"),
380/// Token::Text("world"),
381/// ],
382/// );
383/// ```
384#[inline]
385pub fn skip<'source, Token: Logos<'source>>(_: &mut Lexer<'source, Token>) -> Skip {
386 Skip
387}
388
389#[cfg(doctest)]
390mod test_readme {
391 macro_rules! external_doc_test {
392 ($x:expr) => {
393 #[doc = $x]
394 extern "C" {}
395 };
396 }
397
398 external_doc_test!(include_str!("../../README.md"));
399}