plc_lexer/
lexer.rs

1// Copyright (c) 2020 Ghaith Hachem and Mathias Rieder
2use core::ops::Range;
3use logos::{Filter, Lexer, Logos};
4use plc_ast::ast::{AstId, DirectAccessType, HardwareAccessType};
5use plc_ast::provider::IdProvider;
6use plc_diagnostics::diagnostics::Diagnostic;
7use plc_source::source_location::{SourceLocation, SourceLocationFactory};
8pub use tokens::{Token, TokenClass};
9
10#[cfg(test)]
11mod tests;
12mod tokens;
13
14pub struct ParseSession<'a> {
15    lexer: Lexer<'a, Token>,
16    pub token: Token,
17    pub diagnostics: Vec<Diagnostic>,
18    pub closing_keywords: Vec<Vec<Token>>,
19    /// the token parsed before the current one stored in `token`
20    pub last_token: Token,
21    /// the range of the `last_token`
22    pub last_range: Range<usize>,
23    pub parse_progress: usize,
24    pub id_provider: IdProvider,
25    pub source_range_factory: SourceLocationFactory,
26    pub scope: Option<String>,
27}
28
29#[macro_export]
30macro_rules! expect_token {
31    ($lexer:expr, $token:expr, $return_value:expr) => {
32        if $lexer.token != $token {
33            $lexer.accept_diagnostic(Diagnostic::unexpected_token_found(
34                format!("{:?}", $token).as_str(),
35                $lexer.slice(),
36                $lexer.location(),
37            ));
38            return $return_value;
39        }
40    };
41}
42
43impl<'a> ParseSession<'a> {
44    pub fn new(
45        l: Lexer<'a, Token>,
46        id_provider: IdProvider,
47        source_range_factory: SourceLocationFactory,
48    ) -> ParseSession<'a> {
49        let mut lexer = ParseSession {
50            lexer: l,
51            token: Token::KeywordBy,
52            diagnostics: vec![],
53            closing_keywords: vec![],
54            last_token: Token::End,
55            last_range: 0..0,
56            parse_progress: 0,
57            id_provider,
58            scope: None,
59            source_range_factory,
60        };
61        lexer.advance();
62        lexer
63    }
64
65    pub fn get_src(&self) -> &str {
66        self.lexer.source()
67    }
68
69    pub fn next_id(&mut self) -> AstId {
70        self.id_provider.next_id()
71    }
72
73    /// Tries to consume the given token, returning false if it failed.
74    pub fn try_consume(&mut self, token: Token) -> bool {
75        if self.token == token {
76            self.advance();
77            return true;
78        }
79
80        false
81    }
82
83    /// Returns the token that follows the current one without consuming the
84    /// current token.
85    pub fn peek(&self) -> Token {
86        let mut peeked = self.lexer.clone();
87        peeked.next().unwrap_or(Token::End)
88    }
89
90    pub fn try_consume_or_report(&mut self, token: Token) {
91        if !self.try_consume(token) {
92            self.accept_diagnostic(Diagnostic::missing_token(format!("{token:?}").as_str(), self.location()));
93        }
94    }
95
96    pub fn slice_and_advance(&mut self) -> String {
97        let slice = self.slice().to_string();
98        self.advance();
99        slice
100    }
101
102    pub fn is_end_of_stream(&self) -> bool {
103        self.token == Token::End || self.token == Token::Error
104    }
105
106    pub fn slice_region(&self, range: Range<usize>) -> &str {
107        &self.lexer.source()[range]
108    }
109
110    pub fn advance(&mut self) {
111        self.last_range = self.range();
112        self.last_token = std::mem::replace(&mut self.token, self.lexer.next().unwrap_or(Token::End));
113        self.parse_progress += 1;
114
115        match self.token {
116            Token::KeywordVarInput
117            | Token::KeywordVarOutput
118            | Token::KeywordVarGlobal
119            | Token::KeywordVarInOut
120            | Token::KeywordRef
121            | Token::KeywordVarTemp
122            | Token::KeywordNonRetain
123            | Token::KeywordEndVar
124            | Token::KeywordEndProgram
125            | Token::KeywordEndFunction
126            | Token::KeywordEndCase
127            | Token::KeywordFunctionBlock
128            | Token::KeywordEndFunctionBlock
129            | Token::KeywordEndStruct
130            | Token::KeywordEndAction
131            | Token::KeywordEndActions
132            | Token::KeywordEndIf
133            | Token::KeywordEndFor
134            | Token::KeywordEndRepeat
135            | Token::KeywordEndMethod
136            | Token::KeywordEndClass
137                if !self.slice().to_string().contains('_') =>
138            {
139                self.accept_diagnostic(
140                    Diagnostic::new(format!("the words in {} should be separated by a `_`", self.slice()))
141                        .with_error_code("E013")
142                        .with_location(self.location()),
143                );
144            }
145            _ => {}
146        }
147    }
148
149    pub fn slice(&self) -> &str {
150        self.lexer.slice()
151    }
152
153    pub fn location(&self) -> SourceLocation {
154        self.source_range_factory.create_range(self.range())
155    }
156
157    pub fn last_location(&self) -> SourceLocation {
158        self.source_range_factory.create_range(self.last_range.clone())
159    }
160
161    pub fn range(&self) -> Range<usize> {
162        self.lexer.span()
163    }
164
165    pub fn accept_diagnostic(&mut self, diagnostic: Diagnostic) {
166        self.diagnostics.push(diagnostic);
167    }
168
169    pub fn enter_region(&mut self, end_token: Vec<Token>) {
170        self.closing_keywords.push(end_token);
171    }
172
173    pub fn close_region(&mut self) {
174        if let Some(expected_token) = self.closing_keywords.pop() {
175            if !expected_token.contains(&self.token) {
176                self.accept_diagnostic(Diagnostic::unexpected_token_found(
177                    format!("{:?}", expected_token[0]).as_str(),
178                    format!("'{}'", self.slice()).as_str(),
179                    self.location(),
180                ));
181            } else {
182                self.advance();
183            }
184        }
185    }
186
187    /// returns the level (which corresponds to the position on the `closing_keywords` stack)
188    /// returns `None` if this token does not close an open region
189    fn get_close_region_level(&self, token: &Token) -> Option<usize> {
190        self.closing_keywords.iter().rposition(|it| it.contains(token))
191    }
192
193    /// returns true if the given token closes an open region
194    pub fn closes_open_region(&self, token: &Token) -> bool {
195        token == &Token::End || self.get_close_region_level(token).is_some()
196    }
197
198    pub fn recover_until_close(&mut self) {
199        let mut hit = self.get_close_region_level(&self.token);
200        let start = self.range();
201        let mut end = self.range().end;
202        while self.token != Token::End && hit.is_none() {
203            end = self.range().end;
204            self.advance();
205            hit = self.closing_keywords.iter().rposition(|it| it.contains(&self.token));
206        }
207
208        //Did we recover in the while loop above?
209        if start.end != self.range().end {
210            let range = start.start..end;
211            self.accept_diagnostic(Diagnostic::unexpected_token_found(
212                format!(
213                    "{:?}",
214                    self.closing_keywords.last().and_then(|it| it.first()).unwrap_or(&Token::End) //only show first expected token
215                )
216                .as_str(),
217                format!("'{}'", self.slice_region(range.clone())).as_str(),
218                self.source_range_factory.create_range(range),
219            ));
220        }
221
222        if let Some(hit) = hit {
223            if self.closing_keywords.len() > hit + 1 {
224                let closing = self
225                    .closing_keywords
226                    .last()
227                    .expect("parse-recovery has no closing-keyword to recover from."); //illegal state! invalid use of parser-recovery?
228                let expected_tokens = format!("{closing:?}");
229                self.accept_diagnostic(Diagnostic::missing_token(expected_tokens.as_str(), self.location()));
230            }
231        }
232    }
233}
234
235fn parse_pragma(lexer: &mut Lexer<Token>) -> Filter<()> {
236    let remainder = lexer.remainder();
237    let chars = remainder.chars();
238    let mut traversed = 0;
239    for c in chars {
240        traversed += c.len_utf8();
241        if c == '}' {
242            lexer.bump(traversed);
243            return Filter::Skip;
244        }
245    }
246    Filter::Emit(())
247}
248
249fn parse_comments(lexer: &mut Lexer<Token>) -> Filter<()> {
250    let (open, close) = get_closing_tag(lexer.slice());
251    let remainder = lexer.remainder();
252    let mut unclosed = 1;
253    let chars = remainder.chars();
254
255    let mut prev = ' ';
256    let mut traversed = 0;
257    for c in chars {
258        if c == '*' && prev == open {
259            unclosed += 1;
260            //Make sure the next action does not consume the star
261            prev = ' ';
262        } else if c == close && prev == '*' {
263            unclosed -= 1;
264            prev = c;
265        } else {
266            prev = c;
267        }
268        traversed += c.len_utf8();
269        if unclosed == 0 {
270            lexer.bump(traversed);
271            //This is a well formed comment, treat it as whitespace
272            return Filter::Skip;
273        }
274    }
275    Filter::Emit(())
276}
277
278fn get_closing_tag(open_tag: &str) -> (char, char) {
279    match open_tag {
280        "(*" => ('(', ')'),
281        "/*" => ('/', '/'),
282        _ => unreachable!(),
283    }
284}
285fn parse_access_type(lexer: &mut Lexer<Token>) -> Option<DirectAccessType> {
286    //Percent is at position 0
287    //Find the size from position 1
288    let access = lexer
289        .slice()
290        .chars()
291        .nth(1)
292        .and_then(|c| match c.to_ascii_lowercase() {
293            'x' => Some(DirectAccessType::Bit),
294            'b' => Some(DirectAccessType::Byte),
295            'w' => Some(DirectAccessType::Word),
296            'd' => Some(DirectAccessType::DWord),
297            'l' => Some(DirectAccessType::LWord),
298            _ => None,
299        })
300        .expect("Unknown access type - tokenizer/grammar incomplete?");
301
302    Some(access)
303}
304
305fn parse_hardware_access_type(lexer: &mut Lexer<Token>) -> Option<(HardwareAccessType, DirectAccessType)> {
306    //Percent is at position 0
307    let hardware_type = lexer
308        .slice()
309        .chars()
310        .nth(1)
311        .and_then(|c| match c.to_ascii_lowercase() {
312            'i' => Some(HardwareAccessType::Input),
313            'q' => Some(HardwareAccessType::Output),
314            'm' => Some(HardwareAccessType::Memory),
315            'g' => Some(HardwareAccessType::Global),
316            _ => None,
317        })
318        .expect("Unknown access type - tokenizer/grammar incomplete?");
319    //Find the size from position 2
320    let access = lexer
321        .slice()
322        .chars()
323        .nth(2)
324        .and_then(|c| match c.to_ascii_lowercase() {
325            'x' => Some(DirectAccessType::Bit),
326            'b' => Some(DirectAccessType::Byte),
327            'w' => Some(DirectAccessType::Word),
328            'd' => Some(DirectAccessType::DWord),
329            'l' => Some(DirectAccessType::LWord),
330            '*' => Some(DirectAccessType::Template),
331            _ => None,
332        })
333        .expect("Unknown access type - tokenizer/grammar incomplete?");
334
335    Some((hardware_type, access))
336}
337
338#[cfg(test)]
339pub fn lex(source: &str) -> ParseSession<'_> {
340    ParseSession::new(Token::lexer(source), IdProvider::default(), SourceLocationFactory::internal(source))
341}
342
343pub fn lex_with_ids(
344    source: &str,
345    id_provider: IdProvider,
346    location_factory: SourceLocationFactory,
347) -> ParseSession<'_> {
348    ParseSession::new(Token::lexer(source), id_provider, location_factory)
349}
plc_lexer/lexer.rs

plc_lexer/
lexer.rs
