diff --git a/lrlex/src/lib/ctbuilder.rs b/lrlex/src/lib/ctbuilder.rs index c37540a88..325e6bab4 100644 --- a/lrlex/src/lib/ctbuilder.rs +++ b/lrlex/src/lib/ctbuilder.rs @@ -640,7 +640,7 @@ where ); } - let (missing_from_lexer, missing_from_parser) = match self.rule_ids_map { + let (mut missing_from_lexer, missing_from_parser) = match self.rule_ids_map { Some(ref rim) => { // Convert from HashMap to HashMap<&str, _> let owned_map = rim @@ -660,6 +660,17 @@ where None => (None, None), }; + if let Some(mut mfl) = missing_from_lexer.take() { + for tok in &lexerdef.expected_missing_tokens { + mfl.remove(tok.as_str()); + } + if mfl.is_empty() { + missing_from_lexer = None; + } else { + missing_from_lexer = Some(mfl); + } + } + let mut has_unallowed_missing = false; let err_indent = " ".repeat(ERROR.len()); if !self.allow_missing_terms_in_lexer diff --git a/lrlex/src/lib/lexer.rs b/lrlex/src/lib/lexer.rs index cb216e937..60cd726ca 100644 --- a/lrlex/src/lib/lexer.rs +++ b/lrlex/src/lib/lexer.rs @@ -398,6 +398,7 @@ where rules: Vec>, start_states: Vec, lex_flags: LexFlags, + pub(crate) expected_missing_tokens: Vec, phantom: PhantomData, } @@ -414,6 +415,7 @@ where rules, start_states, lex_flags: DEFAULT_LEX_FLAGS, + expected_missing_tokens: vec![], phantom: PhantomData, } } @@ -430,6 +432,7 @@ where rules: p.rules, start_states: p.start_states, lex_flags: flags, + expected_missing_tokens: p.expected_missing_tokens, phantom: PhantomData, } }) @@ -547,6 +550,7 @@ where rules: p.rules, start_states: p.start_states, lex_flags, + expected_missing_tokens: p.expected_missing_tokens, phantom: PhantomData, }, ) diff --git a/lrlex/src/lib/parser.rs b/lrlex/src/lib/parser.rs index c3236b587..8192f33cc 100644 --- a/lrlex/src/lib/parser.rs +++ b/lrlex/src/lib/parser.rs @@ -115,6 +115,7 @@ where pub(super) rules: Vec>, pub(super) start_states: Vec, pub(super) lex_flags: LexFlags, + pub(super) expected_missing_tokens: Vec, } fn add_duplicate_occurrence( @@ -189,6 +190,7 @@ where false, Span::new(0, 0), )], + expected_missing_tokens: vec![], }; p.parse()?; Ok(p) @@ -293,6 +295,16 @@ where self.declare_start_states(false, i, declaration_len, line_len, errs) } else if RE_EXCLUSIVE_START_STATE_DECLARATION.is_match(declaration) { self.declare_start_states(true, i, declaration_len, line_len, errs) + } else if declaration.starts_with("%expect-missing") { + for tok in self.src[i + declaration_len..i + line_len].split_ascii_whitespace() { + let tok_unquoted = tok.strip_prefix("\"").and_then(|s| s.strip_suffix("\"")); + if let Some(tok_unquoted) = tok_unquoted { + self.expected_missing_tokens.push(tok_unquoted.to_string()); + } else { + Err(self.mk_error(LexErrorKind::InvalidName, i))?; + } + } + Ok(i + line_len) } else { Err(self.mk_error(LexErrorKind::UnknownDeclaration, i)) } diff --git a/lrpar/cttests/src/expect_missing_tokens.test b/lrpar/cttests/src/expect_missing_tokens.test new file mode 100644 index 000000000..a4c536035 --- /dev/null +++ b/lrpar/cttests/src/expect_missing_tokens.test @@ -0,0 +1,16 @@ +name: Test the %expect-missing lex directive +yacckind: Original(YaccOriginalActionKind::NoAction) +grammar: | + %start meta_start + %token START_A START_B + %% + + meta_start: START_A arule | START_B brule; + arule: 'A'; + brule: 'B'; + %% +lexer: | + %expect-missing "START_A" "START_B" + %% + aA "A" + bB "B"