gccrs: Fix token lexed as a float literal

The lexer cannot distinguish the difference between a float literal and a
tuple index in some cases. This means we should fix this while parsing
depending on the context.

gcc/rust/ChangeLog:

	* expand/rust-macro-invoc-lexer.cc (MacroInvocLexer::split_current_token):
	Add implementation for multiple token split.
	* expand/rust-macro-invoc-lexer.h: Add function prototype.
	* expand/rust-proc-macro-invoc-lexer.cc (ProcMacroInvocLexer::split_current_token):
	Add implementation for 2+ token split for procedural macros.
	* expand/rust-proc-macro-invoc-lexer.h: Add function prototype.
	* lex/rust-lex.cc (Lexer::split_current_token): Add function to split a
	token in multiple other tokens.
	* lex/rust-lex.h: Add function prototype for split_current_token.
	* parse/rust-parse-impl.h (Parser::left_denotation): Handle float tuple
	index identified as a float literal.

Signed-off-by: Pierre-Emmanuel Patry <pierre-emmanuel.patry@embecosm.com>

diff --git a/gcc/rust/expand/rust-macro-invoc-lexer.cc b/gcc/rust/expand/rust-macro-invoc-lexer.cc
index aa1e1ff2ebd3b5a7793fd559b8adb75785bd17cf..003c87d62ff52e4cc78fc5e0ac3625b031b8696d 100644 (file)
--- a/gcc/rust/expand/rust-macro-invoc-lexer.cc
+++ b/gcc/rust/expand/rust-macro-invoc-lexer.cc
@@ -49,6 +49,22 @@ MacroInvocLexer::split_current_token (TokenId new_left, TokenId new_right)
                       std::unique_ptr<AST::Token> (new AST::Token (l_tok)));
 }
 
+void
+MacroInvocLexer::split_current_token (std::vector<TokenPtr> new_tokens)
+{
+  rust_assert (new_tokens.size () > 0);
+
+  auto current_pos = token_stream.begin () + offs;
+
+  token_stream.erase (current_pos);
+
+  for (size_t i = 1; i < new_tokens.size (); i++)
+    {
+      token_stream.insert (current_pos + i, std::unique_ptr<AST::Token> (
+                                             new AST::Token (new_tokens[i])));
+    }
+}
+
 std::vector<std::unique_ptr<AST::Token>>
 MacroInvocLexer::get_token_slice (size_t start_idx, size_t end_idx) const
 {

diff --git a/gcc/rust/expand/rust-macro-invoc-lexer.h b/gcc/rust/expand/rust-macro-invoc-lexer.h
index a65816b5ef571067ba71645529d5e03914612aea..4dc6d691c11e0af1092c6474a1848656bde6702e 100644 (file)
--- a/gcc/rust/expand/rust-macro-invoc-lexer.h
+++ b/gcc/rust/expand/rust-macro-invoc-lexer.h
@@ -67,6 +67,8 @@ public:
   // this will only work with "simple" tokens like punctuation.
   void split_current_token (TokenId new_left, TokenId new_right);
 
+  void split_current_token (std::vector<TokenPtr> new_tokens);
+
   std::vector<std::unique_ptr<AST::Token>>
   get_token_slice (size_t start_idx, size_t end_idx) const;
 };

diff --git a/gcc/rust/expand/rust-proc-macro-invoc-lexer.cc b/gcc/rust/expand/rust-proc-macro-invoc-lexer.cc
index 5990dec4325b7bac50c194c6d1d74136169bed33..1c370c10b29965145074ee22f0b0b02602c67358 100644 (file)
--- a/gcc/rust/expand/rust-proc-macro-invoc-lexer.cc
+++ b/gcc/rust/expand/rust-proc-macro-invoc-lexer.cc
@@ -47,4 +47,19 @@ ProcMacroInvocLexer::split_current_token (TokenId new_left, TokenId new_right)
   token_stream.insert (current_pos, r_tok);
 }
 
+void
+ProcMacroInvocLexer::split_current_token (std::vector<TokenPtr> new_tokens)
+{
+  rust_assert (new_tokens.size () > 0);
+
+  auto current_pos = token_stream.begin () + offs;
+
+  token_stream.erase (current_pos);
+
+  for (size_t i = 1; i < new_tokens.size (); i++)
+    {
+      token_stream.insert (current_pos + i, new_tokens[i]);
+    }
+}
+
 } // namespace Rust

diff --git a/gcc/rust/expand/rust-proc-macro-invoc-lexer.h b/gcc/rust/expand/rust-proc-macro-invoc-lexer.h
index 5a11a4c111c198851097b208cd12c8bde40d6a94..94eb6d5bfe294b5fd28cf6d7cf44da4db4076b6b 100644 (file)
--- a/gcc/rust/expand/rust-proc-macro-invoc-lexer.h
+++ b/gcc/rust/expand/rust-proc-macro-invoc-lexer.h
@@ -40,6 +40,8 @@ public:
   // closes (i.e. T<U<X>> where >> is wrongly lexed as one token). Note that
   // this will only work with "simple" tokens like punctuation.
   void split_current_token (TokenId new_left, TokenId new_right);
+
+  void split_current_token (std::vector<TokenPtr> new_tokens);
 };
 } // namespace Rust
 

diff --git a/gcc/rust/lex/rust-lex.cc b/gcc/rust/lex/rust-lex.cc
index 3883f2379fc682769f06c4e9c640f2f972582a9e..2cfc9cec5d41c1066433fbe383dcc7d54984964d 100644 (file)
--- a/gcc/rust/lex/rust-lex.cc
+++ b/gcc/rust/lex/rust-lex.cc
@@ -2529,6 +2529,18 @@ Lexer::split_current_token (TokenId new_left, TokenId new_right)
   token_queue.insert (1, std::move (new_right_tok));
 }
 
+void
+Lexer::split_current_token (std::vector<TokenPtr> new_tokens)
+{
+  rust_assert (new_tokens.size () > 0);
+  token_queue.replace_current_value (new_tokens[0]);
+
+  for (size_t i = 1; i < new_tokens.size (); i++)
+    {
+      token_queue.insert (i, new_tokens[i]);
+    }
+}
+
 void
 Lexer::start_line (int current_line, int current_column)
 {

diff --git a/gcc/rust/lex/rust-lex.h b/gcc/rust/lex/rust-lex.h
index 683e8c67a89f7016fc48063a427a58ed64514343..feaa3e05fc9e1a6e6f729e7eb7c7f0414e583ce5 100644 (file)
--- a/gcc/rust/lex/rust-lex.h
+++ b/gcc/rust/lex/rust-lex.h
@@ -202,6 +202,8 @@ public:
    * this will only work with "simple" tokens like punctuation. */
   void split_current_token (TokenId new_left, TokenId new_right);
 
+  void split_current_token (std::vector<TokenPtr> new_tokens);
+
   Linemap *get_line_map () { return line_map; }
   std::string get_filename () { return std::string (input.get_filename ()); }
 

diff --git a/gcc/rust/parse/rust-parse-impl.h b/gcc/rust/parse/rust-parse-impl.h
index 4738393fd429fd010627e5c3d94d5f0ca6e98c35..eb0310c888f7363fdf2295a7c98249c12c3a6b35 100644 (file)
--- a/gcc/rust/parse/rust-parse-impl.h
+++ b/gcc/rust/parse/rust-parse-impl.h
@@ -12895,6 +12895,28 @@ Parser<ManagedTokenSource>::left_denotation (const_TokenPtr tok,
                                           std::move (outer_attrs),
                                           restrictions);
          }
+       else if (next_tok->get_id () == FLOAT_LITERAL)
+         {
+           // Lexer has misidentified a tuple index as a float literal
+           // eg: `(x, (y, z)).1.0` -> 1.0 has been identified as a float
+           // literal. This means we should split it into three new separate
+           // tokens, the first tuple index, the dot and the second tuple
+           // index.
+           auto current_loc = next_tok->get_locus ();
+           auto str = next_tok->get_str ();
+           auto dot_pos = str.find (".");
+           auto prefix = str.substr (0, dot_pos);
+           auto suffix = str.substr (dot_pos + 1);
+           lexer.split_current_token (
+             {Token::make_int (current_loc, std::move (prefix),
+                               CORETYPE_PURE_DECIMAL),
+              Token::make (DOT, current_loc + 1),
+              Token::make_int (current_loc + 2, std::move (suffix),
+                               CORETYPE_PURE_DECIMAL)});
+           return parse_tuple_index_expr (tok, std::move (left),
+                                          std::move (outer_attrs),
+                                          restrictions);
+         }
        else if (next_tok->get_id () == IDENTIFIER
                 && lexer.peek_token (1)->get_id () != LEFT_PAREN
                 && lexer.peek_token (1)->get_id () != SCOPE_RESOLUTION)