Fix parsing of 'with' inside streaming operator.

verilog/formatting/formatter_test.cc

@@ -1066,6 +1066,10 @@ static constexpr FormatterTestCase kFormatterTestCases[] = {
         "   parameter  int  b={ << 4  {{ << 2 { a }  }} } ;",
         "parameter int b = {<<4{{<<2{a}}}};\n",
     },
+    {
+        "   parameter  int  b={ << {foo with [ 1 +: 4 ] } } ;",
+        "parameter int b = {<<{foo with [1+:4]}};\n",
+    },
 
     // basic module test cases
     {"module foo;endmodule:foo\n",

verilog/formatting/token_annotator.cc

@@ -207,6 +207,9 @@ static WithReason<int> SpacesRequiredBetween(
   if (right_context.IsInsideFirst({NodeEnum::kStreamingConcatenation}, {})) {
     if (left.TokenEnum() == TK_LS || left.TokenEnum() == TK_RS) {
       return {0, "No space around streaming operators"};
+    } else if (left.TokenEnum() == TK_with__followed_by_bracket ||
+               right.TokenEnum() == TK_with__followed_by_bracket) {
+      return {1, "Space around 'with' keyword in streaming concatenation."};
     } else if (left.format_token_enum == FormatTokenType::numeric_literal ||
                left.format_token_enum == FormatTokenType::identifier ||
                left.format_token_enum == FormatTokenType::keyword) {

verilog/formatting/verilog_token.cc

@@ -298,6 +298,7 @@ static const absl::node_hash_map<verilog_tokentype, FTT>& FormatTokenTypeMap() {
       {verilog_tokentype::TK_wildcard, FTT::keyword},
       {verilog_tokentype::TK_with, FTT::keyword},
       {verilog_tokentype::TK_with__covergroup, FTT::keyword},
+      {verilog_tokentype::TK_with__followed_by_bracket, FTT::keyword},
       {verilog_tokentype::TK_within, FTT::keyword},
       {verilog_tokentype::TK_timeprecision_check, FTT::keyword},
       {verilog_tokentype::TK_timeunit_check, FTT::keyword},

verilog/parser/verilog.lex

@@ -673,6 +673,11 @@ wait_order { UpdateLocation(); return TK_wait_order; }
 wildcard { UpdateLocation(); return TK_wildcard; }
 <COVERGROUP>with { UpdateLocation(); return TK_with__covergroup; }
 with { UpdateLocation(); return TK_with; }
+"with"({TraditionalCommentOrSpace}|{EndOfLineComment})*"[" {
+  yyless(4);    // only consume 'with', push rest back to input
+  UpdateLocation();
+  return TK_with__followed_by_bracket;
+}
 within { UpdateLocation(); return TK_within; }
 timeprecision_check { UpdateLocation(); return TK_timeprecision_check; }
 timeunit_check { UpdateLocation(); return TK_timeunit_check; }

verilog/parser/verilog.y

@@ -483,6 +483,7 @@ is not locally defined, so the grammar here uses only generic identifiers.
 %token TK_wait_order "wait_order"
 %token TK_wildcard "wildcard"
 %token TK_with "with"
+%token TK_with__followed_by_bracket "with ["
 %token TK_with__covergroup "with(covergroup)"
 %token TK_within "within"
 /* Fake tokens that are passed once we have an initial token. */
@@ -3159,12 +3160,12 @@ block_item_or_statement_or_null_list_opt
   | /* empty */
     { $$ = MakeTaggedNode(N::kBlockItemStatementList); }
   ;
+
 stream_expression
   : expression
     { $$ = move($1); }
-  /* TODO(fangism):
-  | expression TK_with select_variable_dimension
-   */
+  | expression TK_with__followed_by_bracket select_variable_dimension
+    { $$ = ExtendNode($1, $2, $3); }
   ;
 stream_expression_list
   : stream_expression_list ',' stream_expression

verilog/parser/verilog_lexer_unittest.cc

@@ -1433,6 +1433,9 @@ static std::initializer_list<LexerTestData> kSequenceTests = {
 
 static std::initializer_list<LexerTestData> kContextKeywordTests = {
     {{SymbolIdentifier, "option"}},
+    {{TK_with__followed_by_bracket, "with"},
+     " // hello\n"
+     " /* this bracket is the context we're looking for -> */["},
     {{TK_covergroup, "covergroup"},
      " ",
      {SymbolIdentifier, "blah1"},

verilog/parser/verilog_parser_unittest.cc

@@ -625,6 +625,17 @@ static const ParserTestCaseArray kFunctionTests = {
     "function void unpack_id(utils_pkg::bytestream_t bytes);\n"
     "{<< byte {this.reserved, this.id}} = bytes;\n"
     "endfunction",
+    "function void unpack_id_with(int nums);\n"
+    "{>>8 {foo, bar with \t [ a +: b]}} = nums;\n"
+    "endfunction",
+    "function void unpack_id_with(int nums);\n"
+    "{>>8 {foo, bar with /* some comment */ /* another one */ // more\n"
+    "[ a +: b]}} = nums;\n"
+    "endfunction",
+    "function void unpack_id_with(int nums);\n"
+    "{>>8 {foo, bar with // eol comment\n"
+    "      [ a +: b]}} = nums;\n"
+    "endfunction",
     // concatenation lvalue
     "module foo;\n"
     "  initial begin\n"