Merge branch 'next-token' into main

src/vm/wren_compiler.c

@@ -173,6 +173,9 @@ typedef struct
   // The 1-based line number of [currentChar].
   int currentLine;
 
+  // The upcoming token.
+  Token next;
+
   // The most recently lexed token.
   Token current;
 
@@ -639,13 +642,13 @@ static bool matchChar(Parser* parser, char c)
 // range.
 static void makeToken(Parser* parser, TokenType type)
 {
-  parser->current.type = type;
-  parser->current.start = parser->tokenStart;
-  parser->current.length = (int)(parser->currentChar - parser->tokenStart);
-  parser->current.line = parser->currentLine;
+  parser->next.type = type;
+  parser->next.start = parser->tokenStart;
+  parser->next.length = (int)(parser->currentChar - parser->tokenStart);
+  parser->next.line = parser->currentLine;
 
   // Make line tokens appear on the line containing the "\n".
-  if (type == TOKEN_LINE) parser->current.line--;
+  if (type == TOKEN_LINE) parser->next.line--;
 }
 
 // If the current character is [c], then consumes it and makes a token of type
@@ -719,17 +722,17 @@ static void makeNumber(Parser* parser, bool isHex)
 
   if (isHex)
   {
-    parser->current.value = NUM_VAL((double)strtoll(parser->tokenStart, NULL, 16));
+    parser->next.value = NUM_VAL((double)strtoll(parser->tokenStart, NULL, 16));
   }
   else
   {
-    parser->current.value = NUM_VAL(strtod(parser->tokenStart, NULL));
+    parser->next.value = NUM_VAL(strtod(parser->tokenStart, NULL));
   }
 
   if (errno == ERANGE)
   {
     lexError(parser, "Number literal was too large (%d).", sizeof(long int));
-    parser->current.value = NUM_VAL(0);
+    parser->next.value = NUM_VAL(0);
   }
 
   // We don't check that the entire token is consumed after calling strtoll()
@@ -921,21 +924,23 @@ static void readString(Parser* parser)
     }
   }
 
-  parser->current.value = wrenNewStringLength(parser->vm,
+  parser->next.value = wrenNewStringLength(parser->vm,
                                               (char*)string.data, string.count);
 
   wrenByteBufferClear(parser->vm, &string);
   makeToken(parser, type);
 }
 
-// Lex the next token and store it in [parser.current].
+// Lex the next token and store it in [parser.next].
 static void nextToken(Parser* parser)
 {
   parser->previous = parser->current;
+  parser->current = parser->next;
 
   // If we are out of tokens, don't try to tokenize any more. We *do* still
   // copy the TOKEN_EOF to previous so that code that expects it to be consumed
   // will still work.
+  if (parser->next.type == TOKEN_EOF) return;
   if (parser->current.type == TOKEN_EOF) return;
 
   while (peekChar(parser) != '\0')
@@ -1094,8 +1099,8 @@ static void nextToken(Parser* parser)
             // even though the source code and console output are UTF-8.
             lexError(parser, "Invalid byte 0x%x.", (uint8_t)c);
           }
-          parser->current.type = TOKEN_ERROR;
-          parser->current.length = 0;
+          parser->next.type = TOKEN_ERROR;
+          parser->next.length = 0;
         }
         return;
     }
@@ -1114,6 +1119,12 @@ static TokenType peek(Compiler* compiler)
   return compiler->parser->current.type;
 }
 
+// Returns the type of the current token.
+static TokenType peekNext(Compiler* compiler)
+{
+  return compiler->parser->next.type;
+}
+
 // Consumes the current token if its type is [expected]. Returns true if a
 // token was consumed.
 static bool match(Compiler* compiler, TokenType expected)
@@ -1162,6 +1173,12 @@ static void consumeLine(Compiler* compiler, const char* errorMessage)
   ignoreNewlines(compiler);
 }
 
+static void allowLineBeforeDot(Compiler* compiler) {
+  if (peek(compiler) == TOKEN_LINE && peekNext(compiler) == TOKEN_DOT) {
+    nextToken(compiler->parser);
+  }
+}
+
 // Variables and scopes --------------------------------------------------------
 
 // Emits one single-byte argument. Returns its index.
@@ -1949,6 +1966,7 @@ static void namedCall(Compiler* compiler, bool canAssign, Code instruction)
   else
   {
     methodCall(compiler, instruction, &signature);
+    allowLineBeforeDot(compiler);
   }
 }
 
@@ -2145,6 +2163,8 @@ static void field(Compiler* compiler, bool canAssign)
     loadThis(compiler);
     emitByteArg(compiler, isLoad ? CODE_LOAD_FIELD : CODE_STORE_FIELD, field);
   }
+
+  allowLineBeforeDot(compiler);
 }
 
 // Compiles a read or assignment to [variable].
@@ -2176,6 +2196,8 @@ static void bareName(Compiler* compiler, bool canAssign, Variable variable)
 
   // Emit the load instruction.
   loadVariable(compiler, variable);
+
+  allowLineBeforeDot(compiler);
 }
 
 static void staticField(Compiler* compiler, bool canAssign)
@@ -2360,6 +2382,8 @@ static void subscript(Compiler* compiler, bool canAssign)
   finishArgumentList(compiler, &signature);
   consume(compiler, TOKEN_RIGHT_BRACKET, "Expect ']' after arguments.");
 
+  allowLineBeforeDot(compiler);
+
   if (canAssign && match(compiler, TOKEN_EQ))
   {
     signature.type = SIG_SUBSCRIPT_SETTER;
@@ -3488,19 +3512,21 @@ ObjFn* wrenCompile(WrenVM* vm, ObjModule* module, const char* source,
   parser.numParens = 0;
 
   // Zero-init the current token. This will get copied to previous when
-  // advance() is called below.
-  parser.current.type = TOKEN_ERROR;
-  parser.current.start = source;
-  parser.current.length = 0;
-  parser.current.line = 0;
-  parser.current.value = UNDEFINED_VAL;
+  // nextToken() is called below.
+  parser.next.type = TOKEN_ERROR;
+  parser.next.start = source;
+  parser.next.length = 0;
+  parser.next.line = 0;
+  parser.next.value = UNDEFINED_VAL;
 
   // Ignore leading newlines.
   parser.skipNewlines = true;
   parser.printErrors = printErrors;
   parser.hasError = false;
 
-  // Read the first token.
+  // Read the first token into next
+  nextToken(&parser);
+  // Copy next -> current
   nextToken(&parser);
 
   int numExistingVariables = module->variables.count;

test/language/chained_newline.wren

@@ -0,0 +1,151 @@
+class Test {
+  construct new() {}
+  test0() {
+    System.print("test0")
+    return this
+  }
+  test1() {
+    System.print("test1")
+    return this
+  }
+  test2() {
+    System.print("test2")
+    return this
+  }
+
+  [index] {
+    System.print("testSubscript")
+    return this
+  }
+}
+
+class Tester {
+  construct new() {
+
+    var test = _test = Test.new()
+
+    //test local access
+
+    test.
+      test0().          // expect: test0
+      test1().          // expect: test1
+      test2()           // expect: test2
+
+    test
+      .test0()          // expect: test0
+      .test1()          // expect: test1
+      .test2()          // expect: test2
+
+    test
+      .test0()          // expect: test0
+      .test1().         // expect: test1
+       test2()          // expect: test2
+
+    test[0]             // expect: testSubscript
+      .test0()          // expect: test0
+
+    test[0].            // expect: testSubscript
+      test0()           // expect: test0
+
+    //test field access
+
+    _test.
+      test0().          // expect: test0
+      test1().          // expect: test1
+      test2()           // expect: test2
+
+    _test
+      .test0()          // expect: test0
+      .test1()          // expect: test1
+      .test2()          // expect: test2
+
+    _test
+      .test0().         // expect: test0
+       test1().         // expect: test1
+       test2()          // expect: test2
+
+    _test[0]            // expect: testSubscript
+      .test0()          // expect: test0
+
+    _test[0].           // expect: testSubscript
+      test0()           // expect: test0
+
+  }
+
+  getter { _test }
+  method() { _test }
+
+} //Tester
+
+//access via methods/getter
+
+var external = Tester.new()
+
+external.getter.
+  test0().              // expect: test0
+  test1().              // expect: test1
+  test2()               // expect: test2
+
+external.getter
+  .test0()              // expect: test0
+  .test1()              // expect: test1
+  .test2()              // expect: test2
+
+external.getter.
+   test0()              // expect: test0
+  .test1()              // expect: test1
+  .test2()              // expect: test2
+
+external.getter[0].     // expect: testSubscript
+   test0()              // expect: test0
+
+external.getter[0]      // expect: testSubscript
+   .test0()             // expect: test0
+
+external.method().
+  test0().              // expect: test0
+  test1().              // expect: test1
+  test2()               // expect: test2
+
+external.method()
+  .test0()              // expect: test0
+  .test1()              // expect: test1
+  .test2()              // expect: test2
+
+external.method().
+  test0()               // expect: test0
+  .test1().             // expect: test1
+  test2()               // expect: test2
+
+external.method()[0].   // expect: testSubscript
+  test0()               // expect: test0
+
+external.method()[0]    // expect: testSubscript
+  .test0()              // expect: test0
+
+
+//regular access in module scope
+
+var other = Test.new()
+
+other.
+  test0().              // expect: test0
+  test1().              // expect: test1
+  test2()               // expect: test2
+
+other
+  .test0()              // expect: test0
+  .test1()              // expect: test1
+  .test2()              // expect: test2
+
+other
+  .test0().             // expect: test0
+   test1()              // expect: test1
+  .test2()              // expect: test2
+
+
+other[0]                // expect: testSubscript
+  .test0()              // expect: test0
+
+other[0].               // expect: testSubscript
+  test0()               // expect: test0