Tokenize yield from as T_YIELD+T_FROM

Fixes GH-14926.

Author: [email protected]

Zend/tests/gh14926.phpt

@@ -0,0 +1,22 @@
+--TEST--
+yield from is tokenized separately
+--FILE--
+<?php
+
+function tokens($str) {
+	$t = token_get_all("<?php $str");
+	array_shift($t);
+	foreach ($t as $t) {
+		echo is_array($t) ? $t[1] : $t, "|";
+	}
+	print "\n";
+}
+
+tokens("yield from \$foo;");
+tokens("yield # comment\n/* other comment */ from \$foo;");
+
+?>
+--EXPECT--
+yield| |from| |$foo|;|
+yield| |# comment|
+|/* other comment */| |from| |$foo|;|

Zend/zend_language_parser.y

@@ -60,7 +60,7 @@ static YYSIZE_T zend_yytnamerr(char*, const char*);
 %precedence T_PRINT
 %precedence T_YIELD
 %precedence T_DOUBLE_ARROW
-%precedence T_YIELD_FROM
+%precedence T_FROM
 %precedence '=' T_PLUS_EQUAL T_MINUS_EQUAL T_MUL_EQUAL T_DIV_EQUAL T_CONCAT_EQUAL T_MOD_EQUAL T_AND_EQUAL T_OR_EQUAL T_XOR_EQUAL T_SL_EQUAL T_SR_EQUAL T_POW_EQUAL T_COALESCE_EQUAL
 %left '?' ':'
 %right T_COALESCE
@@ -109,7 +109,7 @@ static YYSIZE_T zend_yytnamerr(char*, const char*);
 %token <ident> T_LOGICAL_AND   "'and'"
 %token <ident> T_PRINT         "'print'"
 %token <ident> T_YIELD         "'yield'"
-%token <ident> T_YIELD_FROM    "'yield from'"
+%token <ident> T_FROM          "'from'"
 %token <ident> T_INSTANCEOF    "'instanceof'"
 %token <ident> T_NEW           "'new'"
 %token <ident> T_CLONE         "'clone'"
@@ -1312,7 +1312,7 @@ expr:
 	|	T_YIELD { $$ = zend_ast_create(ZEND_AST_YIELD, NULL, NULL); CG(extra_fn_flags) |= ZEND_ACC_GENERATOR; }
 	|	T_YIELD expr { $$ = zend_ast_create(ZEND_AST_YIELD, $2, NULL); CG(extra_fn_flags) |= ZEND_ACC_GENERATOR; }
 	|	T_YIELD expr T_DOUBLE_ARROW expr { $$ = zend_ast_create(ZEND_AST_YIELD, $4, $2); CG(extra_fn_flags) |= ZEND_ACC_GENERATOR; }
-	|	T_YIELD_FROM expr { $$ = zend_ast_create(ZEND_AST_YIELD_FROM, $2); CG(extra_fn_flags) |= ZEND_ACC_GENERATOR; }
+	|	T_YIELD T_FROM expr { $$ = zend_ast_create(ZEND_AST_YIELD_FROM, $3); CG(extra_fn_flags) |= ZEND_ACC_GENERATOR; }
 	|	T_THROW expr { $$ = zend_ast_create(ZEND_AST_THROW, $2); }
 	|	inline_function { $$ = $1; }
 	|	attributes inline_function { $$ = zend_ast_with_attributes($2, $1); }

Zend/zend_language_scanner.l

@@ -1412,9 +1412,14 @@ OPTIONAL_WHITESPACE_OR_COMMENTS ({WHITESPACE}|{MULTI_LINE_COMMENT}|{SINGLE_LINE_
 }
 
 <ST_IN_SCRIPTING>"yield"{WHITESPACE_OR_COMMENTS}"from"[^a-zA-Z0-9_\x80-\xff] {
-	yyless(yyleng - 1);
-	HANDLE_NEWLINES(yytext, yyleng);
-	RETURN_TOKEN_WITH_IDENT(T_YIELD_FROM);
+	yyless(5);
+	yy_push_state(ST_LOOKING_FOR_FROM);
+	RETURN_TOKEN_WITH_IDENT(T_YIELD);
+}
+
+<ST_LOOKING_FOR_FROM>"from" {
+	yy_pop_state();
+	RETURN_TOKEN_WITH_IDENT(T_FROM);
 }
 
 <ST_IN_SCRIPTING>"yield" {
@@ -1580,7 +1585,7 @@ OPTIONAL_WHITESPACE_OR_COMMENTS ({WHITESPACE}|{MULTI_LINE_COMMENT}|{SINGLE_LINE_
 	RETURN_TOKEN(T_NULLSAFE_OBJECT_OPERATOR);
 }
 
-<ST_IN_SCRIPTING,ST_LOOKING_FOR_PROPERTY>{WHITESPACE}+ {
+<ST_IN_SCRIPTING,ST_LOOKING_FOR_PROPERTY,ST_LOOKING_FOR_FROM>{WHITESPACE}+ {
 	goto return_whitespace;
 }
 
@@ -2383,7 +2388,7 @@ inline_char_handler:
 }
 
 
-<ST_IN_SCRIPTING,ST_LOOKING_FOR_PROPERTY>"#"|"//" {
+<ST_IN_SCRIPTING,ST_LOOKING_FOR_PROPERTY,ST_LOOKING_FOR_FROM>"#"|"//" {
 	while (YYCURSOR < YYLIMIT) {
 		switch (*YYCURSOR++) {
 			case '\r':
@@ -2407,7 +2412,7 @@ inline_char_handler:
 	RETURN_OR_SKIP_TOKEN(T_COMMENT);
 }
 
-<ST_IN_SCRIPTING,ST_LOOKING_FOR_PROPERTY>"/*"|"/**"{WHITESPACE} {
+<ST_IN_SCRIPTING,ST_LOOKING_FOR_PROPERTY,ST_LOOKING_FOR_FROM>"/*"|"/**"{WHITESPACE} {
 	int doc_com;
 
 	if (yyleng > 2) {
@@ -2443,7 +2448,7 @@ inline_char_handler:
 	RETURN_OR_SKIP_TOKEN(T_COMMENT);
 }
 
-<ST_LOOKING_FOR_PROPERTY>{ANY_CHAR} {
+<ST_LOOKING_FOR_PROPERTY,ST_LOOKING_FOR_FROM>{ANY_CHAR} {
 	yyless(0);
 	yy_pop_state();
 	goto restart;

ext/tokenizer/tokenizer_data.c

@@ -114,9 +114,9 @@
 const T_YIELD = UNKNOWN;
 /**
  * @var int
- * @cvalue T_YIELD_FROM
+ * @cvalue T_FROM
  */
-const T_YIELD_FROM = UNKNOWN;
+const T_FROM = UNKNOWN;
 /**
  * @var int
  * @cvalue T_INSTANCEOF