From 51278fde66abf62ddb74d0aab8882455effe3b58 Mon Sep 17 00:00:00 2001 From: jrfnl Date: Wed, 31 Jul 2024 03:20:37 +0200 Subject: [PATCH] Tokenizer: apply tab replacement to heredoc/nowdoc opener _You learn something new every day ;-)_ While probably exceedingly rare to be found in actual codebases, the PHP tokenizer apparently allows for whitespace between the `<<<` and the heredoc/nowdoc identifier. See: https://3v4l.org/NUHZd Both spaces as well as tabs are allowed. New lines are not allowed. Comments are also not allowed. See: https://3v4l.org/7PIEK The PHPCS `Tokenizer` did not execute tab replacement on these tokens leading to unexpected `'content'` and incorrect `'length'` values in the `File::$tokens` array, which in turn could lead to incorrect sniff results and incorrect fixes. This commit adds the `T_START_HEREDOC`/`T_START_NOWDOC` tokens to the array of tokens for which to do tab replacement to make them more consistent with the rest of PHPCS. Includes unit tests safeguarding this change. Ref: https://externals.io/message/124462#124518 --- src/Tokenizers/Tokenizer.php | 2 + .../Tokenizer/HeredocNowdocOpenerTest.inc | 31 +++++ .../Tokenizer/HeredocNowdocOpenerTest.php | 121 ++++++++++++++++++ 3 files changed, 154 insertions(+) create mode 100644 tests/Core/Tokenizer/Tokenizer/HeredocNowdocOpenerTest.inc create mode 100644 tests/Core/Tokenizer/Tokenizer/HeredocNowdocOpenerTest.php diff --git a/src/Tokenizers/Tokenizer.php b/src/Tokenizers/Tokenizer.php index ba41f8da88..68238d3023 100644 --- a/src/Tokenizers/Tokenizer.php +++ b/src/Tokenizers/Tokenizer.php @@ -193,6 +193,8 @@ private function createPositionMap() T_DOC_COMMENT_STRING => true, T_CONSTANT_ENCAPSED_STRING => true, T_DOUBLE_QUOTED_STRING => true, + T_START_HEREDOC => true, + T_START_NOWDOC => true, T_HEREDOC => true, T_NOWDOC => true, T_END_HEREDOC => true, diff --git a/tests/Core/Tokenizer/Tokenizer/HeredocNowdocOpenerTest.inc b/tests/Core/Tokenizer/Tokenizer/HeredocNowdocOpenerTest.inc new file mode 100644 index 0000000000..dc2b2f2dc2 --- /dev/null +++ b/tests/Core/Tokenizer/Tokenizer/HeredocNowdocOpenerTest.inc @@ -0,0 +1,31 @@ + + * @copyright 2024 PHPCSStandards and contributors + * @license https://github.com/PHPCSStandards/PHP_CodeSniffer/blob/master/licence.txt BSD Licence + */ + +namespace PHP_CodeSniffer\Tests\Core\Tokenizer\Tokenizer; + +use PHP_CodeSniffer\Tests\Core\Tokenizer\AbstractTokenizerTestCase; + +/** + * Heredoc/nowdoc opener token test. + */ +final class HeredocNowdocOpenerTest extends AbstractTokenizerTestCase +{ + + + /** + * Verify that spaces/tabs in a heredoc/nowdoc opener token get the tab replacement treatment. + * + * @param string $testMarker The comment prefacing the target token. + * @param array $expected Expectations for the token array. + * + * @dataProvider dataHeredocNowdocOpenerTabReplacement + * @covers PHP_CodeSniffer\Tokenizers\Tokenizer::createPositionMap + * + * @return void + */ + public function testHeredocNowdocOpenerTabReplacement($testMarker, $expected) + { + $tokens = $this->phpcsFile->getTokens(); + $opener = $this->getTargetToken($testMarker, [T_START_HEREDOC, T_START_NOWDOC]); + + foreach ($expected as $key => $value) { + if ($key === 'orig_content' && $value === null) { + $this->assertArrayNotHasKey($key, $tokens[$opener], "Unexpected 'orig_content' key found in the token array."); + continue; + } + + $this->assertArrayHasKey($key, $tokens[$opener], "Key $key not found in the token array."); + $this->assertSame($value, $tokens[$opener][$key], "Value for key $key does not match expectation."); + } + + }//end testHeredocNowdocOpenerTabReplacement() + + + /** + * Data provider. + * + * @see testHeredocNowdocOpenerTabReplacement() + * + * @return array>> + */ + public static function dataHeredocNowdocOpenerTabReplacement() + { + return [ + 'Heredoc opener without space' => [ + 'testMarker' => '/* testHeredocOpenerNoSpace */', + 'expected' => [ + 'length' => 6, + 'content' => '<< null, + ], + ], + 'Nowdoc opener without space' => [ + 'testMarker' => '/* testNowdocOpenerNoSpace */', + 'expected' => [ + 'length' => 8, + 'content' => "<<<'EOD' +", + 'orig_content' => null, + ], + ], + 'Heredoc opener with space(s)' => [ + 'testMarker' => '/* testHeredocOpenerHasSpace */', + 'expected' => [ + 'length' => 7, + 'content' => '<<< END +', + 'orig_content' => null, + ], + ], + 'Nowdoc opener with space(s)' => [ + 'testMarker' => '/* testNowdocOpenerHasSpace */', + 'expected' => [ + 'length' => 21, + 'content' => "<<< 'END' +", + 'orig_content' => null, + ], + ], + 'Heredoc opener with tab(s)' => [ + 'testMarker' => '/* testHeredocOpenerHasTab */', + 'expected' => [ + 'length' => 18, + 'content' => '<<< "END" +', + 'orig_content' => '<<< "END" +', + ], + ], + 'Nowdoc opener with tab(s)' => [ + 'testMarker' => '/* testNowdocOpenerHasTab */', + 'expected' => [ + 'length' => 11, + 'content' => "<<< 'END' +", + 'orig_content' => "<<< 'END' +", + ], + ], + ]; + + }//end dataHeredocNowdocOpenerTabReplacement() + + +}//end class