RegularParser now properly takes into account escaped tokens

.github/workflows/test-old.yaml

@@ -20,7 +20,7 @@ jobs:
         uses: 'shivammathur/setup-php@v2'
         with:
           php-version: '${{ matrix.php }}'
-          tools: 'composer:v1'
+          tools: 'composer:v2'
           coverage: 'xdebug'
       - name: 'PHP'
         run: 'php -v'

.github/workflows/test.yaml

@@ -11,12 +11,11 @@ jobs:
     runs-on: '${{ matrix.os }}'
     strategy:
       matrix:
-        php: ['7.4', '8.0', '8.1', '8.2', '8.3']
+        php: ['7.4', '8.0', '8.1', '8.2', '8.3', '8.4', '8.5']
         os: ['ubuntu-latest']
         failure: [false]
         include:
-          - { php: '8.4', os: 'ubuntu-latest', failure: true } # Psalm does not support PHP 8.4 yet
-          - { php: '8.5', os: 'ubuntu-latest', failure: true } # '8.5' means 'nightly'
+          - { php: '8.6', os: 'ubuntu-latest', failure: true } # '8.6' means 'nightly'
     steps:
       - name: 'Checkout'
         uses: 'actions/checkout@v4'
@@ -38,7 +37,7 @@ jobs:
       - name: 'Psalm'
         run: |
           composer remove --dev -W 'phpunit/phpunit'
-          composer require --dev -W 'vimeo/psalm=^5.0' 'nikic/php-parser=^4.0'
+          composer require --dev -W 'vimeo/psalm=>=5.0' 'nikic/php-parser=>=4.0'
           php vendor/bin/psalm --shepherd --php-version=${{ matrix.php }}
         continue-on-error: '${{ matrix.failure }}'
       - name: 'Infection'

Makefile

@@ -1,4 +1,4 @@
-PHP_VERSION ?= 8.0
+PHP_VERSION ?= 8.5
 PHP := docker-compose run --rm php-${PHP_VERSION}
 
 php-version:

docker-compose.yaml

@@ -18,3 +18,8 @@ services:
   php-7.3: { <<: *php, build: { context: docker/php,     args: { PHP_VERSION: 7.3   } } }
   php-7.4: { <<: *php, build: { context: docker/php,     args: { PHP_VERSION: 7.4   } } }
   php-8.0: { <<: *php, build: { context: docker/php,     args: { PHP_VERSION: 8.0   } } }
+  php-8.1: { <<: *php, build: { context: docker/php,     args: { PHP_VERSION: 8.1   } } }
+  php-8.2: { <<: *php, build: { context: docker/php,     args: { PHP_VERSION: 8.2   } } }
+  php-8.3: { <<: *php, build: { context: docker/php,     args: { PHP_VERSION: 8.3   } } }
+  php-8.4: { <<: *php, build: { context: docker/php,     args: { PHP_VERSION: 8.4   } } }
+  php-8.5: { <<: *php, build: { context: docker/php,     args: { PHP_VERSION: 8.5   } } }

docker/php/Dockerfile

@@ -2,7 +2,7 @@ ARG PHP_VERSION=8.0
 FROM php:$PHP_VERSION
 
 RUN php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');" \
- && php -r "if (hash_file('sha384', 'composer-setup.php') === '906a84df04cea2aa72f40b5f787e49f22d4c2f19492ac310e8cba5b96ac8b64115ac402c8cd292b8a03482574915d1a8') { echo 'Installer verified'; } else { echo 'Installer corrupt'; unlink('composer-setup.php'); } echo PHP_EOL;" \
+ && php -r "if (hash_file('sha384', 'composer-setup.php') === 'c8b085408188070d5f52bcfe4ecfbee5f727afa458b2573b8eaaf77b3419b0bf2768dc67c86944da1544f06fa544fd47') { echo 'Installer verified'; } else { echo 'Installer corrupt'; unlink('composer-setup.php'); } echo PHP_EOL;" \
  && php composer-setup.php \
  && php -r "unlink('composer-setup.php');" \
  && mv composer.phar /usr/local/bin/composer

psalm.xml

@@ -16,4 +16,9 @@
         </ignoreFiles>
     </projectFiles>
 
+    <issueHandlers>
+        <MissingOverrideAttribute errorLevel="suppress" />
+        <UnusedPsalmSuppress errorLevel="suppress" />
+    </issueHandlers>
+
 </psalm>

src/Event/FilterShortcodesEvent.php

@@ -10,6 +10,7 @@
  * is used directly in processor.
  *
  * @author Tomasz Kowalczyk <[email protected]>
+ * @psalm-suppress ClassMustBeFinal
  */
 class FilterShortcodesEvent
 {
@@ -24,7 +25,7 @@
      */
     public function __construct(array $shortcodes, $parent = null)
     {
         if(null !== $parent && false === $parent instanceof ProcessedShortcode) {
             throw new \LogicException('Parameter $parent must be an instance of ProcessedShortcode.');
         }
 

src/Event/ReplaceShortcodesEvent.php

@@ -10,6 +10,7 @@
  * results in the source text.
  *
  * @author Tomasz Kowalczyk <[email protected]>
+ * @psalm-suppress ClassMustBeFinal
  */
 class ReplaceShortcodesEvent
 {
@@ -29,7 +30,7 @@
      */
     public function __construct($text, array $replacements, $shortcode = null)
     {
         if(null !== $shortcode && false === $shortcode instanceof ShortcodeInterface) {
             throw new \LogicException('Parameter $shortcode must be an instance of ShortcodeInterface.');
         }
 

src/Parser/RegexParser.php

@@ -46,6 +46,7 @@ public function parse($text)
         // loop instead of array_map to pass the arguments explicitly
         $shortcodes = array();
         foreach($matches[0] as $match) {
+            /** @psalm-suppress PossiblyFalseArgument */
             $offset = mb_strlen(substr($text, 0, $match[1]), 'utf-8');
             $shortcodes[] = $this->parseSingle($match[0], $offset);
         }
@@ -108,11 +109,13 @@ private function parseValue($value)
      * @param string $value
      *
      * @return string
+     * @psalm-suppress InvalidFalsableReturnType
      */
     private function extractValue($value)
     {
         $length = strlen($this->syntax->getParameterValueDelimiter());
 
+        /** @psalm-suppress FalsableReturnStatement */
         return $this->isDelimitedValue($value) ? substr($value, $length, -1 * $length) : $value;
     }
 

src/Parser/RegularParser.php

@@ -9,6 +9,8 @@
 
 /**
  * @author Tomasz Kowalczyk <[email protected]>
+ * @psalm-suppress PossiblyUndefinedArrayOffset
+ * @psalm-suppress PossiblyUndefinedVariable
  */
 final class RegularParser implements ParserInterface
 {
@@ -79,6 +81,7 @@ public function parse($text)
                 }
             }
         }
+        /** @psalm-suppress PossiblyFalseArgument */
         ini_set('xdebug.max_nesting_level', $nestingLevel);
 
         return $shortcodes;
@@ -269,7 +272,9 @@ private function getBacktrack()
     {
         $position = array_pop($this->backtracks);
         $backtrack = '';
+        /** @psalm-suppress PossiblyNullOperand */
         for($i = $position; $i < $this->position; $i++) {
+            /** @psalm-suppress PossiblyNullArrayOffset */
             $backtrack .= $this->tokens[$i][1];
         }
 
@@ -285,13 +290,17 @@ private function backtrack($modifyPosition = true)
     {
         $position = array_pop($this->backtracks);
         if($modifyPosition) {
+            /** @psalm-suppress PossiblyNullPropertyAssignmentValue */
             $this->position = $position;
         }
 
         $backtrack = '';
+        /** @psalm-suppress PossiblyNullOperand */
         for($i = $position; $i < $this->lastBacktrack; $i++) {
+            /** @psalm-suppress PossiblyNullArrayOffset */
             $backtrack .= $this->tokens[$i][1];
         }
+        /** @psalm-suppress PossiblyNullPropertyAssignmentValue */
         $this->lastBacktrack = $position;
 
         return $backtrack;
@@ -339,6 +348,7 @@ private function match($type, $ws)
      * @param string $text
      *
      * @psalm-return list<array{0:int,1:string,2:int}>
+     * @psalm-suppress MixedReturnTypeCoercion
      */
     private function tokenize($text)
     {
@@ -362,21 +372,23 @@ private function tokenize($text)
                 default: { throw new \RuntimeException('Invalid token.'); }
             }
             $tokens[] = array($type, $token, $position);
+            /** @psalm-suppress MixedArgument */
             $position += mb_strlen($token, 'utf-8');
         }
 
+        /** @psalm-suppress MixedReturnTypeCoercion */
         return $tokens;
     }
 
     /** @return non-empty-string */
     private function prepareLexer(SyntaxInterface $syntax)
     {
         // FIXME: for some reason Psalm does not understand the `@psalm-var callable() $var` annotation
-        /** @psalm-suppress MissingClosureParamType, MissingClosureReturnType */
+        /** @psalm-suppress MissingClosureParamType,MissingClosureReturnType,PossiblyNullOperand */
         $group = function($text, $group) {
             return '(?<'.(string)$group.'>'.preg_replace('/(.)/us', '\\\\$0', (string)$text).')';
         };
-        /** @psalm-suppress MissingClosureParamType, MissingClosureReturnType */
+        /** @psalm-suppress MissingClosureParamType,MissingClosureReturnType */
         $quote = function($text) {
             return preg_replace('/(.)/us', '\\\\$0', (string)$text);
         };
@@ -388,6 +400,7 @@ private function prepareLexer(SyntaxInterface $syntax)
                 $quote($syntax->getClosingTagMarker()),
                 $quote($syntax->getParameterValueSeparator()),
                 $quote($syntax->getParameterValueDelimiter()),
+                '\\\\',
                 '\s+',
             )).').)+)',
             '(?<ws>\s+)',

src/Parser/WordpressParser.php

@@ -22,6 +22,8 @@
  * @see https://core.trac.wordpress.org/browser/tags/4.3.1/src/wp-includes/shortcodes.php#L239
  * @see https://core.trac.wordpress.org/browser/tags/4.3.1/src/wp-includes/shortcodes.php#L448
  * @psalm-suppress RiskyTruthyFalsyComparison
+ * @psalm-suppress PossiblyNullArgument
+ * @psalm-suppress PossiblyFalseArgument
  *
  * @author Tomasz Kowalczyk <[email protected]>
  */

src/Serializer/JsonSerializer.php

@@ -6,6 +6,8 @@
 
 /**
  * @author Tomasz Kowalczyk <[email protected]>
+ * @psalm-suppress FalsableReturnStatement
+ * @psalm-suppress InvalidFalsableReturnType
  */
 final class JsonSerializer implements SerializerInterface
 {

src/Serializer/YamlSerializer.php

@@ -7,6 +7,9 @@
 
 /**
  * @author Tomasz Kowalczyk <[email protected]>
+ * @psalm-suppress ReservedWord
+ * @psalm-suppress InvalidReturnStatement
+ * @psalm-suppress InvalidReturnType
  */
 final class YamlSerializer implements SerializerInterface
 {

src/ShortcodeFacade.php

@@ -20,6 +20,7 @@
 
 /**
  * @author Tomasz Kowalczyk <[email protected]>
+ * @psalm-suppress ClassMustBeFinal
  */
 class ShortcodeFacade
 {

tests/ParserTest.php

@@ -1,6 +1,7 @@
 <?php
 namespace Thunder\Shortcode\Tests;
 
+use PHPUnit\Framework\Attributes\DataProvider;
 use Thunder\Shortcode\HandlerContainer\HandlerContainer;
 use Thunder\Shortcode\Parser\RegularParser;
 use Thunder\Shortcode\Parser\ParserInterface;
@@ -23,6 +24,7 @@ final class ParserTest extends AbstractTestCase
      *
      * @dataProvider provideShortcodes
      */
+    #[DataProvider('provideShortcodes')]
     public function testParser(ParserInterface $parser, $code, array $expected)
     {
         $this->assertShortcodes($parser->parse($code), $expected);
@@ -276,6 +278,26 @@ public function testIssue77()
         ));
     }
 
+    public function testIssue119()
+    {
+        $cases = array(
+            '[a k="\"y"]inner[/a]' => new ParsedShortcode(new Shortcode('a', array('k' => '\"y'), 'inner', null), '[a k="\"y"]inner[/a]', 0),
+            '[a k=" \"y"]inner[/a]' => new ParsedShortcode(new Shortcode('a', array('k' => ' \"y'), 'inner', null), '[a k=" \"y"]inner[/a]', 0),
+            '[a k=" x\"y"]inner[/a]' => new ParsedShortcode(new Shortcode('a', array('k' => ' x\"y'), 'inner', null), '[a k=" x\"y"]inner[/a]', 0),
+            '[a k="x\"y"]inner[/a]' => new ParsedShortcode(new Shortcode('a', array('k' => 'x\"y'), 'inner', null), '[a k="x\"y"]inner[/a]', 0),
+            '[mention id=1 name="foo\"ff\""][/mention]' => new ParsedShortcode(new Shortcode('mention', array('id' => '1', 'name' => 'foo\"ff\"'), '', null), '[mention id=1 name="foo\"ff\""][/mention]', 0),
+        );
+        $parser = new RegularParser();
+        foreach($cases as $input => $expected) {
+            $this->assertShortcodes($parser->parse($input), array($expected));
+        }
+
+        $this->assertShortcodes($parser->parse('[a k="x\"y"]inner[/a] [mention id=1 name="foo\"ff\""][/mention]'), array(
+            new ParsedShortcode(new Shortcode('a', array('k' => 'x\"y'), 'inner', null), '[a k="x\"y"]inner[/a]', 0),
+            new ParsedShortcode(new Shortcode('mention', array('id' => '1', 'name' => 'foo\"ff\"'), '', null), '[mention id=1 name="foo\"ff\""][/mention]', 22),
+        ));
+    }
+
     public function testWordPress()
     {
         $parser = new WordpressParser();

tests/ProcessorTest.php

@@ -1,6 +1,7 @@
 <?php
 namespace Thunder\Shortcode\Tests;
 
+use PHPUnit\Framework\Attributes\DataProvider;
 use Thunder\Shortcode\Handler\ContentHandler;
 use Thunder\Shortcode\Handler\DeclareHandler;
 use Thunder\Shortcode\Handler\EmailHandler;
@@ -62,6 +63,7 @@ public function testReplaceWithoutContentOffset()
      *
      * @dataProvider provideTexts
      */
+    #[DataProvider('provideTexts')]
     public function testProcessorProcess($text, $result)
     {
         $processor = new Processor(new RegexParser(), $this->getHandlers());
@@ -164,6 +166,7 @@ public function testProcessorShortcodePositions()
     /**
      * @dataProvider provideBuiltInTests
      */
+    #[DataProvider('provideBuiltInTests')]
     public function testBuiltInHandlers($text, $result)
     {
         $handlers = new HandlerContainer();

tests/SerializerTest.php

@@ -1,6 +1,7 @@
 <?php
 namespace Thunder\Shortcode\Tests;
 
+use PHPUnit\Framework\Attributes\DataProvider;
 use Thunder\Shortcode\Serializer\JsonSerializer;
 use Thunder\Shortcode\Serializer\SerializerInterface;
 use Thunder\Shortcode\Serializer\TextSerializer;
@@ -18,6 +19,7 @@ final class SerializerTest extends AbstractTestCase
     /**
      * @dataProvider provideShortcodes
      */
+    #[DataProvider('provideShortcodes')]
     public function testSerializer(SerializerInterface $serializer, ShortcodeInterface $test)
     {
         $result = $serializer->serialize($test);
@@ -61,6 +63,7 @@ public static function provideShortcodes()
     /**
      * @dataProvider provideUnserialized
      */
+    #[DataProvider('provideUnserialized')]
     public function testUnserialize(SerializerInterface $serializer, ShortcodeInterface $test, $text)
     {
         $tested = $serializer->unserialize($text);
@@ -89,6 +92,7 @@ public static function provideUnserialized()
     /**
      * @dataProvider provideExceptions
      */
+    #[DataProvider('provideExceptions')]
     public function testSerializerExceptions(SerializerInterface $serializer, $value, $exceptionClass)
     {
         $this->willThrowException($exceptionClass);

tests/ShortcodeTest.php

@@ -1,6 +1,7 @@
 <?php
 namespace Thunder\Shortcode\Tests;
 
+use PHPUnit\Framework\Attributes\DataProvider;
 use Thunder\Shortcode\HandlerContainer\HandlerContainer;
 use Thunder\Shortcode\Parser\RegexParser;
 use Thunder\Shortcode\Processor\Processor;
@@ -18,6 +19,7 @@ final class ShortcodeTest extends AbstractTestCase
     /**
      * @dataProvider provideShortcodes
      */
+    #[DataProvider('provideShortcodes')]
     public function testShortcode($expected, $name, array $args, $content)
     {
         $s = new Shortcode($name, $args, $content);

tests/SyntaxTest.php

@@ -1,6 +1,7 @@
 <?php
 namespace Thunder\Shortcode\Tests;
 
+use PHPUnit\Framework\Attributes\DataProvider;
 use Thunder\Shortcode\Syntax\Syntax;
 use Thunder\Shortcode\Syntax\CommonSyntax;
 use Thunder\Shortcode\Syntax\SyntaxBuilder;
@@ -14,6 +15,7 @@ final class SyntaxTest extends AbstractTestCase
     /**
      * @dataProvider provideSyntaxes
      */
+    #[DataProvider('provideSyntaxes')]
     public function testSyntax(SyntaxInterface $syntax, $open, $close, $slash, $parameter, $value)
     {
         static::assertSame($open, $syntax->getOpeningTag());