Merge pull request #465 from greg0ire/drop-lexer-1

Drop Lexer 1

Author: greg0ire

UPGRADE.md

@@ -0,0 +1,6 @@
+# Upgrade from 1.0.x to 2.0.x
+
+`DocLexer::peek()` and `DocLexer::glimpse` now return
+`Doctrine\Common\Lexer\Token` objects. When using `doctrine/lexer` 2, these
+implement `ArrayAccess` as a way for you to still be able to treat them as
+arrays in some ways.

composer.json

@@ -34,7 +34,7 @@
     "require": {
         "php": "^7.1 || ^8.0",
         "ext-tokenizer": "*",
-        "doctrine/lexer": "^1 || ^2",
+        "doctrine/lexer": "^2",
         "psr/cache": "^1 || ^2 || ^3"
     },
     "require-dev": {

lib/Doctrine/Common/Annotations/DocLexer.php

@@ -70,7 +70,7 @@ public function nextTokenIsAdjacent(): bool
     {
         return $this->token === null
             || ($this->lookahead !== null
-                && ($this->lookahead['position'] - $this->token['position']) === strlen($this->token['value']));
+                && ($this->lookahead->position - $this->token->position) === strlen($this->token->value));
     }
 
     /**
@@ -128,16 +128,4 @@ protected function getType(&$value)
 
         return $type;
     }
-
-    /** @return array{value: int|string, type:self::T_*|null, position:int} */
-    public function peek(): ?array
-    {
-        $token = parent::peek();
-
-        if ($token === null) {
-            return null;
-        }
-
-        return (array) $token;
-    }
 }

lib/Doctrine/Common/Annotations/DocParser.php

@@ -454,7 +454,7 @@ private function syntaxError(string $expected, ?array $token = null): Annotation
         $message  = sprintf('Expected %s, got ', $expected);
         $message .= $this->lexer->lookahead === null
             ? 'end of string'
-            : sprintf("'%s' at position %s", $token['value'], $token['position']);
+            : sprintf("'%s' at position %s", $token->value, $token->position);
 
         if (strlen($this->context)) {
             $message .= ' in ' . $this->context;
@@ -684,16 +684,16 @@ private function Annotations(): array
         $annotations = [];
 
         while ($this->lexer->lookahead !== null) {
-            if ($this->lexer->lookahead['type'] !== DocLexer::T_AT) {
+            if ($this->lexer->lookahead->type !== DocLexer::T_AT) {
                 $this->lexer->moveNext();
                 continue;
             }
 
             // make sure the @ is preceded by non-catchable pattern
             if (
                 $this->lexer->token !== null &&
-                $this->lexer->lookahead['position'] === $this->lexer->token['position'] + strlen(
-                    $this->lexer->token['value']
+                $this->lexer->lookahead->position === $this->lexer->token->position + strlen(
+                    $this->lexer->token->value
                 )
             ) {
                 $this->lexer->moveNext();
@@ -705,12 +705,12 @@ private function Annotations(): array
             $peek = $this->lexer->glimpse();
             if (
                 ($peek === null)
-                || ($peek['type'] !== DocLexer::T_NAMESPACE_SEPARATOR && ! in_array(
-                    $peek['type'],
+                || ($peek->type !== DocLexer::T_NAMESPACE_SEPARATOR && ! in_array(
+                    $peek->type,
                     self::$classIdentifiers,
                     true
                 ))
-                || $peek['position'] !== $this->lexer->lookahead['position'] + 1
+                || $peek->position !== $this->lexer->lookahead->position + 1
             ) {
                 $this->lexer->moveNext();
                 continue;
@@ -1186,18 +1186,18 @@ private function Identifier(): string
 
         $this->lexer->moveNext();
 
-        $className = $this->lexer->token['value'];
+        $className = $this->lexer->token->value;
 
         while (
             $this->lexer->lookahead !== null &&
-            $this->lexer->lookahead['position'] === ($this->lexer->token['position'] +
-            strlen($this->lexer->token['value'])) &&
+            $this->lexer->lookahead->position === ($this->lexer->token->position +
+            strlen($this->lexer->token->value)) &&
             $this->lexer->isNextToken(DocLexer::T_NAMESPACE_SEPARATOR)
         ) {
             $this->match(DocLexer::T_NAMESPACE_SEPARATOR);
             $this->matchAny(self::$classIdentifiers);
 
-            $className .= '\\' . $this->lexer->token['value'];
+            $className .= '\\' . $this->lexer->token->value;
         }
 
         return $className;
@@ -1215,7 +1215,7 @@ private function Value()
     {
         $peek = $this->lexer->glimpse();
 
-        if ($peek['type'] === DocLexer::T_EQUALS) {
+        if ($peek->type === DocLexer::T_EQUALS) {
             return $this->FieldAssignment();
         }
 
@@ -1244,21 +1244,21 @@ private function PlainValue()
             return $this->Constant();
         }
 
-        switch ($this->lexer->lookahead['type']) {
+        switch ($this->lexer->lookahead->type) {
             case DocLexer::T_STRING:
                 $this->match(DocLexer::T_STRING);
 
-                return $this->lexer->token['value'];
+                return $this->lexer->token->value;
 
             case DocLexer::T_INTEGER:
                 $this->match(DocLexer::T_INTEGER);
 
-                return (int) $this->lexer->token['value'];
+                return (int) $this->lexer->token->value;
 
             case DocLexer::T_FLOAT:
                 $this->match(DocLexer::T_FLOAT);
 
-                return (float) $this->lexer->token['value'];
+                return (float) $this->lexer->token->value;
 
             case DocLexer::T_TRUE:
                 $this->match(DocLexer::T_TRUE);
@@ -1290,7 +1290,7 @@ private function PlainValue()
     private function FieldAssignment(): stdClass
     {
         $this->match(DocLexer::T_IDENTIFIER);
-        $fieldName = $this->lexer->token['value'];
+        $fieldName = $this->lexer->token->value;
 
         $this->match(DocLexer::T_EQUALS);
 
@@ -1365,14 +1365,14 @@ private function ArrayEntry(): array
         $peek = $this->lexer->glimpse();
 
         if (
-            $peek['type'] === DocLexer::T_EQUALS
-                || $peek['type'] === DocLexer::T_COLON
+            $peek->type === DocLexer::T_EQUALS
+                || $peek->type === DocLexer::T_COLON
         ) {
             if ($this->lexer->isNextToken(DocLexer::T_IDENTIFIER)) {
                 $key = $this->Constant();
             } else {
                 $this->matchAny([DocLexer::T_INTEGER, DocLexer::T_STRING]);
-                $key = $this->lexer->token['value'];
+                $key = $this->lexer->token->value;
             }
 
             $this->matchAny([DocLexer::T_EQUALS, DocLexer::T_COLON]);

tests/Doctrine/Tests/Common/Annotations/DocLexerTest.php

@@ -3,6 +3,7 @@
 namespace Doctrine\Tests\Common\Annotations;
 
 use Doctrine\Common\Annotations\DocLexer;
+use Doctrine\Common\Lexer\Token;
 use PHPUnit\Framework\TestCase;
 
 use function str_repeat;
@@ -20,12 +21,12 @@ public function testMarkerAnnotation(): void
         self::assertTrue($lexer->moveNext());
         self::assertNull($lexer->token);
         self::assertNotNull($lexer->lookahead);
-        self::assertEquals('@', $lexer->lookahead['value']);
+        self::assertEquals('@', $lexer->lookahead->value);
 
         self::assertTrue($lexer->moveNext());
         self::assertNotNull($lexer->token);
-        self::assertEquals('@', $lexer->token['value']);
-        self::assertEquals('Name', $lexer->lookahead['value']);
+        self::assertEquals('@', $lexer->token->value);
+        self::assertEquals('Name', $lexer->lookahead->value);
 
         self::assertFalse($lexer->moveNext());
     }
@@ -114,9 +115,9 @@ public function testScannerTokenizesDocBlockWhitConstants(): void
         foreach ($tokens as $expected) {
             $lexer->moveNext();
             $lookahead = $lexer->lookahead;
-            self::assertEquals($expected['value'], $lookahead['value']);
-            self::assertEquals($expected['type'], $lookahead['type']);
-            self::assertEquals($expected['position'], $lookahead['position']);
+            self::assertEquals($expected['value'], $lookahead->value);
+            self::assertEquals($expected['type'], $lookahead->type);
+            self::assertEquals($expected['position'], $lookahead->position);
         }
 
         self::assertFalse($lexer->moveNext());
@@ -155,9 +156,9 @@ public function testScannerTokenizesDocBlockWhitInvalidIdentifier(): void
         foreach ($tokens as $expected) {
             $lexer->moveNext();
             $lookahead = $lexer->lookahead;
-            self::assertEquals($expected['value'], $lookahead['value']);
-            self::assertEquals($expected['type'], $lookahead['type']);
-            self::assertEquals($expected['position'], $lookahead['position']);
+            self::assertEquals($expected['value'], $lookahead->value);
+            self::assertEquals($expected['type'], $lookahead->type);
+            self::assertEquals($expected['position'], $lookahead->position);
         }
 
         self::assertFalse($lexer->moveNext());
@@ -172,7 +173,7 @@ public function testWithinDoubleQuotesVeryVeryLongStringWillNotOverflowPregSplit
 
         $lexer->setInput('"' . str_repeat('.', 20240) . '"');
 
-        self::assertIsArray($lexer->glimpse());
+        self::assertInstanceOf(Token::class, $lexer->glimpse());
     }
 
     /**
@@ -216,9 +217,9 @@ public function testRecognizesDoubleQuotesEscapeSequence(): void
         foreach ($tokens as $expected) {
             $lexer->moveNext();
             $lookahead = $lexer->lookahead;
-            self::assertEquals($expected['value'], $lookahead['value']);
-            self::assertEquals($expected['type'], $lookahead['type']);
-            self::assertEquals($expected['position'], $lookahead['position']);
+            self::assertEquals($expected['value'], $lookahead->value);
+            self::assertEquals($expected['type'], $lookahead->type);
+            self::assertEquals($expected['position'], $lookahead->position);
         }
 
         self::assertFalse($lexer->moveNext());
@@ -296,9 +297,9 @@ private function expectDocblockTokens(string $docBlock, array $expectedTokens):
             $lookahead = $lexer->lookahead;
 
             $actualTokens[] = [
-                'value' => $lookahead['value'],
-                'type' => $lookahead['type'],
-                'position' => $lookahead['position'],
+                'value' => $lookahead->value,
+                'type' => $lookahead->type,
+                'position' => $lookahead->position,
             ];
         }