Refactor the lexer to avoid altering the input

By maintaining an offset in the input for lexing rather than removing
the matched part from the input, performance is better, as PHP does not
need to copy big substrings anymore.

Author: stof

src/Seld/JsonLint/Lexer.php

@@ -20,20 +20,20 @@ class Lexer
 {
     private $EOF = 1;
     private $rules = array(
-        0 => '/^\s+/',
-        1 => '/^-?([0-9]|[1-9][0-9]+)(\.[0-9]+)?([eE][+-]?[0-9]+)?\b/',
-        2 => '{^"(?>\\\\["bfnrt/\\\\]|\\\\u[a-fA-F0-9]{4}|[^\0-\x1f\\\\"]++)*+"}',
-        3 => '/^\{/',
-        4 => '/^\}/',
-        5 => '/^\[/',
-        6 => '/^\]/',
-        7 => '/^,/',
-        8 => '/^:/',
-        9 => '/^true\b/',
-        10 => '/^false\b/',
-        11 => '/^null\b/',
-        12 => '/^$/',
-        13 => '/^./',
+        0 => '/\G\s+/',
+        1 => '/\G-?([0-9]|[1-9][0-9]+)(\.[0-9]+)?([eE][+-]?[0-9]+)?\b/',
+        2 => '{\G"(?>\\\\["bfnrt/\\\\]|\\\\u[a-fA-F0-9]{4}|[^\0-\x1f\\\\"]++)*+"}',
+        3 => '/\G\{/',
+        4 => '/\G\}/',
+        5 => '/\G\[/',
+        6 => '/\G\]/',
+        7 => '/\G,/',
+        8 => '/\G:/',
+        9 => '/\Gtrue\b/',
+        10 => '/\Gfalse\b/',
+        11 => '/\Gnull\b/',
+        12 => '/\G$/',
+        13 => '/\G./',
     );
 
     private $conditions = array(
@@ -47,7 +47,7 @@ class Lexer
     private $input;
     private $more;
     private $done;
-    private $matched;
+    private $offset;
 
     public $match;
     public $yylineno;
@@ -70,8 +70,9 @@ public function setInput($input)
         $this->input = $input;
         $this->more = false;
         $this->done = false;
+        $this->offset = 0;
         $this->yylineno = $this->yyleng = 0;
-        $this->yytext = $this->matched = $this->match = '';
+        $this->yytext = $this->match = '';
         $this->conditionStack = array('INITIAL');
         $this->yylloc = array('first_line' => 1, 'first_column' => 0, 'last_line' => 1, 'last_column' => 0);
 
@@ -88,16 +89,16 @@ public function showPosition()
 
     public function getPastInput()
     {
-        $past = substr($this->matched, 0, strlen($this->matched) - strlen($this->match));
+        $pastLength = $this->offset - strlen($this->match);
 
-        return (strlen($past) > 20 ? '...' : '') . substr($past, -20);
+        return ($pastLength > 20 ? '...' : '') . substr($this->input, max(0, $pastLength - 20), min(20, $pastLength));
     }
 
     public function getUpcomingInput()
     {
         $next = $this->match;
         if (strlen($next) < 20) {
-            $next .= substr($this->input, 0, 20 - strlen($next));
+            $next .= substr($this->input, $this->offset, 20 - strlen($next));
         }
 
         return substr($next, 0, 20) . (strlen($next) > 20 ? '...' : '');
@@ -113,7 +114,7 @@ private function next()
         if ($this->done) {
             return $this->EOF;
         }
-        if ($this->input === '') {
+        if ($this->offset === \strlen($this->input)) {
             $this->done = true;
         }
 
@@ -131,7 +132,7 @@ private function next()
         $rulesLen = count($rules);
 
         for ($i=0; $i < $rulesLen; $i++) {
-            if (preg_match($this->rules[$rules[$i]], $this->input, $match)) {
+            if (preg_match($this->rules[$rules[$i]], $this->input, $match, 0, $this->offset)) {
                 preg_match_all('/\n.*/', $match[0], $lines);
                 $lines = $lines[0];
                 if ($lines) {
@@ -148,8 +149,7 @@ private function next()
                 $this->match .= $match[0];
                 $this->yyleng = strlen($this->yytext);
                 $this->more = false;
-                $this->input = substr($this->input, strlen($match[0]));
-                $this->matched .= $match[0];
+                $this->offset += \strlen($match[0]);
                 $token = $this->performAction($rules[$i], $this->conditionStack[count($this->conditionStack)-1]);
                 if ($token) {
                     return $token;
@@ -159,7 +159,7 @@ private function next()
             }
         }
 
-        if ($this->input === "") {
+        if ($this->offset === \strlen($this->input)) {
             return $this->EOF;
         }