[Utils] Add token-based duplicate code detector with CI gate

.github/workflows/duplicated_code.yaml

@@ -0,0 +1,29 @@
+name: Duplicated Code
+
+on:
+    pull_request: null
+    push:
+        branches:
+            - "main"
+
+env:
+    COMPOSER_ROOT_VERSION: "dev-main"
+
+jobs:
+    duplicated_code:
+        runs-on: ubuntu-latest
+        timeout-minutes: 5
+
+        steps:
+            -   uses: actions/checkout@v5
+
+            -
+                uses: shivammathur/setup-php@v2
+                with:
+                    php-version: 8.4
+                    coverage: none
+
+            -   run: composer install --no-progress --ansi
+
+            # fails when a large copy-pasted block (>= 400 tokens) is added
+            -   run: composer duplicated-code

composer.json

@@ -70,7 +70,8 @@
                 "src"
             ],
             "Rector\\Utils\\": "utils",
-            "Rector\\Utils\\PHPStan\\": "utils/phpstan/src"
+            "Rector\\Utils\\PHPStan\\": "utils/phpstan/src",
+            "Rector\\Utils\\Duplicates\\": "utils/duplicates/src"
         },
         "files": [
             "src/functions/node_helper.php"
@@ -83,6 +84,7 @@
                 "tests"
             ],
             "Rector\\Utils\\PHPStan\\Tests\\": "utils/phpstan/tests",
+            "Rector\\Utils\\Duplicates\\Tests\\": "utils/duplicates/tests",
             "E2e\\Parallel\\Reflection\\Resolver\\": "e2e/parallel-reflection-resolver/src/",
             "Rector\\Scripts\\": "scripts/src"
         },
@@ -104,11 +106,12 @@
             "@phpstan",
             "@test"
         ],
-        "test": "vendor/bin/fastunit tests rules-tests utils/phpstan/tests",
+        "test": "vendor/bin/fastunit tests rules-tests utils/phpstan/tests utils/duplicates/tests",
         "check-cs": "vendor/bin/ecs check --ansi",
         "fix-cs": "vendor/bin/ecs check --fix --ansi",
         "phpstan": "vendor/bin/phpstan analyse --ansi --memory-limit=512M",
         "rector": "bin/rector process --ansi",
+        "duplicated-code": "php utils/duplicates/bin/duplicates.php --min-lines 5 --min-tokens 400 src rules",
         "preload": "php build/build-preload.php .",
         "release": "vendor/bin/rng --from-commit X --to-commit Y --remote-repository rectorphp/rector-symfony --remote-repository rectorphp/rector-doctrine --remote-repository rectorphp/rector-phpunit"
     },

utils/duplicates/README.md

@@ -0,0 +1,19 @@
+# Duplicates
+
+Token-based copy-paste detector, a small clone of [phpcpd](https://github.com/phpcpd-next/phpcpd).
+
+It tokenizes PHP with `token_get_all()`, strips whitespace and comments, then finds exact duplicate token spans with a Rabin-Karp rolling hash.
+
+## Usage
+
+```bash
+php utils/duplicates/bin/duplicates.php src rules
+```
+
+Options:
+
+- `--min-lines` minimum lines of a clone (default `5`)
+- `--min-tokens` minimum tokens of a clone (default `70`)
+- `--fuzzy` ignore variable names when matching
+
+Exit code is `1` when clones are found, `0` otherwise.

utils/duplicates/bin/duplicates.php

@@ -0,0 +1,109 @@
+#!/usr/bin/env php
+<?php
+
+declare(strict_types=1);
+
+use Rector\Utils\Duplicates\CloneDetector;
+use Symfony\Component\Finder\Finder;
+
+$autoloadFile = __DIR__ . '/../../../vendor/autoload.php';
+if (! file_exists($autoloadFile)) {
+    fwrite(STDERR, 'Composer autoload not found, run "composer install" first.' . PHP_EOL);
+    exit(1);
+}
+
+require_once $autoloadFile;
+
+$minLines = 5;
+$minTokens = 70;
+$fuzzy = false;
+$paths = [];
+
+$args = array_slice($argv, 1);
+for ($i = 0; $i < count($args); ++$i) {
+    $arg = $args[$i];
+
+    if ($arg === '--fuzzy') {
+        $fuzzy = true;
+    } elseif ($arg === '--min-lines') {
+        $minLines = (int) ($args[++$i] ?? $minLines);
+    } elseif ($arg === '--min-tokens') {
+        $minTokens = (int) ($args[++$i] ?? $minTokens);
+    } else {
+        $paths[] = $arg;
+    }
+}
+
+if ($paths === []) {
+    fwrite(STDERR, 'Usage: php bin/duplicates.php [--min-lines N] [--min-tokens N] [--fuzzy] <path>...' . PHP_EOL);
+    exit(1);
+}
+
+$filePaths = [];
+foreach ($paths as $path) {
+    if (is_file($path)) {
+        $filePaths[] = $path;
+        continue;
+    }
+
+    if (! is_dir($path)) {
+        continue;
+    }
+
+    $finder = new Finder()
+        ->files()
+        ->in($path)
+        ->name('*.php');
+
+    foreach ($finder as $fileInfo) {
+        $realPath = $fileInfo->getRealPath();
+        if ($realPath === false) {
+            continue;
+        }
+
+        $filePaths[] = $realPath;
+    }
+}
+
+if ($filePaths === []) {
+    echo 'No PHP files found in the given paths.' . PHP_EOL;
+    exit(0);
+}
+
+$cloneDetector = new CloneDetector($minLines, $minTokens, $fuzzy);
+$clones = $cloneDetector->detect($filePaths);
+
+if ($clones === []) {
+    echo sprintf('[OK] No duplicates found in %d files', count($filePaths)) . PHP_EOL;
+    exit(0);
+}
+
+$duplicatedLines = 0;
+foreach ($clones as $clone) {
+    echo sprintf(
+        '  - %s:%d-%d (%d lines, %d tokens)',
+        $clone->firstFile->filePath,
+        $clone->firstFile->startLine,
+        $clone->firstFile->endLine,
+        $clone->lines,
+        $clone->tokens
+    ) . PHP_EOL;
+    echo sprintf(
+        '    %s:%d-%d',
+        $clone->secondFile->filePath,
+        $clone->secondFile->startLine,
+        $clone->secondFile->endLine
+    ) . PHP_EOL;
+    echo PHP_EOL;
+
+    $duplicatedLines += $clone->lines;
+}
+
+fwrite(STDERR, sprintf(
+    '[ERROR] Found %d clones with %d duplicated lines in %d scanned files',
+    count($clones),
+    $duplicatedLines,
+    count($filePaths)
+) . PHP_EOL);
+
+exit(1);

utils/duplicates/src/CloneDetector.php

@@ -0,0 +1,196 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Rector\Utils\Duplicates;
+
+use Rector\Utils\Duplicates\ValueObject\CodeClone;
+use Rector\Utils\Duplicates\ValueObject\CodeCloneFile;
+
+/**
+ * Token-based copy-paste detector using a Rabin-Karp rolling hash over the
+ * normalized token stream, mirroring the classic phpcpd behaviour.
+ * @see \Rector\Utils\Duplicates\Tests\CloneDetectorTest
+ */
+final class CloneDetector
+{
+    /**
+     * Tokens that carry no structural meaning for clone detection.
+     *
+     * @var array<int, true>
+     */
+    private const array IGNORED_TOKENS = [
+        T_INLINE_HTML => true,
+        T_COMMENT => true,
+        T_DOC_COMMENT => true,
+        T_OPEN_TAG => true,
+        T_OPEN_TAG_WITH_ECHO => true,
+        T_CLOSE_TAG => true,
+        T_WHITESPACE => true,
+    ];
+
+    /**
+     * Bytes contributed by each kept token to the signature: 1 type byte + 4 CRC32 bytes.
+     */
+    private const int BYTES_PER_TOKEN = 5;
+
+    /**
+     * First seen location per window hash: hash => [filePath, tokenIndex].
+     *
+     * @var array<string, array{string, int}>
+     */
+    private array $hashes = [];
+
+    /**
+     * Per-file token line numbers, kept for span line lookup.
+     *
+     * @var array<string, int[]>
+     */
+    private array $fileLines = [];
+
+    /**
+     * @var CodeClone[]
+     */
+    private array $clones = [];
+
+    public function __construct(
+        private readonly int $minLines,
+        private readonly int $minTokens,
+        private readonly bool $fuzzy
+    ) {
+    }
+
+    /**
+     * @param string[] $filePaths
+     * @return CodeClone[]
+     */
+    public function detect(array $filePaths): array
+    {
+        foreach ($filePaths as $filePath) {
+            $this->processFile($filePath);
+        }
+
+        return $this->clones;
+    }
+
+    private function processFile(string $filePath): void
+    {
+        $source = file_get_contents($filePath);
+        if ($source === false) {
+            return;
+        }
+
+        [$signature, $lines] = $this->tokenize($source);
+        $this->fileLines[$filePath] = $lines;
+
+        $tokenCount = count($lines);
+        if ($tokenCount < $this->minTokens) {
+            return;
+        }
+
+        $lastWindow = $tokenCount - $this->minTokens;
+        $windowBytes = $this->minTokens * self::BYTES_PER_TOKEN;
+
+        $found = false;
+        $firstToken = 0;
+        $originFile = '';
+        $originToken = 0;
+
+        for ($i = 0; $i <= $lastWindow; ++$i) {
+            $hash = substr(md5(substr($signature, $i * self::BYTES_PER_TOKEN, $windowBytes), true), 0, 8);
+
+            if (isset($this->hashes[$hash])) {
+                if (! $found) {
+                    $found = true;
+                    $firstToken = $i;
+                    [$originFile, $originToken] = $this->hashes[$hash];
+                }
+
+                continue;
+            }
+
+            if ($found) {
+                $this->recordClone($originFile, $originToken, $filePath, $firstToken, $i);
+                $found = false;
+            }
+
+            $this->hashes[$hash] = [$filePath, $i];
+        }
+
+        if ($found) {
+            $this->recordClone($originFile, $originToken, $filePath, $firstToken, $lastWindow + 1);
+        }
+    }
+
+    private function recordClone(
+        string $originFile,
+        int $originToken,
+        string $currentFile,
+        int $currentToken,
+        int $mismatchWindow
+    ): void {
+        $windowCount = $mismatchWindow - $currentToken;
+        $tokenSpan = $windowCount + $this->minTokens - 1;
+
+        $originLines = $this->fileLines[$originFile];
+        $currentLines = $this->fileLines[$currentFile];
+
+        $originStartLine = $originLines[$originToken];
+        $originEndLine = $originLines[min($originToken + $tokenSpan - 1, count($originLines) - 1)];
+
+        $currentStartLine = $currentLines[$currentToken];
+        $currentEndLine = $currentLines[min($currentToken + $tokenSpan - 1, count($currentLines) - 1)];
+
+        $numberOfLines = $originEndLine - $originStartLine + 1;
+        if ($numberOfLines < $this->minLines) {
+            return;
+        }
+
+        $this->clones[] = new CodeClone(
+            new CodeCloneFile($originFile, $originStartLine, $originEndLine),
+            new CodeCloneFile($currentFile, $currentStartLine, $currentEndLine),
+            $numberOfLines,
+            $tokenSpan
+        );
+    }
+
+    /**
+     * Builds the token signature and the parallel line map for a source string.
+     *
+     * @return array{string, int[]}
+     */
+    private function tokenize(string $source): array
+    {
+        $signature = '';
+        $lines = [];
+        $currentLine = 1;
+
+        foreach (token_get_all($source) as $token) {
+            if (is_array($token)) {
+                $tokenId = $token[0];
+                $tokenText = $token[1];
+                $currentLine = $token[2];
+
+                if (isset(self::IGNORED_TOKENS[$tokenId])) {
+                    $currentLine += substr_count($tokenText, "\n");
+                    continue;
+                }
+
+                if ($this->fuzzy && $tokenId === T_VARIABLE) {
+                    $tokenText = '$';
+                }
+
+                $signature .= chr($tokenId & 255) . pack('N*', crc32($tokenText));
+                $lines[] = $currentLine;
+                $currentLine += substr_count($tokenText, "\n");
+
+                continue;
+            }
+
+            $signature .= chr(0) . pack('N*', crc32($token));
+            $lines[] = $currentLine;
+        }
+
+        return [$signature, $lines];
+    }
+}