From 080dca67bf55185d60f5c9175270697c2983ca47 Mon Sep 17 00:00:00 2001 From: Wouter Wolters Date: Mon, 17 Aug 2026 18:48:43 +0200 Subject: [PATCH 1/2] [TASK] Speed up functional data set imports CSV data set imports currently perform full Doctrine schema introspection for every imported table. PostgreSQL additionally queries several system catalog tables before synchronizing each auto-increment sequence. Use TYPO3's cached schema information to obtain column types and identify the auto-increment column. Skip sequence handling for tables without such a column. Pass the known column name to the sequence reset helper and use PG_GET_SERIAL_SEQUENCE() to synchronize PostgreSQL sequences in a single query. Retain the existing catalog lookup as a fallback for callers that do not provide column metadata. In a complete PostgreSQL functional suite run with the same ten-way split as CI, aggregate runtime decreased from 2:26:36.598 to 2:14:04.534, saving 12:32.064 or 8.55%. The slowest chunk improved from 16:04.463 to 14:47.040, saving 8.03%. All 12,484 tests passed. Releases: main, 10, 9 --- .../Framework/DataHandling/DataSet.php | 16 +++++++++---- Classes/Core/Testbase.php | 23 +++++++++++++++---- 2 files changed, 30 insertions(+), 9 deletions(-) diff --git a/Classes/Core/Functional/Framework/DataHandling/DataSet.php b/Classes/Core/Functional/Framework/DataHandling/DataSet.php index 02273a20..5dbdb751 100644 --- a/Classes/Core/Functional/Framework/DataHandling/DataSet.php +++ b/Classes/Core/Functional/Framework/DataHandling/DataSet.php @@ -69,14 +69,20 @@ public static function import(string $path): void foreach ($dataSet->getTableNames() as $tableName) { $connection = GeneralUtility::makeInstance(ConnectionPool::class)->getConnectionForTable($tableName); $platform = $connection->getDatabasePlatform(); - // @todo Check if we can use the cached schema information here instead. - $tableDetails = $connection->createSchemaManager()->introspectTable($tableName); + $columnInfos = $connection->getSchemaInformation()->listTableColumnInfos($tableName); + $autoIncrementColumnName = null; + foreach ($columnInfos as $columnInfo) { + if ($columnInfo->autoincrement) { + $autoIncrementColumnName = $columnInfo->name; + break; + } + } foreach ($dataSet->getElements($tableName) as $element) { // Some DBMS like postgresql are picky about inserting blob types with correct cast, setting // types correctly (like Connection::PARAM_LOB) allows doctrine to create valid SQL $types = []; foreach ($element as $columnName => $columnValue) { - $types[$columnName] = $columnType = $tableDetails->getColumn($columnName)->getType(); + $types[$columnName] = $columnType = $columnInfos[$columnName]->getType(); // JSON-Field data is converted (json-encode'd) within $connection->insert(), and since json field // data can only be provided json encoded in the csv dataset files, we need to decode them here. if ($columnValue !== null && $columnType instanceof JsonType) { @@ -86,7 +92,9 @@ public static function import(string $path): void // Insert the row $connection->insert($tableName, $element, $types); } - Testbase::resetTableSequences($connection, $tableName); + if ($autoIncrementColumnName !== null) { + Testbase::resetTableSequences($connection, $tableName, $autoIncrementColumnName); + } } } diff --git a/Classes/Core/Testbase.php b/Classes/Core/Testbase.php index 4c736e64..02dac9c5 100644 --- a/Classes/Core/Testbase.php +++ b/Classes/Core/Testbase.php @@ -914,14 +914,27 @@ public function createDatabaseStructure(ContainerInterface $container): void } /** - * Perform post processing of database tables after an insert has been performed. - * Doing this once per insert is rather slow, but due to the soft reference behavior - * this needs to be done after every row to ensure consistent results. + * Synchronize an auto-increment sequence after inserting records with explicit IDs. */ - public static function resetTableSequences(Connection $connection, string $tableName): void - { + public static function resetTableSequences( + Connection $connection, + string $tableName, + ?string $autoIncrementColumnName = null + ): void { $platform = $connection->getDatabasePlatform(); if ($platform instanceof DoctrinePostgreSQLPlatform) { + if ($autoIncrementColumnName !== null) { + $connection->executeStatement( + sprintf( + 'SELECT SETVAL(PG_GET_SERIAL_SEQUENCE(%s, %s), COALESCE(MAX(%s), 0)+1, FALSE) FROM %s', + $connection->quote($connection->quoteIdentifier($tableName)), + $connection->quote($autoIncrementColumnName), + $connection->quoteIdentifier($autoIncrementColumnName), + $connection->quoteIdentifier($tableName) + ) + ); + return; + } $queryBuilder = $connection->createQueryBuilder(); $queryBuilder->getRestrictions()->removeAll(); $row = $queryBuilder->select('PGT.schemaname', 'S.relname', 'C.attname', 'T.relname AS tablename') From 58049c2bd52512e5b454778f8a742431a29b40a1 Mon Sep 17 00:00:00 2001 From: Wouter Wolters Date: Mon, 17 Aug 2026 20:22:16 +0200 Subject: [PATCH 2/2] [TASK] Bulk insert functional test data sets Functional CSV data sets currently execute one INSERT statement for every row. Large fixtures therefore spend most of their setup time on database round trips. Resolve column types once per table, keep the required JSON conversion, and pass all rows to Connection::bulkInsert(). The connection automatically splits statements at the platform parameter limit, while empty data sets and sequence resets retain their existing behavior. For the 743-row RootlineUtility fixture, this reduces runtime by 28% on PostgreSQL, 48% on SQLite, and 80% on MariaDB. Releases: main, 10, 9 --- .../Framework/DataHandling/DataSet.php | 23 ++++++++++++------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/Classes/Core/Functional/Framework/DataHandling/DataSet.php b/Classes/Core/Functional/Framework/DataHandling/DataSet.php index 5dbdb751..43615e6b 100644 --- a/Classes/Core/Functional/Framework/DataHandling/DataSet.php +++ b/Classes/Core/Functional/Framework/DataHandling/DataSet.php @@ -77,20 +77,27 @@ public static function import(string $path): void break; } } - foreach ($dataSet->getElements($tableName) as $element) { + $fields = $dataSet->getFields($tableName); + $elements = $dataSet->getElements($tableName); + if ($fields !== null && $elements !== []) { // Some DBMS like postgresql are picky about inserting blob types with correct cast, setting // types correctly (like Connection::PARAM_LOB) allows doctrine to create valid SQL $types = []; - foreach ($element as $columnName => $columnValue) { + foreach ($fields as $columnName) { $types[$columnName] = $columnType = $columnInfos[$columnName]->getType(); - // JSON-Field data is converted (json-encode'd) within $connection->insert(), and since json field - // data can only be provided json encoded in the csv dataset files, we need to decode them here. - if ($columnValue !== null && $columnType instanceof JsonType) { - $element[$columnName] = $columnType->convertToPHPValue($columnValue, $platform); + // JSON-Field data is converted (json-encode'd) within $connection->bulkInsert(), and since json + // field data can only be provided json encoded in the csv dataset files, we need to decode them + // here. + if ($columnType instanceof JsonType) { + foreach ($elements as &$element) { + if ($element[$columnName] !== null) { + $element[$columnName] = $columnType->convertToPHPValue($element[$columnName], $platform); + } + } + unset($element); } } - // Insert the row - $connection->insert($tableName, $element, $types); + $connection->bulkInsert($tableName, $elements, $fields, $types); } if ($autoIncrementColumnName !== null) { Testbase::resetTableSequences($connection, $tableName, $autoIncrementColumnName);