TYPO3 DataHandler: Faster Bulk Imports

<?php
namespace TYPO3\CMS\Core\DataHandling;

class DataHandler
{
    // ============================================
    // PERFORMANCE-CRITICAL PROPERTIES
    // ============================================
    
    /**
     * Disable logging to sys_log table
     * IMPACT: Eliminates one DB insert per record
     * NOTE: Untyped in TYPO3 v12; typed declarations added in v13+
     */
    public $enableLogging = true;

    /**
     * Bypass ALL permission checks
     * IMPACT: Skips user/group permission evaluation
     * WARNING: Only use in trusted CLI context!
     */
    public $bypassAccessCheckForRecords = false;

    /**
     * Skip post-save record verification
     * IMPACT: Removes one SELECT query per record
     */
    public $checkStoredRecords = true;

    /**
     * Run as admin user (bypass all restrictions)
     * IMPACT: Skips permission system entirely
     * NOTE: Set AFTER calling start(), as start() overwrites
     *       this from $this->BE_USER->user['admin'].
     */
    public $admin;

    // ============================================
    // WORKSPACE & VERSIONING
    // ============================================

    /**
     * Bypass workspace restrictions
     * IMPACT: When true, skips workspace overlay logic
     * NOTE: Workspace versioning is primarily controlled via
     *       TCA 'versioningWS' per table, not this flag.
     */
    public $bypassWorkspaceRestrictions = false;

    // ============================================
    // REFERENCE INDEX
    // ============================================

    /**
     * NOTE: updateRefIndex is a METHOD, not a settable property.
     * Reference index updates cannot be toggled via a flag.
     * To defer ref index updates, use the ReferenceIndex API directly
     * after your import is complete.
     */
    
    // ============================================
    // STATE ACCUMULATION (The Problem!)
    // ============================================
    
    /**
     * Error log - grows with each operation
     * PROBLEM: Unbounded array growth in long-running imports
     */
    public array $errorLog = [];
    
    /**
     * Copy mapping array - tracks copied records
     * PROBLEM: Memory consumption increases linearly
     */
    public array $copyMappingArray_merged = [];
    
    /**
     * Remap stack for NEW placeholder IDs
     * PROBLEM: Grows with number of new records
     */
    public array $remapStack = [];
}

use TYPO3\CMS\Core\DataHandling\DataHandler;
use TYPO3\CMS\Core\Utility\GeneralUtility;

function getOptimizedDataHandler(): DataHandler
{
    $dataHandler = GeneralUtility::makeInstance(DataHandler::class);
    
    // Performance Optimization (set before start())
    $dataHandler->enableLogging = false;              // -5-10% time
    $dataHandler->bypassAccessCheckForRecords = true; // -3-5% time
    $dataHandler->checkStoredRecords = false;         // -2-3% time

    // NOTE: $dataHandler->admin must be set AFTER calling start(),
    // because start() unconditionally overwrites it from BE_USER.
    // Reference index updates are handled by the updateRefIndex()
    // method, not a settable property. To defer ref index updates,
    // use the ReferenceIndex API directly after your import.

    return $dataHandler;
}

// Temporarily disable versioning for import duration
$GLOBALS['TCA']['tx_news_domain_model_news']['ctrl']['versioningWS'] = false;

// Perform bulk import...

// Re-enable afterwards (if needed)
$GLOBALS['TCA']['tx_news_domain_model_news']['ctrl']['versioningWS'] = true;

use TYPO3\CMS\Core\Database\ConnectionPool;
use TYPO3\CMS\Core\Utility\GeneralUtility;

function bulkImportWithOptimization(array $records, string $tableName): void
{
    // Disable versioning
    $GLOBALS['TCA'][$tableName]['ctrl']['versioningWS'] = false;
    
    // Get database connection for transaction control
    $connection = GeneralUtility::makeInstance(ConnectionPool::class)
        ->getConnectionForTable($tableName);
    
    // Start transaction
    $connection->beginTransaction();
    
    try {
        // Process in batches of 500
        $chunks = array_chunk($records, 500);
        
        foreach ($chunks as $chunk) {
            // NEW DataHandler instance per batch - critical!
            $dataHandler = GeneralUtility::makeInstance(DataHandler::class);
            
            // Apply optimizations (before start)
            $dataHandler->enableLogging = false;
            $dataHandler->bypassAccessCheckForRecords = true;
            $dataHandler->checkStoredRecords = false;

            // Build datamap
            $dataMap = [];
            foreach ($chunk as $record) {
                $dataMap[$tableName]['NEW' . uniqid()] = $record;
            }

            // Execute
            $dataHandler->start($dataMap, []);
            // Set admin AFTER start() — start() overwrites it from BE_USER
            $dataHandler->admin = true;
            $dataHandler->process_datamap();
            
            // Check for errors
            if (!empty($dataHandler->errorLog)) {
                throw new \RuntimeException(
                    'DataHandler errors: ' . implode(', ', $dataHandler->errorLog)
                );
            }
            
            // Explicit cleanup
            unset($dataHandler);
        }
        
        // Commit transaction
        $connection->commit();
        
        // Single cache flush after everything
        $cacheManager = GeneralUtility::makeInstance(\TYPO3\CMS\Core\Cache\CacheManager::class);
        $cacheManager->flushCaches();
        
    } catch (\Exception $e) {
        // Rollback on error
        $connection->rollBack();
        throw $e;
    }
}

name: typo3-datahandler-benchmark
type: typo3
docroot: public
php_version: "8.2"
webserver_type: apache-fpm
router_http_port: "8080"
router_https_port: "8443"
database:
  type: mariadb
  version: "10.11"
xdebug_enabled: false
web_environment:
  - TYPO3_CONTEXT=Development
php:
  ini:
    memory_limit: '2G'

<?php
declare(strict_types=1);

namespace App\Command;

use Faker\Factory;
use Symfony\Component\Console\Attribute\AsCommand;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Input\InputArgument;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Output\OutputInterface;
use Symfony\Component\Console\Style\SymfonyStyle;
use TYPO3\CMS\Core\Core\Environment;

#[AsCommand(
    name: 'data:generate',
    description: 'Generates fake news records for benchmarking'
)]
class GenerateDataCommand extends Command
{
    protected function configure(): void
    {
        $this->addArgument('count', InputArgument::OPTIONAL, 'Number of records', '10000');
        $this->addArgument('pid', InputArgument::OPTIONAL, 'Target PID', '1');
    }

    protected function execute(InputInterface $input, OutputInterface $output): int
    {
        $io = new SymfonyStyle($input, $output);
        $count = (int)$input->getArgument('count');
        $pid = (int)$input->getArgument('pid');
        $faker = Factory::create();
        $records = [];

        $io->progressStart($count);

        for ($i = 0; $i < $count; $i++) {
            $records[] = [
                'pid' => $pid,
                'title' => $faker->sentence(),
                'teaser' => $faker->paragraph(),
                'bodytext' => $faker->paragraphs(3, true),
            ];
            $io->progressAdvance();
        }

        $io->progressFinish();

        $filePath = Environment::getProjectPath() . '/var/data/news_records.json';
        file_put_contents($filePath, json_encode($records, JSON_PRETTY_PRINT));

        $io->success(sprintf('%d records generated: %s', $count, $filePath));

        return Command::SUCCESS;
    }
}

<?php
declare(strict_types=1);

namespace App\Command;

use Symfony\Component\Console\Attribute\AsCommand;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Output\OutputInterface;
use Symfony\Component\Console\Style\SymfonyStyle;
use TYPO3\CMS\Core\Core\Bootstrap;
use TYPO3\CMS\Core\Core\Environment;
use TYPO3\CMS\Core\DataHandling\DataHandler;
use TYPO3\CMS\Core\Utility\GeneralUtility;

#[AsCommand(
    name: 'data:import',
    description: 'Imports news records with benchmarking'
)]
class ImportDataCommand extends Command
{
    protected function configure(): void
    {
        $this->addOption('batch-size', null, InputOption::VALUE_REQUIRED, 'Records per batch', 0);
        $this->addOption('disable-logging', null, InputOption::VALUE_NONE, 'Disable logging');
        $this->addOption('bypass-permissions', null, InputOption::VALUE_NONE, 'Bypass permissions');
        $this->addOption('disable-versioning', null, InputOption::VALUE_NONE, 'Disable versioning');
    }

    protected function execute(InputInterface $input, OutputInterface $output): int
    {
        $io = new SymfonyStyle($input, $output);
        $startTime = hrtime(true);

        Bootstrap::initializeBackendAuthentication();

        $filePath = Environment::getProjectPath() . '/var/data/news_records.json';
        if (!file_exists($filePath)) {
            $io->error('Data file not found. Run data:generate first.');
            return Command::FAILURE;
        }

        $records = json_decode(file_get_contents($filePath), true);
        $tableName = 'tx_news_domain_model_news';

        if ($input->getOption('disable-versioning')) {
            $GLOBALS['TCA'][$tableName]['ctrl']['versioningWS'] = false;
        }

        $batchSize = (int)$input->getOption('batch-size');
        if ($batchSize > 0) {
            $this->runBatchedImport($records, $tableName, $batchSize, $input, $io);
        } else {
            $this->runSingleImport($records, $tableName, $input, $io);
        }

        $executionTime = (hrtime(true) - $startTime) / 1e9;
        $peakMemory = memory_get_peak_usage(true) / 1024 / 1024;

        $io->table(
            ['Metric', 'Value'],
            [
                ['Execution Time', sprintf('%.2f seconds', $executionTime)],
                ['Peak Memory', sprintf('%.2f MB', $peakMemory)],
                ['Throughput', sprintf('%.2f rec/s', count($records) / $executionTime)],
            ]
        );

        return Command::SUCCESS;
    }

    private function runBatchedImport(
        array $records,
        string $tableName,
        int $batchSize,
        InputInterface $input,
        SymfonyStyle $io
    ): void {
        $chunks = array_chunk($records, $batchSize);
        $io->progressStart(count($chunks));

        foreach ($chunks as $chunk) {
            $dataMap = [];
            foreach ($chunk as $record) {
                $dataMap[$tableName]['NEW' . uniqid()] = $record;
            }

            $dataHandler = $this->getDataHandlerInstance($input);
            $dataHandler->start($dataMap, []);
            $dataHandler->admin = true; // Must be set after start()
            $dataHandler->process_datamap();

            unset($dataHandler);
            $io->progressAdvance();
        }

        $io->progressFinish();
    }

    private function getDataHandlerInstance(InputInterface $input): DataHandler
    {
        $dataHandler = GeneralUtility::makeInstance(DataHandler::class);
        $dataHandler->checkStoredRecords = false;

        if ($input->getOption('disable-logging')) {
            $dataHandler->enableLogging = false;
        }
        if ($input->getOption('bypass-permissions')) {
            $dataHandler->bypassAccessCheckForRecords = true;
        }
        
        return $dataHandler;
    }
}