Progress and Performance Tracking

.env.example

@@ -1 +1,3 @@
 APP_KEY=
+# DEV_PHP_VERSION=8.5-dev-macos
+DEV_PHP_VERSION=8.5-dev

Makefile

@@ -122,3 +122,6 @@ docker-test: docker-build
 
 docker-shell: docker-build
 	docker run --rm -it -v "$$(pwd)":/workspace --entrypoint sh $(DOCKER_IMAGE)
+
+
+include Makefile.dev.mk

Makefile.dev.mk

@@ -0,0 +1,17 @@
+up:
+	docker compose up -d
+
+down:
+	docker compose down
+
+restart: down up
+
+bash:
+	docker compose exec php bash
+
+test:
+	docker compose exec php bash -c 'XDEBUG_MODE=coverage composer test'
+
+check:
+	docker compose exec php bash -c 'composer lint && composer test:types'
+

app/Data/Cloning/StatsLoopData.php

@@ -0,0 +1,27 @@
+<?php
+
+declare(strict_types=1);
+
+namespace App\Data\Cloning;
+
+/**
+ * Snapshot of a single chunk-loop iteration inside `transferTable()`.
+ *
+ * `rowsDone` and `rowsSkipped` count rows for **this loop only**, not
+ * cumulative — inspect the parent `StatsTableTransferData` for
+ * cumulative progress.
+ */
+final readonly class StatsLoopData
+{
+    public function __construct(
+        public int $loopIndex,
+        public int $chunkRows,
+        public float $selectSeconds,
+        public float $transformSeconds,
+        public float $insertSeconds,
+        public float $overallSeconds,
+        public int $rowsDone,
+        public int $rowsSkipped,
+        public int $totalRows,
+    ) {}
+}

app/Data/Cloning/StatsLoopSnapshotData.php

@@ -0,0 +1,28 @@
+<?php
+
+declare(strict_types=1);
+
+namespace App\Data\Cloning;
+
+/**
+ * Immutable scalar snapshot captured after a chunk loop is recorded.
+ * Consumers can iterate `StatsTableTransferData::$statsOverTime` to observe
+ * how cumulative progress and per-phase throughput evolve across loops.
+ *
+ * Only scalars are stored (not references to the mutable aggregates), so a
+ * snapshot is intrinsically immutable and cheap to retain per loop.
+ */
+final readonly class StatsLoopSnapshotData
+{
+    public function __construct(
+        public int $loopIndex,
+        public int $loopsRecorded,
+        public int $rowsDoneCumulative,
+        public int $rowsSkippedCumulative,
+        public ?float $percentComplete,
+        public ?float $selectPacePerMillion,
+        public ?float $transformPacePerMillion,
+        public ?float $insertPacePerMillion,
+        public ?float $loopPacePerMillion,
+    ) {}
+}

app/Data/Cloning/StatsPhaseAggregateData.php

@@ -0,0 +1,85 @@
+<?php
+
+declare(strict_types=1);
+
+namespace App\Data\Cloning;
+
+/**
+ * Mutable running aggregate for a single per-chunk timing phase. Updated
+ * in O(1) via `record()`; derived throughput figures are exposed as
+ * virtual (computed) properties.
+ */
+final class StatsPhaseAggregateData
+{
+    public private(set) int $count = 0;
+
+    public private(set) float $sum = 0.0;
+
+    public private(set) ?float $min = null;
+
+    public private(set) ?float $max = null;
+
+    public private(set) ?float $last = null;
+
+    public private(set) ?int $lastRows = null;
+
+    public private(set) int $rowsProcessed = 0;
+
+    /** Mean seconds per sample, or null when no sample has been recorded. */
+    public ?float $averageSeconds {
+        get => $this->count > 0 ? $this->sum / $this->count : null;
+    }
+
+    /**
+     * Aggregate seconds per row across all samples,
+     * or null when no rows have been processed.
+     */
+    public ?float $pace {
+        get => $this->rowsProcessed > 0
+            ? ($this->sum / $this->rowsProcessed)
+            : null;
+    }
+
+    /**
+     * Aggregate seconds per 1,000,000 rows across all samples,
+     * or null when no rows have been processed.
+     */
+    public ?float $pacePerMillion {
+        get => ($pace = $this->pace) !== null
+            ? $pace * 1_000_000.0
+            : null;
+    }
+
+    public ?float $latestPace {
+        get => $this->last !== null && $this->lastRows !== null && $this->lastRows > 0
+            ? $this->last / $this->lastRows
+            : null;
+    }
+
+    /** Latest-sample seconds per 1,000,000 rows, or null when unavailable. */
+    public ?float $latestPacePerMillion {
+        get => ($pace = $this->latestPace) !== null
+            ? $pace * 1_000_000.0
+            : null;
+    }
+
+    public static function withRecord(float $seconds, int $rows): self
+    {
+        $instance = new self;
+
+        $instance->record($seconds, $rows);
+
+        return $instance;
+    }
+
+    public function record(float $seconds, int $rows): void
+    {
+        $this->count++;
+        $this->sum += $seconds;
+        $this->min = $this->min === null ? $seconds : min($this->min, $seconds);
+        $this->max = $this->max === null ? $seconds : max($this->max, $seconds);
+        $this->last = $seconds;
+        $this->lastRows = $rows;
+        $this->rowsProcessed += max(0, $rows);
+    }
+}

app/Data/Cloning/StatsTableTransferData.php

@@ -0,0 +1,173 @@
+<?php
+
+declare(strict_types=1);
+
+namespace App\Data\Cloning;
+
+use Illuminate\Support\Collection;
+
+/**
+ * Per-table timing container. One instance travels through every chunk
+ * loop of a single `transferTable()` invocation.
+ *
+ * Loops are pushed via `recordLoop(StatsLoopData)`; per-phase running
+ * aggregates (`selectAggregate` / `transformAggregate` / `insertAggregate`)
+ * are updated in O(1) so the derived throughput figures never re-scan
+ * `$loops`.
+ */
+final class StatsTableTransferData
+{
+    public private(set) ?TableRunPhase $status = null;
+
+    /** Wall-clock seconds for each one-shot phase; null until (and unless) it runs. */
+    public private(set) ?float $countingRowsSeconds = null;
+
+    public private(set) ?float $disableFkSeconds = null;
+
+    public private(set) ?float $clearTableSeconds = null;
+
+    /** @var Collection<int, StatsLoopData> */
+    public private(set) Collection $loops;
+
+    /** @var Collection<int, StatsLoopSnapshotData> */
+    public private(set) Collection $statsOverTime;
+
+    public private(set) int $totalRows = 0;
+
+    public private(set) int $rowsDone = 0;
+
+    public private(set) int $rowsSkipped = 0;
+
+    /** Rows accounted for so far (transferred + skipped). */
+    public int $rowsProcessed {
+        get => $this->rowsDone + $this->rowsSkipped;
+    }
+
+    /** Rows still outstanding against `totalRows` (never negative). */
+    public int $rowsRemaining {
+        get => max(0, $this->totalRows - $this->rowsProcessed);
+    }
+
+    /** Completion ratio in the range [0, 100], or null when the total is unknown. */
+    public ?float $percentComplete {
+        get => $this->totalRows > 0
+            ? min(100.0, ($this->rowsProcessed / $this->totalRows) * 100.0)
+            : null;
+    }
+
+    /**
+     * Estimated wall-clock seconds until this table finishes, from the latest
+     * loop pace × outstanding rows. 0.0 once no rows remain, and 0.0 until a
+     * row total and at least one completed loop are known.
+     */
+    public float $estimatedSecondsRemaining {
+        get {
+            if ($this->rowsRemaining <= 0) {
+                return 0.0;
+            }
+
+            return $this->loopAggregate->latestPace * $this->rowsRemaining;
+        }
+    }
+
+    public private(set) StatsPhaseAggregateData $selectAggregate;
+
+    public private(set) StatsPhaseAggregateData $transformAggregate;
+
+    public private(set) StatsPhaseAggregateData $insertAggregate;
+
+    /**
+     * Per-loop aggregate over each chunk's wall-clock time
+     * (`StatsLoopData::$overallSeconds`, rows = chunkRows). Useful for
+     * loop throughput including inter-phase overhead.
+     */
+    public private(set) StatsPhaseAggregateData $loopAggregate;
+
+    public function __construct()
+    {
+        $this->loops = new Collection;
+        $this->statsOverTime = new Collection;
+        $this->selectAggregate = new StatsPhaseAggregateData;
+        $this->transformAggregate = new StatsPhaseAggregateData;
+        $this->insertAggregate = new StatsPhaseAggregateData;
+        $this->loopAggregate = new StatsPhaseAggregateData;
+    }
+
+    public function setStatus(TableRunPhase $status): void
+    {
+        $this->status = $status;
+    }
+
+    public function setTotalRows(int $totalRows): void
+    {
+        $this->totalRows = max(0, $totalRows);
+    }
+
+    public function recordCountingRows(float $seconds): void
+    {
+        $this->countingRowsSeconds = $seconds;
+    }
+
+    public function recordDisableFk(float $seconds): void
+    {
+        $this->disableFkSeconds = $seconds;
+    }
+
+    public function recordClearTable(float $seconds): void
+    {
+        $this->clearTableSeconds = $seconds;
+    }
+
+    /**
+     * Append a completed chunk loop, update running aggregates in O(1),
+     * and append a stats-over-time snapshot.
+     */
+    public function recordLoop(StatsLoopData $loop): void
+    {
+        $this->loops->push($loop);
+
+        $this->selectAggregate->record($loop->selectSeconds, $loop->chunkRows);
+        $this->transformAggregate->record($loop->transformSeconds, $loop->chunkRows);
+        $this->insertAggregate->record($loop->insertSeconds, $loop->chunkRows);
+        $this->loopAggregate->record($loop->overallSeconds, $loop->chunkRows);
+
+        $this->rowsDone += $loop->rowsDone;
+        $this->rowsSkipped += $loop->rowsSkipped;
+
+        $this->statsOverTime->push(new StatsLoopSnapshotData(
+            loopIndex: $loop->loopIndex,
+            loopsRecorded: $this->loops->count(),
+            rowsDoneCumulative: $this->rowsDone,
+            rowsSkippedCumulative: $this->rowsSkipped,
+            percentComplete: $this->percentComplete,
+            selectPacePerMillion: $this->selectAggregate->pacePerMillion,
+            transformPacePerMillion: $this->transformAggregate->pacePerMillion,
+            insertPacePerMillion: $this->insertAggregate->pacePerMillion,
+            loopPacePerMillion: $this->loopAggregate->pacePerMillion,
+        ));
+    }
+
+    public function aggregate(TableRunPhase $phase): StatsPhaseAggregateData
+    {
+        return match ($phase) {
+            TableRunPhase::CountingRows => $this->oneShotAggregate($this->countingRowsSeconds),
+            TableRunPhase::DisableFkChecks => $this->oneShotAggregate($this->disableFkSeconds),
+            TableRunPhase::Clear => $this->oneShotAggregate($this->clearTableSeconds),
+            TableRunPhase::Select => $this->selectAggregate,
+            TableRunPhase::Transform => $this->transformAggregate,
+            TableRunPhase::Insert => $this->insertAggregate,
+            TableRunPhase::Loop => $this->loopAggregate,
+        };
+    }
+
+    /**
+     * Wrap a one-shot phase duration as a single-sample aggregate. A phase that
+     * never ran (null) yields an empty (count-0) aggregate so consumers can skip it.
+     */
+    private function oneShotAggregate(?float $seconds): StatsPhaseAggregateData
+    {
+        return $seconds === null
+            ? new StatsPhaseAggregateData
+            : StatsPhaseAggregateData::withRecord($seconds, $this->totalRows);
+    }
+}

app/Data/Cloning/TableRunPhase.php

@@ -0,0 +1,47 @@
+<?php
+
+declare(strict_types=1);
+
+namespace App\Data\Cloning;
+
+/**
+ * The distinct phases tracked during a single table transfer.
+ *
+ * `CountingRows` / `DisableFkChecks` / `Clear` are one-shot phases that run once
+ * before the chunk loop. `Select` / `Transform` / `Insert` are the per-chunk
+ * sub-phases, and `Loop` is the whole-loop wall-clock. `cases()` yields them in
+ * render order (one-shot first, then the per-chunk sub-phases, then `Loop`).
+ */
+enum TableRunPhase: string
+{
+    case CountingRows = 'countingRows';
+    case DisableFkChecks = 'disablingFkChecks';
+    case Clear = 'clearingData';
+
+    case Select = 'selectData';
+    case Transform = 'transformingData';
+    case Insert = 'insertingData';
+    case Loop = 'loop';
+
+    public function isOneShot(): bool
+    {
+        return match ($this) {
+            self::CountingRows, self::DisableFkChecks, self::Clear => true,
+            default => false,
+        };
+    }
+
+    /** Human-readable, present-continuous label for live progress display. */
+    public function label(): string
+    {
+        return match ($this) {
+            self::CountingRows => 'counting rows',
+            self::DisableFkChecks => 'disabling FK checks',
+            self::Clear => 'clearing data',
+            self::Select => 'selecting',
+            self::Transform => 'transforming',
+            self::Insert => 'inserting',
+            self::Loop => 'writing',
+        };
+    }
+}