diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..d718920 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,14 @@ +root = true + +[*] +charset = utf-8 +end_of_line = lf +insert_final_newline = true +indent_style = space +indent_size = 4 +trim_trailing_whitespace = true +max_line_length = 120 +tab_width = 4 + +[*.{yml,yaml}] +indent_size = 2 diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..a8aa16c --- /dev/null +++ b/.gitattributes @@ -0,0 +1,17 @@ +# Path-based git attributes +# https://www.kernel.org/pub/software/scm/git/docs/gitattributes.html + +# Ignore all test and documentation with "export-ignore". +/.github export-ignore +/.gitattributes export-ignore +/.gitignore export-ignore +/phpunit.xml.dist export-ignore +/art export-ignore +/docs export-ignore +/tests export-ignore +/.editorconfig export-ignore +/.php_cs.dist.php export-ignore +/phpstan* export-ignore +/psalm* export-ignore +/CHANGELOG.md export-ignore +/CONTRIBUTING.md export-ignore diff --git a/.github/ISSUE_TEMPLATE/1_Bug_report.md b/.github/ISSUE_TEMPLATE/1_Bug_report.md new file mode 100644 index 0000000..4fb5e16 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/1_Bug_report.md @@ -0,0 +1,18 @@ +--- +name: "Bug report" +about: "Report something that's broken." +--- + + + + +- Package Version: #.#.# +- Laravel Version: #.#.# +- PHP Version: #.#.# + +### Description: + + +### Steps To Reproduce: + + diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..cea9fde --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,5 @@ +blank_issues_enabled: false +contact_links: + - name: Support Questions & Other + url: https://github.com/InteractionDesignFoundation/laravel-db-toolkit/discussions + about: 'If you have a question or need help using the library, click:' diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..d8e269c --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,11 @@ +version: 2 +updates: + - package-ecosystem: composer + directory: "/" + schedule: + interval: monthly + open-pull-requests-limit: 50 + labels: + - "dependencies" + assignees: + - ixdf-bot diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml new file mode 100644 index 0000000..4a15a91 --- /dev/null +++ b/.github/workflows/run-tests.yml @@ -0,0 +1,42 @@ +name: run-tests + +on: [ push, pull_request ] + +jobs: + test: + runs-on: ubuntu-latest + strategy: + fail-fast: true + matrix: + php: [ 8.1, 8.2, 8.3 ] + laravel: [ 10.* ] + testbench: [ 8.* ] + dependency-version: [ prefer-stable ] + + name: P${{ matrix.php }} - L${{ matrix.laravel }} - ${{ matrix.dependency-version }} + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup PHP + uses: shivammathur/setup-php@v2 + with: + php-version: ${{ matrix.php }} + extensions: dom, curl, libxml, mbstring, zip, pcntl, pdo, sqlite, pdo_sqlite, bcmath, soap, intl, gd, exif, iconv, imagick, fileinfo + coverage: none + + - name: Setup problem matchers + run: | + echo "::add-matcher::${{ runner.tool_cache }}/php.json" + echo "::add-matcher::${{ runner.tool_cache }}/phpunit.json" + + - name: Install dependencies + run: | + composer require "laravel/framework:${{ matrix.laravel }}" "orchestra/testbench:${{ matrix.testbench }}" --no-interaction --no-update + composer update --${{ matrix.dependency-version }} --prefer-dist --no-interaction + env: + COMPOSER_AUTH: ${{ secrets.COMPOSER_AUTH }} + + - name: Execute tests + run: composer test diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..321d425 --- /dev/null +++ b/.gitignore @@ -0,0 +1,12 @@ +/node_modules +yarn.lock +package-lock.json + +/vendor +composer.phar +composer.lock +auth.json + +phpunit.xml +.phpunit.result.cache +.phpunit.cache diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..1592aea --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2019 The Interaction Design Foundation + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..713a34f --- /dev/null +++ b/README.md @@ -0,0 +1,44 @@ +# Laravel Database Toolkit + +![logo with ants-engineers repairing a database](./art/logo-sm.jpg) + +[![Latest Stable Version](https://poser.pugx.org/interaction-design-foundation/laravel-db-toolkit/v)](https://packagist.org/packages/interaction-design-foundation/laravel-db-toolkit) +[![Total Downloads](https://poser.pugx.org/interaction-design-foundation/laravel-db-toolkit/downloads)](https://packagist.org/packages/interaction-design-foundation/laravel-db-toolkit) +[![License](https://poser.pugx.org/interaction-design-foundation/laravel-db-toolkit/license)](https://packagist.org/packages/interaction-design-foundation/laravel-db-toolkit) + +The package contains few Laravel console commands that validate database schema and data and report about potential issues. + + +## Installation + +You can install the package in to your Laravel app via composer: + +```bash +composer require interaction-design-foundation/laravel-db-toolkit +``` + + +## Usage + +```shell +# Find invalid data created in non-strict SQL mode. +php artisan database:find-invalid-values + +# Find risky auto-incremental columns on databases which values are close to max possible values. +php artisan database:find-risky-columns +``` + + +### Changelog + +Please see [Releases](https://github.com/InteractionDesignFoundation/laravel-db-toolkit/releases) for more information on what has changed recently. + + +## Contributing + +Please see [CONTRIBUTING](CONTRIBUTING.md) for details. + + +## License + +The MIT License (MIT). Please see [License File](LICENSE) for more information. diff --git a/art/logo-sm.jpg b/art/logo-sm.jpg new file mode 100644 index 0000000..54a0599 Binary files /dev/null and b/art/logo-sm.jpg differ diff --git a/art/logo.png b/art/logo.png new file mode 100644 index 0000000..ae1be4c Binary files /dev/null and b/art/logo.png differ diff --git a/composer.json b/composer.json new file mode 100644 index 0000000..3d8ad84 --- /dev/null +++ b/composer.json @@ -0,0 +1,51 @@ +{ + "name": "interaction-design-foundation/laravel-db-toolkit", + "description": "A set of console commands for SQL databases", + "license": "MIT", + "type": "library", + "keywords": [ + "laravel", + "database", + "console commands" + ], + "require": { + "php": "^8.1", + "doctrine/dbal": "^3.7 || ^4.0", + "laravel/framework": "^10.2" + }, + "require-dev": { + "interaction-design-foundation/coding-standard": "^0.3", + "phpunit/phpunit": "^10.1" + }, + "minimum-stability": "dev", + "prefer-stable": true, + "autoload": { + "psr-4": { + "InteractionDesignFoundation\\LaravelDatabaseToolkit\\": "src/" + } + }, + "autoload-dev": { + "psr-4": { + "InteractionDesignFoundation\\LaravelDatabaseToolkit\\Tests\\": "tests/" + } + }, + "config": { + "allow-plugins": { + "dealerdirect/phpcodesniffer-composer-installer": true + }, + "sort-packages": true + }, + "extra": { + "laravel": { + "providers": [ + "InteractionDesignFoundation\\LaravelDatabaseToolkit\\DatabaseToolkitServiceProvider" + ] + } + }, + "scripts": { + "cs:check": "phpcbf -p -s --colors --report-full --report-summary", + "cs:fix": "phpcbf -p --colors", + "cs": "@cs:fix", + "test": "phpunit --colors=always" + } +} diff --git a/phpunit.xml.dist b/phpunit.xml.dist new file mode 100644 index 0000000..a64c7e8 --- /dev/null +++ b/phpunit.xml.dist @@ -0,0 +1,25 @@ + + + + + + tests + + + + + + + src/ + + + diff --git a/src/Console/Commands/FindInvalidDatabaseValues.php b/src/Console/Commands/FindInvalidDatabaseValues.php new file mode 100644 index 0000000..5e716c0 --- /dev/null +++ b/src/Console/Commands/FindInvalidDatabaseValues.php @@ -0,0 +1,168 @@ +getConnection($connections); + $schema = $connection->getDoctrineSchemaManager(); + if (!$connection instanceof MySqlConnection) { + throw new \InvalidArgumentException('Command supports MySQL DBs only.'); + } + + $this->registerTypeMappings($schema->getDatabasePlatform()); + + foreach ($schema->listTables() as $table) { + foreach ($table->getColumns() as $column) { + $this->processColumn($column, $table, $connection); + } + } + + if ($this->valuesWithIssuesFound === 0) { + return self::SUCCESS; + } + + $this->error("Found {$this->valuesWithIssuesFound} Database values with issues."); + + return self::FAILURE; + } + + private function processColumn(Column $column, Table $table, Connection $connection): void + { + $this->info("{$table->getName()}.{$column->getName()}:\t{$column->getType()->getName()}", 'vvv'); + + if ($this->shouldRunCheckType(self::CHECK_TYPE_NULL)) { + $this->checkNullOnNotNullableColumn($column, $connection, $table); + } + + if ($this->shouldRunCheckType(self::CHECK_TYPE_DATETIME)) { + $this->checkForInvalidDatetimeValues($column, $connection, $table); + } + + if ($this->shouldRunCheckType(self::CHECK_TYPE_LONG_TEXT)) { + $this->checkForTooLongTextTypeValues($column, $connection, $table); + } + + if ($this->shouldRunCheckType(self::CHECK_TYPE_LONG_STRING)) { + $this->checkForTooLongStringTypeValues($column, $connection, $table); + } + } + + private function getConnection(ConnectionResolverInterface $connections): Connection + { + $connectionName = $this->argument('connection'); + if ($connectionName === 'default') { + $connectionName = config('database.default'); + } + + $connection = $connections->connection($connectionName); + assert($connection instanceof Connection); + + return $connection; + } + + private function checkNullOnNotNullableColumn(Column $column, Connection $connection, Table $table): void + { + if ($column->getNotnull()) { + $columnName = $column->getName(); + + $nullsOnNotNullableColumnCount = DB::selectOne("SELECT COUNT(`{$columnName}`) AS `count` FROM {$connection->getDatabaseName()}.`{$table->getName()}` WHERE `{$columnName}` IS NULL")->count; + if ($nullsOnNotNullableColumnCount > 0) { + $this->error("{$table->getName()}.{$columnName} has {$nullsOnNotNullableColumnCount} NULLs but the column is not nullable."); + $this->valuesWithIssuesFound += $nullsOnNotNullableColumnCount; + } else { + $this->comment("\t".self::CHECK_TYPE_NULL.': OK', 'vvv'); + } + } + } + + private function checkForInvalidDatetimeValues(Column $column, Connection $connection, Table $table): void + { + $integerProbablyUsedForTimestamp = in_array($column->getType()->getName(), [Types::INTEGER, Types::BIGINT], true) && (str_contains($column->getName(), 'timestamp') || str_ends_with($column->getName(), '_at')); + if ( + $integerProbablyUsedForTimestamp + || in_array($column->getType()->getName(), [Types::DATE_MUTABLE, Types::DATE_IMMUTABLE, Types::DATETIME_MUTABLE, Types::DATETIME_IMMUTABLE, Types::DATETIMETZ_MUTABLE, Types::DATETIMETZ_IMMUTABLE], true) + ) { + $columnName = $column->getName(); + + $invalidDatetimeRecordsCount = DB::selectOne("SELECT COUNT(`{$columnName}`) AS `count` FROM {$connection->getDatabaseName()}.`{$table->getName()}` WHERE `{$columnName}` <= 1")->count; + if ($invalidDatetimeRecordsCount > 0) { + $this->error("{$table->getName()}.{$columnName} has {$invalidDatetimeRecordsCount} invalid datetime values."); + $this->valuesWithIssuesFound += $invalidDatetimeRecordsCount; + } else { + $this->comment("\t".self::CHECK_TYPE_DATETIME.': OK', 'vvv'); + } + } + } + + private function checkForTooLongTextTypeValues(Column $column, Connection $connection, Table $table): void + { + if ($column->getType()->getName() === Types::TEXT) { + $columnName = $column->getName(); + + $tooLongTextValuesCount = DB::selectOne("SELECT COUNT(`{$columnName}`) AS `count` FROM {$connection->getDatabaseName()}.`{$table->getName()}` WHERE LENGTH(`{$columnName}`) > @@max_allowed_packet;")->count; + if ($tooLongTextValuesCount > 0) { + $this->error("{$table->getName()}.{$columnName} has {$tooLongTextValuesCount} too long text values."); + $this->valuesWithIssuesFound += $tooLongTextValuesCount; + } else { + $this->comment("\t".self::CHECK_TYPE_LONG_TEXT.': OK', 'vvv'); + } + } + } + + private function checkForTooLongStringTypeValues(Column $column, Connection $connection, Table $table): void + { + if (in_array($column->getType()->getName(), [Types::STRING, Types::ASCII_STRING], true)) { + $columnName = $column->getName(); + + $maxLength = $column->getLength(); + + if (is_int($maxLength) && $maxLength !== 0) { + $tooLongStringValuesCount = DB::selectOne("SELECT COUNT(`{$columnName}`) AS `count` FROM {$connection->getDatabaseName()}.`{$table->getName()}` WHERE LENGTH(`{$columnName}`) > {$maxLength};")->count; + if ($tooLongStringValuesCount > 0) { + $this->error("{$table->getName()}.{$columnName} has {$tooLongStringValuesCount} too long string values (longer than {$maxLength} chars)."); + $this->valuesWithIssuesFound += $tooLongStringValuesCount; + } else { + $this->comment("\t".self::CHECK_TYPE_LONG_STRING.': OK', 'vvv'); + } + } else { + $this->warn("Could not find max length for {$table->getName()}.{$columnName} column."); + } + } + } + + private function shouldRunCheckType(string $type): bool + { + $checks = $this->option('check'); + + return $checks === [] + || (is_array($checks) && in_array($type, $checks, true)); + } +} diff --git a/src/Console/Commands/FindRiskyDatabaseColumns.php b/src/Console/Commands/FindRiskyDatabaseColumns.php new file mode 100644 index 0000000..48b7cb6 --- /dev/null +++ b/src/Console/Commands/FindRiskyDatabaseColumns.php @@ -0,0 +1,221 @@ + */ + private array $columnMinsAndMaxs = [ + 'integer' => [ + 'min' => -2_147_483_648, + 'max' => 2_147_483_647, + ], + 'unsigned integer' => [ + 'min' => 0, + 'max' => 4_294_967_295, + ], + 'bigint' => [ + 'min' => -9_223_372_036_854_775_808, + 'max' => 9_223_372_036_854_775_807, + ], + 'unsigned bigint' => [ + 'min' => 0, + 'max' => 18_446_744_073_709_551_615, + ], + 'tinyint' => [ + 'min' => -128, + 'max' => 127, + ], + 'unsigned tinyint' => [ + 'min' => 0, + 'max' => 255, + ], + 'smallint' => [ + 'min' => -32_768, + 'max' => 32_767, + ], + 'unsigned smallint' => [ + 'min' => 0, + 'max' => 65_535, + ], + 'mediumint' => [ + 'min' => -8_388_608, + 'max' => 8_388_607, + ], + 'unsigned mediumint' => [ + 'min' => 0, + 'max' => 16_777_215, + ], + 'decimal' => [ + 'min' => -99999999999999999999999999999.99999999999999999999999999999, + 'max' => 99999999999999999999999999999.99999999999999999999999999999, + ], + 'unsigned decimal' => [ + 'min' => 0, + 'max' => 99999999999999999999999999999.99999999999999999999999999999, + ], + ]; + + public function handle(ConnectionResolverInterface $connections): int + { + $thresholdAlarmPercentage = (float) $this->option('threshold'); + + $connection = $this->getConnection($connections); + $schema = $connection->getDoctrineSchemaManager(); + if (! $connection instanceof MySqlConnection) { + throw new \InvalidArgumentException('Command supports MySQL DBs only.'); + } + + $this->registerTypeMappings($schema->getDatabasePlatform()); + + $outputTable = []; + + foreach ($schema->listTables() as $table) { + $riskyColumnsInfo = $this->processTable($table, $connection, $thresholdAlarmPercentage); + if (is_array($riskyColumnsInfo)) { + $outputTable = [...$outputTable, ...$riskyColumnsInfo]; + } + } + + if (count($outputTable) === 0) { + $this->info('No issues found.'); + return self::SUCCESS; + } + + $this->error(sprintf('%d auto-incremental column(s) found where %s%% of the total possible values have already been used.', count($outputTable), $thresholdAlarmPercentage), 'quiet'); + + $keys = array_column($outputTable, 'percentage'); + array_multisort($keys, \SORT_DESC, $outputTable); + + $this->table(['Table', 'Column', 'Type', 'Size', 'Cur. Val', 'Max. Val', 'Occupancy (%)'], $outputTable); + + return self::FAILURE; + } + + /** @return list>|null */ + private function processTable(Table $table, Connection $connection, float $thresholdAlarmPercentage): ?array + { + $this->comment("Table {$connection->getDatabaseName()}.{$table->getName()}: checking...", 'v'); + + $tableSize = $this->getTableSize($connection, $table->getName()); + if ($tableSize === null) { + $tableSize = -1; // not critical info, we can skip this issue + } + + /** @var \Illuminate\Support\Collection $columns */ + $columns = collect($table->getColumns()) + ->filter(static fn(Column $column): bool => $column->getAutoincrement()); + + $riskyColumnsInfo = []; + + foreach ($columns as $column) { + $columnName = $column->getName(); + $columnType = $column->getType()->getName(); + if ($column->getUnsigned()) { + $columnType = "unsigned {$columnType}"; + } + + $this->comment("\t{$columnName} is autoincrement.", 'vvv'); + + $maxValueForColumnKey = $this->getMaxValueForColumn($columnType); + $currentHighestValue = $this->getCurrentHighestValueForColumn($connection->getDatabaseName(), $table->getName(), $columnName); + + $percentageUsed = round($currentHighestValue / $maxValueForColumnKey * 100, 4); + + if ($percentageUsed >= $thresholdAlarmPercentage) { + $this->error("{$connection->getDatabaseName()}.{$table->getName()}.{$columnName} is full for {$percentageUsed}% (threshold for allowed usage is {$thresholdAlarmPercentage}%)", 'quiet'); + + $riskyColumnsInfo[] = [ + 'table' => "{$connection->getDatabaseName()}.{$table->getName()}", + 'column' => $columnName, + 'type' => $columnType, + 'size' => $this->formatBytes($tableSize, 2), + 'current' => number_format($currentHighestValue), + 'max' => number_format($maxValueForColumnKey), + 'percentage' => sprintf('%s', round($percentageUsed, 4)), + ]; + } + } + + $this->comment("Table {$connection->getDatabaseName()}.{$table->getName()}: OK", 'vv'); + + return count($riskyColumnsInfo) > 0 + ? $riskyColumnsInfo + : null; + } + + private function getConnection(ConnectionResolverInterface $connections): Connection + { + $connectionName = $this->argument('connection'); + if ($connectionName === 'default') { + $connectionName = config('database.default'); + } + + $connection = $connections->connection($connectionName); + assert($connection instanceof Connection); + + return $connection; + } + + private function getMaxValueForColumn(string $columnType): int | float + { + if (array_key_exists($columnType, $this->columnMinsAndMaxs)) { + return $this->columnMinsAndMaxs[$columnType]['max']; + } + + throw new \RuntimeException("Could not find max value for `{$columnType}` column type."); + } + + private function getCurrentHighestValueForColumn(string $database, string $tableName, string $columnName): int + { + $currentHighestValue = DB::select("SELECT MAX(`{$columnName}`) FROM `{$database}`.`{$tableName}`"); + $currentHighestFieldName = array_keys((array) $currentHighestValue[0])[0]; + + return $currentHighestValue[0]->{$currentHighestFieldName} ?? 0; + } + + private function formatBytes(int $size, int $precision): string + { + if ($size === 0) { + return '0'; + } + + $base = log($size) / log(1024); + $suffixes = [' bytes', ' KB', ' MB', ' GB', ' TB']; + $index = (int) floor($base); + if (! array_key_exists($index, $suffixes)) { + throw new \RuntimeException('Unknown size unit.'); + } + + $suffix = $suffixes[$index]; + return round(1024 ** ($base - floor($base)), $precision).$suffix; + } +} diff --git a/src/DatabaseToolkitServiceProvider.php b/src/DatabaseToolkitServiceProvider.php new file mode 100644 index 0000000..25a4b86 --- /dev/null +++ b/src/DatabaseToolkitServiceProvider.php @@ -0,0 +1,24 @@ +app->runningInConsole()) { + $this->commands([ + FindInvalidDatabaseValues::class, + FindRiskyDatabaseColumns::class, + ]); + } + } +} diff --git a/tests/Console/Commands/FindRiskyDatabaseColumnsTest.php b/tests/Console/Commands/FindRiskyDatabaseColumnsTest.php new file mode 100644 index 0000000..62adaaa --- /dev/null +++ b/tests/Console/Commands/FindRiskyDatabaseColumnsTest.php @@ -0,0 +1,37 @@ +artisan(FindRiskyDatabaseColumns::class); + + assert($pendingCommand instanceof PendingCommand); + $pendingCommand->assertExitCode(0); + } + + #[Test] + public function it_fails_on_very_low_threshold(): void + { + /** We can create {@see \App\Modules\Logging\Models\VisitorActivity} instances, but they are slow to create (too many indexes) */ + QuoteFactory::new()->createOne(['id' => 42_000]); + + $pendingCommand = $this->artisan(FindRiskyDatabaseColumns::class, ['--threshold' => 0.001]); + + assert($pendingCommand instanceof PendingCommand); + $pendingCommand->assertExitCode(1); + $pendingCommand->expectsOutputToContain(get_table_name(Quote::class).'.id is full for '); // check line like "ixdf.quote.id is 0.0148% full (threshold for allowed usage is 0.0000001%)\n" + } +}