Initial commit

This commit is contained in:
root
2026-01-19 17:44:46 +01:00
commit 823af8b11d
8721 changed files with 1130846 additions and 0 deletions

View File

@@ -0,0 +1,488 @@
<?php
/************************************************************************
* This file is part of EspoCRM.
*
* EspoCRM Open Source CRM application.
* Copyright (C) 2014-2025 EspoCRM, Inc.
* Website: https://www.espocrm.com
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
* The interactive user interfaces in modified source and object code versions
* of this program must display Appropriate Legal Notices, as required under
* Section 5 of the GNU Affero General Public License version 3.
*
* In accordance with Section 7(b) of the GNU Affero General Public License version 3,
* these Appropriate Legal Notices must retain the display of the "EspoCRM" word.
************************************************************************/
namespace Espo\Core\Utils\Database\Schema;
use Espo\Core\InjectableFactory;
use Espo\Core\Utils\Database\ConfigDataProvider;
use Espo\Core\Utils\Database\MetadataProvider as MetadataProvider;
use Espo\Core\Utils\Log;
use Espo\Core\Utils\Util;
use Espo\ORM\Defs\AttributeDefs;
use Espo\ORM\Defs\EntityDefs;
use Espo\ORM\Defs\IndexDefs;
use Espo\ORM\Defs\Params\AttributeParam;
use Espo\ORM\Defs\Params\EntityParam;
use Espo\ORM\Defs\Params\RelationParam;
use Espo\ORM\Defs\RelationDefs;
use Espo\ORM\Entity;
use Doctrine\DBAL\Types\Types;
use Doctrine\DBAL\Schema\SchemaException;
use Doctrine\DBAL\Schema\Table;
use Doctrine\DBAL\Schema\Schema as DbalSchema;
use Doctrine\DBAL\Types\Type as DbalType;
use Espo\ORM\Type\AttributeType;
/**
* Schema representation builder.
*/
class Builder
{
private const ATTR_ID = 'id';
private const ATTR_DELETED = 'deleted';
private int $idLength;
private string $idDbType;
/** @var string[] */
private $typeList;
private ColumnPreparator $columnPreparator;
public function __construct(
private Log $log,
private InjectableFactory $injectableFactory,
ConfigDataProvider $configDataProvider,
ColumnPreparatorFactory $columnPreparatorFactory,
MetadataProvider $metadataProvider
) {
$this->typeList = array_keys(DbalType::getTypesMap());
$platform = $configDataProvider->getPlatform();
$this->columnPreparator = $columnPreparatorFactory->create($platform);
$this->idLength = $metadataProvider->getIdLength();
$this->idDbType = $metadataProvider->getIdDbType();
}
/**
* Build a schema representation for an ORM metadata.
*
* @param array<string, mixed> $ormMeta Raw ORM metadata.
* @param ?string[] $entityTypeList Specific entity types.
* @throws SchemaException
*/
public function build(array $ormMeta, ?array $entityTypeList = null): DbalSchema
{
$this->log->debug('Schema\Builder - Start');
$ormMeta = $this->amendMetadata($ormMeta, $entityTypeList);
$tables = [];
$schema = new DbalSchema();
foreach ($ormMeta as $entityType => $entityParams) {
$entityDefs = EntityDefs::fromRaw($entityParams, $entityType);
$this->buildEntity($entityDefs, $schema, $tables);
}
foreach ($ormMeta as $entityType => $entityParams) {
foreach (($entityParams[EntityParam::RELATIONS] ?? []) as $relationName => $relationParams) {
$relationDefs = RelationDefs::fromRaw($relationParams, $relationName);
if ($relationDefs->getType() !== Entity::MANY_MANY) {
continue;
}
$this->buildManyMany($entityType, $relationDefs, $schema, $tables);
}
}
$this->log->debug('Schema\Builder - End');
return $schema;
}
/**
* @param array<string, Table> $tables
* @throws SchemaException
*/
private function buildEntity(EntityDefs $entityDefs, DbalSchema $schema, array &$tables): void
{
if ($entityDefs->getParam('skipRebuild')) {
return;
}
$entityType = $entityDefs->getName();
$modifier = $this->getEntityDefsModifier($entityDefs);
if ($modifier) {
$modifiedEntityDefs = $modifier->modify($entityDefs);
$entityDefs = EntityDefs::fromRaw($modifiedEntityDefs->toAssoc(), $entityType);
}
$this->log->debug("Schema\Builder: Entity $entityType");
$tableName = Util::toUnderScore($entityType);
if ($schema->hasTable($tableName)) {
$tables[$entityType] ??= $schema->getTable($tableName);
$this->log->debug('Schema\Builder: Table [' . $tableName . '] exists.');
return;
}
$table = $schema->createTable($tableName);
$tables[$entityType] = $table;
/** @var array<string, mixed> $tableParams */
$tableParams = $entityDefs->getParam('params') ?? [];
foreach ($tableParams as $paramName => $paramValue) {
$table->addOption($paramName, $paramValue);
}
$primaryColumns = [];
foreach ($entityDefs->getAttributeList() as $attributeDefs) {
if (
$attributeDefs->isNotStorable() ||
$attributeDefs->getType() === Entity::FOREIGN
) {
continue;
}
$column = $this->columnPreparator->prepare($attributeDefs);
if ($attributeDefs->getType() === Entity::ID) {
$primaryColumns[] = $column->getName();
}
if (!in_array($column->getType(), $this->typeList)) {
$this->log->warning(
'Schema\Builder: Column type [' . $column->getType() . '] not supported, ' .
$entityType . ':' . $attributeDefs->getName()
);
continue;
}
if ($table->hasColumn($column->getName())) {
continue;
}
$this->addColumn($table, $column);
}
$table->setPrimaryKey($primaryColumns);
$this->addIndexes($table, $entityDefs->getIndexList());
}
private function getEntityDefsModifier(EntityDefs $entityDefs): ?EntityDefsModifier
{
/** @var ?class-string<EntityDefsModifier> $modifierClassName */
$modifierClassName = $entityDefs->getParam('modifierClassName');
if (!$modifierClassName) {
return null;
}
return $this->injectableFactory->create($modifierClassName);
}
/**
* @param array<string, mixed> $ormMeta
* @param ?string[] $entityTypeList
* @return array<string, mixed>
*/
private function amendMetadata(array $ormMeta, ?array $entityTypeList): array
{
if (isset($ormMeta['unsetIgnore'])) {
$protectedOrmMeta = [];
foreach ($ormMeta['unsetIgnore'] as $protectedKey) {
$protectedOrmMeta = Util::merge(
$protectedOrmMeta,
Util::fillArrayKeys($protectedKey, Util::getValueByKey($ormMeta, $protectedKey))
);
}
unset($ormMeta['unsetIgnore']);
}
// Unset some keys.
if (isset($ormMeta['unset'])) {
/** @var array<string, mixed> $ormMeta */
$ormMeta = Util::unsetInArray($ormMeta, $ormMeta['unset']);
unset($ormMeta['unset']);
}
if (isset($protectedOrmMeta)) {
/** @var array<string, mixed> $ormMeta */
$ormMeta = Util::merge($ormMeta, $protectedOrmMeta);
}
if (isset($entityTypeList)) {
$dependentEntityTypeList = $this->getDependentEntityTypeList($entityTypeList, $ormMeta);
$this->log->debug(
'Schema\Builder: Rebuild for entity types: [' .
implode(', ', $entityTypeList) . '] with dependent entity types: [' .
implode(', ', $dependentEntityTypeList) . ']'
);
$ormMeta = array_intersect_key($ormMeta, array_flip($dependentEntityTypeList));
}
return $ormMeta;
}
/**
* @throws SchemaException
*/
private function addColumn(Table $table, Column $column): void
{
$table->addColumn(
$column->getName(),
$column->getType(),
self::convertColumn($column)
);
}
/**
* Prepare a relation table for the manyMany relation.
*
* @param string $entityType
* @param array<string, Table> $tables
* @throws SchemaException
*/
private function buildManyMany(
string $entityType,
RelationDefs $relationDefs,
DbalSchema $schema,
array &$tables
): void {
$relationshipName = $relationDefs->getRelationshipName();
if (isset($tables[$relationshipName])) {
return;
}
$tableName = Util::toUnderScore($relationshipName);
$this->log->debug("Schema\Builder: ManyMany for $entityType.{$relationDefs->getName()}");
if ($schema->hasTable($tableName)) {
$this->log->debug('Schema\Builder: Table [' . $tableName . '] exists.');
$tables[$relationshipName] ??= $schema->getTable($tableName);
return;
}
$table = $schema->createTable($tableName);
$idColumn = $this->columnPreparator->prepare(
AttributeDefs::fromRaw([
AttributeParam::DB_TYPE => Types::BIGINT,
'type' => Entity::ID,
AttributeParam::LEN => 20,
'autoincrement' => true,
], self::ATTR_ID)
);
$this->addColumn($table, $idColumn);
if (!$relationDefs->hasMidKey() || !$relationDefs->getForeignMidKey()) {
$this->log->error('Schema\Builder: Relationship midKeys are empty.', [
'entityType' => $entityType,
'relationName' => $relationDefs->getName(),
]);
return;
}
$midKeys = [
$relationDefs->getMidKey(),
$relationDefs->getForeignMidKey(),
];
foreach ($midKeys as $midKey) {
$column = $this->columnPreparator->prepare(
AttributeDefs::fromRaw([
'type' => Entity::FOREIGN_ID,
AttributeParam::DB_TYPE => $this->idDbType,
AttributeParam::LEN => $this->idLength,
], $midKey)
);
$this->addColumn($table, $column);
}
/** @var array<string, array<string, mixed>> $additionalColumns */
$additionalColumns = $relationDefs->getParam(RelationParam::ADDITIONAL_COLUMNS) ?? [];
foreach ($additionalColumns as $fieldName => $fieldParams) {
if ($fieldParams['type'] === AttributeType::FOREIGN_ID) {
$fieldParams = array_merge([
AttributeParam::DB_TYPE => $this->idDbType,
AttributeParam::LEN => $this->idLength,
], $fieldParams);
}
$column = $this->columnPreparator->prepare(AttributeDefs::fromRaw($fieldParams, $fieldName));
$this->addColumn($table, $column);
}
$deletedColumn = $this->columnPreparator->prepare(
AttributeDefs::fromRaw([
'type' => Entity::BOOL,
'default' => false,
], self::ATTR_DELETED)
);
$this->addColumn($table, $deletedColumn);
$table->setPrimaryKey([self::ATTR_ID]);
$this->addIndexes($table, $relationDefs->getIndexList());
$tables[$relationshipName] = $table;
}
/**
* @param IndexDefs[] $indexDefsList
* @throws SchemaException
*/
private function addIndexes(Table $table, array $indexDefsList): void
{
foreach ($indexDefsList as $indexDefs) {
$columns = array_map(
fn($item) => Util::toUnderScore($item),
$indexDefs->getColumnList()
);
if ($indexDefs->isUnique()) {
$table->addUniqueIndex($columns, $indexDefs->getKey());
continue;
}
$table->addIndex($columns, $indexDefs->getKey(), $indexDefs->getFlagList());
}
}
/**
* @todo Move to a class. Add unit test.
* @return array<string, mixed>
*/
private static function convertColumn(Column $column): array
{
$result = [
'notnull' => $column->isNotNull(),
];
if ($column->getLength() !== null) {
$result['length'] = $column->getLength();
}
if ($column->getDefault() !== null) {
$result['default'] = $column->getDefault();
}
if ($column->getAutoincrement() !== null) {
$result['autoincrement'] = $column->getAutoincrement();
}
if ($column->getPrecision() !== null) {
$result['precision'] = $column->getPrecision();
}
if ($column->getScale() !== null) {
$result['scale'] = $column->getScale();
}
if ($column->getUnsigned() !== null) {
$result['unsigned'] = $column->getUnsigned();
}
if ($column->getFixed() !== null) {
$result['fixed'] = $column->getFixed();
}
// Can't use customSchemaOptions as it causes unwanted ALTER TABLE.
$result['platformOptions'] = [];
if ($column->getCollation()) {
$result['platformOptions']['collation'] = $column->getCollation();
}
if ($column->getCharset()) {
$result['platformOptions']['charset'] = $column->getCharset();
}
return $result;
}
/**
* @param string[] $entityTypeList
* @param array<string, mixed> $ormMeta
* @param string[] $depList
* @return string[]
*/
private function getDependentEntityTypeList(array $entityTypeList, array $ormMeta, array $depList = []): array
{
foreach ($entityTypeList as $entityType) {
if (in_array($entityType, $depList)) {
continue;
}
$depList[] = $entityType;
$entityDefs = EntityDefs::fromRaw($ormMeta[$entityType] ?? [], $entityType);
foreach ($entityDefs->getRelationList() as $relationDefs) {
if (!$relationDefs->hasForeignEntityType()) {
continue;
}
$itemEntityType = $relationDefs->getForeignEntityType();
if (in_array($itemEntityType, $depList)) {
continue;
}
$depList = $this->getDependentEntityTypeList([$itemEntityType], $ormMeta, $depList);
}
}
return $depList;
}
}

View File

@@ -0,0 +1,203 @@
<?php
/************************************************************************
* This file is part of EspoCRM.
*
* EspoCRM Open Source CRM application.
* Copyright (C) 2014-2025 EspoCRM, Inc.
* Website: https://www.espocrm.com
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
* The interactive user interfaces in modified source and object code versions
* of this program must display Appropriate Legal Notices, as required under
* Section 5 of the GNU Affero General Public License version 3.
*
* In accordance with Section 7(b) of the GNU Affero General Public License version 3,
* these Appropriate Legal Notices must retain the display of the "EspoCRM" word.
************************************************************************/
namespace Espo\Core\Utils\Database\Schema;
/**
* A DB column parameters.
*/
class Column
{
private bool $notNull = false;
private ?int $length = null;
private mixed $default = null;
private ?bool $autoincrement = null;
private ?int $precision = null;
private ?int $scale = null;
private ?bool $unsigned = null;
private ?bool $fixed = null;
private ?string $collation = null;
private ?string $charset = null;
private function __construct(
private string $name,
private string $type
) {}
public static function create(string $name, string $type): self
{
return new self($name, $type);
}
public function getName(): string
{
return $this->name;
}
public function getType(): string
{
return $this->type;
}
public function isNotNull(): bool
{
return $this->notNull;
}
public function getLength(): ?int
{
return $this->length;
}
public function getDefault(): mixed
{
return $this->default;
}
public function getAutoincrement(): ?bool
{
return $this->autoincrement;
}
public function getUnsigned(): ?bool
{
return $this->unsigned;
}
public function getPrecision(): ?int
{
return $this->precision;
}
public function getScale(): ?int
{
return $this->scale;
}
public function getFixed(): ?bool
{
return $this->fixed;
}
public function getCollation(): ?string
{
return $this->collation;
}
public function getCharset(): ?string
{
return $this->charset;
}
public function withNotNull(bool $notNull = true): self
{
$obj = clone $this;
$obj->notNull = $notNull;
return $obj;
}
public function withLength(?int $length): self
{
$obj = clone $this;
$obj->length = $length;
return $obj;
}
public function withDefault(mixed $default): self
{
$obj = clone $this;
$obj->default = $default;
return $obj;
}
public function withAutoincrement(?bool $autoincrement = true): self
{
$obj = clone $this;
$obj->autoincrement = $autoincrement;
return $obj;
}
/**
* Unsigned. Supported only by MySQL.
*/
public function withUnsigned(?bool $unsigned = true): self
{
$obj = clone $this;
$obj->unsigned = $unsigned;
return $obj;
}
public function withPrecision(?int $precision): self
{
$obj = clone $this;
$obj->precision = $precision;
return $obj;
}
public function withScale(?int $scale): self
{
$obj = clone $this;
$obj->scale = $scale;
return $obj;
}
/**
* Fixed length. For string and binary types.
*/
public function withFixed(?bool $fixed = true): self
{
$obj = clone $this;
$obj->fixed = $fixed;
return $obj;
}
public function withCollation(?string $collation): self
{
$obj = clone $this;
$obj->collation = $collation;
return $obj;
}
public function withCharset(?string $charset): self
{
$obj = clone $this;
$obj->charset = $charset;
return $obj;
}
}

View File

@@ -0,0 +1,37 @@
<?php
/************************************************************************
* This file is part of EspoCRM.
*
* EspoCRM Open Source CRM application.
* Copyright (C) 2014-2025 EspoCRM, Inc.
* Website: https://www.espocrm.com
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
* The interactive user interfaces in modified source and object code versions
* of this program must display Appropriate Legal Notices, as required under
* Section 5 of the GNU Affero General Public License version 3.
*
* In accordance with Section 7(b) of the GNU Affero General Public License version 3,
* these Appropriate Legal Notices must retain the display of the "EspoCRM" word.
************************************************************************/
namespace Espo\Core\Utils\Database\Schema;
use Espo\ORM\Defs\AttributeDefs;
interface ColumnPreparator
{
public function prepare(AttributeDefs $defs): Column;
}

View File

@@ -0,0 +1,62 @@
<?php
/************************************************************************
* This file is part of EspoCRM.
*
* EspoCRM Open Source CRM application.
* Copyright (C) 2014-2025 EspoCRM, Inc.
* Website: https://www.espocrm.com
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
* The interactive user interfaces in modified source and object code versions
* of this program must display Appropriate Legal Notices, as required under
* Section 5 of the GNU Affero General Public License version 3.
*
* In accordance with Section 7(b) of the GNU Affero General Public License version 3,
* these Appropriate Legal Notices must retain the display of the "EspoCRM" word.
************************************************************************/
namespace Espo\Core\Utils\Database\Schema;
use Espo\Core\Binding\BindingContainerBuilder;
use Espo\Core\InjectableFactory;
use Espo\Core\Utils\Database\Helper;
use Espo\Core\Utils\Metadata;
use RuntimeException;
class ColumnPreparatorFactory
{
public function __construct(
private Metadata $metadata,
private InjectableFactory $injectableFactory,
private Helper $helper
) {}
public function create(string $platform): ColumnPreparator
{
/** @var ?class-string<ColumnPreparator> $className */
$className = $this->metadata
->get(['app', 'databasePlatforms', $platform, 'columnPreparatorClassName']);
if (!$className) {
throw new RuntimeException("No Column-Preparator for {$platform}.");
}
$binding = BindingContainerBuilder::create()
->bindInstance(Helper::class, $this->helper)
->build();
return $this->injectableFactory->createWithBinding($className, $binding);
}
}

View File

@@ -0,0 +1,247 @@
<?php
/************************************************************************
* This file is part of EspoCRM.
*
* EspoCRM Open Source CRM application.
* Copyright (C) 2014-2025 EspoCRM, Inc.
* Website: https://www.espocrm.com
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
* The interactive user interfaces in modified source and object code versions
* of this program must display Appropriate Legal Notices, as required under
* Section 5 of the GNU Affero General Public License version 3.
*
* In accordance with Section 7(b) of the GNU Affero General Public License version 3,
* these Appropriate Legal Notices must retain the display of the "EspoCRM" word.
************************************************************************/
namespace Espo\Core\Utils\Database\Schema\ColumnPreparators;
use Doctrine\DBAL\Types\Types;
use Espo\Core\Utils\Database\Dbal\Types\LongtextType;
use Espo\Core\Utils\Database\Dbal\Types\MediumtextType;
use Espo\Core\Utils\Database\Helper;
use Espo\Core\Utils\Database\Schema\Column;
use Espo\Core\Utils\Database\Schema\ColumnPreparator;
use Espo\Core\Utils\Util;
use Espo\ORM\Defs\AttributeDefs;
use Espo\ORM\Defs\Params\AttributeParam;
use Espo\ORM\Entity;
class MysqlColumnPreparator implements ColumnPreparator
{
private const PARAM_DB_TYPE = AttributeParam::DB_TYPE;
private const PARAM_DEFAULT = AttributeParam::DEFAULT;
private const PARAM_NOT_NULL = AttributeParam::NOT_NULL;
private const PARAM_AUTOINCREMENT = 'autoincrement';
private const PARAM_PRECISION = 'precision';
private const PARAM_SCALE = 'scale';
private const PARAM_BINARY = 'binary';
public const TYPE_MYSQL = 'MySQL';
public const TYPE_MARIADB = 'MariaDB';
private const MB4_INDEX_LENGTH_LIMIT = 3072;
private const DEFAULT_INDEX_LIMIT = 1000;
/** @var string[] */
private array $mediumTextTypeList = [
Entity::TEXT,
Entity::JSON_OBJECT,
Entity::JSON_ARRAY,
];
/** @var array<string, string> */
private array $columnTypeMap = [
Entity::BOOL => Types::BOOLEAN,
Entity::INT => Types::INTEGER,
Entity::VARCHAR => Types::STRING,
];
private ?int $maxIndexLength = null;
public function __construct(
private Helper $helper
) {}
public function prepare(AttributeDefs $defs): Column
{
$dbType = $defs->getParam(self::PARAM_DB_TYPE);
$type = $defs->getType();
$length = $defs->getLength();
$default = $defs->getParam(self::PARAM_DEFAULT);
$notNull = $defs->getParam(self::PARAM_NOT_NULL);
$autoincrement = $defs->getParam(self::PARAM_AUTOINCREMENT);
$precision = $defs->getParam(self::PARAM_PRECISION);
$scale = $defs->getParam(self::PARAM_SCALE);
$binary = $defs->getParam(self::PARAM_BINARY);
$columnType = $dbType ?? $type;
if (in_array($type, $this->mediumTextTypeList) && !$dbType) {
$columnType = MediumtextType::NAME;
}
$columnType = $this->columnTypeMap[$columnType] ?? $columnType;
$columnName = Util::toUnderScore($defs->getName());
$column = Column::create($columnName, strtolower($columnType));
if ($length !== null) {
$column = $column->withLength($length);
}
if ($default !== null) {
$column = $column->withDefault($default);
}
if ($notNull !== null) {
$column = $column->withNotNull($notNull);
}
if ($autoincrement !== null) {
$column = $column->withAutoincrement($autoincrement);
}
if ($precision !== null) {
$column = $column->withPrecision($precision);
}
if ($scale !== null) {
$column = $column->withScale($scale);
}
$mb3 = false;
switch ($type) {
case Entity::ID:
case Entity::FOREIGN_ID:
case Entity::FOREIGN_TYPE:
$mb3 = $this->getMaxIndexLength() < self::MB4_INDEX_LENGTH_LIMIT;
break;
case Entity::TEXT:
$column = $column->withDefault(null);
break;
case Entity::JSON_ARRAY:
$default = is_array($default) ? json_encode($default) : null;
$column = $column->withDefault($default);
break;
case Entity::BOOL:
$default = intval($default ?? false);
$column = $column->withDefault($default);
break;
}
if ($type !== Entity::ID && $autoincrement) {
$column = $column
->withNotNull()
->withUnsigned();
}
if (
!in_array($columnType, [
Types::STRING,
Types::TEXT,
MediumtextType::NAME,
LongtextType::NAME,
])
) {
return $column;
}
$collation = $binary ?
'utf8mb4_bin' :
'utf8mb4_unicode_ci';
$charset = 'utf8mb4';
if ($mb3) {
$collation = $binary ?
'utf8mb3_bin' :
'utf8mb3_unicode_ci';
$charset = 'utf8mb3';
}
return $column
->withCollation($collation)
->withCharset($charset);
}
private function getMaxIndexLength(): int
{
if (!isset($this->maxIndexLength)) {
$this->maxIndexLength = $this->detectMaxIndexLength();
}
return $this->maxIndexLength;
}
/**
* Get maximum index length.
*/
private function detectMaxIndexLength(): int
{
$tableEngine = $this->getTableEngine();
if (!$tableEngine) {
return self::DEFAULT_INDEX_LIMIT;
}
return match ($tableEngine) {
'InnoDB' => 3072,
default => 1000,
};
}
/**
* Get a table or default engine.
*/
private function getTableEngine(): ?string
{
$databaseType = $this->helper->getType();
if (!in_array($databaseType, [self::TYPE_MYSQL, self::TYPE_MARIADB])) {
return null;
}
$query = "SHOW TABLE STATUS WHERE Engine = 'MyISAM'";
$vars = [];
$pdo = $this->helper->getPDO();
$sth = $pdo->prepare($query);
$sth->execute($vars);
$result = $sth->fetchColumn();
if (!empty($result)) {
return 'MyISAM';
}
return 'InnoDB';
}
}

View File

@@ -0,0 +1,155 @@
<?php
/************************************************************************
* This file is part of EspoCRM.
*
* EspoCRM Open Source CRM application.
* Copyright (C) 2014-2025 EspoCRM, Inc.
* Website: https://www.espocrm.com
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
* The interactive user interfaces in modified source and object code versions
* of this program must display Appropriate Legal Notices, as required under
* Section 5 of the GNU Affero General Public License version 3.
*
* In accordance with Section 7(b) of the GNU Affero General Public License version 3,
* these Appropriate Legal Notices must retain the display of the "EspoCRM" word.
************************************************************************/
namespace Espo\Core\Utils\Database\Schema\ColumnPreparators;
use Doctrine\DBAL\Types\Types;
use Espo\Core\Utils\Database\Schema\Column;
use Espo\Core\Utils\Database\Schema\ColumnPreparator;
use Espo\Core\Utils\Util;
use Espo\ORM\Defs\AttributeDefs;
use Espo\ORM\Defs\Params\AttributeParam;
use Espo\ORM\Entity;
class PostgresqlColumnPreparator implements ColumnPreparator
{
private const PARAM_DB_TYPE = AttributeParam::DB_TYPE;
private const PARAM_DEFAULT = AttributeParam::DEFAULT;
private const PARAM_NOT_NULL = AttributeParam::NOT_NULL;
private const PARAM_AUTOINCREMENT = 'autoincrement';
private const PARAM_PRECISION = 'precision';
private const PARAM_SCALE = 'scale';
/** @var string[] */
private array $textTypeList = [
Entity::TEXT,
Entity::JSON_OBJECT,
Entity::JSON_ARRAY,
];
/** @var array<string, string> */
private array $columnTypeMap = [
Entity::BOOL => Types::BOOLEAN,
Entity::INT => Types::INTEGER,
Entity::VARCHAR => Types::STRING,
// DBAL reverse engineers as blob.
Types::BINARY => Types::BLOB,
];
public function __construct() {}
public function prepare(AttributeDefs $defs): Column
{
$dbType = $defs->getParam(self::PARAM_DB_TYPE);
$type = $defs->getType();
$length = $defs->getLength();
$default = $defs->getParam(self::PARAM_DEFAULT);
$notNull = $defs->getParam(self::PARAM_NOT_NULL);
$autoincrement = $defs->getParam(self::PARAM_AUTOINCREMENT);
$precision = $defs->getParam(self::PARAM_PRECISION);
$scale = $defs->getParam(self::PARAM_SCALE);
$columnType = $dbType ?? $type;
if (in_array($type, $this->textTypeList) && !$dbType) {
$columnType = Types::TEXT;
}
$columnType = $this->columnTypeMap[$columnType] ?? $columnType;
$columnName = Util::toUnderScore($defs->getName());
$column = Column::create($columnName, strtolower($columnType));
if ($length !== null) {
$column = $column->withLength($length);
}
if ($default !== null) {
$column = $column->withDefault($default);
}
if ($notNull !== null) {
$column = $column->withNotNull($notNull);
}
if ($autoincrement !== null) {
$column = $column->withAutoincrement($autoincrement);
}
if ($precision !== null) {
$column = $column->withPrecision($precision);
}
if ($scale !== null) {
$column = $column->withScale($scale);
}
switch ($type) {
case Entity::TEXT:
$column = $column->withDefault(null);
break;
case Entity::JSON_ARRAY:
$default = is_array($default) ? json_encode($default) : null;
$column = $column->withDefault($default);
break;
case Entity::BOOL:
$default = intval($default ?? false);
$column = $column->withDefault($default);
break;
}
if ($type !== Entity::ID && $autoincrement) {
$column = $column
->withNotNull()
->withUnsigned();
}
return $column;
// @todo Revise. Comparator would detect the column as changed if charset is set.
/*if (
!in_array($columnType, [
Types::STRING,
Types::TEXT,
])
) {
return $column;
}
return $column->withCharset('UTF8');*/
}
}

View File

@@ -0,0 +1,375 @@
<?php
/************************************************************************
* This file is part of EspoCRM.
*
* EspoCRM Open Source CRM application.
* Copyright (C) 2014-2025 EspoCRM, Inc.
* Website: https://www.espocrm.com
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
* The interactive user interfaces in modified source and object code versions
* of this program must display Appropriate Legal Notices, as required under
* Section 5 of the GNU Affero General Public License version 3.
*
* In accordance with Section 7(b) of the GNU Affero General Public License version 3,
* these Appropriate Legal Notices must retain the display of the "EspoCRM" word.
************************************************************************/
namespace Espo\Core\Utils\Database\Schema;
use Doctrine\DBAL\Exception as DbalException;
use Doctrine\DBAL\Schema\Column as Column;
use Doctrine\DBAL\Schema\ColumnDiff;
use Doctrine\DBAL\Schema\Schema;
use Doctrine\DBAL\Schema\SchemaDiff;
use Doctrine\DBAL\Schema\TableDiff;
use Doctrine\DBAL\Types\TextType;
use Doctrine\DBAL\Types\Type;
use Doctrine\DBAL\Types\Types;
use Espo\Core\Utils\Database\Dbal\Types\LongtextType;
use Espo\Core\Utils\Database\Dbal\Types\MediumtextType;
class DiffModifier
{
/**
* @param RebuildMode::* $mode
* @throws DbalException
*/
public function modify(
SchemaDiff $diff,
Schema $schema,
bool $secondRun = false,
string $mode = RebuildMode::SOFT
): bool {
$reRun = false;
$isHard = $mode === RebuildMode::HARD;
$diff = $this->handleRemovedSequences($diff, $schema);
$diff->removedTables = [];
foreach ($diff->changedTables as $tableDiff) {
$reRun = $this->amendTableDiff($tableDiff, $secondRun, $isHard) || $reRun;
}
return $reRun;
}
/**
* @throws DbalException
*/
private function amendTableDiff(TableDiff $tableDiff, bool $secondRun, bool $isHard): bool
{
$reRun = false;
/**
* @todo Leave only for MariaDB?
* MariaDB supports RENAME INDEX as of v10.5.
* Find out how long does it take to rename for different databases.
*/
if (!$isHard) {
// Prevent index renaming as an operation may take a lot of time.
$tableDiff->renamedIndexes = [];
}
foreach ($tableDiff->removedColumns as $name => $column) {
$reRun = $this->moveRemovedAutoincrementColumnToChanged($tableDiff, $column, $name) || $reRun;
}
if (!$isHard) {
// Prevent column removal to prevent data loss.
$tableDiff->removedColumns = [];
}
// Prevent column renaming as a not desired behavior.
foreach ($tableDiff->renamedColumns as $renamedColumn) {
$addedName = strtolower($renamedColumn->getName());
$tableDiff->addedColumns[$addedName] = $renamedColumn;
}
$tableDiff->renamedColumns = [];
foreach ($tableDiff->addedColumns as $column) {
// Suppress autoincrement as need having a unique index first.
$reRun = $this->amendAddedColumnAutoincrement($column) || $reRun;
}
foreach ($tableDiff->changedColumns as $name => $columnDiff) {
if (!$isHard) {
// Prevent decreasing length for string columns to prevent data loss.
$this->amendColumnDiffLength($tableDiff, $columnDiff, $name);
// Prevent longtext => mediumtext to prevent data loss.
$this->amendColumnDiffTextType($tableDiff, $columnDiff, $name);
// Prevent changing collation.
$this->amendColumnDiffCollation($tableDiff, $columnDiff, $name);
// Prevent changing charset.
$this->amendColumnDiffCharset($tableDiff, $columnDiff, $name);
}
// Prevent setting autoincrement in first run.
if (!$secondRun) {
$reRun = $this->amendColumnDiffAutoincrement($tableDiff, $columnDiff, $name) || $reRun;
}
}
return $reRun;
}
private function amendColumnDiffLength(TableDiff $tableDiff, ColumnDiff $columnDiff, string $name): void
{
$fromColumn = $columnDiff->fromColumn;
$column = $columnDiff->column;
if (!$fromColumn) {
return;
}
if (!in_array('length', $columnDiff->changedProperties)) {
return;
}
$fromLength = $fromColumn->getLength() ?? 255;
$length = $column->getLength() ?? 255;
if ($fromLength <= $length) {
return;
}
$column->setLength($fromLength);
self::unsetChangedColumnProperty($tableDiff, $columnDiff, $name, 'length');
}
/**
* @throws DbalException
*/
private function amendColumnDiffTextType(TableDiff $tableDiff, ColumnDiff $columnDiff, string $name): void
{
$fromColumn = $columnDiff->fromColumn;
$column = $columnDiff->column;
if (!$fromColumn) {
return;
}
if (!in_array('type', $columnDiff->changedProperties)) {
return;
}
$fromType = $fromColumn->getType();
$type = $column->getType();
if (
!$fromType instanceof TextType ||
!$type instanceof TextType
) {
return;
}
$typePriority = [
Types::TEXT,
MediumtextType::NAME,
LongtextType::NAME,
];
$fromIndex = array_search($fromType->getName(), $typePriority);
$index = array_search($type->getName(), $typePriority);
if ($index >= $fromIndex) {
return;
}
$column->setType(Type::getType($fromType->getName()));
self::unsetChangedColumnProperty($tableDiff, $columnDiff, $name, 'type');
}
private function amendColumnDiffCollation(TableDiff $tableDiff, ColumnDiff $columnDiff, string $name): void
{
$fromColumn = $columnDiff->fromColumn;
$column = $columnDiff->column;
if (!$fromColumn) {
return;
}
if (!in_array('collation', $columnDiff->changedProperties)) {
return;
}
$fromCollation = $fromColumn->getPlatformOption('collation');
if (!$fromCollation) {
return;
}
$column->setPlatformOption('collation', $fromCollation);
self::unsetChangedColumnProperty($tableDiff, $columnDiff, $name, 'collation');
}
private function amendColumnDiffCharset(TableDiff $tableDiff, ColumnDiff $columnDiff, string $name): void
{
$fromColumn = $columnDiff->fromColumn;
$column = $columnDiff->column;
if (!$fromColumn) {
return;
}
if (!in_array('charset', $columnDiff->changedProperties)) {
return;
}
$fromCharset = $fromColumn->getPlatformOption('charset');
if (!$fromCharset) {
return;
}
$column->setPlatformOption('charset', $fromCharset);
self::unsetChangedColumnProperty($tableDiff, $columnDiff, $name, 'charset');
}
private function amendColumnDiffAutoincrement(TableDiff $tableDiff, ColumnDiff $columnDiff, string $name): bool
{
$fromColumn = $columnDiff->fromColumn;
$column = $columnDiff->column;
if (!$fromColumn) {
return false;
}
if (!in_array('autoincrement', $columnDiff->changedProperties)) {
return false;
}
$column
->setAutoincrement(false)
->setNotnull(false)
->setDefault(null);
if ($name === 'id') {
$column->setNotnull(true);
}
self::unsetChangedColumnProperty($tableDiff, $columnDiff, $name, 'autoincrement');
return true;
}
private function amendAddedColumnAutoincrement(Column $column): bool
{
if (!$column->getAutoincrement()) {
return false;
}
$column
->setAutoincrement(false)
->setNotnull(false)
->setDefault(null);
return true;
}
private function moveRemovedAutoincrementColumnToChanged(TableDiff $tableDiff, Column $column, string $name): bool
{
if (!$column->getAutoincrement()) {
return false;
}
$newColumn = clone $column;
$newColumn
->setAutoincrement(false)
->setNotnull(false)
->setDefault(null);
$changedProperties = [
'autoincrement',
'notnull',
'default',
];
$tableDiff->changedColumns[$name] = new ColumnDiff($name, $newColumn, $changedProperties, $column);
foreach ($tableDiff->removedIndexes as $indexName => $index) {
if ($index->getColumns() === [$name]) {
unset($tableDiff->removedIndexes[$indexName]);
}
}
return true;
}
private static function unsetChangedColumnProperty(
TableDiff $tableDiff,
ColumnDiff $columnDiff,
string $name,
string $property
): void {
if (count($columnDiff->changedProperties) === 1) {
unset($tableDiff->changedColumns[$name]);
}
$columnDiff->changedProperties = array_diff($columnDiff->changedProperties, [$property]);
}
/**
* DBAL does not handle autoincrement columns that are not primary keys,
* making them dropped.
*/
private function handleRemovedSequences(SchemaDiff $diff, Schema $schema): SchemaDiff
{
$droppedSequences = $diff->getDroppedSequences();
if ($droppedSequences === []) {
return $diff;
}
foreach ($droppedSequences as $i => $sequence) {
foreach ($schema->getTables() as $table) {
$namespace = $table->getNamespaceName();
$tableName = $table->getShortestName($namespace);
foreach ($table->getColumns() as $column) {
if (!$column->getAutoincrement()) {
continue;
}
$sequenceName = $sequence->getShortestName($namespace);
$tableSequenceName = sprintf('%s_%s_seq', $tableName, $column->getShortestName($namespace));
if ($tableSequenceName !== $sequenceName) {
continue;
}
unset($droppedSequences[$i]);
continue 3;
}
}
}
$diff->removedSequences = array_values($droppedSequences);
return $diff;
}
}

View File

@@ -0,0 +1,41 @@
<?php
/************************************************************************
* This file is part of EspoCRM.
*
* EspoCRM Open Source CRM application.
* Copyright (C) 2014-2025 EspoCRM, Inc.
* Website: https://www.espocrm.com
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
* The interactive user interfaces in modified source and object code versions
* of this program must display Appropriate Legal Notices, as required under
* Section 5 of the GNU Affero General Public License version 3.
*
* In accordance with Section 7(b) of the GNU Affero General Public License version 3,
* these Appropriate Legal Notices must retain the display of the "EspoCRM" word.
************************************************************************/
namespace Espo\Core\Utils\Database\Schema;
use Espo\Core\Utils\Database\Orm\Defs\EntityDefs;
use Espo\ORM\Defs\EntityDefs as OrmEntityDefs;
/**
* Modifies definitions before building a schema.
*/
interface EntityDefsModifier
{
public function modify(OrmEntityDefs $entityDefs): EntityDefs;
}

View File

@@ -0,0 +1,69 @@
<?php
/************************************************************************
* This file is part of EspoCRM.
*
* EspoCRM Open Source CRM application.
* Copyright (C) 2014-2025 EspoCRM, Inc.
* Website: https://www.espocrm.com
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
* The interactive user interfaces in modified source and object code versions
* of this program must display Appropriate Legal Notices, as required under
* Section 5 of the GNU Affero General Public License version 3.
*
* In accordance with Section 7(b) of the GNU Affero General Public License version 3,
* these Appropriate Legal Notices must retain the display of the "EspoCRM" word.
************************************************************************/
namespace Espo\Core\Utils\Database\Schema\EntityDefsModifiers;
use Espo\Core\Utils\Database\Orm\Defs\AttributeDefs;
use Espo\Core\Utils\Database\Orm\Defs\EntityDefs;
use Espo\Core\Utils\Database\Schema\EntityDefsModifier;
use Espo\ORM\Defs\EntityDefs as OrmEntityDefs;
use Espo\ORM\Defs\Params\AttributeParam;
use Espo\ORM\Type\AttributeType;
/**
* A single JSON column instead of multiple field columns.
*/
class JsonData implements EntityDefsModifier
{
public function modify(OrmEntityDefs $entityDefs): EntityDefs
{
$sourceIdAttribute = $entityDefs->getAttribute('id');
$idAttribute = AttributeDefs::create('id')
->withType(AttributeType::ID);
$length = $sourceIdAttribute->getLength();
$dbType = $sourceIdAttribute->getParam(AttributeParam::DB_TYPE);
if ($length) {
$idAttribute = $idAttribute->withLength($length);
}
if ($dbType) {
$idAttribute = $idAttribute->withDbType($dbType);
}
return EntityDefs::create()
->withAttribute($idAttribute)
->withAttribute(
AttributeDefs::create('data')
->withType(AttributeType::JSON_OBJECT)
);
}
}

View File

@@ -0,0 +1,77 @@
<?php
/************************************************************************
* This file is part of EspoCRM.
*
* EspoCRM Open Source CRM application.
* Copyright (C) 2014-2025 EspoCRM, Inc.
* Website: https://www.espocrm.com
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
* The interactive user interfaces in modified source and object code versions
* of this program must display Appropriate Legal Notices, as required under
* Section 5 of the GNU Affero General Public License version 3.
*
* In accordance with Section 7(b) of the GNU Affero General Public License version 3,
* these Appropriate Legal Notices must retain the display of the "EspoCRM" word.
************************************************************************/
namespace Espo\Core\Utils\Database\Schema;
use Doctrine\DBAL\Types\Type;
use Espo\Core\Utils\Database\ConfigDataProvider;
use Espo\Core\Utils\Metadata;
class MetadataProvider
{
public function __construct(
private ConfigDataProvider $configDataProvider,
private Metadata $metadata
) {}
private function getPlatform(): string
{
return $this->configDataProvider->getPlatform();
}
/**
* @return class-string<RebuildAction>[]
*/
public function getPreRebuildActionClassNameList(): array
{
/** @var class-string<RebuildAction>[] */
return $this->metadata
->get(['app', 'databasePlatforms', $this->getPlatform(), 'preRebuildActionClassNameList']) ?? [];
}
/**
* @return class-string<RebuildAction>[]
*/
public function getPostRebuildActionClassNameList(): array
{
/** @var class-string<RebuildAction>[] */
return $this->metadata
->get(['app', 'databasePlatforms', $this->getPlatform(), 'postRebuildActionClassNameList']) ?? [];
}
/**
* @return array<string, class-string<Type>>
*/
public function getDbalTypeClassNameMap(): array
{
/** @var array<string, class-string<Type>> */
return $this->metadata
->get(['app', 'databasePlatforms', $this->getPlatform(), 'dbalTypeClassNameMap']) ?? [];
}
}

View File

@@ -0,0 +1,37 @@
<?php
/************************************************************************
* This file is part of EspoCRM.
*
* EspoCRM Open Source CRM application.
* Copyright (C) 2014-2025 EspoCRM, Inc.
* Website: https://www.espocrm.com
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
* The interactive user interfaces in modified source and object code versions
* of this program must display Appropriate Legal Notices, as required under
* Section 5 of the GNU Affero General Public License version 3.
*
* In accordance with Section 7(b) of the GNU Affero General Public License version 3,
* these Appropriate Legal Notices must retain the display of the "EspoCRM" word.
************************************************************************/
namespace Espo\Core\Utils\Database\Schema;
use Doctrine\DBAL\Schema\Schema as DbalSchema;
interface RebuildAction
{
public function process(DbalSchema $oldSchema, DbalSchema $newSchema): void;
}

View File

@@ -0,0 +1,96 @@
<?php
/************************************************************************
* This file is part of EspoCRM.
*
* EspoCRM Open Source CRM application.
* Copyright (C) 2014-2025 EspoCRM, Inc.
* Website: https://www.espocrm.com
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
* The interactive user interfaces in modified source and object code versions
* of this program must display Appropriate Legal Notices, as required under
* Section 5 of the GNU Affero General Public License version 3.
*
* In accordance with Section 7(b) of the GNU Affero General Public License version 3,
* these Appropriate Legal Notices must retain the display of the "EspoCRM" word.
************************************************************************/
namespace Espo\Core\Utils\Database\Schema\RebuildActions;
use Doctrine\DBAL\Exception as DbalException;
use Doctrine\DBAL\Schema\Schema as DbalSchema;
use Espo\Core\Utils\Database\Helper;
use Espo\Core\Utils\Database\Schema\RebuildAction;
use Espo\Core\Utils\Log;
use Exception;
class PrepareForFulltextIndex implements RebuildAction
{
public function __construct(
private Helper $helper,
private Log $log
) {}
/**
* @throws DbalException
*/
public function process(DbalSchema $oldSchema, DbalSchema $newSchema): void
{
if ($oldSchema->getTables() === []) {
return;
}
$connection = $this->helper->getDbalConnection();
$pdo = $this->helper->getPDO();
foreach ($newSchema->getTables() as $table) {
$tableName = $table->getName();
$indexes = $table->getIndexes();
foreach ($indexes as $index) {
if (!$index->hasFlag('fulltext')) {
continue;
}
$columns = $index->getColumns();
foreach ($columns as $columnName) {
$sql = "SHOW FULL COLUMNS FROM `" . $tableName . "` WHERE Field = " . $pdo->quote($columnName);
try {
/** @var array{Type: string, Collation: string} $row */
$row = $connection->fetchAssociative($sql);
} catch (Exception) {
continue;
}
switch (strtoupper($row['Type'])) {
case 'LONGTEXT':
$alterSql =
"ALTER TABLE `{$tableName}` " .
"MODIFY `{$columnName}` MEDIUMTEXT COLLATE " . $row['Collation'];
$this->log->info('SCHEMA, Execute Query: ' . $alterSql);
$connection->executeQuery($alterSql);
break;
}
}
}
}
}
}

View File

@@ -0,0 +1,36 @@
<?php
/************************************************************************
* This file is part of EspoCRM.
*
* EspoCRM Open Source CRM application.
* Copyright (C) 2014-2025 EspoCRM, Inc.
* Website: https://www.espocrm.com
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
* The interactive user interfaces in modified source and object code versions
* of this program must display Appropriate Legal Notices, as required under
* Section 5 of the GNU Affero General Public License version 3.
*
* In accordance with Section 7(b) of the GNU Affero General Public License version 3,
* these Appropriate Legal Notices must retain the display of the "EspoCRM" word.
************************************************************************/
namespace Espo\Core\Utils\Database\Schema;
class RebuildMode
{
public const SOFT = 'soft';
public const HARD = 'hard';
}

View File

@@ -0,0 +1,255 @@
<?php
/************************************************************************
* This file is part of EspoCRM.
*
* EspoCRM Open Source CRM application.
* Copyright (C) 2014-2025 EspoCRM, Inc.
* Website: https://www.espocrm.com
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
* The interactive user interfaces in modified source and object code versions
* of this program must display Appropriate Legal Notices, as required under
* Section 5 of the GNU Affero General Public License version 3.
*
* In accordance with Section 7(b) of the GNU Affero General Public License version 3,
* these Appropriate Legal Notices must retain the display of the "EspoCRM" word.
************************************************************************/
namespace Espo\Core\Utils\Database\Schema;
use Doctrine\DBAL\Connection as DbalConnection;
use Doctrine\DBAL\Exception as DbalException;
use Doctrine\DBAL\Platforms\AbstractPlatform;
use Doctrine\DBAL\Schema\AbstractSchemaManager;
use Doctrine\DBAL\Schema\Comparator;
use Doctrine\DBAL\Schema\Schema;
use Doctrine\DBAL\Schema\SchemaDiff;
use Doctrine\DBAL\Schema\SchemaException;
use Doctrine\DBAL\Types\Type;
use Espo\Core\Binding\BindingContainerBuilder;
use Espo\Core\InjectableFactory;
use Espo\Core\Utils\Database\Helper;
use Espo\Core\Utils\Log;
use Espo\Core\Utils\Metadata\OrmMetadataData;
use Throwable;
/**
* A database schema manager.
*/
class SchemaManager
{
/** @var AbstractSchemaManager<AbstractPlatform> */
private AbstractSchemaManager $schemaManager;
private Comparator $comparator;
private Builder $builder;
/**
* @throws DbalException
*/
public function __construct(
private OrmMetadataData $ormMetadataData,
private Log $log,
private Helper $helper,
private MetadataProvider $metadataProvider,
private DiffModifier $diffModifier,
private InjectableFactory $injectableFactory
) {
$this->schemaManager = $this->getDbalConnection()
->getDatabasePlatform()
->createSchemaManager($this->getDbalConnection());
// Not using a platform specific comparator as it unsets a collation and charset if
// they match a table default.
//$this->comparator = $this->schemaManager->createComparator();
$this->comparator = new Comparator($this->getPlatform());
$this->initFieldTypes();
$this->builder = $this->injectableFactory->createWithBinding(
Builder::class,
BindingContainerBuilder::create()
->bindInstance(Helper::class, $this->helper)
->build()
);
}
public function getDatabaseHelper(): Helper
{
return $this->helper;
}
/**
* @throws DbalException
*/
private function getPlatform(): AbstractPlatform
{
return $this->getDbalConnection()->getDatabasePlatform();
}
private function getDbalConnection(): DbalConnection
{
return $this->helper->getDbalConnection();
}
/**
* @throws DbalException
*/
private function initFieldTypes(): void
{
foreach ($this->metadataProvider->getDbalTypeClassNameMap() as $type => $className) {
Type::hasType($type) ?
Type::overrideType($type, $className) :
Type::addType($type, $className);
$this->getDbalConnection()
->getDatabasePlatform()
->registerDoctrineTypeMapping($type, $type);
}
}
/**
* Rebuild database schema. Creates and alters needed tables and columns.
* Does not remove columns, does not decrease column lengths.
*
* @param ?string[] $entityTypeList Specific entity types.
* @param RebuildMode::* $mode A mode.
* @throws SchemaException
* @throws DbalException
* @todo Catch and re-throw exceptions.
*/
public function rebuild(?array $entityTypeList = null, string $mode = RebuildMode::SOFT): bool
{
$fromSchema = $this->introspectSchema();
$schema = $this->builder->build($this->ormMetadataData->getData(), $entityTypeList);
try {
$this->processPreRebuildActions($fromSchema, $schema);
} catch (Throwable $e) {
$this->log->alert('Rebuild database pre-rebuild error: '. $e->getMessage());
return false;
}
$diff = $this->comparator->compareSchemas($fromSchema, $schema);
$needReRun = $this->diffModifier->modify($diff, $schema, false, $mode);
$sql = $this->composeDiffSql($diff);
$result = $this->runSql($sql);
if (!$result) {
return false;
}
if ($needReRun) {
// Needed to handle auto-increment column creation/removal/change.
// As an auto-increment column requires having a unique index, but
// Doctrine DBAL does not handle this.
$intermediateSchema = $this->introspectSchema();
$schema = $this->builder->build($this->ormMetadataData->getData(), $entityTypeList);
$diff = $this->comparator->compareSchemas($intermediateSchema, $schema);
$this->diffModifier->modify($diff, $schema, true);
$sql = $this->composeDiffSql($diff);
$result = $this->runSql($sql);
}
if (!$result) {
return false;
}
try {
$this->processPostRebuildActions($fromSchema, $schema);
} catch (Throwable $e) {
$this->log->alert('Rebuild database post-rebuild error: ' . $e->getMessage());
return false;
}
return true;
}
/**
* @param string[] $queries
* @return bool
*/
private function runSql(array $queries): bool
{
$result = true;
$connection = $this->getDbalConnection();
foreach ($queries as $sql) {
$this->log->info('Schema, query: '. $sql);
try {
$connection->executeQuery($sql);
} catch (Throwable $e) {
$this->log->alert('Rebuild database error: ' . $e->getMessage());
$result = false;
}
}
return $result;
}
/**
* Introspect and return a current database schema.
*
* @throws DbalException
*/
private function introspectSchema(): Schema
{
return $this->schemaManager->introspectSchema();
}
/**
* @return string[]
* @throws DbalException
*/
private function composeDiffSql(SchemaDiff $diff): array
{
return $this->getPlatform()->getAlterSchemaSQL($diff);
}
private function processPreRebuildActions(Schema $actualSchema, Schema $schema): void
{
$binding = BindingContainerBuilder::create()
->bindInstance(Helper::class, $this->helper)
->build();
foreach ($this->metadataProvider->getPreRebuildActionClassNameList() as $className) {
$action = $this->injectableFactory->createWithBinding($className, $binding);
$action->process($actualSchema, $schema);
}
}
private function processPostRebuildActions(Schema $actualSchema, Schema $schema): void
{
$binding = BindingContainerBuilder::create()
->bindInstance(Helper::class, $this->helper)
->build();
foreach ($this->metadataProvider->getPostRebuildActionClassNameList() as $className) {
$action = $this->injectableFactory->createWithBinding($className, $binding);
$action->process($actualSchema, $schema);
}
}
}

View File

@@ -0,0 +1,64 @@
<?php
/************************************************************************
* This file is part of EspoCRM.
*
* EspoCRM Open Source CRM application.
* Copyright (C) 2014-2025 EspoCRM, Inc.
* Website: https://www.espocrm.com
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
* The interactive user interfaces in modified source and object code versions
* of this program must display Appropriate Legal Notices, as required under
* Section 5 of the GNU Affero General Public License version 3.
*
* In accordance with Section 7(b) of the GNU Affero General Public License version 3,
* these Appropriate Legal Notices must retain the display of the "EspoCRM" word.
************************************************************************/
namespace Espo\Core\Utils\Database\Schema;
use Espo\Core\InjectableFactory;
use Espo\Core\Utils\Database\Helper;
use Doctrine\DBAL\Schema\SchemaException;
class SchemaManagerProxy
{
private ?SchemaManager $schemaManager = null;
public function __construct(private InjectableFactory $injectableFactory) {}
private function getSchemaManager(): SchemaManager
{
$this->schemaManager ??= $this->injectableFactory->create(SchemaManager::class);
return $this->schemaManager;
}
/**
* @param ?string[] $entityTypeList
* @param RebuildMode::* $mode
* @throws SchemaException
*/
public function rebuild(?array $entityTypeList = null, string $mode = RebuildMode::SOFT): bool
{
return $this->getSchemaManager()->rebuild($entityTypeList, $mode);
}
public function getDatabaseHelper(): Helper
{
return $this->getSchemaManager()->getDatabaseHelper();
}
}

View File

@@ -0,0 +1,237 @@
<?php
/************************************************************************
* This file is part of EspoCRM.
*
* EspoCRM Open Source CRM application.
* Copyright (C) 2014-2025 EspoCRM, Inc.
* Website: https://www.espocrm.com
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
* The interactive user interfaces in modified source and object code versions
* of this program must display Appropriate Legal Notices, as required under
* Section 5 of the GNU Affero General Public License version 3.
*
* In accordance with Section 7(b) of the GNU Affero General Public License version 3,
* these Appropriate Legal Notices must retain the display of the "EspoCRM" word.
************************************************************************/
namespace Espo\Core\Utils\Database\Schema;
use Espo\Core\ORM\Type\FieldType;
use Espo\Core\Utils\Util;
use Espo\ORM\Defs\IndexDefs;
use Espo\ORM\Defs\Params\AttributeParam;
use Espo\ORM\Defs\Params\EntityParam;
use Espo\ORM\Defs\Params\IndexParam;
class Utils
{
/**
* Get indexes in specific format.
* @deprecated
*
* @param array<string, mixed> $defs
* @param string[] $ignoreFlags @todo Remove parameter?
* @return array<string, array<string, mixed>>
*/
public static function getIndexes(array $defs, array $ignoreFlags = []): array
{
$indexList = [];
foreach ($defs as $entityType => $entityParams) {
$indexes = $entityParams[EntityParam::INDEXES] ?? [];
foreach ($indexes as $indexName => $indexParams) {
$indexDefs = IndexDefs::fromRaw($indexParams, $indexName);
$tableIndexName = $indexParams[IndexParam::KEY] ?? null;
if (!$tableIndexName) {
continue;
}
$columns = $indexDefs->getColumnList();
$flags = $indexDefs->getFlagList();
if ($flags !== []) {
$skipIndex = false;
foreach ($ignoreFlags as $ignoreFlag) {
if (($flagKey = array_search($ignoreFlag, $flags)) !== false) {
unset($flags[$flagKey]);
$skipIndex = true;
}
}
if ($skipIndex && empty($flags)) {
continue;
}
$indexList[$entityType][$tableIndexName][IndexParam::FLAGS] = $flags;
}
if ($columns !== []) {
$indexType = self::getIndexTypeByIndexDefs($indexDefs);
// @todo Revise, may to be removed.
$indexList[$entityType][$tableIndexName][IndexParam::TYPE] = $indexType;
$indexList[$entityType][$tableIndexName][IndexParam::COLUMNS] = array_map(
fn ($item) => Util::toUnderScore($item),
$columns
);
}
}
}
/** @var array<string, array<string, mixed>> */
return $indexList; /** @phpstan-ignore-line */
}
private static function getIndexTypeByIndexDefs(IndexDefs $indexDefs): string
{
if ($indexDefs->isUnique()) {
return 'unique';
}
if (in_array('fulltext', $indexDefs->getFlagList())) {
return 'fulltext';
}
return 'index';
}
/**
* @deprecated
*
* @param array<string, mixed> $ormMeta
* @param int $indexMaxLength
* @param ?array<string, mixed> $indexList
* @param int $characterLength
* @return array<string, mixed>
*/
public static function getFieldListExceededIndexMaxLength(
array $ormMeta,
$indexMaxLength = 1000,
?array $indexList = null,
$characterLength = 4
) {
$permittedFieldTypeList = [
FieldType::VARCHAR,
];
$fields = [];
if (!isset($indexList)) {
$indexList = self::getIndexes($ormMeta, ['fulltext']);
}
foreach ($indexList as $entityName => $indexes) {
foreach ($indexes as $indexName => $indexParams) {
$columnList = $indexParams['columns'];
$indexLength = 0;
foreach ($columnList as $columnName) {
$fieldName = Util::toCamelCase($columnName);
if (!isset($ormMeta[$entityName]['fields'][$fieldName])) {
continue;
}
$indexLength += self::getFieldLength(
$ormMeta[$entityName]['fields'][$fieldName],
$characterLength
);
}
if ($indexLength > $indexMaxLength) {
foreach ($columnList as $columnName) {
$fieldName = Util::toCamelCase($columnName);
if (!isset($ormMeta[$entityName]['fields'][$fieldName])) {
continue;
}
$fieldType = self::getFieldType($ormMeta[$entityName]['fields'][$fieldName]);
if (in_array($fieldType, $permittedFieldTypeList)) {
if (!isset($fields[$entityName]) || !in_array($fieldName, $fields[$entityName])) {
$fields[$entityName][] = $fieldName;
}
}
}
}
}
}
return $fields;
}
/**
* @param array<string, mixed> $ormFieldDefs
* @param int $characterLength
* @return int
*/
private static function getFieldLength(array $ormFieldDefs, $characterLength = 4)
{
$length = 0;
if (isset($ormFieldDefs[AttributeParam::NOT_STORABLE]) && $ormFieldDefs[AttributeParam::NOT_STORABLE]) {
return $length;
}
$defaultLength = [
'datetime' => 8,
'time' => 4,
'int' => 4,
'bool' => 1,
'float' => 4,
'varchar' => 255,
];
$type = self::getDbFieldType($ormFieldDefs);
$length = $defaultLength[$type] ?? $length;
switch ($type) {
case 'varchar':
$length = $length * $characterLength;
break;
}
return $length;
}
/**
* @param array<string, mixed> $ormFieldDefs
* @return string
*/
private static function getDbFieldType(array $ormFieldDefs)
{
return $ormFieldDefs[AttributeParam::DB_TYPE] ?? $ormFieldDefs['type'];
}
/**
* @param array<string, mixed> $ormFieldDefs
*/
private static function getFieldType(array $ormFieldDefs): string
{
return $ormFieldDefs['type'] ?? self::getDbFieldType($ormFieldDefs);
}
}