Initial commit

This commit is contained in:
root
2026-01-19 17:44:46 +01:00
commit 823af8b11d
8721 changed files with 1130846 additions and 0 deletions

View File

@@ -0,0 +1,338 @@
<?php
/************************************************************************
* This file is part of EspoCRM.
*
* EspoCRM Open Source CRM application.
* Copyright (C) 2014-2025 EspoCRM, Inc.
* Website: https://www.espocrm.com
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
* The interactive user interfaces in modified source and object code versions
* of this program must display Appropriate Legal Notices, as required under
* Section 5 of the GNU Affero General Public License version 3.
*
* In accordance with Section 7(b) of the GNU Affero General Public License version 3,
* these Appropriate Legal Notices must retain the display of the "EspoCRM" word.
************************************************************************/
namespace Espo\Core\Webhook;
use Espo\Core\Name\Field;
use Espo\Core\ORM\Entity;
use Espo\Core\ORM\EntityManager;
use Espo\Core\Utils\Config;
use Espo\Core\Utils\Config\SystemConfig;
use Espo\Core\Utils\DataCache;
use Espo\Core\Utils\FieldUtil;
use Espo\Core\Utils\Log;
use Espo\Entities\Webhook;
use Espo\Entities\WebhookEventQueueItem;
use Espo\ORM\Name\Attribute;
use RuntimeException;
use stdClass;
/**
* Processes events. Holds an information about existing events.
*/
class Manager
{
private string $cacheKey = 'webhooks';
/** @var string[] */
protected $skipAttributeList = [
Field::IS_FOLLOWED,
Field::IS_STARRED,
Field::FOLLOWERS . 'Ids',
Field::FOLLOWERS . 'Names',
Field::MODIFIED_AT,
Field::MODIFIED_BY,
Field::STREAM_UPDATED_AT,
Field::VERSION_NUMBER,
];
/** @var ?array<string, bool> */
private $data = null;
public function __construct(
private Config $config,
private DataCache $dataCache,
private EntityManager $entityManager,
private FieldUtil $fieldUtil,
private Log $log,
private SystemConfig $systemConfig,
) {
$this->loadData();
}
private function loadData(): void
{
if ($this->systemConfig->useCache() && $this->dataCache->has($this->cacheKey)) {
/** @var array<string, bool> $data */
$data = $this->dataCache->get($this->cacheKey);
$this->data = $data;
}
if (is_null($this->data)) {
$this->data = $this->buildData();
if ($this->systemConfig->useCache()) {
$this->storeDataToCache();
}
}
}
private function storeDataToCache(): void
{
if ($this->data === null) {
throw new RuntimeException("No data to store.");
}
$this->dataCache->store($this->cacheKey, $this->data);
}
/**
* @return array<string, bool>
*/
private function buildData(): array
{
$data = [];
$list = $this->entityManager
->getRDBRepositoryByClass(Webhook::class)
->select(['event'])
->group(['event'])
->where([
'isActive' => true,
'event!=' => null,
])
->find();
foreach ($list as $webhook) {
$event = $webhook->getEvent();
$data[$event] = true;
}
return $data;
}
/**
* Add an event. To cache the information that at least one webhook for this event exists.
*/
public function addEvent(string $event): void
{
$this->data[$event] = true;
if ($this->systemConfig->useCache()) {
$this->storeDataToCache();
}
}
/**
* Remove an event. If no webhooks with this event left, then it will be removed from the cache.
*/
public function removeEvent(string $event): void
{
$one = !$this->entityManager
->getRDBRepositoryByClass(Webhook::class)
->select([Attribute::ID])
->where([
'event' => $event,
'isActive' => true,
])
->findOne();
if (!$one) {
return;
}
unset($this->data[$event]);
if ($this->systemConfig->useCache()) {
$this->storeDataToCache();
}
}
private function eventExists(string $event): bool
{
return isset($this->data[$event]);
}
private function logDebugEvent(string $event, Entity $entity): void
{
$this->log->debug("Webhook: {event} on record {id}.", [
'id' => $entity->getId(),
'event' => $event,
]);
}
/**
* Process 'create' event.
*/
public function processCreate(Entity $entity, Options $options): void
{
$event = "{$entity->getEntityType()}.create";
if (!$this->eventExists($event)) {
return;
}
$item = $this->entityManager->getRDBRepositoryByClass(WebhookEventQueueItem::class)->getNew();
$item
->setEvent($event)
->setTarget($entity)
->setData($entity->getValueMap())
->setUserId($options->userId);
$this->entityManager->saveEntity($item);
$this->logDebugEvent($event, $entity);
}
/**
* Process 'delete' event.
*/
public function processDelete(Entity $entity, Options $options): void
{
$event = "{$entity->getEntityType()}.delete";
if (!$this->eventExists($event)) {
return;
}
$item = $this->entityManager->getRDBRepositoryByClass(WebhookEventQueueItem::class)->getNew();
$item
->setEvent($event)
->setTarget($entity)
->setData(['id' => $entity->getId()])
->setUserId($options->userId);
$this->entityManager->saveEntity($item);
$this->logDebugEvent($event, $entity);
}
/**
* Process 'update' event.
*/
public function processUpdate(Entity $entity, Options $options): void
{
$event = "{$entity->getEntityType()}.update";
$data = (object) [];
foreach ($entity->getAttributeList() as $attribute) {
if (in_array($attribute, $this->skipAttributeList)) {
continue;
}
if ($entity->isAttributeChanged($attribute)) {
$data->$attribute = $entity->get($attribute);
}
}
if (!count(get_object_vars($data))) {
return;
}
$data->id = $entity->getId();
if ($this->eventExists($event)) {
$item = $this->entityManager->getRDBRepositoryByClass(WebhookEventQueueItem::class)->getNew();
$item
->setEvent($event)
->setTarget($entity)
->setData($data)
->setUserId($options->userId);
$this->entityManager->saveEntity($item);
$this->logDebugEvent($event, $entity);
}
$this->processUpdateFields($entity, $data, $options);
}
private function processUpdateFields(Entity $entity, stdClass $data, Options $options): void
{
foreach ($this->fieldUtil->getEntityTypeFieldList($entity->getEntityType()) as $field) {
$itemEvent = "{$entity->getEntityType()}.fieldUpdate.$field";
if (
!$this->eventExists($itemEvent) ||
!$this->isFieldChanged($entity, $field, $data)
) {
continue;
}
$this->processUpdateField($entity, $field, $itemEvent, $options);
}
}
private function isFieldChanged(Entity $entity, string $field, stdClass $data): bool
{
$attributes = $this->fieldUtil->getActualAttributeList($entity->getEntityType(), $field);
$isChanged = false;
foreach ($attributes as $attribute) {
if (in_array($attribute, $this->skipAttributeList)) {
continue;
}
if (property_exists($data, $attribute)) {
$isChanged = true;
break;
}
}
return $isChanged;
}
private function processUpdateField(Entity $entity, string $field, string $itemEvent, Options $options): void
{
$itemData = (object) [];
$itemData->id = $entity->getId();
$attributeList = $this->fieldUtil->getAttributeList($entity->getEntityType(), $field);
foreach ($attributeList as $attribute) {
if (in_array($attribute, $this->skipAttributeList)) {
continue;
}
$itemData->$attribute = $entity->get($attribute);
}
$item = $this->entityManager->getRDBRepositoryByClass(WebhookEventQueueItem::class)->getNew();
$item
->setEvent($itemEvent)
->setTarget($entity)
->setData($itemData)
->setUserId($options->userId);
$this->entityManager->saveEntity($item);
$this->logDebugEvent($itemEvent, $entity);
}
}

View File

@@ -0,0 +1,40 @@
<?php
/************************************************************************
* This file is part of EspoCRM.
*
* EspoCRM Open Source CRM application.
* Copyright (C) 2014-2025 EspoCRM, Inc.
* Website: https://www.espocrm.com
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
* The interactive user interfaces in modified source and object code versions
* of this program must display Appropriate Legal Notices, as required under
* Section 5 of the GNU Affero General Public License version 3.
*
* In accordance with Section 7(b) of the GNU Affero General Public License version 3,
* these Appropriate Legal Notices must retain the display of the "EspoCRM" word.
************************************************************************/
namespace Espo\Core\Webhook;
readonly class Options
{
/**
* @param ?string $userId A user initiated an event.
*/
public function __construct(
public ?string $userId = null,
) {}
}

View File

@@ -0,0 +1,412 @@
<?php
/************************************************************************
* This file is part of EspoCRM.
*
* EspoCRM Open Source CRM application.
* Copyright (C) 2014-2025 EspoCRM, Inc.
* Website: https://www.espocrm.com
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
* The interactive user interfaces in modified source and object code versions
* of this program must display Appropriate Legal Notices, as required under
* Section 5 of the GNU Affero General Public License version 3.
*
* In accordance with Section 7(b) of the GNU Affero General Public License version 3,
* these Appropriate Legal Notices must retain the display of the "EspoCRM" word.
************************************************************************/
namespace Espo\Core\Webhook;
use Espo\Core\Field\DateTime;
use Espo\Core\Field\LinkParent;
use Espo\Core\Name\Field;
use Espo\Entities\User;
use Espo\Entities\Webhook;
use Espo\Entities\WebhookEventQueueItem;
use Espo\Entities\WebhookQueueItem;
use Espo\Core\AclManager;
use Espo\Core\Utils\Config;
use Espo\Core\Utils\DateTime as DateTimeUtil;
use Espo\Core\Utils\Log;
use Espo\ORM\EntityManager;
use Espo\ORM\Name\Attribute;
use Espo\ORM\Query\Part\Condition as Cond;
use Espo\ORM\Query\SelectBuilder;
use Exception;
use stdClass;
/**
* Groups occurred events into portions and sends them. A portion contains
* multiple events of the same webhook.
*/
class Queue
{
private const EVENT_PORTION_SIZE = 20;
private const PORTION_SIZE = 20;
private const BATCH_SIZE = 50;
private const MAX_ATTEMPT_NUMBER = 4;
private const FAIL_ATTEMPT_PERIOD = '10 minutes';
public function __construct(
private Sender $sender,
private Config $config,
private EntityManager $entityManager,
private AclManager $aclManager,
private Log $log,
) {}
public function process(): void
{
$this->processEvents();
$this->processSending();
}
protected function processEvents(): void
{
$portionSize = $this->config->get('webhookQueueEventPortionSize', self::EVENT_PORTION_SIZE);
$items = $this->entityManager
->getRDBRepositoryByClass(WebhookEventQueueItem::class)
->where(['isProcessed' => false])
->order('number')
->limit(0, $portionSize)
->find();
foreach ($items as $item) {
$this->createQueueFromEvent($item);
$item->setIsProcessed();
$this->entityManager->saveEntity($item);
}
}
protected function createQueueFromEvent(WebhookEventQueueItem $eventItem): void
{
if (!$eventItem->getTargetId() || !$eventItem->getTargetType()) {
return;
}
$target = LinkParent::create($eventItem->getTargetType(), $eventItem->getTargetId());
$webhooks = $this->entityManager
->getRDBRepositoryByClass(Webhook::class)
->where([
'event' => $eventItem->getEvent(),
'isActive' => true,
])
->order(Field::CREATED_AT)
->find();
foreach ($webhooks as $webhook) {
$this->createItem($eventItem, $webhook, $target);
}
}
protected function processSending(): void
{
$portionSize = $this->config->get('webhookQueuePortionSize', self::PORTION_SIZE);
$groupedItems = $this->entityManager
->getRDBRepositoryByClass(WebhookQueueItem::class)
->select([
'webhookId',
'number',
])
->where(
Cond::in(
Cond::column('number'),
$this->entityManager
->getQueryBuilder()
->select('MIN:(number)')
->from(WebhookQueueItem::ENTITY_TYPE)
->where([
'status' => WebhookQueueItem::STATUS_PENDING,
'OR' => [
['processAt' => null],
['processAt<=' => DateTimeUtil::getSystemNowString()],
],
])
->group('webhookId')
->build()
)
)
->limit(0, $portionSize)
->order('number')
->find();
foreach ($groupedItems as $groupItem) {
$this->processSendingGroup($groupItem->getWebhookId());
}
}
private function processSendingGroup(string $webhookId): void
{
$batchSize = $this->config->get('webhookBatchSize', self::BATCH_SIZE);
$items = $this->entityManager
->getRDBRepositoryByClass(WebhookQueueItem::class)
->where([
'webhookId' => $webhookId,
'status' => WebhookQueueItem::STATUS_PENDING,
'OR' => [
['processAt' => null],
['processAt<=' => DateTimeUtil::getSystemNowString()],
],
])
->order('number')
->limit(0, $batchSize)
->find();
$webhook = $this->entityManager->getRDBRepositoryByClass(Webhook::class)->getById($webhookId);
if (!$webhook || !$webhook->isActive()) {
foreach ($items as $item) {
$this->deleteQueueItem($item);
}
return;
}
$forbiddenAttributeList = [];
$user = null;
if ($webhook->getUserId()) {
$user = $this->entityManager->getRDBRepositoryByClass(User::class)->getById($webhook->getUserId());
if (!$user) {
foreach ($items as $item) {
$this->deleteQueueItem($item);
}
return;
}
$forbiddenAttributeList = $this->aclManager
->getScopeForbiddenAttributeList($user, $webhook->getTargetEntityType());
}
$actualItemList = [];
$dataList = [];
foreach ($items as $item) {
$data = $this->prepareItemData($item, $user, $forbiddenAttributeList);
if ($data === null) {
continue;
}
$actualItemList[] = $item;
$dataList[] = $data;
}
if ($dataList === []) {
return;
}
$this->send($webhook, $dataList, $actualItemList);
}
/**
* @param string[] $forbiddenAttributeList
*/
private function prepareItemData(WebhookQueueItem $item, ?User $user, array $forbiddenAttributeList): ?stdClass
{
$targetType = $item->getTargetType();
if (!$targetType) {
$this->deleteQueueItem($item);
return null;
}
$target = null;
if ($this->entityManager->hasRepository($targetType)) {
$query = SelectBuilder::create()
->from($targetType)
->withDeleted()
->build();
$target = $this->entityManager
->getRDBRepository($targetType)
->clone($query)
->where([Attribute::ID => $item->getTargetId()])
->findOne();
}
if (!$target) {
$this->deleteQueueItem($item);
return null;
}
if ($user) {
if (!$this->aclManager->check($user, $target)) {
$this->deleteQueueItem($item);
return null;
}
}
$data = $item->getData();
foreach ($forbiddenAttributeList as $attribute) {
unset($data->$attribute);
}
return $data;
}
/**
* @param stdClass[] $dataList
* @param WebhookQueueItem[] $itemList
*/
private function send(Webhook $webhook, array $dataList, array $itemList): void
{
try {
$code = $this->sender->send($webhook, $dataList);
} catch (Exception $e) {
$this->failQueueItemList($itemList, true);
$this->log->error("Webhook Queue: Webhook '{$webhook->getId()}' sending failed. Error: {$e->getMessage()}");
return;
}
if ($code >= 200 && $code < 400) {
$this->succeedQueueItemList($itemList);
} else if ($code === 410) {
$this->dropWebhook($webhook);
} else if (in_array($code, [0, 401, 403, 404, 405, 408, 500, 503])) {
$this->failQueueItemList($itemList);
} else if ($code >= 400 && $code < 500) {
$this->failQueueItemList($itemList, true);
} else {
$this->failQueueItemList($itemList, true);
}
$this->logSending($webhook, $code);
}
protected function logSending(Webhook $webhook, int $code): void
{
$this->log->debug("Webhook Queue: Webhook '{$webhook->getId()}' sent, response code: $code.");
}
/**
* @param WebhookQueueItem[] $itemList
*/
protected function failQueueItemList(array $itemList, bool $force = false): void
{
foreach ($itemList as $item) {
$this->failQueueItem($item, $force);
}
}
/**
* @param WebhookQueueItem[] $itemList
*/
protected function succeedQueueItemList(array $itemList): void
{
foreach ($itemList as $item) {
$this->succeedQueueItem($item);
}
}
protected function deleteQueueItem(WebhookQueueItem $item): void
{
$this->entityManager
->getRDBRepository(WebhookQueueItem::ENTITY_TYPE)
->deleteFromDb($item->getId());
}
protected function dropWebhook(Webhook $webhook): void
{
$items = $this->entityManager
->getRDBRepositoryByClass(WebhookQueueItem::class)
->where([
'status' => WebhookQueueItem::STATUS_PENDING,
'webhookId' => $webhook->getId(),
])
->order('number')
->find();
foreach ($items as $item) {
$this->deleteQueueItem($item);
}
$this->entityManager->removeEntity($webhook);
}
protected function succeedQueueItem(WebhookQueueItem $item): void
{
$item
->setAttempts($item->getAttempts() + 1)
->setStatus(WebhookQueueItem::STATUS_SUCCESS)
->setProcessedAt(DateTime::createNow());
$this->entityManager->saveEntity($item);
}
protected function failQueueItem(WebhookQueueItem $item, bool $force = false): void
{
$attempts = $item->getAttempts() + 1;
$maxAttemptsNumber = $this->config->get('webhookMaxAttemptNumber', self::MAX_ATTEMPT_NUMBER);
$period = $this->config->get('webhookFailAttemptPeriod', self::FAIL_ATTEMPT_PERIOD);
if ($force) {
$maxAttemptsNumber = 0;
}
$processAt = DateTime::createNow()->modify($period);
$item->setAttempts($attempts);
$item->setProcessAt($processAt);
if ($attempts >= $maxAttemptsNumber) {
$item->setStatus(WebhookQueueItem::STATUS_FAILED);
$item->setProcessAt(null);
}
$this->entityManager->saveEntity($item);
}
private function createItem(WebhookEventQueueItem $eventItem, Webhook $webhook, LinkParent $target): void
{
if (
$webhook->skipOwn() &&
$webhook->getUserId() &&
$eventItem->getUserId() === $webhook->getUserId()
) {
return;
}
$item = $this->entityManager->getRDBRepositoryByClass(WebhookQueueItem::class)->getNew();
$item
->setEvent($eventItem->getEvent())
->setWebhook($webhook)
->setTarget($target)
->setStatus(WebhookQueueItem::STATUS_PENDING)
->setAttempts(0)
->setData($eventItem->getData());
$this->entityManager->saveEntity($item);
}
}

View File

@@ -0,0 +1,147 @@
<?php
/************************************************************************
* This file is part of EspoCRM.
*
* EspoCRM Open Source CRM application.
* Copyright (C) 2014-2025 EspoCRM, Inc.
* Website: https://www.espocrm.com
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
* The interactive user interfaces in modified source and object code versions
* of this program must display Appropriate Legal Notices, as required under
* Section 5 of the GNU Affero General Public License version 3.
*
* In accordance with Section 7(b) of the GNU Affero General Public License version 3,
* these Appropriate Legal Notices must retain the display of the "EspoCRM" word.
************************************************************************/
namespace Espo\Core\Webhook;
use Espo\Core\Exceptions\Error;
use Espo\Core\Utils\Config;
use Espo\Core\Utils\Json;
use Espo\Entities\Webhook;
/**
* Sends a portion.
*/
class Sender
{
private const CONNECT_TIMEOUT = 5;
private const TIMEOUT = 10;
public function __construct(private Config $config)
{}
/**
* @param array<int, mixed> $dataList
* @throws Error
*/
public function send(Webhook $webhook, array $dataList): int
{
$payload = Json::encode($dataList);
$signature = null;
$legacySignature = null;
$secretKey = $webhook->getSecretKey();
if ($secretKey) {
$signature = $this->buildSignature($webhook, $payload, $secretKey);
$legacySignature = $this->buildSignatureLegacy($webhook, $payload, $secretKey);
}
$connectTimeout = $this->config->get('webhookConnectTimeout', self::CONNECT_TIMEOUT);
$timeout = $this->config->get('webhookTimeout', self::TIMEOUT);
$headerList = [];
$headerList[] = 'Content-Type: application/json';
$headerList[] = 'Content-Length: ' . strlen($payload);
if ($signature) {
$headerList[] = 'Signature: ' . $signature;
}
if ($legacySignature) {
$headerList[] = 'X-Signature: ' . $legacySignature;
}
$url = $webhook->getUrl();
if (!$url) {
throw new Error("Webhook does not have URL.");
}
$handler = curl_init($url);
if ($handler === false) {
throw new Error("Could not init CURL for URL {$url}.");
}
curl_setopt($handler, \CURLOPT_RETURNTRANSFER, true);
curl_setopt($handler, \CURLOPT_FOLLOWLOCATION, true);
curl_setopt($handler, \CURLOPT_SSL_VERIFYPEER, true);
curl_setopt($handler, \CURLOPT_HEADER, true);
curl_setopt($handler, \CURLOPT_CUSTOMREQUEST, 'POST');
curl_setopt($handler, \CURLOPT_CONNECTTIMEOUT, $connectTimeout);
curl_setopt($handler, \CURLOPT_TIMEOUT, $timeout);
curl_setopt($handler, \CURLOPT_PROTOCOLS, \CURLPROTO_HTTPS | \CURLPROTO_HTTP);
curl_setopt($handler, \CURLOPT_REDIR_PROTOCOLS, \CURLPROTO_HTTPS);
curl_setopt($handler, \CURLOPT_HTTPHEADER, $headerList);
curl_setopt($handler, \CURLOPT_POSTFIELDS, $payload);
curl_exec($handler);
$code = curl_getinfo($handler, \CURLINFO_HTTP_CODE);
if (!is_numeric($code)) {
$code = 0;
}
if (!is_int($code)) {
$code = intval($code);
}
$errorNumber = curl_errno($handler);
if (
$errorNumber &&
in_array($errorNumber, [\CURLE_OPERATION_TIMEDOUT, \CURLE_OPERATION_TIMEOUTED])
) {
$code = 408;
}
curl_close($handler);
return $code;
}
private function buildSignature(Webhook $webhook, string $payload, string $secretKey): string
{
$webhookId = $webhook->getId();
$hash = hash_hmac('sha256', $payload, $secretKey);
return base64_encode("$webhookId:$hash");
}
/**
* @todo Remove in v11.0.
*/
private function buildSignatureLegacy(Webhook $webhook, string $payload, string $secretKey): string
{
return base64_encode($webhook->getId() . ':' . hash_hmac('sha256', $payload, $secretKey, true));
}
}