ExternalImport-ImageTransformation.php

<?php
declare(strict_types=1);
namespace Vendor\Extension\ExternalImport\Transformation;

use Cobweb\ExternalImport\ImporterAwareInterface;
use Cobweb\ExternalImport\ImporterAwareTrait;
use TYPO3\CMS\Core\Context\Context;
use TYPO3\CMS\Core\Database\Connection;
use TYPO3\CMS\Core\Database\ConnectionPool;
use TYPO3\CMS\Core\Database\Query\QueryBuilder;
use TYPO3\CMS\Core\Resource\Folder;
use TYPO3\CMS\Core\Resource\ResourceFactory;
use TYPO3\CMS\Core\Utility\GeneralUtility;

/**
 * Transformation class for External Import to store multiple assets during import
 * Use this class together with EXT:external_import
 * See https://github.com/cobwebch/external_import
 *
 * @package TYPO3
 * @copyright Copyright and license information, please read the LICENSE.txt
 * @license http://www.gnu.org/licenses/gpl.html GNU General Public License, version 2 or later
 */
class ImageTransformation implements ImporterAwareInterface
{
    use ImporterAwareTrait;

    /**
     * @var string Used to return a dummy identifier in preview mode
     */
    static public $previewMessage = 'Preview mode. Image not handled, nor saved.';

    /**
     * @var ResourceFactory
     */
    protected $resourceFactory;

    /**
     * @var Folder[] Array of Folder objects
     */
    protected $storageFolders = [];

    public function __construct()
    {
        $this->resourceFactory = GeneralUtility::makeInstance(ResourceFactory::class);
    }

    /**
     * Gets multiple images from a string containing URI's and saves it into the given file storage and path.
     * This is adjusted for a special usage
     *      - where images from URL's are no longer available
     *      - multiple images are available for FAL field of a record
     *
     * The parameters array is expected to contain the following information:
     *
     *      - "storage": a combined FAL identifier to a folder (e.g. "1:imported_images")
     *      - "importSourcePath": Import source folder on host or any URL path like http://www.forumdfael.dk/regfoto/
     *      - "defaultExtension": a file extension in case it cannot be found in the URI (e.g. "jpg")
     *      - "referenceUid": the reference field like 'referenceUid' in general configuration
     *      - "updateTable": table name of record which gets created or updated
     *      - "updateField": field name of record which gets created or updated
     *
     * @param array $record The full record that is being transformed
     * @param string $index The index of the field to transform
     * @param array $parameters Additional parameters from the TCA
     * @return mixed Uid of the saved sys_file record (or a message in preview mode)
     * @throws \TYPO3\CMS\Core\Resource\Exception\ResourceDoesNotExistException
     * @throws \TYPO3\CMS\Core\Resource\Exception\ExistingTargetFileNameException
     */
    public function importAssetsFromCommaseparatedStringOfUrls(array $record, string $index, array $parameters)
    {
        // In preview mode, we don't want to handle or save images and just return a dummy string
        if ($this->importer->isPreview()) {
            return self::$previewMessage;
        }
        // Get urls from commaseparated string
        if (empty($record[$index])) {
            // If there's no value to handle, return null
            return null;
        } else {
            $urls = GeneralUtility::trimExplode(',', $record[$index]);
        }
        // Save images from urls and get file uids
        if (!empty($urls)) {
            $fileUids = [];
            foreach ($urls as $url) {
                $fileUid = $this->saveImageFromUri($url, $parameters, true);
                if ($fileUid) {
                    $fileUids[$fileUid] = $fileUid;
                }
            }
        }
        // Set file references for current record
        if (!empty($fileUids)) {
            $insertUids = [];
            $currentDBRecord = $this->getRecordBy($parameters['updateTable'], $parameters['referenceUid'], $record[$parameters['referenceUid']]);
            if (isset($currentDBRecord['uid']) && (int)$currentDBRecord['uid']) {
                $i = 1;
                foreach ($fileUids as $fileUid) {
                    if ((int)$fileUid) {
                        $insertUid = $this->setFileReference($fileUid, $currentDBRecord['uid'], $currentDBRecord['pid'], $i, $parameters);
                        if ($insertUid) {
                            $insertUids[] = $insertUid;
                        }
                        $i++;
                    }
                }
            }
        }
        // Set counter for current record assets
        if (!empty($insertUids)) {
            // Update foreign count reference
            $currentDBRecord = $this->getRecordBy($parameters['updateTable'], $parameters['referenceUid'], $record[$parameters['referenceUid']]);
            $countExist = (int)$currentDBRecord[$parameters['updateField']];
            $countInserts = (int)count($insertUids);
            $countNew = $countExist + $countInserts;
            $this->updateForeignCountReference($countNew, $parameters['updateTable'], $parameters['updateField'], $currentDBRecord['uid']);
        }
        // Leave this transformation without any value (avoid update) 
        return null;
    }

    /**
     * Gets multiple images from a HTML string containing URI's and saves it into the given file storage and path.
     *
     * The parameters array is expected to contain the following information:
     *
     *      - "storage": a combined FAL identifier to a folder (e.g. "1:imported_images")
     *      - "importSourcePath": Import source folder on host or any URL path like http://www.domain.tld/assets/
     *      - "defaultExtension": a file extension in case it cannot be found in the URI (e.g. "jpg")
     *      - "referenceUid": the reference field like 'referenceUid' in general configuration
     *      - "updateTable": table name of record which gets created or updated
     *      - "updateField": field name of record which gets created or updated
     *      - "downloadFileExtensions": comma separated list of file extension to download, if missing, all are downloaded
     *
     * @param array $record The full record that is being transformed
     * @param string $index The index of the field to transform
     * @param array $parameters Additional parameters from the TCA
     * @return mixed Uid of the saved sys_file record (or a message in preview mode)
     * @throws \TYPO3\CMS\Core\Resource\Exception\ResourceDoesNotExistException
     * @throws \TYPO3\CMS\Core\Resource\Exception\ExistingTargetFileNameException
     */
    public function importAssetsFromHtml(array $record, string $index, array $parameters)
    {
        // If there's no value to handle, return null
        if (empty($record[$index])) {
            return null;
        }
        // In preview mode, we don't want to handle or save images and just return a dummy string
        if ($this->importer->isPreview()) {
            return self::$previewMessage;
        }
        // Parse html
        $html = $record[$index];
        $doc = new \DOMDocument();
        $doc->loadHTML($html);
        $xpath = new \DOMXPath($doc);
        // Get HTML <img src="?" />
        $images = [];
        $imageDOMNodeList = $xpath->query('//img/@src');
        if (!empty($imageDOMNodeList)) {
            foreach($imageDOMNodeList as $imageDOMNode) {
                if ($imageDOMNode->value) {
                    $images[] = $imageDOMNode->value;
                }
            }
        }
        // Get HTML <a href="?" />
        $downloads = [];
        $downloadDOMNodeList = $xpath->query('//a/@href');
        if (!empty($downloadDOMNodeList)) {
            foreach($downloadDOMNodeList as $downloadDOMNode) {
                if ($downloadDOMNode->value) {
                    $download = $downloadDOMNode->value;
                    $downloadUrlParts = parse_url($download);
                    if (isset($downloadUrlParts['path'])) {
                        $downloadPathParts = pathinfo($downloadUrlParts['path']);
                        if (!empty($downloadPathParts['extension'])) {
                            if (isset($parameters['downloadFileExtensions'])) {
                                $downloadFileExtensions = GeneralUtility::trimExplode(',', $parameters['downloadFileExtensions']);
                                if (in_array($downloadPathParts['extension'],$downloadFileExtensions)) {
                                    $downloads[] = $downloadDOMNode->value;
                                }
                            }
                        }
                    }
                }
            }
        }
        // Collect all sources
        $sources = [];
        if (!empty($images)) {
            $sources = array_merge_recursive($sources, $images);
        }
        if (!empty($downloads)) {
            $sources = array_merge_recursive($sources, $downloads);
        }
        // Handle sources, insert sys_file_references
        if (!empty($sources)) {
            $insertedReferences = [];
            $sources = array_unique($sources);
            $i = 1;
            foreach ($sources as $src) {
                $urlParts = parse_url($src);
                if (isset($urlParts['scheme']) && isset($urlParts['host'])) {
                    // for path like src="https://domain.tld/assets/.."
                    if ($urlParts['host'] == 'www.domain.tld') {
                        $respectImportSourcePath = true;
                    } else {
                        $respectImportSourcePath = false;
                    }
                    $url = $src;
                } else {
                    // @todo - path like src="/assets/.."
                    $url = null;
                }
                // Save images from urls and get file uids
                if (!empty($url)) {
                    $fileUid = $this->saveImageFromUri($url, $parameters, $respectImportSourcePath);
                }
                // Set file references for current record
                if (!empty($fileUid)) {
                    $currentDBRecord = $this->getRecordBy($parameters['updateTable'], $parameters['referenceUid'],
                        $record[$parameters['referenceUid']]);
                    $insertUid = $this->setFileReference($fileUid, $currentDBRecord['uid'], $currentDBRecord['pid'],
                        $i, $parameters);
                    if ($insertUid) {
                        $insertedReferences[] = $insertUid;
                    }
                    $i++;
                }
            }
        }
        // Set counter for current record assets
        if (!empty($insertedReferences)) {
            // Update foreign count reference
            $currentDBRecord = $this->getRecordBy($parameters['updateTable'], $parameters['referenceUid'], $record[$parameters['referenceUid']]);
            $countExist = (int)$currentDBRecord[$parameters['updateField']];
            $countInserts = (int)count($insertedReferences);
            $countNew = $countExist + $countInserts;
            $this->updateForeignCountReference($countNew, $parameters['updateTable'], $parameters['updateField'], $currentDBRecord['uid']);
        }
        // Leave this transformation without any value (avoid update) 
        return null;
    }

    /**
     * Gets an image from a URI and saves it into the given file storage and path.
     * This is adjusted for a special usage, where images from URL's are no longer available
     *
     * The parameters array is expected to contain the following information:
     *
     *      - "storage": a combined FAL identifier to a folder (e.g. "1:imported_images")
     *      - "defaultExtension": a file extension in case it cannot be found in the URI (e.g. "jpg")
     *
     * @param string $url The full record that is being transformed
     * @param array $parameters Additional parameters from the TCA
     * @param array $respectImportSourcePath Force using importSourcePath configuration for any urls
     * @return mixed Uid of the saved sys_file record (or a message in preview mode)
     * @throws \TYPO3\CMS\Core\Resource\Exception\ResourceDoesNotExistException
     * @throws \TYPO3\CMS\Core\Resource\Exception\ExistingTargetFileNameException
     */
    protected function saveImageFromUri($url = '', array $parameters, $respectImportSourcePath = true)
    {
        // If there's no value to handle, return null
        if (empty($url)) {
            return null;
        }

        // In preview mode, we don't want to handle or save images and just return a dummy string
        if ($this->importer->isPreview()) {
            return self::$previewMessage;
        }

        // Throw an exception if storage is not defined
        if (empty($parameters['storage'])) {
            throw new \InvalidArgumentException(
                'No storage given for importing files',
                1602170223
            );
        }

        // Ensure the storage folder is loaded (we keep a local cache of folder objects for efficiency)
        if ($this->storageFolders[$parameters['storage']] === null) {
            $this->storageFolders[$parameters['storage']] = $this->resourceFactory->getFolderObjectFromCombinedIdentifier(
                $parameters['storage']
            );
        }
        // Assemble a file name
        $urlParts = parse_url($url);
        $pathParts = pathinfo($urlParts['path']);
        $fileName = $pathParts['basename'];
        if (empty($pathParts['extension'])) {
            if (isset($parameters['defaultExtension'])) {
                $fileName .= '.' . $parameters['defaultExtension'];
            } else {
                throw new \InvalidArgumentException(
                    sprintf(
                        'No extension could be found for imported file %s',
                        $url
                    ),
                    1602170422
                );
            }
        }

        $fileName = $this->storageFolders[$parameters['storage']]->getStorage()->sanitizeFileName(
            $fileName,
            $this->storageFolders[$parameters['storage']]
        );
        // Check if the file already exists
        if ($this->storageFolders[$parameters['storage']]->hasFile($fileName)) {
            $fileObject = $this->resourceFactory->getFileObjectFromCombinedIdentifier(
                $parameters['storage'] . '/' . $fileName
            );
            // If the file does not yet exist locally, grab it from the remote server and add it to predefined storage
        } else {
            $temporaryFile = GeneralUtility::tempnam('external_import_uploads');
            // Adjust import source if path given
            if (isset($parameters['importSourcePath']) && $respectImportSourcePath) {
                $fileUrl = $this->getBaseUri() . ltrim(rtrim($parameters['importSourcePath'], '/'), '/') . '/' . $fileName;
            } else {
                $fileUrl = $url;
            }
            $file = GeneralUtility::getUrl($fileUrl, 0, null, $report);
            // If the file could not be fetched, report and throw an exception
            if ($file === false) {
                // Avoid exeception, to import further
                return null;
                #$error = sprintf(
                #    'File %s could not be fetched.',
                #    $url
                #);
                #if (isset($report['message'])) {
                #    $error .= ' ' . sprintf(
                #            'Reason: %s (code: %s)',
                #            $report['message'],
                #            $report['error'] ?? 0
                #        );
                #}
                #throw new \TYPO3\CMS\Core\Resource\Exception\ResourceDoesNotExistException(
                #    $error,
                #    1613555057
                #);
            }
            GeneralUtility::writeFileToTypo3tempDir(
                $temporaryFile,
                $file
            );
            $fileObject = $this->storageFolders[$parameters['storage']]->addFile(
                $temporaryFile,
                $fileName
            );
        }
        // Return the file's ID
        return $fileObject->getUid();
    }

    /**
     * Get record by field and value
     *
     * @return array|null
     */
    protected function getRecordBy($tableName, $fieldName, $fieldvalue)
    {
        /** @var QueryBuilder $queryBuilder */
        $queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)
            ->getQueryBuilderForTable($tableName);
        // Remove all restrictions (ignore hidden, which is already set during imports)
        $queryBuilder
            ->getRestrictions()
            ->removeAll();
        return $queryBuilder
            ->select('*')
            ->from($tableName)
            ->where(
                $queryBuilder->expr()->eq($fieldName, $fieldvalue)
            )
            ->execute()
            ->fetch();
    }

    /**
     * Get baseUri
     *
     * @return string
     */
    protected function getBaseUri()
    {
        $baseUri = GeneralUtility::getIndpEnv('TYPO3_SITE_URL');
        if (substr($baseUri, -1) !== '/') {
            $baseUri .= '/';
        }
        return $baseUri;
    }

    /**
     * Set file reference for current record file
     *
     * @param int $uid_local
     * @param int $uid_foreign
     * @param int $pid
     * @param int $sorting
     * @param array $parameters Additional parameters from the TCA
     * @return int|null
     */
    protected function setFileReference($uid_local, $uid_foreign, $pid, $sorting, $parameters)
    {
        $context = GeneralUtility::makeInstance(Context::class);
        $tableName = 'sys_file_reference';
        $insertFields = [
            'pid' => (int)$pid,
            'crdate' => $GLOBALS['EXEC_TIME'],
            'tstamp' => $GLOBALS['EXEC_TIME'],
            'cruser_id' => $context->getAspect('backend.user')->get('id'),
            'uid_local' => (int)$uid_local,
            'uid_foreign' => (int)$uid_foreign,
            'tablenames' => $parameters['updateTable'],
            'fieldname' => $parameters['updateField'],
            'sorting_foreign' => (int)$sorting,
            'table_local' => 'sys_file'
        ];
        /** @var Connection $connection */
        $connection = GeneralUtility::makeInstance(ConnectionPool::class)
            ->getConnectionForTable($tableName);
        try {
            $connection->insert($tableName, $insertFields);
            $sql_insert_id = $connection->lastInsertId($tableName);
            return $sql_insert_id;
        } catch (\Doctrine\DBAL\DBALException $e) {
            #$insertErrorMessage = $e->getMessage();
        }
    }

    /**
     * Update count of foreign field reference for current record
     *
     * @param int $count
     * @param string $tableName
     * @param string $fieldName
     * @param int $recordUid
     * @return void
     */
    protected function updateForeignCountReference($count, $tableName, $fieldName, $recordUid)
    {
        $updateFields = [
            $fieldName => (int)$count
        ];
        /** @var Connection $connection */
        $connection = GeneralUtility::makeInstance(ConnectionPool::class)
            ->getConnectionForTable($tableName);
        try {
            $connection->update($tableName, $updateFields, ['uid' => (int)$recordUid]);
        } catch (\Doctrine\DBAL\DBALException $e) {
            #$updateErrorMessage = $e->getMessage();
        }
    }
}

Hi,
I can't get your Class to work. Here my TCA:

'external' => [
                '0' => [
                    'field' => 'image',
                    'multipleRows' => true,
                    'transformations' => [
                        10 => [
                            'userFunction' => [
                                'class' => \Vendor\Extension\ExternalImport\Transformation\ImageTransformation::class,
                                'method' => 'importAssetsFromCommaseparatedStringOfUrls',
                                'parameters' => [
                                    'storage' => '2:productImages',
                                    'importSourcePath' => 'http://domain.de/fileadmin/user_upload/',
                                    'referenceUid' => 'uid in import data',
                                    'updateTable' => 'field to import data',
                                    'updateField' => 'image',
                                    'defaultExtension' => 'jpg'
                                ]
                            ]
                        ]
                    ],
                    'children' => [
                        'table' => 'sys_file_reference',
                        'columns' => [
                            'uid_local' => [
                                'field' => 'image'
                            ],
                            'uid_foreign' => [
                                'field' => '__parent.id__'
                            ],
                            'title' => [
                                'field' => ''
                            ],
                            'tablenames' => [
                                'value' => 'my_table'
                            ],
                            'fieldname' => [
                                'value' => 'image'
                            ],
                            'table_local' => [
                                'value' => 'sys_file'
                            ]
                        ],
                        'controlColumnsForUpdate' => 'uid_local, uid_foreign, tablenames, fieldname, table_local',
                        'controlColumnsForDelete' => 'uid_foreign, tablenames, fieldname, table_local'
                    ]
                ]
            ] 

Where is the mistake?

Do you have copied this class over to your own extension

ext/your_extension/Classes/ExternalImport/Transformation/ImageTransformation.php

and renamed Vendor\Extension to your requirements, both in PHP class and TCA?

You can adjust Vendor/Extension and the Path as you like, but Vendor/Extension is important to match with your extension.

Hi, yes, I have access to your class.
In preview mode your script seems to work. The step StoreDataStep shows the splitted entries in sys_file_reference.
But after running the synchronisation, nothing happens ...

Then I would start to debug inbetween code to find out, what is going wrong and when.

It looks like your script doesn't work with the Chrildren configuration. In the "StoreStepData" preview, the comma-separated URLs are divided into the data to be imported.

Only '30' has comma-separated images ...

I try to find out why ...

Hi,
createNamedParameter() in function getRecordBy will do the Job for me. My referenceUid ist a string.

    /**
     * Get record by field and value
     *
     * @return array|null
     */
    protected function getRecordBy($tableName, $fieldName, $fieldvalue)
    {
        /** @var QueryBuilder $queryBuilder */
        $queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)
            ->getQueryBuilderForTable($tableName);
        // Remove all restrictions (ignore hidden, which is already set during imports)
        $queryBuilder
            ->getRestrictions()
            ->removeAll();
        return $queryBuilder
            ->select('*')
            ->from($tableName)
            ->where(
                $queryBuilder->expr()->eq($fieldName, $queryBuilder->createNamedParameter($fieldvalue))
            )
            ->execute()
            ->fetch();
    }

createNamedParameter () is always a good choice;)

And a chrildren configuration is not needed

btw: thanks for your script

Thx @stephangrass for your improvements. I just published like you, to support cobwebch/external_import