Import customers into Magento from command line using a CSV file

Magento Customer Importer

<?php

define('BUNCH_SIZE_TO_INSERT', 20);

$magentoRootPath = './public';
require_once $magentoRootPath . '/shell/abstract.php';

class CustomerImportFromCsv extends Mage_Shell_Abstract
{
    protected $_headerArray = array();
    protected $_attributes = array();
    protected $_attributeRows = array();
    protected $_customFieldNamesEqualsTo;
    protected $_nextEntityId;
    protected $_entityTypeId;
    protected $_passId;
    protected $_passTable;
    protected $_connection;
    protected $_resource;
    protected $_table;
    protected $_indexValueAttributes = array('group_id');

    protected $_defaultValuesForBuiltInImport = array(
        '_website' => 'uk',
        'group_id' => 1,
        'firstname' => ' ',
        'middlename' => ' ',
        'lastname' => ' ',
        'email' => '[email protected]',
        'reward_update_notification' => '1',
        'reward_warning_notification' => '1',
    );

    protected $_defaultValuesForCustomImport = array(
        'group_id' => '1',
        'store_id' => '1',
        'entity_type_id' => '1',
        'attribute_set_id' => '0',
        'website_id' => '1',
        'is_active' => '1',
        'email' => '[email protected]',
    );

    public function __construct()
    {
        parent::__construct();

        Mage::setIsDeveloperMode(true);
        ini_set('display_errors', 1);
        echo "\n";

        $entityType             = Mage::getSingleton('eav/config')->getEntityType('customer');
        $this->_entityTypeId    = $entityType->getEntityTypeId();
        $this->_connection      = Mage::getSingleton('core/resource')->getConnection('write');
        $this->_resource        = Mage::getModel('customer/customer');
        $this->_table           = $this->_resource->getResource()->getEntityTable();
        
        $this->_initAttributes();

        if ($this->getArg('h') || $this->getArg('help') || count($this->_args) == 0) {
            echo $this->usageHelp();
            return 1;
        }

        $this->behavior = ($this->getArg('b') ? $this->getArg('b') : ($this->getArg('behavior') ? $this->getArg('behavior') : 'replace'));
        $this->behavior === true ? $this->behavior = 'replace': '';
        $this->_customFieldNamesEqualsTo = $this->getFieldMappings();
    }

    public function run()
    {
        if ($filename = $this->getCustomersSourceFile()) {
            $files = $this->splitFile($filename, ($this->getArg('l') ? $this->getArg('l') : ($this->getArg('linecount') ? $this->getArg('linecount') : true)));    

            echo "Start customers '$this->behavior' action from $filename\n";
            Mage::log("Start customers '$this->behavior' action from $filename", Zend_Log::DEBUG);

            $count = 0;
            $files_ = array();
            foreach ($files as $file) {
                if (empty($this->_customFieldNamesEqualsTo)) {
                    $count += $this->_importWithBuiltInFunctionality($file);
                } else {
                    $count += $this->_importFromCustomFormattedCsv($file);
                }
            }
            
            $this->unlinkFiles($files);
            $this->unlinkFiles($files_);
            echo "\nDone (processed rows count: " . $count . ")\n";
        }
    }

    protected function _initAttributes()
    {
        $collection = Mage::getResourceModel('customer/attribute_collection')->addSystemHiddenFilterWithPasswordHash();
        foreach ($collection as $attribute) {
            $this->_attributes[$attribute->getAttributeCode()] = array(
                'id'          => $attribute->getId(),
                'is_required' => $attribute->getIsRequired(),
                'is_static'   => $attribute->isStatic(),
                'rules'       => $attribute->getValidateRules() ? unserialize($attribute->getValidateRules()) : null,
                'type'        => Mage_ImportExport_Model_Import::getAttributeType($attribute),
                'options'     => $this->getAttributeOptions($attribute)
            );
        }
        return $this;
    }

    public function getCustomersSourceFile()
    {
        $option1 = $this->getArg('f');
        $option2 = $this->getArg('filename');
        if ($option1 || $option2) {
            if ($option1 && $option1 != 1) {
                if (file_exists($option1)) {
                    return $option1;
                } else {
                    echo "Skipping customers source file $option1\n";
                }
            }
            if ($option2 && $option2 != 1) {
                if (file_exists($option2)) {
                    return $option2;
                } else {
                    echo "Skipping customers source file $option2\n";
                }
            }
            if ($default = 'customers.csv') {
                if (file_exists($default)) {
                    return $default;
                } else {
                    echo "Skipping default customers source file $default\n";
                }
            } else {
                echo "No default customers source file found in config.xml\n";
            }
        }
    }

    public function getFieldMappings()
    {
        $mappings = array();
        $option1 = $this->getArg('m');
        $option2 = $this->getArg('map');

        if ($option1 || $option2) {
            if ($option1 && $option1 != 1) {
                parse_str(str_replace(array(',', ' '), array('&', ''), trim($option1)), $mappings);
            }
            if ($option2 && $option2 != 1) {
                parse_str(str_replace(array(',', ' '), array('&', ''), trim($option2)), $mappings);
            }
        }
        return $mappings;
    }

    public function initCustomersImportModel()
    {
        $customersimport = Mage::getModel('importexport/import');
        $customersimport->setEntity('customer');
        $customersimport->setBehavior($this->behavior);
        
        return $customersimport;
    }

    public function usageHelp()
    {
        $tmpFolder = substr(
            Mage::getConfig()->getOptions()->getTmpDir(),
            strlen(Mage::getConfig()->getOptions()->getBaseDir()) + 1
        );
        
        return <<<HELP
SYNOPSIS
    php -f customerImportFromCsv.php
    php -f customerImportFromCsv.php [-- [OPTIONS...]]

DESCRIPTION
    This can import customers from CSV file.
    
    Imports:
            file.csv: multiple-row csv files, supports multiselect fields
        Multiselect fields:
            Use commas to separate values, but make sure not to put asterisks in column header, ie:
                sku ,*_category      ,color
                TEST,"CAT1/CAT2,CAT3","red,yellow"

OPTIONS
    -h
    -help
                            print this usage and exit

    -b
    -behavior
                            set import behavior for csv file: append|delete|replace (default)

    -f [file]
    -filename [file]
                            import customers from csv file, if source file not specified or not found uses defined in config.xml

    -l <n>
    -linecount <n>
                            split import file into pieces of n lines length ($tmpFolder must be writable)

    -m
    -map
                            set custom fields' mapping when you use custom formatted CSV file

EXAMPLES
                            php -f customerImportFromCsv.php -- -f customers.csv
                            php -f customerImportFromCsv.php -- -f customers.csv -m "u_email_address=email, u_first_name=firstname, u_last_name=lastname"

HELP;
    }

    public function splitFile($filename, $linecount)
    {
        $extension = pathinfo($filename, PATHINFO_EXTENSION);
        $count = 0;
        $i = 0;
        $j = 0;
        $files = array();
        $h = fopen($filename, 'r');     
        $header = fgets($h);

        $defaultValueIndex = array();
        $this->_headerArray = explode(',', trim($header));

        foreach (array_keys($this->_defaultValuesForBuiltInImport) as $requiredField) {
            if (FALSE !== $indexInHeader = array_search($requiredField, $this->_headerArray)) {
                $defaultValueIndex[$requiredField] = $indexInHeader;
            } else {
                $defaultValueIndex[$requiredField] = count($this->_headerArray);
                $this->_headerArray[] = $requiredField;
            }
        }
        $header = implode(',', $this->_headerArray) . "\n";

        while ($line = fgets($h)) {
            // first iteration / no "attr-only" line / line count has been reached
            if ($i == 0 || (substr($line, 0 ,1) != ',' && substr($line, 0 ,2) != '""' && substr($line, 0 ,2) != "''" && $i >= $linecount && !($linecount === true))) {
                if (defined('t')) {
                    fclose($t);
                }
                $files[] = Mage::getConfig()->getOptions()->getTmpDir() . DS . pathinfo($filename, PATHINFO_FILENAME) . '.tmp.' . $j . '.' . $extension;
                $t = fopen($files[$j++], 'w');
                fwrite($t, $header);
                $i = 0;
            }
            $line = $this->_convertCsvLineToValid($line, $this->_headerArray) . "\n";
            fwrite($t, $line);
            $count++;
            $i++;
        }
        fclose($h);
        if (defined('t')) {
            fclose($t);
        }
        
        return $files;
    }

    public function unlinkFiles($files)
    {
        foreach ($files as $file) {
            unlink($file);
        }
    }

    private function _convertCsvLineToValid($line, $headerArray)
    {
        $lineArray = explode(',', trim($line));
        foreach ($this->_defaultValuesForBuiltInImport as $requiredField => $defaultValue) {
            $requiredFieldIndex = array_search($requiredField, $headerArray);
            if (empty($lineArray[$requiredFieldIndex])) {
                $lineArray[$requiredFieldIndex] = $defaultValue;
            }
        }
        return implode(',', $lineArray);
    }

    private function _importWithBuiltInFunctionality($file)
    {
        $count = 0;
        $customersimport = $this->initCustomersImportModel();
        $validationResult = $customersimport->validateSource($file);
        $processedRowsCount = $customersimport->getProcessedRowsCount();
        
        if ($processedRowsCount > 0) {
            // get only one field each time to prevent disruption because some line errors
            while (!$validationResult) {
                echo "\n";
                $errorLines = array();
                foreach ($customersimport->getErrors() as $type => $lines) {
                    echo ":::: " . $type . " in line(s) " . implode(", ", $lines) . " ::::\n";
                    $errorLines = array_merge($errorLines, $lines);
                }
                echo "\n";
                $file_ = str_replace('tmp.', 'tmp._', $file);
                $files_[] = $file_;
                $h = fopen($file, 'r');
                $t = fopen($file_, 'w');
                $i = 0;
                $error = false;
                while ($line = fgets($h)) {
                    if ((substr($line, 0 ,1) != ',' && substr($line, 0 ,2) != '""' && substr($line, 0 ,2) != "''")) {
                        $error = false;
                    }
                    if (!$error) {
                        if (!in_array($i, $errorLines)) {
                            fwrite($t, $line);
                        } else {
                            echo ":: Line " . $i . " :: " . $line;
                            $error = true;
                        }
                    }
                    $i++;
                }
                fclose($h);
                fclose($t);
                
                $customersimport = $this->initCustomersImportModel();
                $validationResult = $customersimport->validateSource($file_);
                $processedRowsCount = $customersimport->getProcessedRowsCount();
                $file = $file_;
            }
        
            $count += $processedRowsCount;
            $customersimport->importSource();
        }

        return $count;
    }

    private function _importFromCustomFormattedCsv($filePath)
    {
        $count = 0;
        $entityRows = $this->_attributeRows = array();
        $this->_nextEntityId   = Mage::getResourceHelper('importexport')->getNextAutoincrement($this->_table);
        $this->_passId         = $this->_resource->getAttribute('password_hash')->getId();
        $this->_passTable      = $this->_resource->getAttribute('password_hash')->getBackend()->getTable();

        $file = fopen($filePath, 'r');
        $header = fgets($file);
        $error = false;
        while ($line = fgets($file)) {
            if ((substr($line, 0 ,1) != ',' && substr($line, 0 ,2) != '""' && substr($line, 0 ,2) != "''")) {
                $error = false;
            }
            if (!$error) {
                $count++;
                $entityRows[] = $this->_createEntityArrayFromKnownFields($line);
                $this->_loadAttributesFromKnownFields($line);
            }

            if ($count > 0 && $count % BUNCH_SIZE_TO_INSERT == 0) {
                $this->_connection->insertMultiple($this->_table, $entityRows);
                $this->_saveCustomerAttributes($this->_attributeRows);
                $entityRows = array();
                $this->_attributeRows = array();
            }
        }
        fclose($file);

        if ($count > 0 && ! empty($entityRows)) {
            $this->_connection->insertMultiple($this->_table, $entityRows);
            $this->_saveCustomerAttributes($this->_attributeRows);
        }
        
        return $count;
    }

    protected function _saveCustomerAttributes(array $attributesData)
    {
        foreach ($attributesData as $tableName => $data) {
            $tableData = array();

            foreach ($data as $customerId => $attrData) {
                foreach ($attrData as $attributeId => $value) {
                    $tableData[] = array(
                        'entity_id'      => $customerId,
                        'entity_type_id' => $this->_entityTypeId,
                        'attribute_id'   => $attributeId,
                        'value'          => $value
                    );
                }
            }
            $this->_connection->insertOnDuplicate($tableName, $tableData, array('value'));
        }
        return $this;
    }

    private function _createEntityArrayFromKnownFields($line)
    {
        $row = $tmpRow = array();
        $lineArray = explode(',', trim($line));
        foreach ($this->_customFieldNamesEqualsTo as $customFieldName => $correspondingFieldName) {
            $customFieldIndex = array_search($customFieldName, $this->_headerArray);
            $tmpRow[$correspondingFieldName] = $lineArray[$customFieldIndex];
        }

        foreach ($this->_defaultValuesForCustomImport as $fieldName => $defaultValue) {
            $row[$fieldName] = (empty($tmpRow[$fieldName]) || $tmpRow[$fieldName] === "NULL")
                                ? $defaultValue
                                : $tmpRow[$fieldName];
        }

        return $row;
    }

    private function _loadAttributesFromKnownFields($line)
    {
        $row = $tmpRow = array();
        $entityId = $this->_nextEntityId++;
        $lineArray = explode(',', trim($line));
        foreach ($this->_customFieldNamesEqualsTo as $customFieldName => $correspondingFieldName) {
            $customFieldIndex = array_search($customFieldName, $this->_headerArray);
            $tmpRow[$correspondingFieldName] = $lineArray[$customFieldIndex];
        }

        foreach (array_merge($this->_headerArray, array_keys($tmpRow)) as $index => $fieldName) {
            if (empty($lineArray[$index])) {
                $lineArray[$index] = '';
            }
            $row[$fieldName] = empty($tmpRow[$fieldName]) ? $lineArray[$index] : $tmpRow[$fieldName];
            if ($row[$fieldName] == 'NULL' && array_key_exists($fieldName, $this->_defaultValuesForBuiltInImport)) {
                $row[$fieldName] = $this->_defaultValuesForBuiltInImport[$fieldName];
            }
        }

        // attribute values
        foreach (array_intersect_key($row, $this->_attributes) as $attrCode => $value) {
            if (!$this->_attributes[$attrCode]['is_static'] && strlen($value)) {
                /** @var $attribute Mage_Customer_Model_Attribute */
                $attribute  = $this->_resource->getAttribute($attrCode);
                $backModel  = $attribute->getBackendModel();
                $attrParams = $this->_attributes[$attrCode];

                if ('select' == $attrParams['type']) {
                    $value = $attrParams['options'][strtolower($value)];
                } elseif ('datetime' == $attrParams['type']) {
                    $value = gmstrftime($strftimeFormat, strtotime($value));
                } elseif ($backModel) {
                    $attribute->getBackend()->beforeSave($this->_resource->setData($attrCode, $value));
                    $value = $this->_resource->getData($attrCode);
                }
                $this->_attributeRows[$attribute->getBackend()->getTable()][$entityId][$attrParams['id']] = $value;

                // restore 'backend_model' to avoid default setting
                $attribute->setBackendModel($backModel);
            }
        }
        // password change/set
        if (isset($row['password']) && strlen($row['password'])) {
            $this->_attributeRows[$this->_passTable][$entityId][$this->_passId] = $this->_resource->hashPassword($row['password']);
        }
    }

    public function getAttributeOptions(Mage_Eav_Model_Entity_Attribute_Abstract $attribute, $indexValAttrs = array())
    {
        $options = array();

        if ($attribute->usesSource()) {
            // merge global entity index value attributes
            $indexValAttrs = array_merge($indexValAttrs, $this->_indexValueAttributes);

            // should attribute has index (option value) instead of a label?
            $index = in_array($attribute->getAttributeCode(), $indexValAttrs) ? 'value' : 'label';

            // only default (admin) store values used
            $attribute->setStoreId(Mage_Catalog_Model_Abstract::DEFAULT_STORE_ID);

            try {
                foreach ($attribute->getSource()->getAllOptions(false) as $option) {
                    $value = is_array($option['value']) ? $option['value'] : array($option);
                    foreach ($value as $innerOption) {
                        if (strlen($innerOption['value'])) { // skip ' -- Please Select -- ' option
                            $options[strtolower($innerOption[$index])] = $innerOption['value'];
                        }
                    }
                }
            } catch (Exception $e) {
                // ignore exceptions connected with source models
            }
        }
        return $options;
    }
}

$main = new CustomerImportFromCsv();
$main->run();