Migrating Drupal 7 File Entities to Drupal 8 Media Entities

FileEntity.php

<?php
// modules/custom/my_custom_module/src/Plugin/migrate/source/FileEntity.php
namespace Drupal\my_custom_module\Plugin\migrate\source;

use Drupal\Core\Database\Query\Condition;
use Drupal\migrate\Row;
use Drupal\migrate_drupal\Plugin\migrate\source\d7\FieldableEntity;

/**
 * Drupal 7 file_entity source from database.
 *
 * @MigrateSource(
 *   id = "file_entity",
 *   source_provider = "file"
 * )
 */
class FileEntity extends FieldableEntity {

  /**
   * {@inheritdoc}
   */
  public function query() {
    $query = $this->select('file_managed', 'f')
      ->fields('f')
      ->orderBy('f.fid');
    if (isset($this->configuration['type'])) {
      $query->condition('f.type', $this->configuration['type']);
    }
    // Filter by scheme(s), if configured.
    if (isset($this->configuration['scheme'])) {
      $schemes = array();
      // Accept either a single scheme, or a list.
      foreach ((array) $this->configuration['scheme'] as $scheme) {
        $schemes[] = rtrim($scheme) . '://';
      }
      $schemes = array_map([$this->getDatabase(), 'escapeLike'], $schemes);

      // The uri LIKE 'public://%' OR uri LIKE 'private://%'.
      $conditions = new Condition('OR');
      foreach ($schemes as $scheme) {
        $conditions->condition('uri', $scheme . '%', 'LIKE');
      }
      $query->condition($conditions);
    }

    return $query;
  }

  /**
   * {@inheritdoc}
   */
  public function prepareRow(Row $row) {
    // Get Field API field values.
    foreach (array_keys($this->getFields('file', $row->getSourceProperty('type'))) as $field) {
      $fid = $row->getSourceProperty('fid');
      $row->setSourceProperty($field, $this->getFieldValues('file', $field, $fid));
    }
    return parent::prepareRow($row);
  }

  /**
   * {@inheritdoc}
   */
  public function fields() {
    return array(
      'fid' => $this->t('File ID'),
      'uid' => $this->t('The {users}.uid who added the file. If set to 0, this file was added by an anonymous user.'),
      'filename' => $this->t('File name'),
      'uri' => $this->t('The URI to access the file'),
      'filemime' => $this->t('File MIME Type'),
      'status' => $this->t('The published status of a file.'),
      'timestamp' => $this->t('The time that the file was added.'),
      'type' => $this->t('The type of this file.'),
    );
  }

  /**
   * {@inheritdoc}
   */
  public function getIds() {
    $ids['fid']['type'] = 'integer';
    return $ids;
  }

}

my_files.yml

# modules/custom/my_custom_module/migrations/my_files.yml
id: my_files
label: Files
migration_tags:
  - Custom
source:
  plugin: d7_file
  constants:
    source_base_path: 'sites/default/files/'
    old_files_path: 'sites/default/files/migration-files'
process:
  filename: filename
  source_full_path:
    -
      plugin: concat
      delimiter: /
      source:
        - constants/old_files_path
        - filepath
    -
      plugin: urlencode
  uri:
    -
      plugin: skip_youtube_files
      source:
        - '@source_full_path'
        - uri
    -
      plugin: file_copy
  filemime: filemime
  # filesize is dynamically computed when file entities are saved, so there is
  # no point in migrating it.
  # filesize: filesize
  status: status
  # Drupal 7 didn't keep track of the file's creation or update time -- all it
  # had was the vague "timestamp" column. So we'll use it for both.
  created: timestamp
  changed: timestamp
  fid: fid

  uid:
    -
      plugin: skip_on_empty
      method: process
      source: uid
    -
      plugin: migration
      migration: my_users

destination:
  plugin: entity:file
  migration_dependencies:
    required:
      - my_users

my_media_audio.yml

# modules/custom/my_custom_module/migrations/my_media_audio.yml
id: my_media_audio
label: Media Audio
migration_tags:
  - Custom

source:
  plugin: file_entity
  type: audio
  constants:
    bundle: 'audio'

process:
  mid: fid
  bundle: 'constants/bundle'

  langcode:
    plugin: default_value
    source: language
    default_value: "und"

  name: filename

  uid:
    -
      plugin: skip_on_empty
      method: process
      source: uid
    -
      plugin: migration
      migration: my_users

  status: status

  # Drupal 7 didn't keep track of the file's creation or update time -- all it
  # had was the vague "timestamp" column. So we'll use it for both.
  created: timestamp
  changed: timestamp

  # File field see media_entity.bundle.audio.yml.
  field_media_audio/target_id: fid
  # Title field.
  field_title: field_title
  # Transcript field.
  field_transcript: field_transcript

destination:
  plugin: entity:media
  migration_dependencies:
    required:
      - my_files
      - my_users

To migrate media images, I moved the dedicated field entity fields for alt and title to the image field.
modules/custom/my_custom_module/migrations/my_media_audio.yml
id: my_media_image
label: Files
migration_tags:
  - Custom

source:
  plugin: file_entity
  type: image
  constants:
    bundle: 'image'

process:
  mid: fid
  bundle: 'constants/bundle'

  langcode:
    plugin: default_value
    source: language
    default_value: "und"

  name: filename

  uid:
    -
      plugin: skip_on_empty
      method: process
      source: uid
    -
      plugin: migration
      migration: my_users

  status: status

  # Drupal 7 didn't keep track of the file's creation or update time -- all it
  # had was the vague "timestamp" column. So we'll use it for both.
  created: timestamp
  changed: timestamp

  # Image field see media_entity.bundle.image.yml.
  field_media_image/target_id: fid
  field_media_image/alt: field_file_image_alt_text/0/value
  field_media_image/title: field_file_image_title_text/0/value

  # Description field.
  field_description: field_image_description
  # Caption field.
  field_caption: field_caption

destination:
  plugin: entity:media
  migration_dependencies:
    required:
      - my_files
      - my_users

my_media_local_video.yml

# modules/custom/my_custom_module/migrations/my_media_local_video.yml
id: my_media_local_video
label: Files
migration_tags:
  - Custom

source:
  plugin: file_entity
  type: video
  # See output of SELECT DISTINCT(SUBSTRING_INDEX(uri, ':', 1))  FROM file_managed WHERE type = 'video';
  scheme:
    - "public"
  constants:
    bundle: 'local_video'

process:
  mid: fid
  bundle: 'constants/bundle'

  langcode:
    plugin: default_value
    source: language
    default_value: "und"

  name: filename

  uid:
    -
      plugin: skip_on_empty
      method: process
      source: uid
    -
      plugin: migration
      migration: my_users

  status: status

  # Drupal 7 didn't keep track of the file's creation or update time -- all it
  # had was the vague "timestamp" column. So we'll use it for both.
  created: timestamp
  changed: timestamp

  # File field see media_entity.bundle.local_video.yml.
  field_media_video/target_id: fid
  # Title field.
  field_title: field_video_title
  # Transcript field.
  field_transcript: field_transcript

destination:
  plugin: entity:media
  migration_dependencies:
    required:
      - my_files
      - my_users

my_media_video.yml

# modules/custom/my_custom_module/migrations/my_media_video.yml
id: my_media_video
label: Files
migration_tags:
  - Custom

source:
  plugin: file_entity
  type: video
  # See output of SELECT DISTINCT(SUBSTRING_INDEX(uri, ':', 1))  FROM file_managed WHERE type = 'video';
  scheme:
    - "youtube"
  constants:
    bundle: 'video'

process:
  mid: fid
  bundle: 'constants/bundle'

  langcode:
    plugin: default_value
    source: language
    default_value: "und"

  name: filename

  uid:
    -
      plugin: skip_on_empty
      method: process
      source: uid
    -
      plugin: migration
      migration: my_users

  status: status

  # Drupal 7 didn't keep track of the file's creation or update time -- all it
  # had was the vague "timestamp" column. So we'll use it for both.
  created: timestamp
  changed: timestamp

  # Embed field see media_entity.bundle.video.yml.
  field_media_video_embed_field:
    plugin: youtube
    source: uri
  # Title field.
  field_title: field_video_title
  # Transcript field.
  field_transcript: field_transcript

destination:
  plugin: entity:media
  migration_dependencies:
    required:
      - my_files
      - my_users

SkipYoutubeVideos.php

<?php
// modules/custom/my_custom_module/src/Plugin/migrate/process/SkipYoutubeVideos.php
namespace Drupal\my_custom_module\Plugin\migrate\process;

use Drupal\migrate\MigrateExecutableInterface;
use Drupal\migrate\MigrateSkipRowException;
use Drupal\migrate\ProcessPluginBase;
use Drupal\migrate\Row;

/**
 * Skip youtube videos.
 *
 * @MigrateProcessPlugin(
 *   id = "skip_youtube_files"
 * )
 */
class SkipYoutubeVideos extends ProcessPluginBase {

  /**
   * {@inheritdoc}
   */
  public function transform($value, MigrateExecutableInterface $migrate_executable, Row $row, $destination_property) {
    if (parse_url(end($value), PHP_URL_SCHEME) == 'youtube') {
      throw new MigrateSkipRowException();
    }
    return $value;
  }

}

Youtube.php

<?php
// modules/custom/my_custom_module/src/Plugin/migrate/process/Youtube.php
namespace Drupal\my_module\Plugin\migrate\process;

use Drupal\migrate\MigrateExecutableInterface;
use Drupal\migrate\ProcessPluginBase;
use Drupal\migrate\Row;

/**
 * Custom process plugin to convert youtube scheme uri to video url.
 *
 * @MigrateProcessPlugin(
 *   id = "youtube"
 * )
 */
class Youtube extends ProcessPluginBase {

  const SCHEME = 'youtube://';
  const BASE_URL = 'http://youtube.com/watch?';

  /**
   * {@inheritdoc}
   */
  public function transform($value, MigrateExecutableInterface $migrate_executable, Row $row, $destination_property) {
    // Convert youtube scheme uri to video url.
    if (strpos($value, static::SCHEME) !== FALSE) {
      $value = static::BASE_URL . implode('=', explode('/', str_replace(static::SCHEME, '', $value), 2));
    }
    else {
      $value = NULL;
    }
    return $value;
  }

}

Thank you @Ellen-Nunes for trying the solution. I'm glad it worked for you.

The issue seems to be unrelated to the above code. I can't say for certain as I don't have all the information but Sep 18, 2017 seems very specific. I'd look to answer the following question:

• Do you have some highwater mark setup in migrations?
• Do you have track_changes setup in migration?
• Is your source plugin adding a date/status filter to the query?
• If you are using multiple schemes then there might be a bug around https://gist.github.com/jibran/8e7cd2319e873858dd49a272227a4fd2#file-fileentity-php-L39-L43. I'd check the source plugin query.
• Is processed row count is less than the actual row count provided by the source plugin? If yes then please check the migration message table for this migration for the specific error.
• The source ID getting ignored can be run individually using drush mim my_migration --idlist=1,2,3, provided by the https://www.drupal.org/project/migrate_tools, to make sure it is getting processed and if it processed then please try to debug that by putting the breakpoint in \Drupal\migrate\MigrateExecutable::processRow.

Thank you for the quick reply! No highwater, track_changes, or date/status filter. The entity:file migration total and imported counts are both about 6,000 and include ALL files (those uploaded before and after Sept 18, 2017). The entity:media migration total and imported counts are exactly the same (3748) and do not include any files uploaded after Sept 18, 2017.

It's as if the 2,000 or so image files uploaded after Sep 18, 2017 are not being "seen" by the migration as files with type=image. So it does seem to be a problem with the scheme or file type.

After digging I found our Drupal 7 source site had an outdated beta version of the File Entity module and several Media modules installed, but disabled. Uninstalling made things worse, so I instead updated File Entity, then uploaded one new image. Migrate-status now shows one more in its total (3749). But, that still leaves the problem of how to fix all the image files uploaded in the past 2 years!

Would it be possible to change the plugin to use mimetype or "all" or something else instead of "type" which appears to be corrupted for our site?

Sorry for my ignorance, I have few questions:

1. Is this module works for moving Drupal 7 file entities to drupal 8 media entities?
2. Do I have to copy over the drupal 7 files in drupal 8 or migration will pick up the location by itself?
3. What commands needs to be run fater the module has been installed
   I tried drush migrate-import my_files but that didn't work

I am really new at migration, any help will be appreciated. Thanks

Doesn't the Media Migration module (https://www.drupal.org/project/media_migration) make possible the automatic generation of this migration using the command drush migrate:upgrade?