Drupal.org reroll patches with syntax change

phpcs.xml

<?xml version="1.0" encoding="UTF-8"?>
<ruleset name="drupal_core">
    <description>Default PHP CodeSniffer configuration for Drupal core.</description>
    <file>.</file>
    <arg name="extensions" value="engine,inc,install,module,php,profile,test,theme"/>

    <!--Exclude third party code.-->
    <exclude-pattern>./assets/vendor/*</exclude-pattern>

    <!--Exclude test files that are intentionally empty, or intentionally violate coding standards.-->
    <exclude-pattern>./modules/system/tests/fixtures/HtaccessTest</exclude-pattern>

    <rule ref="Generic.Arrays.DisallowLongArraySyntax" />
</ruleset>

update-array-syntax.php

<?php

/**
 * @file
 *
 * Script to use the Drupal.org REST API to download the latest patch files.
 *
 * composer require guzzlehttp/guzzle
 *
 * script location: ~/Sites/d8/scripts/
 * webroot: ~/Sites/d8/public/
 *
 * To run:
 * $ php scripts/update-array-syntax.php
 *
 * Reference:
 *  - https://www.drupal.org/drupalorg/docs/api
 *  - http://docs.guzzlephp.org/en/latest/quickstart.html
 */

use GuzzleHttp\Client;
use GuzzleHttp\Exception\RequestException;
use GuzzleHttp\Psr7;

require 'vendor/autoload.php';

// Where to save cache and patches.
$base_path = __DIR__;
$webroot_relative_to_script = dirname($base_path) . '/public';

// Create some folders to hold things.
$new_folders_messages = `mkdir -p {$base_path}/{patches,new-patches,good-new-patches,bad-new-patches}`;

$client = new Client([
  'base_uri' => 'https://www.drupal.org/api-d7/',
  'connect_timeout' => 120,
  'timeout'  => 120,
]);

$default_query = [
  // Project Issue nodes.
  'type' => 'project_issue',
  // Drupal Core.
  'field_project' => '3060',
  // Project Status.
  'field_issue_status' => 14,
  // Project version.
  'field_issue_version' => '8.3.x-dev',
  'page' => 0,
];

// A list of end point query params sorted in priority.
// @see https://www.drupal.org/node/2776975#comment-11969497
$patch_query_priority = [
  // RTBC first.
  ['field_issue_status' => 14] + $default_query,
  ['field_issue_version' => '8.4.x-dev', 'field_issue_status' => 14] + $default_query,
  // Needs review second.
  ['field_issue_status' => 8] + $default_query,
  ['field_issue_version' => '8.4.x-dev', 'field_issue_status' => 8] + $default_query,
];

// Cache the found patches because the REST endpoints are slow and 5xx'in.
$latest_patches_cache_file = $base_path . '/latest-patches.txt';
$latest_patches_cache = @file_get_contents($latest_patches_cache_file);
if ($latest_patches_cache) {
  $latest_patches = json_decode($latest_patches_cache, TRUE);
}
else {
  $latest_patches = [];
}

$latest_patches_by_branch = [];

$projects_to_collect_latest_patch = [];
// Do a 'light' query for the nids of all the issues we are going to work with.
// this is to avoid full node results for patches we already have.
foreach ($patch_query_priority as $query) {
  // Branch from version number.
  $branch = str_replace('-dev', '', $query['field_issue_version']);
  do {
    try {
      $response = $client->request('GET', 'node.json', [
        'query' => ['full' => 0] + $query,
      ]);
    }
    catch (RequestException $e) {
      print Psr7\str($e->getRequest());
      if ($e->hasResponse()) {
        print Psr7\str($e->getResponse());
      }
      break;
    }

    $query['page']++;
    $data = json_decode($response->getBody());
    foreach ($data->list as $node) {
      $nid = (string) $node->id;
      // If we don't have a patch for this issue in cache, add it to be
      // collected.
      if (!isset($latest_patches[$nid])) {
        $projects_to_collect_latest_patch[$nid] = $nid;
      }
      // Store the patches by branch.
      $latest_patches_by_branch[$branch][$nid] = $nid;
    }
  } while (isset($data->next));
}

foreach ($projects_to_collect_latest_patch as $nid) {
  // If it's in cache don't look up files again.
  if (isset($latest_patches[$nid])) {
    continue;
  }

  // Load information about the node.
  try {
    $response = $client->request('GET', 'node/' . $nid . '.json');
  }
  catch (RequestException $e) {
    print Psr7\str($e->getRequest());
    if ($e->hasResponse()) {
      print Psr7\str($e->getResponse());
    }
    break;
  }
  $node = json_decode($response->getBody());

  if (empty($node->field_issue_files)) {
    continue;
  }

  // Treat the issue files as an array.
  $files = (array) $node->field_issue_files;
  print "$node->nid\n";

  // Reverse search through the files for the last patch.
  $file = end($files);

//    $comments = (array) $node->comments;
//    var_dump($node->nid . ':' . count($comments));

  // Keep track of patches without patches.
  $latest_patches[$node->nid] = FALSE;

  do {
    try {
      // Get the file name and URL by asking for info about the file.
      // (Wish I didn't have to make an extra call for the name and URL).
      $file_response = $client->request('GET', 'file/' . $file->file->id . '.json');
    }
    catch (Exception $e) {
      print "Failed to get file info for node: $nid\n";
      // Unset to avoid caching failures.
      unset($latest_patches[$nid]);
      break;
    }
    $file_data = json_decode($file_response->getBody());

    if (preg_match('/^.*\.(patch|diff)$/i', $file_data->name)) {
      $latest_patches[$node->nid] = $file_data->url;
      break;
    }
  } while ($file = prev($files));
}

foreach ($latest_patches as $nid => $patch) {
  if (empty($patch)) {
    continue;
  }

  $file_path = $base_path . '/patches/' . basename($patch);
  // Don't down the file again.
  if (file_exists($file_path)) {
    $grep_message = `grep array\( {$file_path}`;
    if (!$grep_message) {
      // Don't keep track of the patches that don't have array('s in them.
      $latest_patches[$nid] = FALSE;
    }
    continue;
  }

  try {
    // Download the patch.
    $response = $client->request('GET', $patch, ['sink' => $file_path]);
    // Ensure the downloaded patch has an 'array(' in it to avoid unnessasary
    // processing.
    if (file_exists($file_path)) {
      $grep_message = `grep array\( {$file_path}`;
      if (!$grep_message) {
        // Don't keep track of the patches that don't have array('s in them.
        $latest_patches[$nid] = FALSE;
      }
    }
  }
  catch (Exception $e) {
    print "Failed to download patch: $patch\n";
    // Unset to avoid caching failures.
    unset($latest_patches[$nid]);
  }
}

// Store the collected latest patches from all pages into cache to avoid a few
// API calls on further calls.
$latest_patches_cache = json_encode($latest_patches, JSON_PRETTY_PRINT);
file_put_contents($latest_patches_cache_file, $latest_patches_cache);

// 2. Run the array-fixing script on all patches in the directory, saving the
// fixed patches to a new directory with a suffix.
// Move to the webroot.
chdir($webroot_relative_to_script);
print 'CWD:' . getcwd() . "\n";

// Ensure origin is up-to-date.
$message = `git fetch --all`;

foreach ($latest_patches_by_branch as $branch => $nodes) {
  switch ($branch) {
    case '8.3.x':
      $short_array_commit = 'bd446d0772465b0a2fbbcc7453dcfc655b2c9941';
      break;

    case '8.4.x':
      $short_array_commit = '52e3eec616efab4d5201e30d085a09ba5c4817e6';
      break;
  }
  // Checkout the branch we will be working with and sure it's matching the
  // origin.
  $reset_message = `git checkout --quiet -f {$branch} && git reset --hard origin/{$branch}`;
  foreach ($nodes as $nid) {
    // Skip fixing patches with FALSE because they have no long array syntax.
    if (empty($latest_patches[$nid])) {
      continue;
    }
    $patch_name = basename($latest_patches[$nid]);
    $patch_file = $base_path . '/patches/' . $patch_name;
    $new_patch_file = $base_path . '/new-patches/' . $patch_name;
    $good_new_patch_file = $base_path . '/good-new-patches/' . $patch_name;
    $bad_new_patch_file = $base_path . '/bad-new-patches/' . $patch_name;

    // Reset working directory for the next patch against HEAD.
    $reset_message = `git clean -fd && git reset --hard origin/{$branch}`;

    // See if patch needs to be fixed, apply against head.
    // @todo I think this may yield false skips.
    // $patch_message = `patch --silent -p1 < {$patch_file}`;
    // if (!$patch_message) {
    //   print "Patch applied against HEAD, skipping: $patch_name\n";
    //   continue;
    // }

    // Reset to pre short-array-syntax patch.
    $reset_message = `git clean -fd && git reset --hard {$short_array_commit}^`;
    // See if patch needs to be fixed, apply against head.
    $patch_message = `patch --silent -p1 < {$patch_file}`;
    if ($patch_message) {
      print "Patch doesn't apply before short array commit, skipping: $patch_name\n";
      continue;
    }

    // Get a list of files touched by the patch and run code sniffer against
    // them.
    $git_status_message = `git status --porcelain | sed s/^...//`;
    $paths = explode("\n", $git_status_message);
    foreach ($paths as $path) {
      // Don't check deleted paths.
      if (file_exists($path)) {
        $fix_message = `phpcbf --standard={$base_path}/phpcs.xml {$path}`;
      }
    }

    $git_commit_message = `git add -A . && git commit -m "re-rolled array-syntax"`;
    $spaced_paths = implode(' ', $paths);
    // Create patch against the short array commit to avoid unnecessary
    // conflicts.
    $git_diff_message = `git diff {$short_array_commit} -- {$spaced_paths} > {$new_patch_file}`;

    // Reset back to HEAD of the branch and see if the new patch applies.
    $reset_message = `git clean -fd && git reset --hard origin/{$branch}`;

    // If the apply passes, move the patch to a directory of safe patches;
    // if it fails; move it to a separate directory
    // of patches that need other reroll.
    $patch_message = `patch --silent -p1 < {$new_patch_file}`;
    if ($patch_message) {
      $cp_message = `cp {$new_patch_file} {$bad_new_patch_file}`;
      print "Patch doesn't apply against head, skipping: $patch_name\n";
    }
    else {
      $cp_message = `cp {$new_patch_file} {$good_new_patch_file}`;
      print "WOOT, patch applies! - $patch_name\n";
    }
  }
}