Experimenting with a Hard Webpack Cache

hard-webpack-cache.js

{
  apply: function(compiler) {
    var times = {};
    var cache = {};
    var moduleCache = {};
    try {
      cache = JSON.parse(require('fs').readFileSync(__dirname + '/webpack.resolve.cache.json', 'utf8'));
      require('fs').readdirSync(__dirname + '/webpack.module.cache').forEach(function(name) {
        let subcache = JSON.parse(require('fs').readFileSync(__dirname + '/webpack.module.cache/' + name, 'utf8'));
        let key = Object.keys(subcache)[0];
        moduleCache[key] = subcache[key];
        // console.log(moduleCache[key]);
      });
      // moduleCache = JSON.parse(require('fs').readFileSync(__dirname + '/webpack.module.cache.json', 'utf8'));
    }
    catch (e) {}
    // console.log(Object.keys(moduleCache).length);
    // process.exit(0);
    var ConstDependency = require('webpack/lib/dependencies/ConstDependency');
    var NullFactory = require('webpack/lib/NullFactory');
    var NullDependency = require('webpack/lib/dependencies/NullDependency');
    var NullDependencyTemplate = require('webpack/lib/dependencies/NullDependencyTemplate');
    var ModuleDependency = require('webpack/lib/dependencies/ModuleDependency');
    var RawModule = require('webpack/lib/RawModule');
    var RawSource = require('webpack/lib/RawSource');
    var ContextDependency = require('webpack/lib/dependencies/ContextDependency');
    function CacheModule(sourceStr, identifier) {
      RawModule.call(this, sourceStr, identifier);
      
    }
    CacheModule.prototype = Object.create(RawModule.prototype);
    CacheModule.prototype.constructor = CacheModule;
    function needRebuild(buildTimestamp, fileDependencies, contextDependencies, fileTimestamps, contextTimestamps) {
      var timestamp = 0;
      fileDependencies.forEach(function(file) {
        var ts = fileTimestamps[file];
        if(!ts) timestamp = Infinity;
        if(ts > timestamp) timestamp = ts;
      });
      contextDependencies.forEach(function(context) {
        var ts = contextTimestamps[context];
        if(!ts) timestamp = Infinity;
        if(ts > timestamp) timestamp = ts;
      });
      return timestamp >= buildTimestamp;
    }
    CacheModule.prototype.needRebuild = function(fileTimestamps, contextTimestamps) {
      return needRebuild(this.buildTimestamp, this.fileDependencies, this.contextDependencies, fileTimestamps, contextTimestamps);
    };
    function RawModuleDependency(request) {
      ModuleDependency.call(this, request);
    }
    RawModuleDependency.prototype = Object.create(ModuleDependency.prototype);
    RawModuleDependency.prototype.constructor = RawModuleDependency;
    function RawModuleDependencyTemplate() {
    }
    RawModuleDependencyTemplate.prototype.apply = function() {};
    RawModuleDependencyTemplate.prototype.applyAsTemplateArgument = function() {};
    var fileTimestamps = {};
    compiler.plugin(['watch-run', 'run'], function(compiler, cb) {
      // console.log('run', Boolean(moduleCache));
      compiler.recordsInputPath = compiler.recordsOutputPath = __dirname + '/webpack.records.json';

      // console.log(moduleCache);
      // console.log(moduleCache.fileDependencies);
      if(!moduleCache.fileDependencies) return cb();
      var fs = compiler.inputFileSystem;
      var fileTs = compiler.fileTimestamps = fileTimestamps = {};
      // console.log(moduleCache.fileDependencies);
      require('async').forEach(moduleCache.fileDependencies, function(file, callback) {
        require('fs').stat(file, function(err, stat) {
          if(err) {
            if(err.code === "ENOENT") return callback();
            return callback(err);
          }

          fileTs[file] = stat.mtime || Infinity;
          callback();
        });
      }, cb);
    });
    compiler.plugin('compilation', function(compilation, params) {
      compilation.fileTimestamps = fileTimestamps;

      compilation.dependencyFactories.set(RawModuleDependency, params.normalModuleFactory);
      compilation.dependencyTemplates.set(RawModuleDependency, new RawModuleDependencyTemplate);

      compilation.dependencyFactories.set(ContextDependency, params.contextModuleFactory);
      compilation.dependencyTemplates.set(ContextDependency, new RawModuleDependencyTemplate);

      compilation.dependencyFactories.set(NullDependency, new NullFactory());
      compilation.dependencyTemplates.set(NullDependency, new NullDependencyTemplate);

      params.normalModuleFactory.plugin('resolver', function(fn) {
        return function(request, cb) {
          // console.log(compilation);
          // process.exit(0);
          // console.log(request);
          // process.exit(0);
          let cacheId = JSON.stringify([request.context, request.request]);
          if (cache[cacheId]) {
            console.log('resolve cache hit', cache[cacheId].request);
            let result = cache[cacheId];
            result.parser = compilation.compiler.parser;
            if (moduleCache[result.request]) {
              let cacheItem = moduleCache[result.request];
              // console.log(cacheId, cacheItem.buildTimestamp, fileTimestamps);
              if (!needRebuild(cacheItem.buildTimestamp, cacheItem.fileDependencies, [], fileTimestamps, compiler.contextTimestamps)) {
                console.log('module cache hit', result.request);
                let module = new CacheModule(cacheItem.source, cacheItem.identifier, cacheItem.identifier);
                module.context = cacheItem.context;
                module.request = cacheItem.request;
                module.assets = Object.keys(cacheItem.assets).reduce(function(carry, key) {
                  carry[key] = new RawSource(cacheItem.assets[key]);
                  return carry;
                }, {});
                module.buildTimestamp = cacheItem.buildTimestamp;
                module.fileDependencies = cacheItem.fileDependencies;
                module.contextDependencies = [];
                module.dependencies = cacheItem.dependencies.map(function(req) {
                  if (req.contextDependency) {
                    return new ContextDependency(req.request, req.recursive, req.regExp ? new RegExp(req.regExp) : null);
                  }
                  if (req.constDependency) {
                    return new NullDependency();
                  }
                  return new RawModuleDependency(req.request);
                });
                return cb(null, module);
              }
              // console.log('rebuild', cacheId);
            }
            else {
              console.log('module cache miss', result.request);
            }
            return cb(null, result);
          }
          console.log('cache miss', cacheId);
          // console.log(request.request, Date.now());
          times[request.request] = [Date.now()];
          // console.log('resolver', arguments);
          var originalRequest = request;
          return fn.call(null, request, function(err, request) {
            // console.log(arguments);
            if (err) {
              return cb(err);
            }
            if (request.source) {
              delete times[originalRequest.request];
            }
            else {
              // console.log(request);
              cache[JSON.stringify([request.context, request.rawRequest])] = request;
              times[request.rawRequest][1] = Date.now();
            }
            cb.apply(null, arguments);
          });
        };
      });
      params.normalModuleFactory.plugin('after-resolve', function(request, cb) {
        // console.log(request.rawRequest, Date.now() - times[request.rawRequest][0], Date.now());
        // times[request.rawRequest][1] = Date.now();
        // console.log('after-resolve', request);
        cb(null, request);
      });
    });
    compiler.plugin('after-compile', function(compilation, cb) {
      var startCacheTime = Date.now();
      require('fs').writeFileSync(__dirname + '/webpack.resolve.cache.json', JSON.stringify(cache), 'utf8');
      console.log(Object.keys(times).reduce(function(carry, key) {
        let t = times[key][1] - times[key][0];
        return carry + (isNaN(t) ? 0 : t);
      }, 0));
      var min = Object.keys(times).reduce(function(carry, key) {
        let t = times[key][0];
        return t < carry ? t : carry;
      }, Infinity);
      var max = Object.keys(times).reduce(function(carry, key) {
        let t = times[key][1];
        return t > carry ? t : carry;
      }, 0);
      console.log(min, max, max - min);
      // console.log(stats.endTime - stats.startTime);
      // console.log(stats.compilation.modules.reduce(function(carry, module) {
      //   carry.factory += module.profile.factory;
      //   carry.building += module.profile.building;
      //   carry.dependencies += module.profile.building;
      //   return carry;
      // }, {factory: 0, building: 0, dependencies: 0,}));
      // console.log(stats.compilation.modules.map(function(module) {
      //   return module.dependencies;
      // }));
      compilation.modules.forEach(function(module) {
        if (module.request && module.cacheable && !(module instanceof CacheModule)) {
          // console.log('caching', module.request, module.userRequest, module.rawRequest);
          // if (module.blocks.length) {
          //   console.log(module.blocks);
          //   process.exit(0);
          // }
          // if (module.variables.length) {
          //   console.log(module.variables.map(function(variable) {return variable.dependencies}));
          //   process.exit(0);
          // }
          moduleCache[module.request] = {
            moduleId: module.id,
            context: module.context,
            request: module.request,
            identifier: module.identifier(),
            assets: Object.keys(module.assets || {}).reduce(function(carry, key) {
              carry[key] = module.assets[key].source();
              return carry;
            }, {}),
            buildTimestamp: module.buildTimestamp,
            // source: stats.compilation.moduleTemplate.render(module, stats.compilation.dependencyTemplates, module.chunk).source(),
            source: module.source(compilation.dependencyTemplates, compilation.moduleTemplate.outputOptions, compilation.moduleTemplate.requestShortener).source(),
            dependencies: module.dependencies
            .concat([].concat(module.variables.reduce(function(carry, variable) {carry.push.apply(carry, variable.dependencies); return carry;}, [])))
            .map(function(dep) {return {contextDependency: dep instanceof ContextDependency, constDependency: dep instanceof ConstDependency, request: dep.request, recursive: dep.recursive, regExp: dep.regExp ? dep.regExp.source : null};})
            .filter(function(req) {return req.request || req.constDependency;}),
            fileDependencies: module.fileDependencies,
          };
          // console.log(module.dependencies
          //   .concat([].concat(module.variables.reduce(function(carry, variable) {carry.push.apply(carry, variable.dependencies); return carry;}, []))));
          // console.log(module.dependencies
//               .concat([].concat(module.variables.reduce(function(carry, variable) {carry.push.apply(carry, variable.dependencies); return carry;}, [])))
//               // .map(function(dep) {return {contextDependency: dep instanceof ContextDependency, constDependency: dep instanceof ConstDependency, request: dep.request, recursive: dep.recursive, regExp: dep.regExp ? dep.regExp.toString() : null};})
//               // .filter(function(req) {return req.request || req.constDependency;}));
//               .filter(function(dep) {return dep instanceof ContextDependency}));
require('mkdirp').sync(__dirname + '/webpack.module.cache/');
var hashName = require('crypto').createHash('sha1').update(module.request).digest().hexSlice();
          require('fs').writeFileSync(__dirname + '/webpack.module.cache/' + hashName + '.json', JSON.stringify({[module.request]: moduleCache[module.request]}), 'utf8');
        }
      });
      moduleCache.fileDependencies = compilation.fileDependencies;
      require('fs').writeFileSync(__dirname + '/webpack.module.cache/file-dependencies.json', JSON.stringify({fileDependencies: compilation.fileDependencies}), 'utf8');
      // console.log(moduleCache.fileDependencies);
      // require('fs').writeFileSync(__dirname + '/webpack.module.cache.json', JSON.stringify(moduleCache), 'utf8');
      // require('fs').writeFileSync(__dirname + '/webpack.stats.json', JSON.stringify(compilation.modules.map(function(module) {
      //   return [module.readableIdentifier(compilation.moduleTemplate.requestShortener), module.profile];
      // })), 'utf8');
      console.log('cache out', Date.now() - startCacheTime);
      // if (compilation.assets['animated-preview.html']) {
      //   require('fs').writeFileSync(__dirname + '/webpack.index.html.js', compilation.assets['animated-preview.html'].source());
      // }
      cb();
      // - store raw source
      // - records plugin
      // - store dependency requests
      // - create raw module for found source
      // - add raw dependency dependency factory, bind to normal module factory
      // - add null template to dependency templates
    });
  },
},