Webpack dev server will not restart correctly

Server.js

//I tried editing the webpack dev server prototype to use the middleware callback, but it had the same issue

//new version
Server.prototype.close = function(callback) {
	this.sockets.forEach(function(sock) {
		sock.close();
	});
  this.listeningApp.close(function() {
    //this is only reached the first call to close
    this.middleware.close(callback);
  }.bind(this));
	this.sockets = [];
}

//original
Server.prototype.close = function() {
	this.sockets.forEach(function(sock) {
		sock.close();
	});
	this.sockets = [];
	this.middleware.close();
	this.listeningApp.close();
}

test-server.js

let webpackConfig = require('./webpack.config.js');
//i want to slowly add entry points because i only need a few per testing session but i have too many to run all at once
webpackConfig.entry = {};

let compiler = webpack(webpackConfig);
let webpackServer = new webpackDevServer(compiler, {
  stats: {
    colors: true
  }
});
webpackServer.listen(program.webpackPort);

//app is another express server running on a different port as webpack
app.get('/specs/*?', (req, res, next) => {
  const specPath = req.params[0];
  if(!webpackConfig.entry[specPath]){
    webpackConfig.entry[specPath] = getEntryPointForPath(specPath);
    //this stuff only works the first time. the second time i hit this endpoint with a new spec, it won't restart the dev server
    webpackServer.close();
    compiler = webpack(webpackConfig);
    webpackServer = new webpackDevServer(compiler, {
      stats: {
        colors: true
      }
    });
    webpackServer.listen(program.webpackPort);
  }

  let newUrl = getWebpackUrl(webpackBaseUrl, specPath);
  res.redirect(newUrl);
});