UrlExpander.java

package org.insight.urlexpander;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.URI;
import java.util.concurrent.Phaser;

import org.apache.http.HttpResponse;
import org.apache.http.client.config.CookieSpecs;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.HttpHead;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.concurrent.FutureCallback;
import org.apache.http.impl.client.RedirectLocations;
import org.apache.http.impl.nio.client.CloseableHttpAsyncClient;
import org.apache.http.impl.nio.client.HttpAsyncClients;
import org.apache.http.protocol.BasicHttpContext;
import org.apache.http.protocol.HttpContext;

/*
 * Follow Redirects from short urls using HEAD http requests, avoiding downloading body.
 * 
 * Pipe URLS into STDIN, STDOUT is original URL \t Expanded
 */
public class FollowRedirectHead {

  public final static void main(String[] args) {
    BufferedReader buffer = new BufferedReader(new InputStreamReader(System.in));

    RequestConfig globalConfig = RequestConfig.custom().setCookieSpec(CookieSpecs.IGNORE_COOKIES).build();
    CloseableHttpAsyncClient client = HttpAsyncClients.custom().setDefaultRequestConfig(globalConfig).setMaxConnTotal(1024).build();

    client.start();

    final boolean verbose;
    if (args.length > 0) {
      if (args[0].equalsIgnoreCase("--verbose")) {
        verbose = true;
      } else {
        verbose = false;
      }
    } else {
      verbose = false;
    }

    final Phaser phaser = new Phaser(1); // wait for threads

    while (true) {

      String readin = "";
      try {
        readin = buffer.readLine();
      } catch (IOException e) {
        continue;
      }

      final String processURI = readin;
      if ((processURI == null) || processURI.isEmpty()) {
        phaser.arriveAndAwaitAdvance(); // await any async tasks to complete
        break;
      }

      try {
        final HttpHead request = new HttpHead(processURI);

        request.setConfig(globalConfig);
        final HttpContext context = new BasicHttpContext();

        FutureCallback<HttpResponse> callback = new FutureCallback<HttpResponse>() {
          @Override
          public void completed(final HttpResponse response2) {
            //System.out.println(request.getRequestLine() + "->" + response2.getStatusLine());
            RedirectLocations locations = (RedirectLocations) context.getAttribute(HttpClientContext.REDIRECT_LOCATIONS);
            if (locations != null) {
              if (verbose) {
                String lastLoc = processURI;
                for (URI u : locations.getAll()) {
                  System.out.println(lastLoc + "\t" + u.toString());
                  lastLoc = u.toString();
                }
                System.out.println("");
              } else {
                System.out.println(processURI + "\t" + locations.getAll().get(locations.getAll().size() - 1));
              }
            }
            phaser.arriveAndDeregister();
          }

          @Override
          public void failed(final Exception ex) {
            phaser.arriveAndDeregister();
          }

          @Override
          public void cancelled() {
            phaser.arriveAndDeregister();
          }
        };

        phaser.register();
        client.execute(request, context, callback);
      } catch (IllegalArgumentException e) {
        continue;
      }

    } // loop


    try {
      client.close();
    } catch (IOException e) {
    }

  }
}