Skip to content

Instantly share code, notes, and snippets.

@tingletech

tingletech/merritt.diff

Last active August 29, 2015 14:20
Show Gist options
  • Save tingletech/646df710b4853db9a728 to your computer and use it in GitHub Desktop.
Save tingletech/646df710b4853db9a728 to your computer and use it in GitHub Desktop.
Download ZIP
simple curl clone using org.apache.httpcomponents
Raw
gistfile1.txt
BT-macBookPro:workspace tingle$ ./url.gvy http://user:[email protected]/headers
{
"headers": {
"Authorization": "Basic dXNlcjpwYXNzd2Q=",
"Host": "httpbin.org",
"User-Agent": "Apache-HttpClient/4.3.6 (java 1.5)"
}
}
BT-macBookPro:workspace tingle$ curl http://user:[email protected]/headers
{
"headers": {
"Accept": "*/*",
"Authorization": "Basic dXNlcjpwYXNzd2Q=",
"Host": "httpbin.org",
"User-Agent": "curl/7.30.0"
}
}
BT-macBookPro:workspace tingle$ ./url.gvy --head http://user:[email protected]/headers
HTTP/1.1 200 OK [Server: nginx, Date: Wed, 29 Apr 2015 04:47:33 GMT, Content-Type: application/json, Content-Length: 154, Connection: keep-alive, Access-Control-Allow-Origin: *, Access-Control-Allow-Credentials: true] org.apache.http.conn.BasicManagedEntity@5629510
BT-macBookPro:workspace tingle$ curl --head http://user:[email protected]/headers
HTTP/1.1 200 OK
Server: nginx
Date: Wed, 29 Apr 2015 04:47:39 GMT
Content-Type: application/json
Content-Length: 153
Connection: keep-alive
Access-Control-Allow-Origin: *
Access-Control-Allow-Credentials: true
Raw
merritt.diff
--- a/core/src/main/java/org/cdlib/mrt/utility/HTTPUtil.java Mon Feb 11 14:22:52 2013 -0800
+++ b/core/src/main/java/org/cdlib/mrt/utility/HTTPUtil.java Tue Jun 09 12:00:09 2015 -0700
@@ -51,6 +51,9 @@
import org.apache.http.Header;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.protocol.HTTP;
+import org.apache.http.entity.BasicHttpEntity;
+
+import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
@@ -59,10 +62,11 @@
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.entity.UrlEncodedFormEntity;
-import org.apache.http.impl.client.DefaultHttpClient;
+//import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.message.BasicNameValuePair;
-import org.apache.http.params.BasicHttpParams;
-import org.apache.http.params.HttpParams;
+import org.apache.http.client.config.RequestConfig;
+//import org.apache.http.params.BasicHttpParams;
+//import org.apache.http.params.HttpParams;
import org.apache.http.StatusLine;
import org.apache.http.Header;
/**
@@ -71,7 +75,7 @@
* @author dloy
*/
public class HTTPUtil {
-
+ private static final boolean DEBUG = false;
/**
* Send this manifestFile to mrt store
@@ -88,6 +92,11 @@
int responseCode = response.getStatusLine().getStatusCode();
HttpEntity entity = response.getEntity();
if (entity != null && (responseCode >= 200 && responseCode < 300)) {
+ if (DEBUG) {
+ System.out.println("ContentLength=" + entity.getContentLength());
+ System.out.println("isChunked=" + entity.isChunked());
+ System.out.println("isStreaming=" + entity.isStreaming());
+ }
return entity.getContent();
}
if (responseCode == 404) {
@@ -106,11 +115,104 @@
);
} catch( TException tex ) {
- System.out.println("trace:" + StringUtil.stackTrace(tex));
+ if (DEBUG) System.out.println("trace:" + StringUtil.stackTrace(tex));
throw tex;
} catch( Exception ex ) {
- System.out.println("trace:" + StringUtil.stackTrace(ex));
+ if (DEBUG) System.out.println("trace:" + StringUtil.stackTrace(ex));
+ throw new TException.GENERAL_EXCEPTION("HTTPUTIL: getObject- Exception:" + ex);
+ }
+ }
+
+ /**
+ * Send this manifestFile to mrt store
+ * @param manifestFile
+ * @return
+ * @throws org.cdlib.framework.utility.FrameworkException
+ */
+ public static HttpEntity getObjectEntity(String requestURL, int timeout, long startByte, long endByte)
+ throws TException
+ {
+ try {
+ HttpResponse response = getHttpResponse(requestURL, timeout, startByte, endByte);
+ int responseCode = response.getStatusLine().getStatusCode();
+ HttpEntity entity = response.getEntity();
+ if (entity != null && (responseCode >= 200 && responseCode < 300)) {
+ if (DEBUG) {
+ System.out.println("ContentLength=" + entity.getContentLength());
+ System.out.println("isChunked=" + entity.isChunked());
+ System.out.println("isStreaming=" + entity.isStreaming());
+ }
+ return entity;
+ }
+ if (responseCode == 404) {
+ throw new TException.REQUESTED_ITEM_NOT_FOUND(
+ "HTTPUTIL: getObject- Error during HttpClient processing"
+ + " - timeout:" + timeout
+ + " - URL:" + requestURL
+ + " - responseCode:" + responseCode
+ );
+ }
+ throw new TException.EXTERNAL_SERVICE_UNAVAILABLE(
+ "HTTPUTIL: getObject- Error during HttpClient processing"
+ + " - timeout:" + timeout
+ + " - URL:" + requestURL
+ + " - responseCode:" + responseCode
+ );
+
+ } catch( TException tex ) {
+ if (DEBUG) System.out.println("trace:" + StringUtil.stackTrace(tex));
+ throw tex;
+
+ } catch( Exception ex ) {
+ if (DEBUG) System.out.println("trace:" + StringUtil.stackTrace(ex));
+ throw new TException.GENERAL_EXCEPTION("HTTPUTIL: getObject- Exception:" + ex);
+ }
+ }
+
+
+ /**
+ * Send this manifestFile to mrt store
+ * @param manifestFile
+ * @return
+ * @throws org.cdlib.framework.utility.FrameworkException
+ */
+ public static HttpEntity getObjectEntity(String requestURL, int timeout)
+ throws TException
+ {
+ try {
+ HttpResponse response = getHttpResponse(requestURL, timeout);
+ int responseCode = response.getStatusLine().getStatusCode();
+ HttpEntity entity = response.getEntity();
+ if (entity != null && (responseCode >= 200 && responseCode < 300)) {
+ if (DEBUG) {
+ System.out.println("ContentLength=" + entity.getContentLength());
+ System.out.println("isChunked=" + entity.isChunked());
+ System.out.println("isStreaming=" + entity.isStreaming());
+ }
+ return entity;
+ }
+ if (responseCode == 404) {
+ throw new TException.REQUESTED_ITEM_NOT_FOUND(
+ "HTTPUTIL: getObject- Error during HttpClient processing"
+ + " - timeout:" + timeout
+ + " - URL:" + requestURL
+ + " - responseCode:" + responseCode
+ );
+ }
+ throw new TException.EXTERNAL_SERVICE_UNAVAILABLE(
+ "HTTPUTIL: getObject- Error during HttpClient processing"
+ + " - timeout:" + timeout
+ + " - URL:" + requestURL
+ + " - responseCode:" + responseCode
+ );
+
+ } catch( TException tex ) {
+ if (DEBUG) System.out.println("trace:" + StringUtil.stackTrace(tex));
+ throw tex;
+
+ } catch( Exception ex ) {
+ if (DEBUG) System.out.println("trace:" + StringUtil.stackTrace(ex));
throw new TException.GENERAL_EXCEPTION("HTTPUTIL: getObject- Exception:" + ex);
}
}
@@ -148,6 +250,39 @@
/**
+ * getObject with timeout and retry
+ * @param requestURL build inputStream to this URL
+ * @param timeout milliseconds for timeout
+ * @param retry number of retry attemps
+ * @return InputStream to URL service
+ * @throws org.cdlib.mrt.utility.TException
+ */
+ public static InputStream getObject404(String requestURL, int timeout, int retry)
+ throws TException
+ {
+ InputStream inStream = null;
+ Exception exSave = null;
+ for (int i=0; i < retry; i++) {
+ try {
+ inStream = getObject(requestURL, timeout);
+ return inStream;
+
+ } catch (TException.REQUEST_INVALID tex) {
+ throw tex;
+
+ } catch (TException.REQUESTED_ITEM_NOT_FOUND rinf) {
+ throw rinf;
+
+ } catch (Exception ex) {
+ exSave = ex;
+ }
+ }
+ throw new TException.EXTERNAL_SERVICE_UNAVAILABLE(
+ "HTTPUTIL: getObject"
+ + " - requestURL=" + requestURL, exSave);
+ }
+
+ /**
* Delete
* @param requestURL delete URL
* @param timeout timeout in .001 seconds
@@ -219,15 +354,65 @@
public static HttpResponse getHttpResponse(String requestURL, int timeout)
throws TException
{
- HttpParams params = new BasicHttpParams();
- params.setParameter("http.socket.timeout", new Integer(timeout));
- params.setParameter("http.connection.timeout", new Integer(timeout));
+
+ try {
+ //HttpClient httpclient = new DefaultHttpClient(params);
+
+ RequestConfig requestConfig = RequestConfig.custom().setConnectTimeout(360 * 1000).build();
+ HttpClient httpClient = HttpClientBuilder.create().setDefaultRequestConfig(requestConfig).build();
+ HttpGet httpget = new HttpGet(requestURL);
+ httpget.addHeader("Accept", "*/*");
+ //httpget.addHeader("Transfer-Encoding", "chunked");
+
+ HttpResponse response = httpClient.execute(httpget);
+ if (response != null) {
+ return response;
+ }
+ throw new TException.EXTERNAL_SERVICE_UNAVAILABLE(
+ "HTTPUTIL: getHttpResponse- Error during HttpClient processing");
- //System.out.println("!!!!:" + MESSAGE + "getSerializeObject.requestURL=" + requestURL);
+ } catch( java.net.UnknownHostException uhe) {
+ throw new TException.EXTERNAL_SERVICE_UNAVAILABLE(
+ "HTTPUTIL: Unknown host Exception:" + uhe
+ + " - URL:" + requestURL);
+
+ } catch( IllegalArgumentException iae ) {
+ System.out.println("trace:" + StringUtil.stackTrace(iae));
+ throw new TException.REQUEST_INVALID("HTTPUTIL: getObject- Exception:" + iae);
+
+ } catch( TException tex ) {
+ System.out.println("trace:" + StringUtil.stackTrace(tex));
+ throw tex;
+
+ } catch( Exception ex ) {
+ System.out.println("trace:" + StringUtil.stackTrace(ex));
+ throw new TException.GENERAL_EXCEPTION("HTTPUTIL: getObject- Exception:" + ex);
+ }
+ }
+
+
+ /**
+ * Perform a GET operation
+ * @param requestURL get request
+ * @param timeout connection timeout
+ * @return http response
+ * @throws TException process exception
+ */
+ public static HttpResponse getHttpResponse(String requestURL, int timeout, long startByte, long endByte)
+ throws TException
+ {
+
try {
- HttpClient httpclient = new DefaultHttpClient(params);
+ //HttpClient httpclient = new DefaultHttpClient(params);
+
+ RequestConfig requestConfig = RequestConfig.custom().setConnectTimeout(360 * 1000).build();
+ HttpClient httpClient = HttpClientBuilder.create().setDefaultRequestConfig(requestConfig).build();
HttpGet httpget = new HttpGet(requestURL);
- HttpResponse response = httpclient.execute(httpget);
+ httpget.addHeader("Accept", "*/*");
+ httpget.addHeader("Range", "bytes=" + startByte + "-" + endByte);
+ //httpget.addHeader("Transfer-Encoding", "chunked");
+
+ HttpResponse response = httpClient.execute(httpget);
if (response != null) {
return response;
}
@@ -318,32 +503,42 @@
HttpEntity resEntity = null;
try {
Properties resultProp = new Properties();
- StatusLine statusLine = response.getStatusLine();
if (response == null) {
throw new TException.EXTERNAL_SERVICE_UNAVAILABLE("HTTPUtil.response2Property - No response");
}
+ StatusLine statusLine = response.getStatusLine();
int statusCode = statusLine.getStatusCode();
resultProp.setProperty("response.status", "" + statusCode);
+ resultProp.setProperty("response.line", "" + statusLine);
+ resultProp.setProperty("response.phrase", statusLine.getReasonPhrase());
- resEntity = response.getEntity();
- String responseState = StringUtil.streamToString(resEntity.getContent(), "utf-8");
- if (StringUtil.isNotEmpty(responseState)) {
- resultProp.setProperty("response.value", responseState);
- System.out.println("mrt-response:" + responseState);
- }
Header [] headers = response.getAllHeaders();
for (Header header : headers) {
resultProp.setProperty(
"header." + header.getName(),
header.getValue());
}
- System.out.println(PropertiesUtil.dumpProperties("!!!!sendArchiveMultipart!!!!", resultProp, 100));
+
+ try {
+ resEntity = response.getEntity();
+ if (resEntity == null) return resultProp;
+ } catch (Exception ex) {
+ return resultProp;
+ }
+ String responseState = StringUtil.streamToString(resEntity.getContent(), "utf-8");
+ if (StringUtil.isNotEmpty(responseState)) {
+ resultProp.setProperty("response.value", responseState);
+ if (DEBUG) System.out.println("mrt-response:" + responseState);
+ }
+ if (DEBUG) {
+ System.out.println(PropertiesUtil.dumpProperties("!!!!sendArchiveMultipart!!!!", resultProp, 100));
- System.out.println("----------------------------------------");
- System.out.println(response.getStatusLine());
- if (resEntity != null) {
- System.out.println("Response content length: " + resEntity.getContentLength());
- System.out.println("Chunked?: " + resEntity.isChunked());
+ System.out.println("----------------------------------------");
+ System.out.println(response.getStatusLine());
+ if (resEntity != null) {
+ System.out.println("Response content length: " + resEntity.getContentLength());
+ System.out.println("Chunked?: " + resEntity.isChunked());
+ }
}
return resultProp;
@@ -374,15 +569,11 @@
public static HttpResponse deleteHttpResponse(String requestURL, int timeout)
throws TException
{
- HttpParams params = new BasicHttpParams();
- params.setParameter("http.socket.timeout", new Integer(timeout));
- params.setParameter("http.connection.timeout", new Integer(timeout));
-
- //System.out.println("!!!!:" + MESSAGE + "getSerializeObject.requestURL=" + requestURL);
try {
- HttpClient httpclient = new DefaultHttpClient(params);
+ RequestConfig requestConfig = RequestConfig.custom().setConnectTimeout(300 * 1000).build();
+ HttpClient httpClient = HttpClientBuilder.create().setDefaultRequestConfig(requestConfig).build();
HttpDelete httpDelete = new HttpDelete(requestURL);
- HttpResponse response = httpclient.execute(httpDelete);
+ HttpResponse response = httpClient.execute(httpDelete);
if (response != null) {
return response;
}
@@ -439,13 +630,9 @@
public static HttpResponse postHttpResponse(String requestURL, Properties prop, int timeout)
throws TException
{
- HttpParams params = new BasicHttpParams();
- params.setParameter("http.socket.timeout", new Integer(timeout));
- params.setParameter("http.connection.timeout", new Integer(timeout));
-
- //System.out.println("!!!!:" + MESSAGE + "getSerializeObject.requestURL=" + requestURL);
try {
- HttpClient httpclient = new DefaultHttpClient(params);
+ RequestConfig requestConfig = RequestConfig.custom().setConnectTimeout(300 * 1000).build();
+ HttpClient httpClient = HttpClientBuilder.create().setDefaultRequestConfig(requestConfig).build();
List<BasicNameValuePair> formparams = new ArrayList<BasicNameValuePair>();
Enumeration e = prop.propertyNames();
String key = null;
@@ -463,7 +650,7 @@
HttpPost httppost = new HttpPost(requestURL);
httppost.setHeader("Content-Type", "application/x-www-form-urlencoded");
httppost.setEntity(entityForm);
- HttpResponse response = httpclient.execute(httppost);
+ HttpResponse response = httpClient.execute(httppost);
if (response != null) {
return response;
}
@@ -523,13 +710,9 @@
int timeout)
throws TException
{
- HttpParams params = new BasicHttpParams();
- params.setParameter("http.socket.timeout", new Integer(timeout));
- params.setParameter("http.connection.timeout", new Integer(timeout));
-
- //System.out.println("!!!!:" + MESSAGE + "getSerializeObject.requestURL=" + requestURL);
try {
- HttpClient httpclient = new DefaultHttpClient(params);
+ RequestConfig requestConfig = RequestConfig.custom().setConnectTimeout(300 * 1000).build();
+ HttpClient httpClient = HttpClientBuilder.create().setDefaultRequestConfig(requestConfig).build();
HttpPost httppost = new HttpPost(requestURL);
MultipartEntity reqEntity = new MultipartEntity();
Enumeration e = stringParts.propertyNames();
@@ -556,7 +739,7 @@
httppost.setEntity(reqEntity);
System.out.println("executing request " + httppost.getRequestLine());
- HttpResponse response = httpclient.execute(httppost);
+ HttpResponse response = httpClient.execute(httppost);
return response;
} catch( Exception ex ) {
Raw
url.gvy
#!/usr/bin/env groovy
@Grapes(
@Grab(group='org.apache.httpcomponents', module='httpclient', version='4.3.6')
)
import java.net.URL
import org.apache.http.client.methods.HttpGet
import org.apache.http.impl.client.DefaultHttpClient
import org.apache.http.params.BasicHttpParams
import org.apache.http.impl.client.BasicResponseHandler
if (!args) {
println 'usage: url.grv [--head] "URL"'
System.exit(0)
}
def head = false
def url = ''
if (args[0] == '--head') {
head = true
url = args[1]
} else {
url = args[0]
}
// should do the same thing as
// https://bitbucket.org/merritt/mrt-core/src/e2f6843ddfaf3ce0d4370e87c98a21ce483d5eac/core/src/main/java/org/cdlib/mrt/utility/HTTPUtil.java?at=default#cl-212
params = new BasicHttpParams()
httpclient = new DefaultHttpClient()
httpget = new HttpGet(url)
response = httpclient.execute(httpget)
if (head) {
println(response)
} else {
print(new BasicResponseHandler().handleResponse(response))
}
/*
Copyright © 2015, Regents of the University of California
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in
the documentation and/or other materials provided with the
distribution.
* Neither the name of the University of California nor the names
of its contributors may be used to endorse or promote products
derived from this software without specific prior written
permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
POSSIBILITY OF SUCH DAMAGE.
*/
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment