Created
January 21, 2015 16:03
-
-
Save vmarcinko/217c36115c975c8c464e to your computer and use it in GitHub Desktop.
Nanocube DMP encoder in java
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
public class CategoryEnumInfo { | |
private final String label; | |
private final Byte encodedValue; | |
public CategoryEnumInfo(String label, Byte encodedValue) { | |
this.label = label; | |
this.encodedValue = encodedValue; | |
} | |
public String getLabel() { | |
return label; | |
} | |
public Byte getEncodedValue() { | |
return encodedValue; | |
} | |
@Override | |
public boolean equals(Object o) { | |
if (this == o) { | |
return true; | |
} | |
if (o == null || getClass() != o.getClass()) { | |
return false; | |
} | |
CategoryEnumInfo that = (CategoryEnumInfo) o; | |
if (!label.equals(that.label)) { | |
return false; | |
} | |
if (!encodedValue.equals(that.encodedValue)) { | |
return false; | |
} | |
return true; | |
} | |
@Override | |
public int hashCode() { | |
int result = label.hashCode(); | |
result = 31 * result + encodedValue.hashCode(); | |
return result; | |
} | |
@Override | |
public String toString() { | |
final StringBuilder sb = new StringBuilder("CategoryEnumInfo{"); | |
sb.append("label='").append(label).append('\''); | |
sb.append(", encodedValue=").append(encodedValue); | |
sb.append('}'); | |
return sb.toString(); | |
} | |
} |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import java.io.ByteArrayOutputStream; | |
import java.io.IOException; | |
import java.nio.ByteBuffer; | |
import java.nio.ByteOrder; | |
import java.nio.charset.Charset; | |
import java.text.SimpleDateFormat; | |
import java.util.Date; | |
import java.util.HashMap; | |
import java.util.Map; | |
public class NanocubeDmpEncoder { | |
private static final Charset charset = Charset.forName("UTF8"); | |
private final String name; | |
private final int locationZoom; | |
private final Date timeOffset; | |
private final int timeBinInSecs; | |
private final int timeByteLength; | |
private final Map<String, Map<Object, CategoryEnumInfo>> categoryEnumerations; | |
private final int recordLength; | |
public NanocubeDmpEncoder( | |
String name, | |
int locationZoom, | |
Date timeOffset, | |
int timeBinInSecs, | |
int timeByteLength, | |
Map<String, Map<Object, CategoryEnumInfo>> categoryEnumerations | |
) { | |
if (name == null) { | |
throw new IllegalArgumentException("Name is null"); | |
} | |
if (locationZoom < 1) { | |
throw new IllegalArgumentException("Location zoom cannot be less than 1, but is: " + locationZoom); | |
} | |
if (timeOffset == null) { | |
throw new IllegalArgumentException("TimeOffset is null"); | |
} | |
if (timeBinInSecs < 1) { | |
throw new IllegalArgumentException("Time bin cannot be less than 1 second, but is: " + timeBinInSecs); | |
} | |
if (timeByteLength != 2 && timeByteLength != 4 && timeByteLength != 8) { | |
throw new IllegalArgumentException("Time byte length must be 2, 4 or 8, but is " + timeByteLength); | |
} | |
this.name = name; | |
this.locationZoom = locationZoom; | |
this.timeOffset = timeOffset; | |
this.timeBinInSecs = timeBinInSecs; | |
this.timeByteLength = timeByteLength; | |
this.categoryEnumerations = new HashMap<>(categoryEnumerations); // defensive copying | |
this.recordLength = calculateRecordLength(timeByteLength, categoryEnumerations.size()); | |
} | |
private int calculateRecordLength(int timeByteLength, int categoryCount) { | |
return 12 + categoryCount + timeByteLength; | |
} | |
public int getRecordLength() { | |
return recordLength; | |
} | |
public byte[] encodeHeaders() throws IOException { | |
ByteArrayOutputStream baos = new ByteArrayOutputStream(); | |
encodeString(baos, "name: " + name); | |
encodeString(baos, "encoding: binary"); | |
encodeString(baos, "metadata: location__origin degrees_mercator_quadtree" + locationZoom); | |
encodeString(baos, "field: location nc_dim_quadtree_" + locationZoom); | |
for (Map.Entry<String, Map<Object, CategoryEnumInfo>> entry : categoryEnumerations.entrySet()) { | |
String categoryName = entry.getKey(); | |
Map<Object, CategoryEnumInfo> enumerations = entry.getValue(); | |
encodeString(baos, "field: " + categoryName + " nc_dim_cat_1"); | |
for (CategoryEnumInfo enumInfo : enumerations.values()) { | |
encodeString(baos, "valname: " + categoryName + " " + enumInfo.getEncodedValue() + " " + enumInfo.getLabel()); | |
} | |
} | |
SimpleDateFormat timeOffsetFormat = new SimpleDateFormat("yyyy-MM-dd_HH:mm:ss"); | |
String formattedTimeOffset = timeOffsetFormat.format(timeOffset); | |
encodeString(baos, "metadata: tbin " + formattedTimeOffset + "_" + timeBinInSecs + "s"); | |
encodeString(baos, "field: Time nc_dim_time_" + timeByteLength); | |
encodeString(baos, "field: count nc_var_uint_4"); | |
encodeString(baos, ""); | |
return baos.toByteArray(); | |
} | |
public byte[] encodeRecord(double latitude, double longitude, Date time, int count, Map<String, Object> categoryValues) throws IOException { | |
if (time == null || time.before(timeOffset)) { | |
throw new IllegalArgumentException("Time must be equal or after configured time offset (" + timeOffset + "), but is: " + time); | |
} | |
ByteBuffer byteBuffer = ByteBuffer.allocate(recordLength); | |
byteBuffer.order(ByteOrder.LITTLE_ENDIAN); | |
// location | |
int lonValue = lonToTileX(longitude, locationZoom); | |
byteBuffer.putInt(lonValue); | |
int latValue = latToTileY(latitude, locationZoom); | |
byteBuffer.putInt(latValue); | |
// category values | |
for (Map.Entry<String, Map<Object, CategoryEnumInfo>> entry : categoryEnumerations.entrySet()) { | |
String categoryName = entry.getKey(); | |
Map<Object, CategoryEnumInfo> enumerations = entry.getValue(); | |
Object categoryValue = categoryValues.get(categoryName); | |
CategoryEnumInfo enumInfo = enumerations.get(categoryValue); | |
byteBuffer.put(enumInfo.getEncodedValue()); | |
} | |
// time | |
long binIndex = timeToBinIndex(time); | |
switch (timeByteLength) { | |
case 2: | |
byteBuffer.putShort((short) binIndex); | |
break; | |
case 4: | |
byteBuffer.putInt((int) binIndex); | |
break; | |
default: | |
byteBuffer.putLong(binIndex); | |
break; | |
} | |
// count | |
byteBuffer.putInt(count); | |
return byteBuffer.array(); | |
} | |
private long timeToBinIndex(Date time) { | |
long ageSinceOffsetInMillis = time.getTime() - timeOffset.getTime(); | |
long timeBinInMillis = 1000 * timeBinInSecs; | |
long ageInBins = (int) (ageSinceOffsetInMillis / timeBinInMillis); | |
long maxValue = getTimeMaxValue(); | |
if (ageInBins > maxValue) { | |
throw new IllegalArgumentException("Invalid time: " + time + "; period since time offset (" + timeOffset + ") calculated in time bins (" + ageInBins + " of " + timeBinInSecs + "s bins) exceeds maximum " + timeByteLength + " byte integer value: " + maxValue); | |
} | |
return ageInBins; | |
} | |
private static void encodeString(ByteArrayOutputStream baos, String str) throws IOException { | |
String eolAppended = str + "\n"; | |
baos.write(eolAppended.getBytes(charset)); | |
} | |
private static int lonToTileX(double lonDeg, int zoom) { | |
lonDeg = Math.max(-180, lonDeg); | |
lonDeg = Math.min(180, lonDeg); | |
double n = Math.pow(2, zoom); | |
return (int) (n * ((lonDeg + 180) / 360)); | |
} | |
private static int latToTileY(double latDeg, int zoom) { | |
latDeg = Math.max(-85.0511, latDeg); | |
latDeg = Math.min(85.0511, latDeg); | |
double latRad = latDeg / (double) 180 * Math.PI; | |
double n = Math.pow(2, zoom); | |
double yTile = n * (1 - (Math.log(Math.tan(latRad) + 1.0 / Math.cos(latRad)) / Math.PI)) / 2.0; | |
return (int) (n - 1 - yTile); | |
} | |
private long getTimeMaxValue() { | |
switch (timeByteLength) { | |
case 2: | |
return Short.MAX_VALUE; | |
case 4: | |
return Integer.MAX_VALUE; | |
default: | |
return Long.MAX_VALUE; | |
} | |
} | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment