Last active
December 16, 2015 20:35
-
-
Save echeipesh/812948930d6f5ec7b442 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
trait LayerDeleter[ID] { | |
def delete(id: ID): Unit | |
} | |
class S3LayerDeleter(val store: AttributeStore[JsonFormat]) extends LayerDeleter[LayerId] { | |
/** Implement using S3 DELETE */ | |
def delete(id: ID): Unit | |
} | |
class HadoopLayerDeleter(val store: AttributeStore[JsonFormat]) extends LayerDeleter[LayerId] { | |
/** Implement using Hadoop FileSystem */ | |
def delete(id: ID): Unit | |
} | |
class AccumuloLayerDeleter(val store: AttributeStore[JsonFormat]) extends LayerDeleter[LayerId] { | |
/** Implement using BatchDeleter */ | |
def delete(id: ID): Unit | |
} | |
trait LayerCopier[ID] { | |
def copy(from: ID, to: ID): Unit | |
} | |
class S3LayerCopier(val store: AttributeStore[JsonFormat]) extends LayerCopier[LayerId] { | |
/** Implement using S3 COPY */ | |
// - Copy to different bucket/key/LayerId | |
// - Copy with different index? (Dfferent target file name in COPY) | |
def copy(from: ID, to: ID): Unit | |
def copy(from: LayerId, to: LayerId, key: String) | |
def copy(from: LayerId, to: LayerId, bucket: String, key: String) | |
} | |
class HadoopLayerCopier(val store: AttributeStore[JsonFormat]) extends LayerCopier[LayerId] { | |
/** Implement using Hadoop FileSystem */ | |
// - Copy to different Path/LayerId | |
// - Copy with different index? (Can't do With FileSystem.copy) | |
def copy(from: ID, to: ID): Unit | |
def copy(from: LayerId, to: LayerId, path: Path) | |
} | |
class AccumuloLayerCopier(val store: AttributeStore[JsonFormat]) extends LayerCopier[LayerId] { | |
/** Must read/bulk import */ | |
// - Copy to another table/LayerId | |
// - Copy with different index? (Sure, we have to read it anyway) | |
def copy(from: ID, to: ID): Unit | |
def copy(from: LayerId, to: LayerId, table: String) | |
} | |
trait LayerMover[ID] { | |
def move(from: ID, to: ID): Unit | |
} | |
class S3LayerMover(val store: AttributeStore[JsonFormat]) extends LayerMover[LayerId] { | |
/** Implement using S3 COPY/DELETE command */ | |
def move(from: ID, to: ID): Unit | |
def move(from: LayerId, to: LayerId, key: String) | |
def move(from: LayerId, to: LayerId, bucket: String, key: String) | |
} | |
class HadoopLayerMover(val store: AttributeStore[JsonFormat]) extends LayerMover[LayerId] { | |
/** Implement using Hadoop FileSystem move */ | |
def move(from: ID, to: ID): Unit | |
def move(from: LayerId, to: LayerId, path: Path) | |
} | |
class AccumuloLayerMover(val store: AttributeStore[JsonFormat]) extends LayerMover[LayerId] { | |
/** Maybe possible to use sublcass TransformingIterator, must be on Accumulo CLASS_PATH | |
* Fallback is to copy/delete */ | |
def move(from: ID, to: ID): Unit | |
def move(from: LayerId, to: LayerId, table: String) | |
} | |
trait LayerReindexer[ID] { | |
def reindex(id: ID, keyIndex: KeyIndex[K]) | |
def reindex(id: ID, indexMethod: KeyIndexMethod[K]): Unit | |
} | |
class S3LayerReindexer(val store: AttributeStore[JsonFormat]) extends LayerReindexer[LayerId] { | |
/** Implement using S3 Copy/Delete command */ | |
// TODO: What happens when we reindex in place? How do we not stomp all over ourselves? | |
} | |
class HadoopLayerReindexer(val store: AttributeStore[JsonFormat]) extends LayerReindexer[LayerId] { | |
/** Read the layer, Backup old folder, write with new index, delete backup */ | |
} | |
class AccumuloLayerReindexer(val store: AttributeStore[JsonFormat]) extends LayerReindexer[LayerId] { | |
/** Read the layer, delete old layer, write new layer. Very fragile. Can't backup for inplace reindex */ | |
// TODO: What do we do if we reindex in place? Again we're stomping | |
} | |
trait LayerManager[ID] { | |
def delete(id: ID): Unit | |
def copy(from: ID, to: ID): Unit | |
def move(from: ID, to: ID): Unit | |
def reindex[K](id: ID, indexMethod: KeyIndexMethod[K]) | |
def reindex[K](id: ID, keyIndex: KeyIndex[K]) | |
} | |
class S3LayerManager(val store: AttributeStore[JsonFormat]) extends LayerManager[LayerId] { | |
def copy(from: LayerId, to: LayerId, bucket: String, key: String) | |
def move(from: ID, to: ID, bucket: String, key: String) | |
def reindex[K](id: ID, indexMethod: KeyIndexMethod[K], bucket: String, key: String) | |
def reindex[K](id: ID, keyIndex: KeyIndex[K], bucket: String, key: String) | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment