work-in-progress data source abstraction architecture/design pattern

data-source-abstraction-layer.md

work-in-progress data source abstraction architecture/design pattern

#### Data source abtraction layer

class FileStore
    def write
    def read
    def remove

class S3Store
    def write
    def read
    def remove

class MongoStore
    def get
    def set
    def remove

class DataStore
    def __init__(key)
        self.file_store = FileStore() or S3Store
        self.kv_store = MongoStore
        self._fk = '_file_list'
        self._location = '_location'
        self._file_list = []

    def refresh(): # decorator
        ** before func call**
        # query mongo by key
        # if not found
        #   create blob with key
        #   split mongdb id use and _location
        #   http://highscalability.com/blog/2015/11/4/strategy-avoid-lots-of-little-files.html

    # get value by key
    @refresh
    def get(key)
        return self.kv_store.get(key)
    
    # store (post,patch) value using key
    def set(key,value)
        return self.kv_store.set(key)

    # delete
    @refresh
    def remove(self,key)
        return self.kv_store.remove(key)

    # get file
    @refresh
    def get_file(mypath)
        return self.file_store.read(mypath)

    # store file
    def set_file()
        try:
            self.file_store.write(mypath)
            file_list = self.get(self._fk)
            file_list.append(relpath(mypath))
            self.set(self._fk,file_list)
        except:
            raise 
    # del file
    @refresh
    def remove_file()
        try:
            self.file_store.remove(mypath)
            file_list = self.get(self._fk)
            file_list.pop(relpath(mypath))
            self.set(self._fk,file_list)
        except:
            raise 

    # clean up all files and key/value pairs
    @refresh
    def purge()
        # use unset and unset_files

#### Application

class Application
    def __init__
      self.key
      self.ds = DataStore(self.key)
      
    def refresh ???
      *** before 
      self.ds.get(...)
      self.ds.get_files(...)
      *** function call
      *** after
      self.ds.set(...)
      self.ds.set_files(...)

    @refresh(input=,output=)
    def jobA()
          ??

    @refresh(input=,output=)
    def jobB()
          ??