dedup_skip_identical

#import requests
from ctypes import *
import swiftclient
from swiftclient.service import SwiftService, SwiftUploadObject
#from swiftclient.client import put_object
import os
import glob
from sys import argv

_opts = {
    "auth": 'https://cloud.swiftstack.com/auth/v1.0',
    "auth_version": '1.0',
    "user": "dedup",
    "key": "dedup",
}

def clean_temp(temp_slo_folder):
  files = glob.glob(temp_slo_folder+"*")
  for file in files:
    os.remove(file)

def create_manifest_chunks(p, o):
  lib=cdll.LoadLibrary('./libclsPyVariableChunk.so')
  lib.ProcessFileToVar.restype=c_char_p
  return lib.ProcessFileToVar(p, 0, 2, 64, 0, True, True, o, True)

def upload_manifest(manifest_obj, args):
  try:
    swiftclient.client.put_object(
      args["storage_url"], args["auth_token"], args["manifest_container"],
       args["filename"], manifest_obj,
       query_string="multipart-manifest=put")
    print("upload ok!")
  except Exception, e:
    print(e)

def upload_dedup_chunks(container, obj_path):
  headers = []
  options = {
    'skip_identical': True,
    'header': headers
  }
  with SwiftService(options=_opts) as swift:
    objects = []
    for chunks in os.listdir(obj_path):
      objects.append(swiftclient.service.SwiftUploadObject(obj_path+chunks, chunks))
    resp = swift.upload(container, objects=objects, options=options)
    for r in resp:
      print(r)

if __name__ == '__main__':
  # input file for dedup
  source = argv[1]
  args = {
        "filename": os.path.basename(source),
        "source_file": os.path.abspath(source),
        "chunks_container": "chunks",
        "manifest_container": "dedup",
        "auth_token": "AUTH_tkcb27d60d8b974ffabc2dd049fd9cdae4",
        "storage_url": "https://cloud.swiftstack.com/v1/AUTH_dedup",
        "temp_directory": "/Users/jwang/Documents/tmp/jbox/slo/",
    }
  clean_temp(args["temp_directory"])
  result=create_manifest_chunks(args["source_file"], args["temp_directory"])
  upload_dedup_chunks(args["chunks_container"], args["temp_directory"])
  upload_manifest(result, args)