dedup-POC

from __future__ import print_function
import sys
import os
from ctypes import *
import requests
import subprocess
from subprocess import call
import re
import glob

def get_so(p, o):
  lib=cdll.LoadLibrary('./libclsPyVariableChunk.so')
  lib.ProcessFileToVar.restype=c_char_p
  return lib.ProcessFileToVar(p, 0, 2, 64, 0, True, True, o, True)

def upload_slo(swift_continer, swift_object, swift_auth, content):
  headers = {'Content-Type': 'text/csv','X-Auth-Token': swift_auth,'Cache-Control': 'no-cache',}
  data = content
  request_url = 'https://cloud.swiftstack.com/v1/AUTH_dedup/' + swift_continer +  '/' + swift_object + '?multipart-manifest=put'
  requests.put(request_url, headers=headers, data=data)

def upload_chunks(temp_slo_folder, chunks_container):
  for chunks in os.listdir(temp_slo_folder):
    cmd = ['swift', 'upload', '--changed', '-A', 'https://cloud.swiftstack.com/auth/v1.0', '-U', 'dedup', '-K', 'dedup', chunks_container, "./slo/"+chunks, '--object-name', chunks]
    p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    out, err =  p.communicate()

def get_token():
  cmd = ['swift', '-A', 'https://cloud.swiftstack.com/auth/v1.0', '-U', 'dedup', '-K', 'dedup', 'auth', '|', 'grep', 'OS_AUTH_TOKEN']
  p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
  out, err =  p.communicate()
  return re.search('OS_AUTH_TOKEN=(.*)$', out).group(1)

def clean_temp(temp_slo_folder):
  files = glob.glob(temp_slo_folder+"*")
  for file in files:
    os.remove(file)

if __name__ == '__main__':
  if len(sys.argv) == 6:

    #collect arguments
    dedup_file = sys.argv[1]
    temp_slo_folder = sys.argv[2]
    swift_container = sys.argv[3]
    swift_object = sys.argv[4]
    chunks_container = sys.argv[5]

    # clean out temp slo folder
    clean_temp(temp_slo_folder)

    # generate the json and splite file into chunks under ./slo
    result=get_so(dedup_file, temp_slo_folder)

    # get token
    swift_auth = get_token()

    # upload chunks
    upload_chunks(temp_slo_folder, chunks_container)

    # upload slo
    upload_slo(swift_container, swift_object, swift_auth, result)
  else:
    print("please try $python dedup_upload.py [dedup_file] [temp_slo_folder] [swift_container] [swift_slo_object_name] [swift_dedup_container]")