Skip to content

Instantly share code, notes, and snippets.

@chianingwang
Created November 16, 2017 23:55
Show Gist options
  • Save chianingwang/70e93c6671cec9bfc040ac5dcde5d2a1 to your computer and use it in GitHub Desktop.
Save chianingwang/70e93c6671cec9bfc040ac5dcde5d2a1 to your computer and use it in GitHub Desktop.
code I modified ( not sure it's correct or not )
===s3_sync/provider_factory.py===
from .sync_s3 import SyncS3
from .sync_swift import SyncSwift
def create_provider(sync_settings, max_conns, per_account=False):
provider_type = sync_settings.get('protocol', None)
if not provider_type or provider_type == 's3':
return SyncS3(sync_settings, max_conns, per_account)
elif provider_type == 'swift' or provider_type.startswith('keystone'):
return SyncSwift(sync_settings, max_conns, per_account)
else:
raise NotImplementedError()
===s3_sync/sync_swift.py===
class SyncSwift(BaseSync):
def __init__(self, settings, max_conns=10, per_account=False):
super(SyncSwift, self).__init__(settings, max_conns, per_account)
# Used to verify the remote container in case of per_account uploads
self.verified_container = False
@property
def remote_container(self):
if not self._per_account:
return self.aws_bucket
else:
# In this case the aws_bucket is treated as a prefix
return self.aws_bucket + self.container
def _get_client_factory(self):
# TODO: support LDAP auth
# TODO: support v2 auth
username = self.settings['aws_identity']
key = self.settings['aws_secret']
# Endpoint must be defined for the Swift clusters and should be the
# auth URL
endpoint = self.settings['aws_endpoint']
def swift_client_factory():
if self.settings.get('protocol', None) == 'swift':
return swiftclient.client.Connection(
authurl=endpoint, user=username, key=key, retries=3)
elif self.settings.get('protocol', None) == 'keystone_v1':
return swiftclient.client.Connection(
authurl=endpoint, user=username, key=key, auth_version='1',
retries=3)
elif self.settings.get('protocol', None) == 'keystone_v2':
return swiftclient.client.Connection(
authurl=endpoint, user=username, key=key, auth_version='2',
tenant_name=_tenant_name, retries=3)
elif self.settings.get('protocol', None) == 'keystone_v3':
_os_options = {
'user_domain_name': 'Default',
'project_domain_name': 'Default',
'project_name': 'admin'
}
return swiftclient.client.Connection(
authurl=endpoint, user=username, key=key, auth_version='3',
os_options=_os_options, retries=3)
return swift_client_factory
---------------------------------------------------------------------------------
My Mock code, I know it's wrong and can't find the better way to handle it.
===test/unit/test_provider.py===
import mock
import unittest
from s3_sync.sync_s3 import SyncS3
from s3_sync.sync_swift import SyncSwift
from s3_sync.provide_factory import create_provider ---> import error
class TestProviderFactory(unittest.TestCase):
def setUp(self):
self._sync_settings = {
'account': 'account',
'container': 'container',
'aws_bucket': 'bucket',
'protocol': 'keystone_v1'
}
self._per_account = False
@mock.patch('s3_sync.provide_factory.create_provider')
def test_create_provider(self, mock_create_provider):
mock_create_provider.return_value = mock.Mock()
provider = create_provider(self._sync_settings, max_conns=3,
per_account=self._per_account)
provider.client_pool.get_client():
self.assertEqual(0, base.client_pool.get_semaphore.balance)
self.assertEqual(
0, base.client_pool.client_pool[0].semaphore.balance)
self.assertEqual(1, base.client_pool.get_semaphore.balance)
===test/unit/test_keystone.py===
import json
import mock
import swiftclient
from utils import FakeStream
from swift.common import swob
from s3_sync import utils
from s3_sync.sync_swift import SyncSwift
import unittest
from swiftclient.exceptions import ClientException
from swift.common.internal_client import UnexpectedResponse
class TestKeystone(unittest.TestCase):
def setUp(self):
self.os_options = "'_os_options = {
'user_domain_name': 'Default',
'project_domain_name': 'Default',
'project_name': 'admin'}"
self.aws_bucket = 'bucket'
self.scratch_space = 'scratch'
self.sync_swift = SyncSwift(
{'aws_bucket': self.aws_bucket,
'aws_identity': 'identity',
'aws_secret': 'credential',
'account': 'account',
'container': 'container',
'aws_endpoint': 'http://swift.url/auth/v1.0',
'auth_version':'1',
'tenant_name':'tenant_name',
'os_options':self.os_options})
@mock.patch('s3_sync.sync_swift.swiftclient.client.Connection')
def test_keystone_v1(self, mock_swift):
mock_swift.return_value = mock.Mock()
# in this case, the "bucket" is actually the prefix
aws_bucket = 'sync_'
sync_swift = SyncSwift(
{'aws_bucket': aws_bucket,
'aws_identity': 'identity',
'aws_secret': 'credential',
'account': 'account',
'container': 'container',
'aws_endpoint': 'http://swift.url/auth/v1.0'},
per_account=False)
self.assertEqual('sync_container', sync_swift.remote_container)
@mock.patch('s3_sync.sync_swift.swiftclient.client.Connection')
def test_keystone_v2(self, mock_swift):
mock_swift.return_value = mock.Mock()
# in this case, the "bucket" is actually the prefix
aws_bucket = 'sync_'
sync_swift = SyncSwift(
{'aws_bucket': aws_bucket,
'aws_identity': 'identity',
'aws_secret': 'credential',
'account': 'account',
'container': 'container',
'tenent_name': 'tenant'
'aws_endpoint': 'http://swift.url/auth/v1.0'},
per_account=False)
self.assertEqual('sync_container', sync_swift.remote_container)
@mock.patch('s3_sync.sync_swift.swiftclient.client.Connection')
def test_keystone_v3(self, mock_swift):
mock_swift.return_value = mock.Mock()
# in this case, the "bucket" is actually the prefix
aws_bucket = 'sync_'
sync_swift = SyncSwift(
{'aws_bucket': aws_bucket,
'aws_identity': 'identity',
'aws_secret': 'credential',
'account': 'account',
'container': 'container',
'os_options':self._os_options,
'aws_endpoint': 'http://swift.url/auth/v1.0'},
per_account=False)
self.assertEqual('sync_container', sync_swift.remote_container)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment