Created
November 16, 2017 23:55
-
-
Save chianingwang/70e93c6671cec9bfc040ac5dcde5d2a1 to your computer and use it in GitHub Desktop.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
code I modified ( not sure it's correct or not ) | |
===s3_sync/provider_factory.py=== | |
from .sync_s3 import SyncS3 | |
from .sync_swift import SyncSwift | |
def create_provider(sync_settings, max_conns, per_account=False): | |
provider_type = sync_settings.get('protocol', None) | |
if not provider_type or provider_type == 's3': | |
return SyncS3(sync_settings, max_conns, per_account) | |
elif provider_type == 'swift' or provider_type.startswith('keystone'): | |
return SyncSwift(sync_settings, max_conns, per_account) | |
else: | |
raise NotImplementedError() | |
===s3_sync/sync_swift.py=== | |
class SyncSwift(BaseSync): | |
def __init__(self, settings, max_conns=10, per_account=False): | |
super(SyncSwift, self).__init__(settings, max_conns, per_account) | |
# Used to verify the remote container in case of per_account uploads | |
self.verified_container = False | |
@property | |
def remote_container(self): | |
if not self._per_account: | |
return self.aws_bucket | |
else: | |
# In this case the aws_bucket is treated as a prefix | |
return self.aws_bucket + self.container | |
def _get_client_factory(self): | |
# TODO: support LDAP auth | |
# TODO: support v2 auth | |
username = self.settings['aws_identity'] | |
key = self.settings['aws_secret'] | |
# Endpoint must be defined for the Swift clusters and should be the | |
# auth URL | |
endpoint = self.settings['aws_endpoint'] | |
def swift_client_factory(): | |
if self.settings.get('protocol', None) == 'swift': | |
return swiftclient.client.Connection( | |
authurl=endpoint, user=username, key=key, retries=3) | |
elif self.settings.get('protocol', None) == 'keystone_v1': | |
return swiftclient.client.Connection( | |
authurl=endpoint, user=username, key=key, auth_version='1', | |
retries=3) | |
elif self.settings.get('protocol', None) == 'keystone_v2': | |
return swiftclient.client.Connection( | |
authurl=endpoint, user=username, key=key, auth_version='2', | |
tenant_name=_tenant_name, retries=3) | |
elif self.settings.get('protocol', None) == 'keystone_v3': | |
_os_options = { | |
'user_domain_name': 'Default', | |
'project_domain_name': 'Default', | |
'project_name': 'admin' | |
} | |
return swiftclient.client.Connection( | |
authurl=endpoint, user=username, key=key, auth_version='3', | |
os_options=_os_options, retries=3) | |
return swift_client_factory | |
--------------------------------------------------------------------------------- | |
My Mock code, I know it's wrong and can't find the better way to handle it. | |
===test/unit/test_provider.py=== | |
import mock | |
import unittest | |
from s3_sync.sync_s3 import SyncS3 | |
from s3_sync.sync_swift import SyncSwift | |
from s3_sync.provide_factory import create_provider ---> import error | |
class TestProviderFactory(unittest.TestCase): | |
def setUp(self): | |
self._sync_settings = { | |
'account': 'account', | |
'container': 'container', | |
'aws_bucket': 'bucket', | |
'protocol': 'keystone_v1' | |
} | |
self._per_account = False | |
@mock.patch('s3_sync.provide_factory.create_provider') | |
def test_create_provider(self, mock_create_provider): | |
mock_create_provider.return_value = mock.Mock() | |
provider = create_provider(self._sync_settings, max_conns=3, | |
per_account=self._per_account) | |
provider.client_pool.get_client(): | |
self.assertEqual(0, base.client_pool.get_semaphore.balance) | |
self.assertEqual( | |
0, base.client_pool.client_pool[0].semaphore.balance) | |
self.assertEqual(1, base.client_pool.get_semaphore.balance) | |
===test/unit/test_keystone.py=== | |
import json | |
import mock | |
import swiftclient | |
from utils import FakeStream | |
from swift.common import swob | |
from s3_sync import utils | |
from s3_sync.sync_swift import SyncSwift | |
import unittest | |
from swiftclient.exceptions import ClientException | |
from swift.common.internal_client import UnexpectedResponse | |
class TestKeystone(unittest.TestCase): | |
def setUp(self): | |
self.os_options = "'_os_options = { | |
'user_domain_name': 'Default', | |
'project_domain_name': 'Default', | |
'project_name': 'admin'}" | |
self.aws_bucket = 'bucket' | |
self.scratch_space = 'scratch' | |
self.sync_swift = SyncSwift( | |
{'aws_bucket': self.aws_bucket, | |
'aws_identity': 'identity', | |
'aws_secret': 'credential', | |
'account': 'account', | |
'container': 'container', | |
'aws_endpoint': 'http://swift.url/auth/v1.0', | |
'auth_version':'1', | |
'tenant_name':'tenant_name', | |
'os_options':self.os_options}) | |
@mock.patch('s3_sync.sync_swift.swiftclient.client.Connection') | |
def test_keystone_v1(self, mock_swift): | |
mock_swift.return_value = mock.Mock() | |
# in this case, the "bucket" is actually the prefix | |
aws_bucket = 'sync_' | |
sync_swift = SyncSwift( | |
{'aws_bucket': aws_bucket, | |
'aws_identity': 'identity', | |
'aws_secret': 'credential', | |
'account': 'account', | |
'container': 'container', | |
'aws_endpoint': 'http://swift.url/auth/v1.0'}, | |
per_account=False) | |
self.assertEqual('sync_container', sync_swift.remote_container) | |
@mock.patch('s3_sync.sync_swift.swiftclient.client.Connection') | |
def test_keystone_v2(self, mock_swift): | |
mock_swift.return_value = mock.Mock() | |
# in this case, the "bucket" is actually the prefix | |
aws_bucket = 'sync_' | |
sync_swift = SyncSwift( | |
{'aws_bucket': aws_bucket, | |
'aws_identity': 'identity', | |
'aws_secret': 'credential', | |
'account': 'account', | |
'container': 'container', | |
'tenent_name': 'tenant' | |
'aws_endpoint': 'http://swift.url/auth/v1.0'}, | |
per_account=False) | |
self.assertEqual('sync_container', sync_swift.remote_container) | |
@mock.patch('s3_sync.sync_swift.swiftclient.client.Connection') | |
def test_keystone_v3(self, mock_swift): | |
mock_swift.return_value = mock.Mock() | |
# in this case, the "bucket" is actually the prefix | |
aws_bucket = 'sync_' | |
sync_swift = SyncSwift( | |
{'aws_bucket': aws_bucket, | |
'aws_identity': 'identity', | |
'aws_secret': 'credential', | |
'account': 'account', | |
'container': 'container', | |
'os_options':self._os_options, | |
'aws_endpoint': 'http://swift.url/auth/v1.0'}, | |
per_account=False) | |
self.assertEqual('sync_container', sync_swift.remote_container) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment