Created
December 6, 2017 17:28
-
-
Save chianingwang/2eb5221fad427700c2c3c30e267b028f to your computer and use it in GitHub Desktop.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| (venv) root@developer:~/swift-s3-sync# ./venv/bin/nosetests ./test/integration | |
| ....EEEE | |
| ====================================================================== | |
| ERROR: test_swift_archive (test_s3_sync.TestCloudSync) | |
| ---------------------------------------------------------------------- | |
| Traceback (most recent call last): | |
| File "/root/swift-s3-sync/test/integration/test_s3_sync.py", line 331, in test_swift_archive | |
| expected_location) | |
| File "/root/swift-s3-sync/test/integration/test_s3_sync.py", line 237, in _test_archive | |
| swift_hdrs, listing = wait_for_condition(5, _check_expired) | |
| File "/root/swift-s3-sync/test/integration/test_s3_sync.py", line 32, in wait_for_condition | |
| raise RuntimeError('Timeout expired') | |
| RuntimeError: Timeout expired | |
| -------------------- >> begin captured logging << -------------------- | |
| urllib3.connectionpool: DEBUG: http://localhost:32771 "PUT /v1/AUTH_test/archive-swift/test_archive HTTP/1.1" 201 0 | |
| swiftclient: DEBUG: REQ: curl -i http://localhost:32771/v1/AUTH_test/archive-swift/test_archive -X PUT -H "X-Auth-Token: AUTH_tk23fef78ac..." | |
| swiftclient: DEBUG: RESP STATUS: 201 Created | |
| swiftclient: DEBUG: RESP HEADERS: {u'Content-Length': u'0', u'Last-Modified': u'Wed, 06 Dec 2017 17:23:47 GMT', u'Etag': u'00cf75325f24cbbdbed3b80b26ff500b', u'X-Trans-Id': u'txc739dd8afd19429ea403a-005a2827a2', u'Date': u'Wed, 06 Dec 2017 17:23:46 GMT', u'Content-Type': u'text/html; charset=UTF-8', u'X-Openstack-Request-Id': u'txc739dd8afd19429ea403a-005a2827a2'} | |
| urllib3.connectionpool: DEBUG: http://localhost:32771 "GET /v1/AUTH_test/archive-swift?format=json HTTP/1.1" 200 174 | |
| swiftclient: DEBUG: REQ: curl -i http://localhost:32771/v1/AUTH_test/archive-swift?format=json -X GET -H "Accept-Encoding: gzip" -H "X-Auth-Token: AUTH_tk23fef78ac..." | |
| swiftclient: DEBUG: RESP STATUS: 200 OK | |
| swiftclient: DEBUG: RESP HEADERS: {u'Content-Length': u'174', u'X-Container-Object-Count': u'1', u'Date': u'Wed, 06 Dec 2017 17:23:47 GMT', u'Accept-Ranges': u'bytes', u'X-Storage-Policy': u'Policy-0', u'Last-Modified': u'Wed, 06 Dec 2017 17:23:41 GMT', u'X-Timestamp': u'1512544311.15302', u'X-Trans-Id': u'tx961d9904ddc34a9ba0504-005a2827a2', u'X-Container-Bytes-Used': u'19', u'Content-Type': u'application/json; charset=utf-8', u'X-Openstack-Request-Id': u'tx961d9904ddc34a9ba0504-005a2827a2'} | |
| swiftclient: DEBUG: RESP BODY: [{"hash": "00cf75325f24cbbdbed3b80b26ff500b", "last_modified": "2017-12-06T17:23:46.157450", "bytes": 19, "name": "test_archive", "content_type": "application/octet-stream"}] | |
| urllib3.connectionpool: DEBUG: http://localhost:32771 "GET /v1/AUTH_test/archive-swift?format=json HTTP/1.1" 200 174 | |
| swiftclient: DEBUG: REQ: curl -i http://localhost:32771/v1/AUTH_test/archive-swift?format=json -X GET -H "Accept-Encoding: gzip" -H "X-Auth-Token: AUTH_tk23fef78ac..." | |
| swiftclient: DEBUG: RESP STATUS: 200 OK | |
| swiftclient: DEBUG: RESP HEADERS: {u'Content-Length': u'174', u'X-Container-Object-Count': u'1', u'Date': u'Wed, 06 Dec 2017 17:23:48 GMT', u'Accept-Ranges': u'bytes', u'X-Storage-Policy': u'Policy-0', u'Last-Modified': u'Wed, 06 Dec 2017 17:23:41 GMT', u'X-Timestamp': u'1512544311.15302', u'X-Trans-Id': u'txd7e38f6f2d6c47c1b0587-005a2827a3', u'X-Container-Bytes-Used': u'19', u'Content-Type': u'application/json; charset=utf-8', u'X-Openstack-Request-Id': u'txd7e38f6f2d6c47c1b0587-005a2827a3'} | |
| swiftclient: DEBUG: RESP BODY: [{"hash": "00cf75325f24cbbdbed3b80b26ff500b", "last_modified": "2017-12-06T17:23:46.157450", "bytes": 19, "name": "test_archive", "content_type": "application/octet-stream"}] | |
| urllib3.connectionpool: DEBUG: http://localhost:32771 "GET /v1/AUTH_test/archive-swift?format=json HTTP/1.1" 200 174 | |
| swiftclient: DEBUG: REQ: curl -i http://localhost:32771/v1/AUTH_test/archive-swift?format=json -X GET -H "Accept-Encoding: gzip" -H "X-Auth-Token: AUTH_tk23fef78ac..." | |
| swiftclient: DEBUG: RESP STATUS: 200 OK | |
| swiftclient: DEBUG: RESP HEADERS: {u'Content-Length': u'174', u'X-Container-Object-Count': u'1', u'Date': u'Wed, 06 Dec 2017 17:23:49 GMT', u'Accept-Ranges': u'bytes', u'X-Storage-Policy': u'Policy-0', u'Last-Modified': u'Wed, 06 Dec 2017 17:23:41 GMT', u'X-Timestamp': u'1512544311.15302', u'X-Trans-Id': u'tx8a7ba839b4bb4eae9fdbc-005a2827a4', u'X-Container-Bytes-Used': u'19', u'Content-Type': u'application/json; charset=utf-8', u'X-Openstack-Request-Id': u'tx8a7ba839b4bb4eae9fdbc-005a2827a4'} | |
| swiftclient: DEBUG: RESP BODY: [{"hash": "00cf75325f24cbbdbed3b80b26ff500b", "last_modified": "2017-12-06T17:23:46.157450", "bytes": 19, "name": "test_archive", "content_type": "application/octet-stream"}] | |
| urllib3.connectionpool: DEBUG: http://localhost:32771 "GET /v1/AUTH_test/archive-swift?format=json HTTP/1.1" 200 174 | |
| swiftclient: DEBUG: REQ: curl -i http://localhost:32771/v1/AUTH_test/archive-swift?format=json -X GET -H "Accept-Encoding: gzip" -H "X-Auth-Token: AUTH_tk23fef78ac..." | |
| swiftclient: DEBUG: RESP STATUS: 200 OK | |
| swiftclient: DEBUG: RESP HEADERS: {u'Content-Length': u'174', u'X-Container-Object-Count': u'1', u'Date': u'Wed, 06 Dec 2017 17:23:50 GMT', u'Accept-Ranges': u'bytes', u'X-Storage-Policy': u'Policy-0', u'Last-Modified': u'Wed, 06 Dec 2017 17:23:41 GMT', u'X-Timestamp': u'1512544311.15302', u'X-Trans-Id': u'tx0637e082e129443da3583-005a2827a5', u'X-Container-Bytes-Used': u'19', u'Content-Type': u'application/json; charset=utf-8', u'X-Openstack-Request-Id': u'tx0637e082e129443da3583-005a2827a5'} | |
| swiftclient: DEBUG: RESP BODY: [{"hash": "00cf75325f24cbbdbed3b80b26ff500b", "last_modified": "2017-12-06T17:23:46.157450", "bytes": 19, "name": "test_archive", "content_type": "application/octet-stream"}] | |
| urllib3.connectionpool: DEBUG: http://localhost:32771 "GET /v1/AUTH_test/archive-swift?format=json HTTP/1.1" 200 174 | |
| swiftclient: DEBUG: REQ: curl -i http://localhost:32771/v1/AUTH_test/archive-swift?format=json -X GET -H "Accept-Encoding: gzip" -H "X-Auth-Token: AUTH_tk23fef78ac..." | |
| swiftclient: DEBUG: RESP STATUS: 200 OK | |
| swiftclient: DEBUG: RESP HEADERS: {u'Content-Length': u'174', u'X-Container-Object-Count': u'1', u'Date': u'Wed, 06 Dec 2017 17:23:51 GMT', u'Accept-Ranges': u'bytes', u'X-Storage-Policy': u'Policy-0', u'Last-Modified': u'Wed, 06 Dec 2017 17:23:41 GMT', u'X-Timestamp': u'1512544311.15302', u'X-Trans-Id': u'tx7103ddec05ed4917a9d76-005a2827a6', u'X-Container-Bytes-Used': u'19', u'Content-Type': u'application/json; charset=utf-8', u'X-Openstack-Request-Id': u'tx7103ddec05ed4917a9d76-005a2827a6'} | |
| swiftclient: DEBUG: RESP BODY: [{"hash": "00cf75325f24cbbdbed3b80b26ff500b", "last_modified": "2017-12-06T17:23:46.157450", "bytes": 19, "name": "test_archive", "content_type": "application/octet-stream"}] | |
| --------------------- >> end captured logging << --------------------- | |
| ====================================================================== | |
| ERROR: test_swift_archive_get (test_s3_sync.TestCloudSync) | |
| ---------------------------------------------------------------------- | |
| Traceback (most recent call last): | |
| File "/root/swift-s3-sync/test/integration/test_s3_sync.py", line 457, in test_swift_archive_get | |
| 'get_object', mapping['container'], key, content) | |
| File "/root/swift-s3-sync/test/integration/test_s3_sync.py", line 210, in local_swift | |
| return getattr(self.__class__.swift_src, method)(*args, **kwargs) | |
| File "/root/swift-s3-sync/venv/local/lib/python2.7/site-packages/swiftclient/client.py", line 1786, in get_object | |
| headers=headers) | |
| File "/root/swift-s3-sync/venv/local/lib/python2.7/site-packages/swiftclient/client.py", line 1678, in _retry | |
| service_token=self.service_token, **kwargs) | |
| File "/root/swift-s3-sync/venv/local/lib/python2.7/site-packages/swiftclient/client.py", line 1166, in get_object | |
| raise ClientException.from_response(resp, 'Object GET failed', body) | |
| ClientException: Object GET failed: http://localhost:32771/v1/AUTH_test/swift-restore/test_swift_archive 401 Unauthorized [first 60 chars of response] <html><h1>Unauthorized</h1><p>This server could not verify t | |
| -------------------- >> begin captured logging << -------------------- | |
| urllib3.connectionpool: DEBUG: http://localhost:32771 "PUT /v1/AUTH_test2/swift-restore/test_swift_archive HTTP/1.1" 201 0 | |
| swiftclient: DEBUG: REQ: curl -i http://localhost:32771/v1/AUTH_test2/swift-restore/test_swift_archive -X PUT -H "X-Auth-Token: AUTH_tk721e6f84b..." | |
| swiftclient: DEBUG: RESP STATUS: 201 Created | |
| swiftclient: DEBUG: RESP HEADERS: {u'Content-Length': u'0', u'Last-Modified': u'Wed, 06 Dec 2017 17:23:52 GMT', u'Etag': u'0c27b7f8ce0805cf6cbbd318d7c66e4e', u'X-Trans-Id': u'tx7bcc550e2a4b43d4b36db-005a2827a7', u'Date': u'Wed, 06 Dec 2017 17:23:51 GMT', u'Content-Type': u'text/html; charset=UTF-8', u'X-Openstack-Request-Id': u'tx7bcc550e2a4b43d4b36db-005a2827a7'} | |
| urllib3.connectionpool: DEBUG: http://localhost:32771 "GET /v1/AUTH_test/swift-restore?format=json HTTP/1.1" 200 2 | |
| swiftclient: DEBUG: REQ: curl -i http://localhost:32771/v1/AUTH_test/swift-restore?format=json -X GET -H "Accept-Encoding: gzip" -H "X-Auth-Token: AUTH_tk23fef78ac..." | |
| swiftclient: DEBUG: RESP STATUS: 200 OK | |
| swiftclient: DEBUG: RESP HEADERS: {u'Content-Length': u'2', u'X-Container-Object-Count': u'0', u'Date': u'Wed, 06 Dec 2017 17:23:52 GMT', u'Accept-Ranges': u'bytes', u'X-Storage-Policy': u'Policy-0', u'Last-Modified': u'Wed, 06 Dec 2017 17:23:41 GMT', u'X-Timestamp': u'1512544311.19119', u'X-Trans-Id': u'tx494e2899319147e0bcdbe-005a2827a7', u'X-Container-Bytes-Used': u'0', u'Content-Type': u'application/json; charset=utf-8', u'X-Openstack-Request-Id': u'tx494e2899319147e0bcdbe-005a2827a7'} | |
| swiftclient: DEBUG: RESP BODY: [] | |
| urllib3.connectionpool: DEBUG: http://localhost:32771 "GET /v1/AUTH_test/swift-restore/test_swift_archive HTTP/1.1" 401 131 | |
| swiftclient: INFO: REQ: curl -i http://localhost:32771/v1/AUTH_test/swift-restore/test_swift_archive -X GET -H "X-Auth-Token: AUTH_tk23fef78ac..." | |
| swiftclient: INFO: RESP STATUS: 401 Unauthorized | |
| swiftclient: INFO: RESP HEADERS: {u'Content-Length': u'131', u'X-Openstack-Request-Id': u'txa27ca8c724924dc9af58e-005a2827a8', u'Remote-X-Openstack-Request-Id': u'tx6ed351126c2e4e1d94dcf-005a2827aa', u'X-Trans-Id': u'txa27ca8c724924dc9af58e-005a2827a8', u'Date': u'Wed, 06 Dec 2017 17:23:54 GMT', u'Remote-X-Trans-Id': u'tx6ed351126c2e4e1d94dcf-005a2827aa', u'Content-Type': u'text/html; charset=UTF-8', u'Www-Authenticate': u'Swift realm="http"'} | |
| swiftclient: INFO: RESP BODY: <html><h1>Unauthorized</h1><p>This server could not verify that you are authorized to access the document you requested.</p></html> | |
| urllib3.connectionpool: DEBUG: Starting new HTTP connection (1): localhost | |
| urllib3.connectionpool: DEBUG: http://localhost:32771 "GET /auth/v1.0 HTTP/1.1" 200 0 | |
| swiftclient: DEBUG: REQ: curl -i http://localhost:32771/auth/v1.0 -X GET | |
| swiftclient: DEBUG: RESP STATUS: 200 OK | |
| swiftclient: DEBUG: RESP HEADERS: {u'Content-Length': u'0', u'X-Trans-Id': u'txf45d4cb9894b4baaa65d7-005a2827ab', u'X-Auth-Token-Expires': u'86384', u'X-Auth-Token': u'AUTH_tk23fef78ac...', u'X-Storage-Token': u'AUTH_tk23fef78ac...', u'Date': u'Wed, 06 Dec 2017 17:23:55 GMT', u'X-Storage-Url': u'http://localhost:32771/v1/AUTH_test', u'Content-Type': u'text/html; charset=UTF-8', u'X-Openstack-Request-Id': u'txf45d4cb9894b4baaa65d7-005a2827ab'} | |
| urllib3.connectionpool: DEBUG: Starting new HTTP connection (1): localhost | |
| urllib3.connectionpool: DEBUG: http://localhost:32771 "GET /v1/AUTH_test/swift-restore/test_swift_archive HTTP/1.1" 401 131 | |
| swiftclient: INFO: REQ: curl -i http://localhost:32771/v1/AUTH_test/swift-restore/test_swift_archive -X GET -H "X-Auth-Token: AUTH_tk23fef78ac..." | |
| swiftclient: INFO: RESP STATUS: 401 Unauthorized | |
| swiftclient: INFO: RESP HEADERS: {u'Content-Length': u'131', u'X-Openstack-Request-Id': u'txd85c58a34d7c44958e9de-005a2827ab', u'Remote-X-Openstack-Request-Id': u'tx15f7ffd7061a464a85be4-005a2827ad', u'X-Trans-Id': u'txd85c58a34d7c44958e9de-005a2827ab', u'Date': u'Wed, 06 Dec 2017 17:23:57 GMT', u'Remote-X-Trans-Id': u'tx15f7ffd7061a464a85be4-005a2827ad', u'Content-Type': u'text/html; charset=UTF-8', u'Www-Authenticate': u'Swift realm="http"'} | |
| swiftclient: INFO: RESP BODY: <html><h1>Unauthorized</h1><p>This server could not verify that you are authorized to access the document you requested.</p></html> | |
| --------------------- >> end captured logging << --------------------- | |
| ====================================================================== | |
| ERROR: test_swift_archive_slo_restore (test_s3_sync.TestCloudSync) | |
| ---------------------------------------------------------------------- | |
| Traceback (most recent call last): | |
| File "/root/swift-s3-sync/test/integration/test_s3_sync.py", line 501, in test_swift_archive_slo_restore | |
| 'get_object', mapping['container'], key, content) | |
| File "/root/swift-s3-sync/test/integration/test_s3_sync.py", line 210, in local_swift | |
| return getattr(self.__class__.swift_src, method)(*args, **kwargs) | |
| File "/root/swift-s3-sync/venv/local/lib/python2.7/site-packages/swiftclient/client.py", line 1786, in get_object | |
| headers=headers) | |
| File "/root/swift-s3-sync/venv/local/lib/python2.7/site-packages/swiftclient/client.py", line 1678, in _retry | |
| service_token=self.service_token, **kwargs) | |
| File "/root/swift-s3-sync/venv/local/lib/python2.7/site-packages/swiftclient/client.py", line 1166, in get_object | |
| raise ClientException.from_response(resp, 'Object GET failed', body) | |
| ClientException: Object GET failed: http://localhost:32771/v1/AUTH_test/swift-restore/test_swift_archive 401 Unauthorized [first 60 chars of response] <html><h1>Unauthorized</h1><p>This server could not verify t | |
| -------------------- >> begin captured logging << -------------------- | |
| urllib3.connectionpool: DEBUG: http://localhost:32771 "PUT /v1/AUTH_test2/segments HTTP/1.1" 202 76 | |
| swiftclient: DEBUG: REQ: curl -i http://localhost:32771/v1/AUTH_test2/segments -X PUT -H "Content-Length: 0" -H "X-Auth-Token: AUTH_tk721e6f84b..." | |
| swiftclient: DEBUG: RESP STATUS: 202 Accepted | |
| swiftclient: DEBUG: RESP HEADERS: {u'Date': u'Wed, 06 Dec 2017 17:23:57 GMT', u'Content-Length': u'76', u'Content-Type': u'text/html; charset=UTF-8', u'X-Openstack-Request-Id': u'tx1a7119684a9f4ebebcc45-005a2827ad', u'X-Trans-Id': u'tx1a7119684a9f4ebebcc45-005a2827ad'} | |
| swiftclient: DEBUG: RESP BODY: <html><h1>Accepted</h1><p>The request is accepted for processing.</p></html> | |
| urllib3.connectionpool: DEBUG: http://localhost:32771 "PUT /v1/AUTH_test2/segments/part1 HTTP/1.1" 201 0 | |
| swiftclient: DEBUG: REQ: curl -i http://localhost:32771/v1/AUTH_test2/segments/part1 -X PUT -H "X-Auth-Token: AUTH_tk721e6f84b..." | |
| swiftclient: DEBUG: RESP STATUS: 201 Created | |
| swiftclient: DEBUG: RESP HEADERS: {u'Content-Length': u'0', u'Last-Modified': u'Wed, 06 Dec 2017 17:23:58 GMT', u'Etag': u'd47b127bc2de2d687ddc82dac354c415', u'X-Trans-Id': u'tx612d7526795d4df9bd794-005a2827ad', u'Date': u'Wed, 06 Dec 2017 17:23:57 GMT', u'Content-Type': u'text/html; charset=UTF-8', u'X-Openstack-Request-Id': u'tx612d7526795d4df9bd794-005a2827ad'} | |
| urllib3.connectionpool: DEBUG: http://localhost:32771 "PUT /v1/AUTH_test2/segments/part2 HTTP/1.1" 201 0 | |
| swiftclient: DEBUG: REQ: curl -i http://localhost:32771/v1/AUTH_test2/segments/part2 -X PUT -H "X-Auth-Token: AUTH_tk721e6f84b..." | |
| swiftclient: DEBUG: RESP STATUS: 201 Created | |
| swiftclient: DEBUG: RESP HEADERS: {u'Content-Length': u'0', u'Last-Modified': u'Wed, 06 Dec 2017 17:23:58 GMT', u'Etag': u'd47b127bc2de2d687ddc82dac354c415', u'X-Trans-Id': u'tx1e8299a7c98a4f01af1ba-005a2827ad', u'Date': u'Wed, 06 Dec 2017 17:23:57 GMT', u'Content-Type': u'text/html; charset=UTF-8', u'X-Openstack-Request-Id': u'tx1e8299a7c98a4f01af1ba-005a2827ad'} | |
| urllib3.connectionpool: DEBUG: http://localhost:32771 "PUT /v1/AUTH_test2/swift-restore/test_swift_archive?multipart-manifest=put HTTP/1.1" 201 0 | |
| swiftclient: DEBUG: REQ: curl -i http://localhost:32771/v1/AUTH_test2/swift-restore/test_swift_archive?multipart-manifest=put -X PUT -H "X-Auth-Token: AUTH_tk721e6f84b..." | |
| swiftclient: DEBUG: RESP STATUS: 201 Created | |
| swiftclient: DEBUG: RESP HEADERS: {u'Content-Length': u'0', u'Last-Modified': u'Wed, 06 Dec 2017 17:23:58 GMT', u'Etag': u'"aaea2b5b75ec3f58f23be2f91731a041"', u'X-Trans-Id': u'tx7c211157b41e4ef3b0381-005a2827ad', u'Date': u'Wed, 06 Dec 2017 17:23:58 GMT', u'Content-Type': u'text/html; charset=UTF-8', u'X-Openstack-Request-Id': u'tx7c211157b41e4ef3b0381-005a2827ad'} | |
| urllib3.connectionpool: DEBUG: Starting new HTTP connection (1): localhost | |
| urllib3.connectionpool: DEBUG: http://localhost:32771 "GET /auth/v1.0 HTTP/1.1" 200 0 | |
| swiftclient: DEBUG: REQ: curl -i http://localhost:32771/auth/v1.0 -X GET | |
| swiftclient: DEBUG: RESP STATUS: 200 OK | |
| swiftclient: DEBUG: RESP HEADERS: {u'Content-Length': u'0', u'X-Trans-Id': u'tx6429754c400c469ba07dc-005a2827ae', u'X-Auth-Token-Expires': u'86382', u'X-Auth-Token': u'AUTH_tk23fef78ac...', u'X-Storage-Token': u'AUTH_tk23fef78ac...', u'Date': u'Wed, 06 Dec 2017 17:23:58 GMT', u'X-Storage-Url': u'http://localhost:32771/v1/AUTH_test', u'Content-Type': u'text/html; charset=UTF-8', u'X-Openstack-Request-Id': u'tx6429754c400c469ba07dc-005a2827ae'} | |
| urllib3.connectionpool: DEBUG: Starting new HTTP connection (1): localhost | |
| urllib3.connectionpool: DEBUG: http://localhost:32771 "GET /v1/AUTH_test/swift-restore?format=json HTTP/1.1" 200 2 | |
| swiftclient: DEBUG: REQ: curl -i http://localhost:32771/v1/AUTH_test/swift-restore?format=json -X GET -H "Accept-Encoding: gzip" -H "X-Auth-Token: AUTH_tk23fef78ac..." | |
| swiftclient: DEBUG: RESP STATUS: 200 OK | |
| swiftclient: DEBUG: RESP HEADERS: {u'Content-Length': u'2', u'X-Container-Object-Count': u'0', u'Date': u'Wed, 06 Dec 2017 17:23:59 GMT', u'Accept-Ranges': u'bytes', u'X-Storage-Policy': u'Policy-0', u'Last-Modified': u'Wed, 06 Dec 2017 17:23:41 GMT', u'X-Timestamp': u'1512544311.19119', u'X-Trans-Id': u'tx87f74023476d4bf08888a-005a2827ae', u'X-Container-Bytes-Used': u'0', u'Content-Type': u'application/json; charset=utf-8', u'X-Openstack-Request-Id': u'tx87f74023476d4bf08888a-005a2827ae'} | |
| swiftclient: DEBUG: RESP BODY: [] | |
| urllib3.connectionpool: DEBUG: http://localhost:32771 "GET /v1/AUTH_test/swift-restore/test_swift_archive HTTP/1.1" 401 131 | |
| swiftclient: INFO: REQ: curl -i http://localhost:32771/v1/AUTH_test/swift-restore/test_swift_archive -X GET -H "X-Auth-Token: AUTH_tk23fef78ac..." | |
| swiftclient: INFO: RESP STATUS: 401 Unauthorized | |
| swiftclient: INFO: RESP HEADERS: {u'Content-Length': u'131', u'X-Openstack-Request-Id': u'tx1df11425f3c74bce96cd5-005a2827af', u'Remote-X-Openstack-Request-Id': u'tx6c1a31679a494d49a7975-005a2827b1', u'X-Trans-Id': u'tx1df11425f3c74bce96cd5-005a2827af', u'Date': u'Wed, 06 Dec 2017 17:24:01 GMT', u'Remote-X-Trans-Id': u'tx6c1a31679a494d49a7975-005a2827b1', u'Content-Type': u'text/html; charset=UTF-8', u'Www-Authenticate': u'Swift realm="http"'} | |
| swiftclient: INFO: RESP BODY: <html><h1>Unauthorized</h1><p>This server could not verify that you are authorized to access the document you requested.</p></html> | |
| urllib3.connectionpool: DEBUG: Starting new HTTP connection (1): localhost | |
| urllib3.connectionpool: DEBUG: http://localhost:32771 "GET /auth/v1.0 HTTP/1.1" 200 0 | |
| swiftclient: DEBUG: REQ: curl -i http://localhost:32771/auth/v1.0 -X GET | |
| swiftclient: DEBUG: RESP STATUS: 200 OK | |
| swiftclient: DEBUG: RESP HEADERS: {u'Content-Length': u'0', u'X-Trans-Id': u'txacff8d0b40334a078f0a1-005a2827b2', u'X-Auth-Token-Expires': u'86378', u'X-Auth-Token': u'AUTH_tk23fef78ac...', u'X-Storage-Token': u'AUTH_tk23fef78ac...', u'Date': u'Wed, 06 Dec 2017 17:24:02 GMT', u'X-Storage-Url': u'http://localhost:32771/v1/AUTH_test', u'Content-Type': u'text/html; charset=UTF-8', u'X-Openstack-Request-Id': u'txacff8d0b40334a078f0a1-005a2827b2'} | |
| urllib3.connectionpool: DEBUG: Starting new HTTP connection (1): localhost | |
| urllib3.connectionpool: DEBUG: http://localhost:32771 "GET /v1/AUTH_test/swift-restore/test_swift_archive HTTP/1.1" 401 131 | |
| swiftclient: INFO: REQ: curl -i http://localhost:32771/v1/AUTH_test/swift-restore/test_swift_archive -X GET -H "X-Auth-Token: AUTH_tk23fef78ac..." | |
| swiftclient: INFO: RESP STATUS: 401 Unauthorized | |
| swiftclient: INFO: RESP HEADERS: {u'Content-Length': u'131', u'X-Openstack-Request-Id': u'tx37ecf209d37f48a6b3c3c-005a2827b2', u'Remote-X-Openstack-Request-Id': u'tx49b4571e5a4b4aad81226-005a2827b4', u'X-Trans-Id': u'tx37ecf209d37f48a6b3c3c-005a2827b2', u'Date': u'Wed, 06 Dec 2017 17:24:04 GMT', u'Remote-X-Trans-Id': u'tx49b4571e5a4b4aad81226-005a2827b4', u'Content-Type': u'text/html; charset=UTF-8', u'Www-Authenticate': u'Swift realm="http"'} | |
| swiftclient: INFO: RESP BODY: <html><h1>Unauthorized</h1><p>This server could not verify that you are authorized to access the document you requested.</p></html> | |
| --------------------- >> end captured logging << --------------------- | |
| ====================================================================== | |
| ERROR: test_swift_sync (test_s3_sync.TestCloudSync) | |
| ---------------------------------------------------------------------- | |
| Traceback (most recent call last): | |
| File "/root/swift-s3-sync/test/integration/test_s3_sync.py", line 314, in test_swift_sync | |
| head_resp = wait_for_condition(5, _check_sync) | |
| File "/root/swift-s3-sync/test/integration/test_s3_sync.py", line 32, in wait_for_condition | |
| raise RuntimeError('Timeout expired') | |
| RuntimeError: Timeout expired | |
| -------------------- >> begin captured logging << -------------------- | |
| urllib3.connectionpool: DEBUG: Starting new HTTP connection (1): localhost | |
| urllib3.connectionpool: DEBUG: http://localhost:32771 "GET /auth/v1.0 HTTP/1.1" 200 0 | |
| swiftclient: DEBUG: REQ: curl -i http://localhost:32771/auth/v1.0 -X GET | |
| swiftclient: DEBUG: RESP STATUS: 200 OK | |
| swiftclient: DEBUG: RESP HEADERS: {u'Content-Length': u'0', u'X-Trans-Id': u'tx33215ae5fb71483496a9c-005a2827b4', u'X-Auth-Token-Expires': u'86376', u'X-Auth-Token': u'AUTH_tk23fef78ac...', u'X-Storage-Token': u'AUTH_tk23fef78ac...', u'Date': u'Wed, 06 Dec 2017 17:24:04 GMT', u'X-Storage-Url': u'http://localhost:32771/v1/AUTH_test', u'Content-Type': u'text/html; charset=UTF-8', u'X-Openstack-Request-Id': u'tx33215ae5fb71483496a9c-005a2827b4'} | |
| urllib3.connectionpool: DEBUG: Starting new HTTP connection (1): localhost | |
| urllib3.connectionpool: DEBUG: http://localhost:32771 "PUT /v1/AUTH_test/sync-swift/test_archive HTTP/1.1" 201 0 | |
| swiftclient: DEBUG: REQ: curl -i http://localhost:32771/v1/AUTH_test/sync-swift/test_archive -X PUT -H "X-Auth-Token: AUTH_tk23fef78ac..." | |
| swiftclient: DEBUG: RESP STATUS: 201 Created | |
| swiftclient: DEBUG: RESP HEADERS: {u'Content-Length': u'0', u'Last-Modified': u'Wed, 06 Dec 2017 17:24:05 GMT', u'Etag': u'00cf75325f24cbbdbed3b80b26ff500b', u'X-Trans-Id': u'tx9835642d828e4e23a2581-005a2827b4', u'Date': u'Wed, 06 Dec 2017 17:24:04 GMT', u'Content-Type': u'text/html; charset=UTF-8', u'X-Openstack-Request-Id': u'tx9835642d828e4e23a2581-005a2827b4'} | |
| urllib3.connectionpool: DEBUG: http://localhost:32771 "HEAD /v1/AUTH_test2/swift-sync/test_archive HTTP/1.1" 404 0 | |
| swiftclient: INFO: REQ: curl -i http://localhost:32771/v1/AUTH_test2/swift-sync/test_archive -I -H "X-Auth-Token: AUTH_tk721e6f84b..." | |
| swiftclient: INFO: RESP STATUS: 404 Not Found | |
| swiftclient: INFO: RESP HEADERS: {u'Date': u'Wed, 06 Dec 2017 17:24:04 GMT', u'Content-Length': u'0', u'Content-Type': u'text/html; charset=UTF-8', u'X-Openstack-Request-Id': u'txf98feb501d7d44068235b-005a2827b4', u'X-Trans-Id': u'txf98feb501d7d44068235b-005a2827b4'} | |
| urllib3.connectionpool: DEBUG: http://localhost:32771 "HEAD /v1/AUTH_test2/swift-sync/test_archive HTTP/1.1" 404 0 | |
| swiftclient: INFO: REQ: curl -i http://localhost:32771/v1/AUTH_test2/swift-sync/test_archive -I -H "X-Auth-Token: AUTH_tk721e6f84b..." | |
| swiftclient: INFO: RESP STATUS: 404 Not Found | |
| swiftclient: INFO: RESP HEADERS: {u'Date': u'Wed, 06 Dec 2017 17:24:04 GMT', u'Content-Length': u'0', u'Content-Type': u'text/html; charset=UTF-8', u'X-Openstack-Request-Id': u'txc78da1ca19014518930c3-005a2827b4', u'X-Trans-Id': u'txc78da1ca19014518930c3-005a2827b4'} | |
| urllib3.connectionpool: DEBUG: http://localhost:32771 "HEAD /v1/AUTH_test2/swift-sync/test_archive HTTP/1.1" 404 0 | |
| swiftclient: INFO: REQ: curl -i http://localhost:32771/v1/AUTH_test2/swift-sync/test_archive -I -H "X-Auth-Token: AUTH_tk721e6f84b..." | |
| swiftclient: INFO: RESP STATUS: 404 Not Found | |
| swiftclient: INFO: RESP HEADERS: {u'Date': u'Wed, 06 Dec 2017 17:24:04 GMT', u'Content-Length': u'0', u'Content-Type': u'text/html; charset=UTF-8', u'X-Openstack-Request-Id': u'tx65723f66b21144a8ab761-005a2827b4', u'X-Trans-Id': u'tx65723f66b21144a8ab761-005a2827b4'} | |
| urllib3.connectionpool: DEBUG: http://localhost:32771 "HEAD /v1/AUTH_test2/swift-sync/test_archive HTTP/1.1" 404 0 | |
| swiftclient: INFO: REQ: curl -i http://localhost:32771/v1/AUTH_test2/swift-sync/test_archive -I -H "X-Auth-Token: AUTH_tk721e6f84b..." | |
| swiftclient: INFO: RESP STATUS: 404 Not Found | |
| swiftclient: INFO: RESP HEADERS: {u'Date': u'Wed, 06 Dec 2017 17:24:04 GMT', u'Content-Length': u'0', u'Content-Type': u'text/html; charset=UTF-8', u'X-Openstack-Request-Id': u'txc15e4e5245774ccea9bdf-005a2827b4', u'X-Trans-Id': u'txc15e4e5245774ccea9bdf-005a2827b4'} | |
| urllib3.connectionpool: DEBUG: http://localhost:32771 "HEAD /v1/AUTH_test2/swift-sync/test_archive HTTP/1.1" 404 0 | |
| swiftclient: INFO: REQ: curl -i http://localhost:32771/v1/AUTH_test2/swift-sync/test_archive -I -H "X-Auth-Token: AUTH_tk721e6f84b..." | |
| swiftclient: INFO: RESP STATUS: 404 Not Found | |
| swiftclient: INFO: RESP HEADERS: {u'Date': u'Wed, 06 Dec 2017 17:24:04 GMT', u'Content-Length': u'0', u'Content-Type': u'text/html; charset=UTF-8', u'X-Openstack-Request-Id': u'tx3d50601d5ca14a2095172-005a2827b4', u'X-Trans-Id': u'tx3d50601d5ca14a2095172-005a2827b4'} | |
| urllib3.connectionpool: DEBUG: http://localhost:32771 "HEAD /v1/AUTH_test2/swift-sync/test_archive HTTP/1.1" 404 0 | |
| swiftclient: INFO: REQ: curl -i http://localhost:32771/v1/AUTH_test2/swift-sync/test_archive -I -H "X-Auth-Token: AUTH_tk721e6f84b..." | |
| swiftclient: INFO: RESP STATUS: 404 Not Found | |
| swiftclient: INFO: RESP HEADERS: {u'Date': u'Wed, 06 Dec 2017 17:24:04 GMT', u'Content-Length': u'0', u'Content-Type': u'text/html; charset=UTF-8', u'X-Openstack-Request-Id': u'txc22ec72569d740d292e83-005a2827b4', u'X-Trans-Id': u'txc22ec72569d740d292e83-005a2827b4'} | |
| urllib3.connectionpool: DEBUG: http://localhost:32771 "HEAD /v1/AUTH_test2/swift-sync/test_archive HTTP/1.1" 404 0 | |
| swiftclient: INFO: REQ: curl -i http://localhost:32771/v1/AUTH_test2/swift-sync/test_archive -I -H "X-Auth-Token: AUTH_tk721e6f84b..." | |
| swiftclient: INFO: RESP STATUS: 404 Not Found | |
| swiftclient: INFO: RESP HEADERS: {u'Date': u'Wed, 06 Dec 2017 17:24:04 GMT', u'Content-Length': u'0', u'Content-Type': u'text/html; charset=UTF-8', u'X-Openstack-Request-Id': u'txeaffadbf3f4d4fdab0471-005a2827b4', u'X-Trans-Id': u'txeaffadbf3f4d4fdab0471-005a2827b4'} | |
| urllib3.connectionpool: DEBUG: http://localhost:32771 "HEAD /v1/AUTH_test2/swift-sync/test_archive HTTP/1.1" 404 0 | |
| swiftclient: INFO: REQ: curl -i http://localhost:32771/v1/AUTH_test2/swift-sync/test_archive -I -H "X-Auth-Token: AUTH_tk721e6f84b..." | |
| swiftclient: INFO: RESP STATUS: 404 Not Found | |
| swiftclient: INFO: RESP HEADERS: {u'Date': u'Wed, 06 Dec 2017 17:24:04 GMT', u'Content-Length': u'0', u'Content-Type': u'text/html; charset=UTF-8', u'X-Openstack-Request-Id': u'tx36c65a441f244adbb8d16-005a2827b4', u'X-Trans-Id': u'tx36c65a441f244adbb8d16-005a2827b4'} | |
| urllib3.connectionpool: DEBUG: http://localhost:32771 "HEAD /v1/AUTH_test2/swift-sync/test_archive HTTP/1.1" 404 0 | |
| swiftclient: INFO: REQ: curl -i http://localhost:32771/v1/AUTH_test2/swift-sync/test_archive -I -H "X-Auth-Token: AUTH_tk721e6f84b..." | |
| swiftclient: INFO: RESP STATUS: 404 Not Found | |
| swiftclient: INFO: RESP HEADERS: {u'Date': u'Wed, 06 Dec 2017 17:24:05 GMT', u'Content-Length': u'0', u'Content-Type': u'text/html; charset=UTF-8', u'X-Openstack-Request-Id': u'tx7dd4e538dff24bf2aa686-005a2827b5', u'X-Trans-Id': u'tx7dd4e538dff24bf2aa686-005a2827b5'} | |
| urllib3.connectionpool: DEBUG: http://localhost:32771 "HEAD /v1/AUTH_test2/swift-sync/test_archive HTTP/1.1" 404 0 | |
| swiftclient: INFO: REQ: curl -i http://localhost:32771/v1/AUTH_test2/swift-sync/test_archive -I -H "X-Auth-Token: AUTH_tk721e6f84b..." | |
| swiftclient: INFO: RESP STATUS: 404 Not Found | |
| swiftclient: INFO: RESP HEADERS: {u'Date': u'Wed, 06 Dec 2017 17:24:05 GMT', u'Content-Length': u'0', u'Content-Type': u'text/html; charset=UTF-8', u'X-Openstack-Request-Id': u'txa01085ab33df4f5a8db9e-005a2827b5', u'X-Trans-Id': u'txa01085ab33df4f5a8db9e-005a2827b5'} | |
| urllib3.connectionpool: DEBUG: http://localhost:32771 "HEAD /v1/AUTH_test2/swift-sync/test_archive HTTP/1.1" 404 0 | |
| swiftclient: INFO: REQ: curl -i http://localhost:32771/v1/AUTH_test2/swift-sync/test_archive -I -H "X-Auth-Token: AUTH_tk721e6f84b..." | |
| swiftclient: INFO: RESP STATUS: 404 Not Found | |
| swiftclient: INFO: RESP HEADERS: {u'Date': u'Wed, 06 Dec 2017 17:24:05 GMT', u'Content-Length': u'0', u'Content-Type': u'text/html; charset=UTF-8', u'X-Openstack-Request-Id': u'tx0599b7f1ec1143b4adc55-005a2827b5', u'X-Trans-Id': u'tx0599b7f1ec1143b4adc55-005a2827b5'} | |
| urllib3.connectionpool: DEBUG: http://localhost:32771 "HEAD /v1/AUTH_test2/swift-sync/test_archive HTTP/1.1" 404 0 | |
| swiftclient: INFO: REQ: curl -i http://localhost:32771/v1/AUTH_test2/swift-sync/test_archive -I -H "X-Auth-Token: AUTH_tk721e6f84b..." | |
| swiftclient: INFO: RESP STATUS: 404 Not Found | |
| swiftclient: INFO: RESP HEADERS: {u'Date': u'Wed, 06 Dec 2017 17:24:05 GMT', u'Content-Length': u'0', u'Content-Type': u'text/html; charset=UTF-8', u'X-Openstack-Request-Id': u'tx84176dc197774dda9f24f-005a2827b5', u'X-Trans-Id': u'tx84176dc197774dda9f24f-005a2827b5'} | |
| urllib3.connectionpool: DEBUG: http://localhost:32771 "HEAD /v1/AUTH_test2/swift-sync/test_archive HTTP/1.1" 404 0 | |
| swiftclient: INFO: REQ: curl -i http://localhost:32771/v1/AUTH_test2/swift-sync/test_archive -I -H "X-Auth-Token: AUTH_tk721e6f84b..." | |
| swiftclient: INFO: RESP STATUS: 404 Not Found | |
| swiftclient: INFO: RESP HEADERS: {u'Date': u'Wed, 06 Dec 2017 17:24:05 GMT', u'Content-Length': u'0', u'Content-Type': u'text/html; charset=UTF-8', u'X-Openstack-Request-Id': u'tx4f68f40ad4b24f158dd53-005a2827b5', u'X-Trans-Id': u'tx4f68f40ad4b24f158dd53-005a2827b5'} | |
| urllib3.connectionpool: DEBUG: http://localhost:32771 "HEAD /v1/AUTH_test2/swift-sync/test_archive HTTP/1.1" 404 0 | |
| swiftclient: INFO: REQ: curl -i http://localhost:32771/v1/AUTH_test2/swift-sync/test_archive -I -H "X-Auth-Token: AUTH_tk721e6f84b..." | |
| swiftclient: INFO: RESP STATUS: 404 Not Found | |
| swiftclient: INFO: RESP HEADERS: {u'Date': u'Wed, 06 Dec 2017 17:24:05 GMT', u'Content-Length': u'0', u'Content-Type': u'text/html; charset=UTF-8', u'X-Openstack-Request-Id': u'tx315e8200f5a143e486c6d-005a2827b5', u'X-Trans-Id': u'tx315e8200f5a143e486c6d-005a2827b5'} | |
| urllib3.connectionpool: DEBUG: http://localhost:32771 "HEAD /v1/AUTH_test2/swift-sync/test_archive HTTP/1.1" 404 0 | |
| swiftclient: INFO: REQ: curl -i http://localhost:32771/v1/AUTH_test2/swift-sync/test_archive -I -H "X-Auth-Token: AUTH_tk721e6f84b..." | |
| swiftclient: INFO: RESP STATUS: 404 Not Found | |
| swiftclient: INFO: RESP HEADERS: {u'Date': u'Wed, 06 Dec 2017 17:24:05 GMT', u'Content-Length': u'0', u'Content-Type': u'text/html; charset=UTF-8', u'X-Openstack-Request-Id': u'tx7b04059455e742ac83438-005a2827b5', u'X-Trans-Id': u'tx7b04059455e742ac83438-005a2827b5'} | |
| urllib3.connectionpool: DEBUG: http://localhost:32771 "HEAD /v1/AUTH_test2/swift-sync/test_archive HTTP/1.1" 404 0 | |
| swiftclient: INFO: REQ: curl -i http://localhost:32771/v1/AUTH_test2/swift-sync/test_archive -I -H "X-Auth-Token: AUTH_tk721e6f84b..." | |
| swiftclient: INFO: RESP STATUS: 404 Not Found | |
| swiftclient: INFO: RESP HEADERS: {u'Date': u'Wed, 06 Dec 2017 17:24:05 GMT', u'Content-Length': u'0', u'Content-Type': u'text/html; charset=UTF-8', u'X-Openstack-Request-Id': u'tx3beb3cd0011349afb443e-005a2827b5', u'X-Trans-Id': u'tx3beb3cd0011349afb443e-005a2827b5'} | |
| urllib3.connectionpool: DEBUG: http://localhost:32771 "HEAD /v1/AUTH_test2/swift-sync/test_archive HTTP/1.1" 404 0 | |
| swiftclient: INFO: REQ: curl -i http://localhost:32771/v1/AUTH_test2/swift-sync/test_archive -I -H "X-Auth-Token: AUTH_tk721e6f84b..." | |
| swiftclient: INFO: RESP STATUS: 404 Not Found | |
| swiftclient: INFO: RESP HEADERS: {u'Date': u'Wed, 06 Dec 2017 17:24:05 GMT', u'Content-Length': u'0', u'Content-Type': u'text/html; charset=UTF-8', u'X-Openstack-Request-Id': u'tx8427d67da11d4c8db5146-005a2827b5', u'X-Trans-Id': u'tx8427d67da11d4c8db5146-005a2827b5'} | |
| urllib3.connectionpool: DEBUG: http://localhost:32771 "HEAD /v1/AUTH_test2/swift-sync/test_archive HTTP/1.1" 404 0 | |
| swiftclient: INFO: REQ: curl -i http://localhost:32771/v1/AUTH_test2/swift-sync/test_archive -I -H "X-Auth-Token: AUTH_tk721e6f84b..." | |
| swiftclient: INFO: RESP STATUS: 404 Not Found | |
| swiftclient: INFO: RESP HEADERS: {u'Date': u'Wed, 06 Dec 2017 17:24:06 GMT', u'Content-Length': u'0', u'Content-Type': u'text/html; charset=UTF-8', u'X-Openstack-Request-Id': u'tx697e7b30429944fbb184c-005a2827b6', u'X-Trans-Id': u'tx697e7b30429944fbb184c-005a2827b6'} | |
| urllib3.connectionpool: DEBUG: http://localhost:32771 "HEAD /v1/AUTH_test2/swift-sync/test_archive HTTP/1.1" 404 0 | |
| swiftclient: INFO: REQ: curl -i http://localhost:32771/v1/AUTH_test2/swift-sync/test_archive -I -H "X-Auth-Token: AUTH_tk721e6f84b..." | |
| swiftclient: INFO: RESP STATUS: 404 Not Found | |
| swiftclient: INFO: RESP HEADERS: {u'Date': u'Wed, 06 Dec 2017 17:24:06 GMT', u'Content-Length': u'0', u'Content-Type': u'text/html; charset=UTF-8', u'X-Openstack-Request-Id': u'txc0927a5be02541d0b2acd-005a2827b6', u'X-Trans-Id': u'txc0927a5be02541d0b2acd-005a2827b6'} | |
| urllib3.connectionpool: DEBUG: http://localhost:32771 "HEAD /v1/AUTH_test2/swift-sync/test_archive HTTP/1.1" 404 0 | |
| swiftclient: INFO: REQ: curl -i http://localhost:32771/v1/AUTH_test2/swift-sync/test_archive -I -H "X-Auth-Token: AUTH_tk721e6f84b..." | |
| swiftclient: INFO: RESP STATUS: 404 Not Found | |
| swiftclient: INFO: RESP HEADERS: {u'Date': u'Wed, 06 Dec 2017 17:24:06 GMT', u'Content-Length': u'0', u'Content-Type': u'text/html; charset=UTF-8', u'X-Openstack-Request-Id': u'txba5a9d71f2c64f859592f-005a2827b6', u'X-Trans-Id': u'txba5a9d71f2c64f859592f-005a2827b6'} | |
| urllib3.connectionpool: DEBUG: http://localhost:32771 "HEAD /v1/AUTH_test2/swift-sync/test_archive HTTP/1.1" 404 0 | |
| swiftclient: INFO: REQ: curl -i http://localhost:32771/v1/AUTH_test2/swift-sync/test_archive -I -H "X-Auth-Token: AUTH_tk721e6f84b..." | |
| swiftclient: INFO: RESP STATUS: 404 Not Found | |
| swiftclient: INFO: RESP HEADERS: {u'Date': u'Wed, 06 Dec 2017 17:24:06 GMT', u'Content-Length': u'0', u'Content-Type': u'text/html; charset=UTF-8', u'X-Openstack-Request-Id': u'tx514227ed98314eeea93d9-005a2827b6', u'X-Trans-Id': u'tx514227ed98314eeea93d9-005a2827b6'} | |
| urllib3.connectionpool: DEBUG: http://localhost:32771 "HEAD /v1/AUTH_test2/swift-sync/test_archive HTTP/1.1" 404 0 | |
| swiftclient: INFO: REQ: curl -i http://localhost:32771/v1/AUTH_test2/swift-sync/test_archive -I -H "X-Auth-Token: AUTH_tk721e6f84b..." | |
| swiftclient: INFO: RESP STATUS: 404 Not Found | |
| swiftclient: INFO: RESP HEADERS: {u'Date': u'Wed, 06 Dec 2017 17:24:06 GMT', u'Content-Length': u'0', u'Content-Type': u'text/html; charset=UTF-8', u'X-Openstack-Request-Id': u'txe0e6bf2aad7c419391c11-005a2827b6', u'X-Trans-Id': u'txe0e6bf2aad7c419391c11-005a2827b6'} | |
| urllib3.connectionpool: DEBUG: http://localhost:32771 "HEAD /v1/AUTH_test2/swift-sync/test_archive HTTP/1.1" 404 0 | |
| swiftclient: INFO: REQ: curl -i http://localhost:32771/v1/AUTH_test2/swift-sync/test_archive -I -H "X-Auth-Token: AUTH_tk721e6f84b..." | |
| swiftclient: INFO: RESP STATUS: 404 Not Found | |
| swiftclient: INFO: RESP HEADERS: {u'Date': u'Wed, 06 Dec 2017 17:24:06 GMT', u'Content-Length': u'0', u'Content-Type': u'text/html; charset=UTF-8', u'X-Openstack-Request-Id': u'txa3f05fdd6e054cb7a1e63-005a2827b6', u'X-Trans-Id': u'txa3f05fdd6e054cb7a1e63-005a2827b6'} | |
| urllib3.connectionpool: DEBUG: http://localhost:32771 "HEAD /v1/AUTH_test2/swift-sync/test_archive HTTP/1.1" 404 0 | |
| swiftclient: INFO: REQ: curl -i http://localhost:32771/v1/AUTH_test2/swift-sync/test_archive -I -H "X-Auth-Token: AUTH_tk721e6f84b..." | |
| swiftclient: INFO: RESP STATUS: 404 Not Found | |
| swiftclient: INFO: RESP HEADERS: {u'Date': u'Wed, 06 Dec 2017 17:24:06 GMT', u'Content-Length': u'0', u'Content-Type': u'text/html; charset=UTF-8', u'X-Openstack-Request-Id': u'tx2dba4339cc9b4bf6af24a-005a2827b6', u'X-Trans-Id': u'tx2dba4339cc9b4bf6af24a-005a2827b6'} | |
| urllib3.connectionpool: DEBUG: http://localhost:32771 "HEAD /v1/AUTH_test2/swift-sync/test_archive HTTP/1.1" 404 0 | |
| swiftclient: INFO: REQ: curl -i http://localhost:32771/v1/AUTH_test2/swift-sync/test_archive -I -H "X-Auth-Token: AUTH_tk721e6f84b..." | |
| swiftclient: INFO: RESP STATUS: 404 Not Found | |
| swiftclient: INFO: RESP HEADERS: {u'Date': u'Wed, 06 Dec 2017 17:24:06 GMT', u'Content-Length': u'0', u'Content-Type': u'text/html; charset=UTF-8', u'X-Openstack-Request-Id': u'txba8ebf24015b42558753c-005a2827b6', u'X-Trans-Id': u'txba8ebf24015b42558753c-005a2827b6'} | |
| urllib3.connectionpool: DEBUG: http://localhost:32771 "HEAD /v1/AUTH_test2/swift-sync/test_archive HTTP/1.1" 404 0 | |
| swiftclient: INFO: REQ: curl -i http://localhost:32771/v1/AUTH_test2/swift-sync/test_archive -I -H "X-Auth-Token: AUTH_tk721e6f84b..." | |
| swiftclient: INFO: RESP STATUS: 404 Not Found | |
| swiftclient: INFO: RESP HEADERS: {u'Date': u'Wed, 06 Dec 2017 17:24:06 GMT', u'Content-Length': u'0', u'Content-Type': u'text/html; charset=UTF-8', u'X-Openstack-Request-Id': u'txa40ec2baa7b24384af2b4-005a2827b6', u'X-Trans-Id': u'txa40ec2baa7b24384af2b4-005a2827b6'} | |
| urllib3.connectionpool: DEBUG: http://localhost:32771 "HEAD /v1/AUTH_test2/swift-sync/test_archive HTTP/1.1" 404 0 | |
| swiftclient: INFO: REQ: curl -i http://localhost:32771/v1/AUTH_test2/swift-sync/test_archive -I -H "X-Auth-Token: AUTH_tk721e6f84b..." | |
| swiftclient: INFO: RESP STATUS: 404 Not Found | |
| swiftclient: INFO: RESP HEADERS: {u'Date': u'Wed, 06 Dec 2017 17:24:07 GMT', u'Content-Length': u'0', u'Content-Type': u'text/html; charset=UTF-8', u'X-Openstack-Request-Id': u'tx1e916e3315814cad84c49-005a2827b7', u'X-Trans-Id': u'tx1e916e3315814cad84c49-005a2827b7'} | |
| urllib3.connectionpool: DEBUG: http://localhost:32771 "HEAD /v1/AUTH_test2/swift-sync/test_archive HTTP/1.1" 404 0 | |
| swiftclient: INFO: REQ: curl -i http://localhost:32771/v1/AUTH_test2/swift-sync/test_archive -I -H "X-Auth-Token: AUTH_tk721e6f84b..." | |
| swiftclient: INFO: RESP STATUS: 404 Not Found | |
| swiftclient: INFO: RESP HEADERS: {u'Date': u'Wed, 06 Dec 2017 17:24:07 GMT', u'Content-Length': u'0', u'Content-Type': u'text/html; charset=UTF-8', u'X-Openstack-Request-Id': u'tx7108e5158b504838bf89d-005a2827b7', u'X-Trans-Id': u'tx7108e5158b504838bf89d-005a2827b7'} | |
| urllib3.connectionpool: DEBUG: http://localhost:32771 "HEAD /v1/AUTH_test2/swift-sync/test_archive HTTP/1.1" 404 0 | |
| swiftclient: INFO: REQ: curl -i http://localhost:32771/v1/AUTH_test2/swift-sync/test_archive -I -H "X-Auth-Token: AUTH_tk721e6f84b..." | |
| swiftclient: INFO: RESP STATUS: 404 Not Found | |
| swiftclient: INFO: RESP HEADERS: {u'Date': u'Wed, 06 Dec 2017 17:24:07 GMT', u'Content-Length': u'0', u'Content-Type': u'text/html; charset=UTF-8', u'X-Openstack-Request-Id': u'tx03c33c534782474cb5404-005a2827b7', u'X-Trans-Id': u'tx03c33c534782474cb5404-005a2827b7'} | |
| urllib3.connectionpool: DEBUG: http://localhost:32771 "HEAD /v1/AUTH_test2/swift-sync/test_archive HTTP/1.1" 404 0 | |
| swiftclient: INFO: REQ: curl -i http://localhost:32771/v1/AUTH_test2/swift-sync/test_archive -I -H "X-Auth-Token: AUTH_tk721e6f84b..." | |
| swiftclient: INFO: RESP STATUS: 404 Not Found | |
| swiftclient: INFO: RESP HEADERS: {u'Date': u'Wed, 06 Dec 2017 17:24:07 GMT', u'Content-Length': u'0', u'Content-Type': u'text/html; charset=UTF-8', u'X-Openstack-Request-Id': u'tx22c93a43c8ce4734ba4b0-005a2827b7', u'X-Trans-Id': u'tx22c93a43c8ce4734ba4b0-005a2827b7'} | |
| urllib3.connectionpool: DEBUG: http://localhost:32771 "HEAD /v1/AUTH_test2/swift-sync/test_archive HTTP/1.1" 404 0 | |
| swiftclient: INFO: REQ: curl -i http://localhost:32771/v1/AUTH_test2/swift-sync/test_archive -I -H "X-Auth-Token: AUTH_tk721e6f84b..." | |
| swiftclient: INFO: RESP STATUS: 404 Not Found | |
| swiftclient: INFO: RESP HEADERS: {u'Date': u'Wed, 06 Dec 2017 17:24:07 GMT', u'Content-Length': u'0', u'Content-Type': u'text/html; charset=UTF-8', u'X-Openstack-Request-Id': u'txf0403b299104436bb9271-005a2827b7', u'X-Trans-Id': u'txf0403b299104436bb9271-005a2827b7'} | |
| urllib3.connectionpool: DEBUG: http://localhost:32771 "HEAD /v1/AUTH_test2/swift-sync/test_archive HTTP/1.1" 404 0 | |
| swiftclient: INFO: REQ: curl -i http://localhost:32771/v1/AUTH_test2/swift-sync/test_archive -I -H "X-Auth-Token: AUTH_tk721e6f84b..." | |
| swiftclient: INFO: RESP STATUS: 404 Not Found | |
| swiftclient: INFO: RESP HEADERS: {u'Date': u'Wed, 06 Dec 2017 17:24:07 GMT', u'Content-Length': u'0', u'Content-Type': u'text/html; charset=UTF-8', u'X-Openstack-Request-Id': u'tx73aa20d9402c416cba33a-005a2827b7', u'X-Trans-Id': u'tx73aa20d9402c416cba33a-005a2827b7'} | |
| urllib3.connectionpool: DEBUG: http://localhost:32771 "HEAD /v1/AUTH_test2/swift-sync/test_archive HTTP/1.1" 404 0 | |
| swiftclient: INFO: REQ: curl -i http://localhost:32771/v1/AUTH_test2/swift-sync/test_archive -I -H "X-Auth-Token: AUTH_tk721e6f84b..." | |
| swiftclient: INFO: RESP STATUS: 404 Not Found | |
| swiftclient: INFO: RESP HEADERS: {u'Date': u'Wed, 06 Dec 2017 17:24:07 GMT', u'Content-Length': u'0', u'Content-Type': u'text/html; charset=UTF-8', u'X-Openstack-Request-Id': u'tx20732549cad142eeb0a74-005a2827b7', u'X-Trans-Id': u'tx20732549cad142eeb0a74-005a2827b7'} | |
| urllib3.connectionpool: DEBUG: http://localhost:32771 "HEAD /v1/AUTH_test2/swift-sync/test_archive HTTP/1.1" 404 0 | |
| swiftclient: INFO: REQ: curl -i http://localhost:32771/v1/AUTH_test2/swift-sync/test_archive -I -H "X-Auth-Token: AUTH_tk721e6f84b..." | |
| swiftclient: INFO: RESP STATUS: 404 Not Found | |
| swiftclient: INFO: RESP HEADERS: {u'Date': u'Wed, 06 Dec 2017 17:24:07 GMT', u'Content-Length': u'0', u'Content-Type': u'text/html; charset=UTF-8', u'X-Openstack-Request-Id': u'txedf30105b0f44d9f8602b-005a2827b7', u'X-Trans-Id': u'txedf30105b0f44d9f8602b-005a2827b7'} | |
| urllib3.connectionpool: DEBUG: http://localhost:32771 "HEAD /v1/AUTH_test2/swift-sync/test_archive HTTP/1.1" 404 0 | |
| swiftclient: INFO: REQ: curl -i http://localhost:32771/v1/AUTH_test2/swift-sync/test_archive -I -H "X-Auth-Token: AUTH_tk721e6f84b..." | |
| access_key=s3-sync-test | |
| swiftclient: INFO: RESP STATUS: 404 Not Found | |
| swiftclient: INFO: RESP HEADERS: {u'Date': u'Wed, 06 Dec 2017 17:24:07 GMT', u'Content-Length': u'0', u'Content-Type': u'text/html; charset=UTF-8', u'X-Openstack-Request-Id': u'txfaaa6e6aeb0543d59cd24-005a2827b7', u'X-Trans-Id': u'txfaaa6e6aeb0543d59cd24-005a2827b7'} | |
| urllib3.connectionpool: DEBUG: http://localhost:32771 "HEAD /v1/AUTH_test2/swift-sync/test_archive HTTP/1.1" 404 0 | |
| swiftclient: INFO: REQ: curl -i http://localhost:32771/v1/AUTH_test2/swift-sync/test_archive -I -H "X-Auth-Token: AUTH_tk721e6f84b..." | |
| swiftclient: INFO: RESP STATUS: 404 Not Found | |
| swiftclient: INFO: RESP HEADERS: {u'Date': u'Wed, 06 Dec 2017 17:24:08 GMT', u'Content-Length': u'0', u'Content-Type': u'text/html; charset=UTF-8', u'X-Openstack-Request-Id': u'tx2011c3ffa4c74f689b444-005a2827b8', u'X-Trans-Id': u'tx2011c3ffa4c74f689b444-005a2827b8'} | |
| urllib3.connectionpool: DEBUG: http://localhost:32771 "HEAD /v1/AUTH_test2/swift-sync/test_archive HTTP/1.1" 404 0 | |
| swiftclient: INFO: REQ: curl -i http://localhost:32771/v1/AUTH_test2/swift-sync/test_archive -I -H "X-Auth-Token: AUTH_tk721e6f84b..." | |
| swiftclient: INFO: RESP STATUS: 404 Not Found | |
| swiftclient: INFO: RESP HEADERS: {u'Date': u'Wed, 06 Dec 2017 17:24:08 GMT', u'Content-Length': u'0', u'Content-Type': u'text/html; charset=UTF-8', u'X-Openstack-Request-Id': u'txc7228f0b85d748cebb1c1-005a2827b8', u'X-Trans-Id': u'txc7228f0b85d748cebb1c1-005a2827b8'} | |
| urllib3.connectionpool: DEBUG: http://localhost:32771 "HEAD /v1/AUTH_test2/swift-sync/test_archive HTTP/1.1" 404 0 | |
| swiftclient: INFO: REQ: curl -i http://localhost:32771/v1/AUTH_test2/swift-sync/test_archive -I -H "X-Auth-Token: AUTH_tk721e6f84b..." | |
| swiftclient: INFO: RESP STATUS: 404 Not Found | |
| swiftclient: INFO: RESP HEADERS: {u'Date': u'Wed, 06 Dec 2017 17:24:08 GMT', u'Content-Length': u'0', u'Content-Type': u'text/html; charset=UTF-8', u'X-Openstack-Request-Id': u'txf7b40780e5b1466d92d68-005a2827b8', u'X-Trans-Id': u'txf7b40780e5b1466d92d68-005a2827b8'} | |
| urllib3.connectionpool: DEBUG: http://localhost:32771 "HEAD /v1/AUTH_test2/swift-sync/test_archive HTTP/1.1" 404 0 | |
| swiftclient: INFO: REQ: curl -i http://localhost:32771/v1/AUTH_test2/swift-sync/test_archive -I -H "X-Auth-Token: AUTH_tk721e6f84b..." | |
| swiftclient: INFO: RESP STATUS: 404 Not Found | |
| swiftclient: INFO: RESP HEADERS: {u'Date': u'Wed, 06 Dec 2017 17:24:08 GMT', u'Content-Length': u'0', u'Content-Type': u'text/html; charset=UTF-8', u'X-Openstack-Request-Id': u'tx9dce323c48ca4edbaa885-005a2827b8', u'X-Trans-Id': u'tx9dce323c48ca4edbaa885-005a2827b8'} | |
| urllib3.connectionpool: DEBUG: http://localhost:32771 "HEAD /v1/AUTH_test2/swift-sync/test_archive HTTP/1.1" 404 0 | |
| swiftclient: INFO: REQ: curl -i http://localhost:32771/v1/AUTH_test2/swift-sync/test_archive -I -H "X-Auth-Token: AUTH_tk721e6f84b..." | |
| swiftclient: INFO: RESP STATUS: 404 Not Found | |
| swiftclient: INFO: RESP HEADERS: {u'Date': u'Wed, 06 Dec 2017 17:24:08 GMT', u'Content-Length': u'0', u'Content-Type': u'text/html; charset=UTF-8', u'X-Openstack-Request-Id': u'tx196caae2e6d0430887266-005a2827b8', u'X-Trans-Id': u'tx196caae2e6d0430887266-005a2827b8'} | |
| urllib3.connectionpool: DEBUG: http://localhost:32771 "HEAD /v1/AUTH_test2/swift-sync/test_archive HTTP/1.1" 404 0 | |
| swiftclient: INFO: REQ: curl -i http://localhost:32771/v1/AUTH_test2/swift-sync/test_archive -I -H "X-Auth-Token: AUTH_tk721e6f84b..." | |
| swiftclient: INFO: RESP STATUS: 404 Not Found | |
| swiftclient: INFO: RESP HEADERS: {u'Date': u'Wed, 06 Dec 2017 17:24:08 GMT', u'Content-Length': u'0', u'Content-Type': u'text/html; charset=UTF-8', u'X-Openstack-Request-Id': u'tx1dfa3e6d1ff242a7a611e-005a2827b8', u'X-Trans-Id': u'tx1dfa3e6d1ff242a7a611e-005a2827b8'} | |
| urllib3.connectionpool: DEBUG: http://localhost:32771 "HEAD /v1/AUTH_test2/swift-sync/test_archive HTTP/1.1" 404 0 | |
| swiftclient: INFO: REQ: curl -i http://localhost:32771/v1/AUTH_test2/swift-sync/test_archive -I -H "X-Auth-Token: AUTH_tk721e6f84b..." | |
| swiftclient: INFO: RESP STATUS: 404 Not Found | |
| swiftclient: INFO: RESP HEADERS: {u'Date': u'Wed, 06 Dec 2017 17:24:08 GMT', u'Content-Length': u'0', u'Content-Type': u'text/html; charset=UTF-8', u'X-Openstack-Request-Id': u'tx000b794349e8415790663-005a2827b8', u'X-Trans-Id': u'tx000b794349e8415790663-005a2827b8'} | |
| urllib3.connectionpool: DEBUG: http://localhost:32771 "HEAD /v1/AUTH_test2/swift-sync/test_archive HTTP/1.1" 404 0 | |
| swiftclient: INFO: REQ: curl -i http://localhost:32771/v1/AUTH_test2/swift-sync/test_archive -I -H "X-Auth-Token: AUTH_tk721e6f84b..." | |
| swiftclient: INFO: RESP STATUS: 404 Not Found | |
| swiftclient: INFO: RESP HEADERS: {u'Date': u'Wed, 06 Dec 2017 17:24:08 GMT', u'Content-Length': u'0', u'Content-Type': u'text/html; charset=UTF-8', u'X-Openstack-Request-Id': u'tx0adb83fdf11241dd8eb32-005a2827b8', u'X-Trans-Id': u'tx0adb83fdf11241dd8eb32-005a2827b8'} | |
| urllib3.connectionpool: DEBUG: http://localhost:32771 "HEAD /v1/AUTH_test2/swift-sync/test_archive HTTP/1.1" 404 0 | |
| swiftclient: INFO: REQ: curl -i http://localhost:32771/v1/AUTH_test2/swift-sync/test_archive -I -H "X-Auth-Token: AUTH_tk721e6f84b..." | |
| swiftclient: INFO: RESP STATUS: 404 Not Found | |
| swiftclient: INFO: RESP HEADERS: {u'Date': u'Wed, 06 Dec 2017 17:24:08 GMT', u'Content-Length': u'0', u'Content-Type': u'text/html; charset=UTF-8', u'X-Openstack-Request-Id': u'tx2c01e27306cf4a0482fc9-005a2827b8', u'X-Trans-Id': u'tx2c01e27306cf4a0482fc9-005a2827b8'} | |
| urllib3.connectionpool: DEBUG: http://localhost:32771 "HEAD /v1/AUTH_test2/swift-sync/test_archive HTTP/1.1" 404 0 | |
| swiftclient: INFO: REQ: curl -i http://localhost:32771/v1/AUTH_test2/swift-sync/test_archive -I -H "X-Auth-Token: AUTH_tk721e6f84b..." | |
| swiftclient: INFO: RESP STATUS: 404 Not Found | |
| swiftclient: INFO: RESP HEADERS: {u'Date': u'Wed, 06 Dec 2017 17:24:09 GMT', u'Content-Length': u'0', u'Content-Type': u'text/html; charset=UTF-8', u'X-Openstack-Request-Id': u'tx73448e15804844428b552-005a2827b9', u'X-Trans-Id': u'tx73448e15804844428b552-005a2827b9'} | |
| --------------------- >> end captured logging << --------------------- | |
| ---------------------------------------------------------------------- | |
| Ran 8 tests in 31.612s | |
| FAILED (errors=4) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Here is the log from /var/log/swift-s3-sync.log
[2017-12-06 17:28:27,763] s3-sync [INFO]: Processing 1 rows since row 44 for AUTH_test/sync-swift
[2017-12-06 17:28:28,776] s3-sync [ERROR]: Failed to handle row 46 (test_archive): 'Traceback (most recent call last):\n File "/src/container-crawler/container_crawler/init.py", line 112, in _worker\n handler.handle(row, swift_client)\n File "/swift-s3-sync/s3_sync/sync_container.py", line 90, in handle\n self.provider.delete_object(row['name'], swift_client)\n File "/swift-s3-sync/s3_sync/sync_swift.py", line 201, in delete_object\n headers = swift_client.head_object(self.remote_container, name)\n File "/usr/local/src/python-swiftclient/swiftclient/client.py", line 1772, in head_object\n return self._retry(None, head_object, container, obj, headers=headers)\n File "/usr/local/src/python-swiftclient/swiftclient/client.py", line 1661, in _retry\n self.url, self.token = self.get_auth()\n File "/usr/local/src/python-swiftclient/swiftclient/client.py", line 1613, in get_auth\n timeout=self.timeout)\n File "/usr/local/src/python-swiftclient/swiftclient/client.py", line 643, in get_auth\n timeout=timeout)\n File "/usr/local/src/python-swiftclient/swiftclient/client.py", line 504, in get_auth_1_0\n raise ClientException.from_response(resp, 'Auth GET failed', body)\nClientException: Auth GET failed: http://localhost:8080/auth/v1.0 401 Unauthorized [first 60 chars of response]
Unauthorized
This server could not verify t\n'
[2017-12-06 17:28:28,776] s3-sync [ERROR]: Failed to process AUTH_test/sync-swift with SyncContainer
[2017-12-06 17:28:28,776] s3-sync [ERROR]: Traceback (most recent call last):
File "/src/container-crawler/container_crawler/init.py", line 225, in call_handle_container
self.handle_container(handler)
File "/src/container-crawler/container_crawler/init.py", line 207, in handle_container
handler, items, nodes_count, index)
File "/src/container-crawler/container_crawler/init.py", line 169, in process_items
self.submit_items(handler, owned_rows)
File "/src/container-crawler/container_crawler/init.py", line 164, in submit_items
self._check_errors()
File "/src/container-crawler/container_crawler/init.py", line 139, in _check_errors
raise RuntimeError('Failed to process rows')
RuntimeError: Failed to process rows
[2017-12-06 17:28:28,792] s3-sync [INFO]: Processing 1 rows since row 44 for AUTH_test/sync-swift
[2017-12-06 17:28:29,803] s3-sync [ERROR]: Failed to handle row 46 (test_archive): 'Traceback (most recent call last):\n File "/src/container-crawler/container_crawler/init.py", line 112, in _worker\n handler.handle(row, swift_client)\n File "/swift-s3-sync/s3_sync/sync_container.py", line 90, in handle\n self.provider.delete_object(row['name'], swift_client)\n File "/swift-s3-sync/s3_sync/sync_swift.py", line 201, in delete_object\n headers = swift_client.head_object(self.remote_container, name)\n File "/usr/local/src/python-swiftclient/swiftclient/client.py", line 1772, in head_object\n return self._retry(None, head_object, container, obj, headers=headers)\n File "/usr/local/src/python-swiftclient/swiftclient/client.py", line 1661, in _retry\n self.url, self.token = self.get_auth()\n File "/usr/local/src/python-swiftclient/swiftclient/client.py", line 1613, in get_auth\n timeout=self.timeout)\n File "/usr/local/src/python-swiftclient/swiftclient/client.py", line 643, in get_auth\n timeout=timeout)\n File "/usr/local/src/python-swiftclient/swiftclient/client.py", line 504, in get_auth_1_0\n raise ClientException.from_response(resp, 'Auth GET failed', body)\nClientException: Auth GET failed: http://localhost:8080/auth/v1.0 401 Unauthorized [first 60 chars of response]
Unauthorized
This server could not verify t\n'
[2017-12-06 17:28:29,804] s3-sync [ERROR]: Failed to process AUTH_test/sync-swift with SyncContainer
[2017-12-06 17:28:29,804] s3-sync [ERROR]: Traceback (most recent call last):
File "/src/container-crawler/container_crawler/init.py", line 225, in call_handle_container
self.handle_container(handler)
File "/src/container-crawler/container_crawler/init.py", line 207, in handle_container
handler, items, nodes_count, index)
File "/src/container-crawler/container_crawler/init.py", line 169, in process_items
self.submit_items(handler, owned_rows)
File "/src/container-crawler/container_crawler/init.py", line 164, in submit_items
self._check_errors()
File "/src/container-crawler/container_crawler/init.py", line 139, in _check_errors
raise RuntimeError('Failed to process rows')
RuntimeError: Failed to process rows