Skip to content

Instantly share code, notes, and snippets.

View cbodley's full-sized avatar

Casey Bodley cbodley

  • Red Hat
  • Ann Arbor, MI
View GitHub Profile
@cbodley
cbodley / grep cur_part_id
Created March 3, 2020 18:39
multipart manifest
2020-03-03T13:17:36.641-0500 7f46dd762700 20 RGWObjManifest::operator++(): stripe_ofs=4194304 part_ofs=0 rule->part_size=15728640 cur_part_id=1
2020-03-03T13:17:36.641-0500 7f46dd762700 20 RGWObjManifest::operator++(): stripe_ofs=8388608 part_ofs=0 rule->part_size=15728640 cur_part_id=1
2020-03-03T13:17:36.642-0500 7f46dd762700 20 RGWObjManifest::operator++(): stripe_ofs=12582912 part_ofs=0 rule->part_size=15728640 cur_part_id=1
2020-03-03T13:17:36.642-0500 7f46dd762700 20 RGWObjManifest::operator++(): stripe_ofs=16777216 part_ofs=0 rule->part_size=15728640 cur_part_id=1
2020-03-03T13:17:36.655-0500 7f4694ed1700 20 RGWObjManifest::operator++(): stripe_ofs=19922944 part_ofs=15728640 rule->part_size=15728640 cur_part_id=2
2020-03-03T13:17:36.667-0500 7f47017aa700 20 RGWObjManifest::operator++(): stripe_ofs=24117248 part_ofs=15728640 rule->part_size=15728640 cur_part_id=2
2020-03-03T13:17:36.68
@cbodley
cbodley / split.cc
Created November 22, 2019 17:35
iterator-based string split implementation
#include <string>
#include <gtest/gtest.h>
// a forward iterator over the parts of a split string
class spliterator {
std::string_view str; // full string
std::string_view delims; // delimiters
using size_type = std::string_view::size_type;
size_type pos = 0; // start position of current part
@cbodley
cbodley / list_unordered.py
Created September 18, 2019 15:18
List the contents of an s3 bucket with the AllowUnordered extension
#!/usr/bin/python
import boto3
import sys
# bucket name as first argument
bucketname = sys.argv[1]
# endpoint and keys from vstart
endpoint='http://127.0.0.1:8000'
@cbodley
cbodley / service-2.sdk-extras.json
Created September 18, 2019 14:59
boto3 extension for allow-unordered in s3 ListObjectsRequest
{
"version": 1.0,
"merge": {
"shapes": {
"ListObjectsRequest": {
"members": {
"AllowUnordered": {
"shape":"AllowUnordered",
"documentation":"<p>Allow the listing results to be returned in unsorted order. This may be faster when listing very large buckets.</p>",
"location":"querystring",
diff --git a/src/rgw/rgw_pubsub.h b/src/rgw/rgw_pubsub.h
index d752044aee..94a69d6e35 100644
--- a/src/rgw/rgw_pubsub.h
+++ b/src/rgw/rgw_pubsub.h
@@ -648,7 +648,6 @@ int RGWUserPubSub::write(const rgw_raw_obj& obj, const T& info, RGWObjVersionTra
bufferlist bl;
encode(info, bl);
- auto obj_ctx = store->svc()->sysobj->init_obj_ctx();
int ret = rgw_put_system_obj(obj_ctx, obj.pool, obj.oid,
@cbodley
cbodley / test_cls_log.cc
Created August 9, 2019 21:02
cls_log benchmark
diff --git a/src/test/cls_log/test_cls_log.cc b/src/test/cls_log/test_cls_log.cc
index 868afcaf65..e23322ff25 100644
--- a/src/test/cls_log/test_cls_log.cc
+++ b/src/test/cls_log/test_cls_log.cc
@@ -381,3 +381,34 @@ TEST_F(cls_log, trim_by_marker)
ASSERT_EQ(-ENODATA, do_log_trim(ioctx, oid, from, to));
}
}
+
+static void create_remove(librados::IoCtx& ioctx, std::string& oid, int count)
@cbodley
cbodley / queue.h
Last active July 12, 2019 20:37
cls/queue/queue.h library
struct cls_queue_marker { uint64_t gen; uint64_t offset; };
struct cls_queue_header;
struct cls_queue_entry { cls_queue_marker marker; bufferlist data; };
int queue_marker_encode(const cls_queue_marker& marker, std::string& encoded);
int queue_marker_decode(const std::string& encoded, cls_queue_marker& marker);
int queue_header_read(cls_method_context_t hctx, cls_queue_header&);
int queue_header_write(cls_method_context_t hctx, const cls_queue_header&);
@cbodley
cbodley / gist:a87a4b7976908fb16242e4ff8bde2433
Created June 24, 2019 15:41
vstart fails WITH_ASAN=ON WITH_UBSAN=ON
ceph-mgr dashboard not built - disabling.
** going verbose **
rm -f core*
hostname cezanne
ip 10.17.151.111
port 40239
/home/cbodley/scratch/ceph/build/bin/ceph-authtool --create-keyring --gen-key --name=mon. /home/cbodley/scratch/ceph/build/keyring --cap mon 'allow *'
creating /home/cbodley/scratch/ceph/build/keyring
/home/cbodley/scratch/ceph/src/common/buffer.cc:492:9: runtime error: member access within null pointer of type 'struct raw'
/home/cbodley/scratch/ceph/src/common/buffer.cc:493:9: runtime error: member access within null pointer of type 'struct raw'
@cbodley
cbodley / unordered-8-shard-bucket-list-after.log
Created April 19, 2019 23:37
unordered-8-shard-bucket-list-after.log
2019-04-19 19:15:19.184 7ff71f2ae700 20 HTTP_ACCEPT_ENCODING=identity
2019-04-19 19:15:19.184 7ff71f2ae700 20 HTTP_AUTHORIZATION=AWS4-HMAC-SHA256 Credential=0555b35654ad1656d804/20190419/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=9a9269aa36d87750e5eab3e282f36c5af42b4e70cadd2a124d40a321ce4dce80
2019-04-19 19:15:19.184 7ff71f2ae700 20 HTTP_HOST=cezanne.eng.arb.redhat.com:8000
2019-04-19 19:15:19.184 7ff71f2ae700 20 HTTP_USER_AGENT=Boto3/1.9.103 Python/2.7.15 Linux/4.20.11-100.fc28.x86_64 Botocore/1.12.103
2019-04-19 19:15:19.184 7ff71f2ae700 20 HTTP_VERSION=1.1
2019-04-19 19:15:19.184 7ff71f2ae700 20 HTTP_X_AMZ_CONTENT_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
2019-04-19 19:15:19.184 7ff71f2ae700 20 HTTP_X_AMZ_DATE=20190419T231519Z
2019-04-19 19:15:19.184 7ff71f2ae700 20 QUERY_STRING=max-keys=1000&encoding-type=url&allow-unordered=true
2019-04-19 19:15:19.184 7ff71f2ae700 20 REMOTE_ADDR=10.17.151.111
2019-04-19 19:15:19.184 7ff71f2ae700
@cbodley
cbodley / gist:7783ebc0ba850c1fc679edc19d495e38
Created April 19, 2019 23:32
unordered-8-shard-bucket-list-before.log
2019-04-19 19:01:14.979 7f4196906700 20 HTTP_ACCEPT_ENCODING=identity
2019-04-19 19:01:14.979 7f4196906700 20 HTTP_AUTHORIZATION=AWS4-HMAC-SHA256 Credential=0555b35654ad1656d804/20190419/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=9ddc31d9a97e5ed644337fc6b69b8ae1c92522ef86933e4a8d24f2e9e4c95837
2019-04-19 19:01:14.979 7f4196906700 20 HTTP_HOST=cezanne.eng.arb.redhat.com:8000
2019-04-19 19:01:14.979 7f4196906700 20 HTTP_USER_AGENT=Boto3/1.9.103 Python/2.7.15 Linux/4.20.11-100.fc28.x86_64 Botocore/1.12.103
2019-04-19 19:01:14.979 7f4196906700 20 HTTP_VERSION=1.1
2019-04-19 19:01:14.979 7f4196906700 20 HTTP_X_AMZ_CONTENT_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
2019-04-19 19:01:14.979 7f4196906700 20 HTTP_X_AMZ_DATE=20190419T230114Z
2019-04-19 19:01:14.979 7f4196906700 20 QUERY_STRING=max-keys=1000&encoding-type=url&allow-unordered=true
2019-04-19 19:01:14.979 7f4196906700 20 REMOTE_ADDR=10.17.151.111
2019-04-19 19:01:14.979 7f4196906700