Skip to content

Instantly share code, notes, and snippets.

@robbat2

robbat2/foo.diff

Last active February 17, 2018 00:26
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save robbat2/a602da53e5d07eb02ceb4e55aadbbb1d to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save robbat2/a602da53e5d07eb02ceb4e55aadbbb1d to your computer and use it in GitHub Desktop.
Download ZIP
"radosgw-admin usage show" issue 22963
Raw
foo.diff
$ "radosgw-admin usage show" issue 22963.
$ git log \
-L '^:RGWUsage..show:src/rgw/rgw_usage.cc' \
-L '^:RGWRados..read_usage:src/rgw/rgw_rados.cc' \
-L '^:RGWRados..cls_obj_usage_log_read:src/rgw/rgw_rados.cc' \
-L '^:cls_rgw_usage_log_read:src/cls/rgw/cls_rgw_client.cc' \
-L '^:rgw_user_usage_log_read:src/cls/rgw/cls_rgw.cc' \
-L '^:usage_iterate_range:src/cls/rgw/cls_rgw.cc' \
v10.2.10..ceph/luminous
--
commit 7810567fa37308bfeadf63e65f08adc36bfb886e
Author: Abhishek Lekshmanan <abhishek@suse.com>
Date: Tue Nov 21 16:38:27 2017 +0100
cls/rgw: trim all usage entries in cls_rgw
Currently trim usage will only trim upto 128 omap entries, since we need
to run this in a loop until we're done, actually make the cls return
-ENODATA so that we know when to stop the loop (inspired by a similar
call in cls_log) this involves the following changes
* return -ENODATA when iterate entries goes through and the value of
iter (which is set as the last value of key when succeeded)
* use IoCtx for calling the loop from within cls rather than in rgw
* drop the goto call in rgw_rados since we can return once we're done
processing
Fixes: http://tracker.ceph.com/issues/22234
Signed-off-by: Abhishek Lekshmanan <abhishek@suse.com>
(cherry picked from commit b548a3f3443452210d92cad574bcb73ba6d2ce42)
diff --git a/src/cls/rgw/cls_rgw.cc b/src/cls/rgw/cls_rgw.cc
--- a/src/cls/rgw/cls_rgw.cc
+++ b/src/cls/rgw/cls_rgw.cc
@@ -2904,92 +2904,88 @@
static int usage_iterate_range(cls_method_context_t hctx, uint64_t start, uint64_t end,
string& user, string& key_iter, uint32_t max_entries, bool *truncated,
int (*cb)(cls_method_context_t, const string&, rgw_usage_log_entry&, void *),
void *param)
{
CLS_LOG(10, "usage_iterate_range");
map<string, bufferlist> keys;
#define NUM_KEYS 32
string filter_prefix;
string start_key, end_key;
bool by_user = !user.empty();
uint32_t i = 0;
string user_key;
bool truncated_status = false;
if (!by_user) {
usage_record_prefix_by_time(end, end_key);
} else {
user_key = user;
user_key.append("_");
}
if (key_iter.empty()) {
if (by_user) {
usage_record_prefix_by_user(user, start, start_key);
} else {
usage_record_prefix_by_time(start, start_key);
}
} else {
start_key = key_iter;
}
CLS_LOG(20, "usage_iterate_range start_key=%s", start_key.c_str());
int ret = cls_cxx_map_get_vals(hctx, start_key, filter_prefix, max_entries, &keys, &truncated_status);
if (ret < 0)
return ret;
if (truncated) {
*truncated = truncated_status;
}
map<string, bufferlist>::iterator iter = keys.begin();
if (iter == keys.end())
return 0;
uint32_t num_keys = keys.size();
for (; iter != keys.end(); ++iter,++i) {
const string& key = iter->first;
rgw_usage_log_entry e;
if (!by_user && key.compare(end_key) >= 0) {
CLS_LOG(20, "usage_iterate_range reached key=%s, done", key.c_str());
- if (truncated_status) {
- key_iter = key;
- }
+ key_iter = key;
return 0;
}
if (by_user && key.compare(0, user_key.size(), user_key) != 0) {
CLS_LOG(20, "usage_iterate_range reached key=%s, done", key.c_str());
- if (truncated_status) {
- key_iter = key;
- }
+ key_iter = key;
return 0;
}
ret = usage_record_decode(iter->second, e);
if (ret < 0)
return ret;
if (e.epoch < start)
continue;
/* keys are sorted by epoch, so once we're past end we're done */
if (e.epoch >= end)
return 0;
ret = cb(hctx, key, e, param);
if (ret < 0)
return ret;
if (i == num_keys - 1) {
key_iter = key;
return 0;
}
}
return 0;
}
commit 14a1dcb1ff70b2a468cb47e7f239b98e45c63a88
Author: Mark Kogan <mkogan@redhat.com>
Date: Mon Sep 25 09:53:00 2017 +0300
rgw: update the usage read iterator in truncated scenario
Fixes: http://tracker.ceph.com/issues/21196
Signed-off-by: Mark Kogan <mkogan@redhat.com>
(cherry picked from commit 7306514a813661b77bfcbfc6f534dbabbdd3aa78)
diff --git a/src/cls/rgw/cls_rgw.cc b/src/cls/rgw/cls_rgw.cc
--- a/src/cls/rgw/cls_rgw.cc
+++ b/src/cls/rgw/cls_rgw.cc
@@ -2889,85 +2889,92 @@
static int usage_iterate_range(cls_method_context_t hctx, uint64_t start, uint64_t end,
string& user, string& key_iter, uint32_t max_entries, bool *truncated,
int (*cb)(cls_method_context_t, const string&, rgw_usage_log_entry&, void *),
void *param)
{
CLS_LOG(10, "usage_iterate_range");
map<string, bufferlist> keys;
#define NUM_KEYS 32
string filter_prefix;
string start_key, end_key;
bool by_user = !user.empty();
uint32_t i = 0;
string user_key;
-
- if (truncated)
- *truncated = false;
+ bool truncated_status = false;
if (!by_user) {
usage_record_prefix_by_time(end, end_key);
} else {
user_key = user;
user_key.append("_");
}
if (key_iter.empty()) {
if (by_user) {
usage_record_prefix_by_user(user, start, start_key);
} else {
usage_record_prefix_by_time(start, start_key);
}
} else {
start_key = key_iter;
}
CLS_LOG(20, "usage_iterate_range start_key=%s", start_key.c_str());
- int ret = cls_cxx_map_get_vals(hctx, start_key, filter_prefix, max_entries, &keys, truncated);
+ int ret = cls_cxx_map_get_vals(hctx, start_key, filter_prefix, max_entries, &keys, &truncated_status);
if (ret < 0)
return ret;
-
+ if (truncated) {
+ *truncated = truncated_status;
+ }
+
map<string, bufferlist>::iterator iter = keys.begin();
if (iter == keys.end())
return 0;
uint32_t num_keys = keys.size();
for (; iter != keys.end(); ++iter,++i) {
const string& key = iter->first;
rgw_usage_log_entry e;
if (!by_user && key.compare(end_key) >= 0) {
CLS_LOG(20, "usage_iterate_range reached key=%s, done", key.c_str());
+ if (truncated_status) {
+ key_iter = key;
+ }
return 0;
}
if (by_user && key.compare(0, user_key.size(), user_key) != 0) {
CLS_LOG(20, "usage_iterate_range reached key=%s, done", key.c_str());
+ if (truncated_status) {
+ key_iter = key;
+ }
return 0;
}
ret = usage_record_decode(iter->second, e);
if (ret < 0)
return ret;
if (e.epoch < start)
continue;
/* keys are sorted by epoch, so once we're past end we're done */
if (e.epoch >= end)
return 0;
ret = cb(hctx, key, e, param);
if (ret < 0)
return ret;
if (i == num_keys - 1) {
key_iter = key;
return 0;
}
}
return 0;
}
commit 5334622a8365520fa4247241f97422c044cbf5b2
Author: Yehuda Sadeh <yehuda@redhat.com>
Date: Fri Jul 14 15:28:09 2017 -0700
cls/*: adjust use of cls_cxx_map_get_vals()
Now that objclass call gets a new 'more' param.
Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
diff --git a/src/cls/rgw/cls_rgw.cc b/src/cls/rgw/cls_rgw.cc
--- a/src/cls/rgw/cls_rgw.cc
+++ b/src/cls/rgw/cls_rgw.cc
@@ -2830,90 +2821,85 @@
static int usage_iterate_range(cls_method_context_t hctx, uint64_t start, uint64_t end,
string& user, string& key_iter, uint32_t max_entries, bool *truncated,
int (*cb)(cls_method_context_t, const string&, rgw_usage_log_entry&, void *),
void *param)
{
CLS_LOG(10, "usage_iterate_range");
map<string, bufferlist> keys;
#define NUM_KEYS 32
string filter_prefix;
string start_key, end_key;
bool by_user = !user.empty();
uint32_t i = 0;
string user_key;
if (truncated)
*truncated = false;
if (!by_user) {
usage_record_prefix_by_time(end, end_key);
} else {
user_key = user;
user_key.append("_");
}
if (key_iter.empty()) {
if (by_user) {
usage_record_prefix_by_user(user, start, start_key);
} else {
usage_record_prefix_by_time(start, start_key);
}
} else {
start_key = key_iter;
}
- do {
- CLS_LOG(20, "usage_iterate_range start_key=%s", start_key.c_str());
- int ret = cls_cxx_map_get_vals(hctx, start_key, filter_prefix, NUM_KEYS, &keys);
- if (ret < 0)
- return ret;
+ CLS_LOG(20, "usage_iterate_range start_key=%s", start_key.c_str());
+ int ret = cls_cxx_map_get_vals(hctx, start_key, filter_prefix, max_entries, &keys, truncated);
+ if (ret < 0)
+ return ret;
- map<string, bufferlist>::iterator iter = keys.begin();
- if (iter == keys.end())
- break;
+ map<string, bufferlist>::iterator iter = keys.begin();
+ if (iter == keys.end())
+ return 0;
- for (; iter != keys.end(); ++iter) {
- const string& key = iter->first;
- rgw_usage_log_entry e;
+ uint32_t num_keys = keys.size();
- if (!by_user && key.compare(end_key) >= 0) {
- CLS_LOG(20, "usage_iterate_range reached key=%s, done", key.c_str());
- return 0;
- }
+ for (; iter != keys.end(); ++iter,++i) {
+ const string& key = iter->first;
+ rgw_usage_log_entry e;
- if (by_user && key.compare(0, user_key.size(), user_key) != 0) {
- CLS_LOG(20, "usage_iterate_range reached key=%s, done", key.c_str());
- return 0;
- }
+ if (!by_user && key.compare(end_key) >= 0) {
+ CLS_LOG(20, "usage_iterate_range reached key=%s, done", key.c_str());
+ return 0;
+ }
- ret = usage_record_decode(iter->second, e);
- if (ret < 0)
- return ret;
+ if (by_user && key.compare(0, user_key.size(), user_key) != 0) {
+ CLS_LOG(20, "usage_iterate_range reached key=%s, done", key.c_str());
+ return 0;
+ }
- if (e.epoch < start)
- continue;
+ ret = usage_record_decode(iter->second, e);
+ if (ret < 0)
+ return ret;
- /* keys are sorted by epoch, so once we're past end we're done */
- if (e.epoch >= end)
- return 0;
+ if (e.epoch < start)
+ continue;
- ret = cb(hctx, key, e, param);
- if (ret < 0)
- return ret;
+ /* keys are sorted by epoch, so once we're past end we're done */
+ if (e.epoch >= end)
+ return 0;
+ ret = cb(hctx, key, e, param);
+ if (ret < 0)
+ return ret;
- i++;
- if (max_entries && (i > max_entries)) {
- CLS_LOG(20, "usage_iterate_range reached max_entries (%d), done", max_entries);
- *truncated = true;
- key_iter = key;
- return 0;
- }
+
+ if (i == num_keys - 1) {
+ key_iter = key;
+ return 0;
}
- --iter;
- start_key = iter->first;
- } while (true);
+ }
return 0;
}
commit 4e04bf3a9064821a8bac7c437889e1f75cea7a51
Author: Jiaying Ren <jiaying.ren@umcloud.com>
Date: Wed Jul 5 13:26:16 2017 +0800
rgw: drop unused rgw_pool param, local var and member var
+ drop unused rgw_pool parameter in get_raw_obj_ref() and
get_system_obj_ref(), because each method needs a rgw_raw_obj as
param which includes its own rgw_pool.
+ drop misc unused local vars rgw_pool.
+ drop member variable pool.
Signed-off-by: Jiaying Ren <jiaying.ren@umcloud.com>
diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc
--- a/src/rgw/rgw_rados.cc
+++ b/src/rgw/rgw_rados.cc
@@ -12680,20 +12668,19 @@
int RGWRados::cls_obj_usage_log_read(string& oid, string& user, uint64_t start_epoch, uint64_t end_epoch, uint32_t max_entries,
string& read_iter, map<rgw_user_bucket, rgw_usage_log_entry>& usage, bool *is_truncated)
{
rgw_raw_obj obj(get_zone_params().usage_log_pool, oid);
rgw_rados_ref ref;
- rgw_pool pool;
- int r = get_raw_obj_ref(obj, &ref, &pool);
+ int r = get_raw_obj_ref(obj, &ref);
if (r < 0) {
return r;
}
*is_truncated = false;
r = cls_rgw_usage_log_read(ref.ioctx, ref.oid, user, start_epoch, end_epoch,
max_entries, read_iter, usage, is_truncated);
return r;
}
commit ac77cadc8d212289c86f715dc46178e3853427fd
Merge: 66c2b0c0d6 57f12ec045
Author: Casey Bodley <cbodley@users.noreply.github.com>
Date: Mon Apr 17 09:40:47 2017 -0400
Merge pull request #12536 from ilc/cls_rgw_minor_refactor
cls/rgw: Clean up the "magic string" usage in the cls layer for RGW.
Reviewed-by: Casey Bodley <cbodley@redhat.com>
Reviewed-by: Adam C. Emerson <aemerson@redhat.com>
commit 57f12ec045a8dd119cc5f2fb5531320e196bdc3a
Author: Ira Cooper <ira@samba.org>
Date: Fri Dec 16 13:49:16 2016 -0500
cls/rgw: Clean up the "magic string" usage in the cls layer for RGW.
This set of changes, should make following what is going on across the cls
barrier much easier for people using code browsing tools.
Signed-off-by: Ira Cooper <ira@redhat.com>
diff --git a/src/cls/rgw/cls_rgw_client.cc b/src/cls/rgw/cls_rgw_client.cc
--- a/src/cls/rgw/cls_rgw_client.cc
+++ b/src/cls/rgw/cls_rgw_client.cc
@@ -557,36 +558,36 @@
int cls_rgw_usage_log_read(IoCtx& io_ctx, string& oid, string& user,
uint64_t start_epoch, uint64_t end_epoch, uint32_t max_entries,
string& read_iter, map<rgw_user_bucket, rgw_usage_log_entry>& usage,
bool *is_truncated)
{
if (is_truncated)
*is_truncated = false;
bufferlist in, out;
rgw_cls_usage_log_read_op call;
call.start_epoch = start_epoch;
call.end_epoch = end_epoch;
call.owner = user;
call.max_entries = max_entries;
call.iter = read_iter;
::encode(call, in);
- int r = io_ctx.exec(oid, "rgw", "user_usage_log_read", in, out);
+ int r = io_ctx.exec(oid, RGW_CLASS, RGW_USER_USAGE_LOG_READ, in, out);
if (r < 0)
return r;
try {
rgw_cls_usage_log_read_ret result;
bufferlist::iterator iter = out.begin();
::decode(result, iter);
read_iter = result.next_iter;
if (is_truncated)
*is_truncated = result.truncated;
usage = result.usage;
} catch (buffer::error& e) {
return -EINVAL;
}
return 0;
}
commit 66c2b0c0d66c6019e3f63148eede7802e89d3c51
Author: Yehuda Sadeh <yehuda@redhat.com>
Date: Fri Oct 7 21:03:32 2016 -0700
rgw: introduce rgw_pool, rgw_raw_obj
Pools are represented by rgw_pool (and not rgw_bucket anymore),
and we use rgw_raw_obj to reference rados objs and all 'system'
objects (vs rgw_obj that is used for rgw objects).
Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc
--- a/src/rgw/rgw_rados.cc
+++ b/src/rgw/rgw_rados.cc
@@ -12171,19 +12076,20 @@
int RGWRados::cls_obj_usage_log_read(string& oid, string& user, uint64_t start_epoch, uint64_t end_epoch, uint32_t max_entries,
string& read_iter, map<rgw_user_bucket, rgw_usage_log_entry>& usage, bool *is_truncated)
{
- librados::IoCtx io_ctx;
+ rgw_raw_obj obj(get_zone_params().usage_log_pool, oid);
- *is_truncated = false;
-
- const char *usage_log_pool = get_zone_params().usage_log_pool.name.c_str();
- librados::Rados *rad = get_rados_handle();
- int r = rad->ioctx_create(usage_log_pool, io_ctx);
- if (r < 0)
+ rgw_rados_ref ref;
+ rgw_pool pool;
+ int r = get_raw_obj_ref(obj, &ref, &pool);
+ if (r < 0) {
return r;
+ }
- r = cls_rgw_usage_log_read(io_ctx, oid, user, start_epoch, end_epoch,
+ *is_truncated = false;
+
+ r = cls_rgw_usage_log_read(ref.ioctx, ref.oid, user, start_epoch, end_epoch,
max_entries, read_iter, usage, is_truncated);
return r;
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment