Skip to content

Instantly share code, notes, and snippets.

@arodd

arodd/8.c pg query

Last active July 4, 2019 05:14
Show Gist options
  • Save arodd/c95355a7b55f3e4a94f21bc5e801943d to your computer and use it in GitHub Desktop.
Save arodd/c95355a7b55f3e4a94f21bc5e801943d to your computer and use it in GitHub Desktop.
Download ZIP
ceph - osd failure to start
Raw
8.c pg query
{
"state": "incomplete",
"snap_trimq": "[]",
"snap_trimq_len": 0,
"epoch": 1751,
"up": [
4,
2,
3,
1,
2147483647
],
"acting": [
4,
2,
3,
1,
2147483647
],
"info": {
"pgid": "8.cs0",
"last_update": "1089'53466",
"last_complete": "1089'53466",
"log_tail": "967'51759",
"last_user_version": 0,
"last_backfill": "MIN",
"last_backfill_bitwise": 1,
"purged_snaps": [],
"history": {
"epoch_created": 748,
"epoch_pool_created": 748,
"last_epoch_started": 1300,
"last_interval_started": 1299,
"last_epoch_clean": 1083,
"last_interval_clean": 1082,
"last_epoch_split": 1299,
"last_epoch_marked_full": 0,
"same_up_since": 1745,
"same_interval_since": 1745,
"same_primary_since": 1745,
"last_scrub": "1057'53286",
"last_scrub_stamp": "2019-07-01 03:18:26.190174",
"last_deep_scrub": "1057'53286",
"last_deep_scrub_stamp": "2019-07-01 03:18:26.190174",
"last_clean_scrub_stamp": "2019-07-01 03:18:26.190174"
},
"stats": {
"version": "1089'53466",
"reported_seq": "18",
"reported_epoch": "1751",
"state": "incomplete",
"last_fresh": "2019-07-03 05:01:22.792661",
"last_change": "2019-07-03 03:55:22.658434",
"last_active": "0.000000",
"last_peered": "0.000000",
"last_clean": "0.000000",
"last_became_active": "0.000000",
"last_became_peered": "0.000000",
"last_unstale": "2019-07-03 05:01:22.792661",
"last_undegraded": "2019-07-03 05:01:22.792661",
"last_fullsized": "2019-07-03 05:01:22.792661",
"mapping_epoch": 1745,
"log_start": "967'51759",
"ondisk_log_start": "967'51759",
"created": 748,
"last_epoch_clean": 1083,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "1057'53286",
"last_scrub_stamp": "2019-07-01 03:18:26.190174",
"last_deep_scrub": "1057'53286",
"last_deep_scrub_stamp": "2019-07-01 03:18:26.190174",
"last_clean_scrub_stamp": "2019-07-01 03:18:26.190174",
"log_size": 1707,
"ondisk_log_size": 1707,
"stats_invalid": true,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"manifest_stats_invalid": false,
"snaptrimq_len": 0,
"stat_sum": {
"num_bytes": 0,
"num_objects": 0,
"num_object_clones": 0,
"num_object_copies": 0,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 0,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 0,
"num_whiteouts": 0,
"num_read": 0,
"num_read_kb": 0,
"num_write": 0,
"num_write_kb": 0,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0,
"num_objects_manifest": 0
},
"up": [
4,
2,
3,
1,
2147483647
],
"acting": [
4,
2,
3,
1,
2147483647
],
"blocked_by": [
0
],
"up_primary": 4,
"acting_primary": 4,
"purged_snaps": []
},
"empty": 0,
"dne": 0,
"incomplete": 1,
"last_epoch_started": 1094,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
"peer_info": [
{
"peer": "1(2)",
"pgid": "8.cs2",
"last_update": "1089'53466",
"last_complete": "1089'53466",
"log_tail": "967'51759",
"last_user_version": 53466,
"last_backfill": "MAX",
"last_backfill_bitwise": 0,
"purged_snaps": [],
"history": {
"epoch_created": 748,
"epoch_pool_created": 748,
"last_epoch_started": 1300,
"last_interval_started": 1299,
"last_epoch_clean": 1083,
"last_interval_clean": 1082,
"last_epoch_split": 1299,
"last_epoch_marked_full": 0,
"same_up_since": 1745,
"same_interval_since": 1745,
"same_primary_since": 1745,
"last_scrub": "1057'53286",
"last_scrub_stamp": "2019-07-01 03:18:26.190174",
"last_deep_scrub": "1057'53286",
"last_deep_scrub_stamp": "2019-07-01 03:18:26.190174",
"last_clean_scrub_stamp": "2019-07-01 03:18:26.190174"
},
"stats": {
"version": "1089'53466",
"reported_seq": "515750",
"reported_epoch": "1089",
"state": "active+clean",
"last_fresh": "2019-07-03 00:00:55.949673",
"last_change": "2019-07-02 18:12:25.973469",
"last_active": "2019-07-03 00:00:55.949673",
"last_peered": "2019-07-03 00:00:55.949673",
"last_clean": "2019-07-03 00:00:55.949673",
"last_became_active": "2019-07-02 18:12:25.973256",
"last_became_peered": "2019-07-02 18:12:25.973256",
"last_unstale": "2019-07-03 00:00:55.949673",
"last_undegraded": "2019-07-03 00:00:55.949673",
"last_fullsized": "2019-07-03 00:00:55.949673",
"mapping_epoch": 1745,
"log_start": "967'51759",
"ondisk_log_start": "967'51759",
"created": 748,
"last_epoch_clean": 1083,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "1057'53286",
"last_scrub_stamp": "2019-07-01 03:18:26.190174",
"last_deep_scrub": "1057'53286",
"last_deep_scrub_stamp": "2019-07-01 03:18:26.190174",
"last_clean_scrub_stamp": "2019-07-01 03:18:26.190174",
"log_size": 1707,
"ondisk_log_size": 1707,
"stats_invalid": true,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"manifest_stats_invalid": true,
"snaptrimq_len": 0,
"stat_sum": {
"num_bytes": 26809010770,
"num_objects": 6529,
"num_object_clones": 0,
"num_object_copies": 32648,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 0,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 6529,
"num_whiteouts": 0,
"num_read": 226620,
"num_read_kb": 148056701,
"num_write": 17095,
"num_write_kb": 30657414,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0,
"num_objects_manifest": 0
},
"up": [
4,
2,
3,
1,
2147483647
],
"acting": [
4,
2,
3,
1,
2147483647
],
"blocked_by": [],
"up_primary": 4,
"acting_primary": 4,
"purged_snaps": []
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 1300,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
{
"peer": "1(3)",
"pgid": "8.cs3",
"last_update": "0'0",
"last_complete": "0'0",
"log_tail": "0'0",
"last_user_version": 0,
"last_backfill": "MAX",
"last_backfill_bitwise": 0,
"purged_snaps": [],
"history": {
"epoch_created": 748,
"epoch_pool_created": 748,
"last_epoch_started": 1300,
"last_interval_started": 1299,
"last_epoch_clean": 1083,
"last_interval_clean": 1082,
"last_epoch_split": 1299,
"last_epoch_marked_full": 0,
"same_up_since": 1745,
"same_interval_since": 1745,
"same_primary_since": 1745,
"last_scrub": "1057'53286",
"last_scrub_stamp": "2019-07-01 03:18:26.190174",
"last_deep_scrub": "1057'53286",
"last_deep_scrub_stamp": "2019-07-01 03:18:26.190174",
"last_clean_scrub_stamp": "2019-07-01 03:18:26.190174"
},
"stats": {
"version": "0'0",
"reported_seq": "2",
"reported_epoch": "1712",
"state": "down",
"last_fresh": "2019-07-03 03:06:59.582815",
"last_change": "2019-07-03 03:06:59.582815",
"last_active": "0.000000",
"last_peered": "0.000000",
"last_clean": "0.000000",
"last_became_active": "0.000000",
"last_became_peered": "0.000000",
"last_unstale": "2019-07-03 03:06:59.582815",
"last_undegraded": "2019-07-03 03:06:59.582815",
"last_fullsized": "2019-07-03 03:06:59.582815",
"mapping_epoch": 1745,
"log_start": "0'0",
"ondisk_log_start": "0'0",
"created": 748,
"last_epoch_clean": 1083,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "1057'53286",
"last_scrub_stamp": "2019-07-01 03:18:26.190174",
"last_deep_scrub": "1057'53286",
"last_deep_scrub_stamp": "2019-07-01 03:18:26.190174",
"last_clean_scrub_stamp": "2019-07-01 03:18:26.190174",
"log_size": 0,
"ondisk_log_size": 0,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"manifest_stats_invalid": false,
"snaptrimq_len": 0,
"stat_sum": {
"num_bytes": 0,
"num_objects": 0,
"num_object_clones": 0,
"num_object_copies": 0,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 0,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 0,
"num_whiteouts": 0,
"num_read": 0,
"num_read_kb": 0,
"num_write": 0,
"num_write_kb": 0,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0,
"num_objects_manifest": 0
},
"up": [
4,
2,
3,
1,
2147483647
],
"acting": [
4,
2,
3,
1,
2147483647
],
"blocked_by": [
0,
2,
3,
4
],
"up_primary": 4,
"acting_primary": 4,
"purged_snaps": []
},
"empty": 1,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 0,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
{
"peer": "1(4)",
"pgid": "8.cs4",
"last_update": "1089'53466",
"last_complete": "1089'53466",
"log_tail": "967'51759",
"last_user_version": 0,
"last_backfill": "8:30020acc:::1000016a007.00000401:head",
"last_backfill_bitwise": 1,
"purged_snaps": [],
"history": {
"epoch_created": 748,
"epoch_pool_created": 748,
"last_epoch_started": 1300,
"last_interval_started": 1299,
"last_epoch_clean": 1083,
"last_interval_clean": 1082,
"last_epoch_split": 1299,
"last_epoch_marked_full": 0,
"same_up_since": 1745,
"same_interval_since": 1745,
"same_primary_since": 1745,
"last_scrub": "1057'53286",
"last_scrub_stamp": "2019-07-01 03:18:26.190174",
"last_deep_scrub": "1057'53286",
"last_deep_scrub_stamp": "2019-07-01 03:18:26.190174",
"last_clean_scrub_stamp": "2019-07-01 03:18:26.190174"
},
"stats": {
"version": "0'0",
"reported_seq": "0",
"reported_epoch": "0",
"state": "unknown",
"last_fresh": "0.000000",
"last_change": "0.000000",
"last_active": "0.000000",
"last_peered": "0.000000",
"last_clean": "0.000000",
"last_became_active": "0.000000",
"last_became_peered": "0.000000",
"last_unstale": "0.000000",
"last_undegraded": "0.000000",
"last_fullsized": "0.000000",
"mapping_epoch": 1745,
"log_start": "0'0",
"ondisk_log_start": "0'0",
"created": 0,
"last_epoch_clean": 0,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "0'0",
"last_scrub_stamp": "0.000000",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "0.000000",
"last_clean_scrub_stamp": "0.000000",
"log_size": 0,
"ondisk_log_size": 0,
"stats_invalid": true,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"manifest_stats_invalid": false,
"snaptrimq_len": 0,
"stat_sum": {
"num_bytes": 213909504,
"num_objects": 51,
"num_object_clones": 0,
"num_object_copies": 0,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 6528,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 51,
"num_whiteouts": 0,
"num_read": 0,
"num_read_kb": 0,
"num_write": 0,
"num_write_kb": 0,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0,
"num_objects_manifest": 0
},
"up": [
4,
2,
3,
1,
2147483647
],
"acting": [
4,
2,
3,
1,
2147483647
],
"blocked_by": [],
"up_primary": 4,
"acting_primary": 4,
"purged_snaps": []
},
"empty": 0,
"dne": 0,
"incomplete": 1,
"last_epoch_started": 1094,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
{
"peer": "2(1)",
"pgid": "8.cs1",
"last_update": "1089'53466",
"last_complete": "1089'53466",
"log_tail": "967'51759",
"last_user_version": 0,
"last_backfill": "8:30020acc:::1000016a007.00000401:head",
"last_backfill_bitwise": 1,
"purged_snaps": [],
"history": {
"epoch_created": 748,
"epoch_pool_created": 748,
"last_epoch_started": 1300,
"last_interval_started": 1299,
"last_epoch_clean": 1083,
"last_interval_clean": 1082,
"last_epoch_split": 1299,
"last_epoch_marked_full": 0,
"same_up_since": 1745,
"same_interval_since": 1745,
"same_primary_since": 1745,
"last_scrub": "1057'53286",
"last_scrub_stamp": "2019-07-01 03:18:26.190174",
"last_deep_scrub": "1057'53286",
"last_deep_scrub_stamp": "2019-07-01 03:18:26.190174",
"last_clean_scrub_stamp": "2019-07-01 03:18:26.190174"
},
"stats": {
"version": "1089'53466",
"reported_seq": "10",
"reported_epoch": "1744",
"state": "down",
"last_fresh": "2019-07-03 03:55:21.592613",
"last_change": "2019-07-03 03:55:21.592613",
"last_active": "0.000000",
"last_peered": "0.000000",
"last_clean": "0.000000",
"last_became_active": "0.000000",
"last_became_peered": "0.000000",
"last_unstale": "2019-07-03 03:55:21.592613",
"last_undegraded": "2019-07-03 03:55:21.592613",
"last_fullsized": "2019-07-03 03:55:21.592613",
"mapping_epoch": 1745,
"log_start": "967'51759",
"ondisk_log_start": "967'51759",
"created": 748,
"last_epoch_clean": 1083,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "1057'53286",
"last_scrub_stamp": "2019-07-01 03:18:26.190174",
"last_deep_scrub": "1057'53286",
"last_deep_scrub_stamp": "2019-07-01 03:18:26.190174",
"last_clean_scrub_stamp": "2019-07-01 03:18:26.190174",
"log_size": 1707,
"ondisk_log_size": 1707,
"stats_invalid": true,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"manifest_stats_invalid": false,
"snaptrimq_len": 0,
"stat_sum": {
"num_bytes": 213909504,
"num_objects": 51,
"num_object_clones": 0,
"num_object_copies": 255,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 6528,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 51,
"num_whiteouts": 0,
"num_read": 0,
"num_read_kb": 0,
"num_write": 0,
"num_write_kb": 0,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0,
"num_objects_manifest": 0
},
"up": [
4,
2,
3,
1,
2147483647
],
"acting": [
4,
2,
3,
1,
2147483647
],
"blocked_by": [
0,
4
],
"up_primary": 4,
"acting_primary": 4,
"purged_snaps": []
},
"empty": 0,
"dne": 0,
"incomplete": 1,
"last_epoch_started": 1094,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
{
"peer": "2(4)",
"pgid": "8.cs4",
"last_update": "1089'53466",
"last_complete": "1089'53466",
"log_tail": "967'51759",
"last_user_version": 53466,
"last_backfill": "MAX",
"last_backfill_bitwise": 0,
"purged_snaps": [],
"history": {
"epoch_created": 748,
"epoch_pool_created": 748,
"last_epoch_started": 1300,
"last_interval_started": 1299,
"last_epoch_clean": 1083,
"last_interval_clean": 1082,
"last_epoch_split": 1299,
"last_epoch_marked_full": 0,
"same_up_since": 1745,
"same_interval_since": 1745,
"same_primary_since": 1745,
"last_scrub": "1057'53286",
"last_scrub_stamp": "2019-07-01 03:18:26.190174",
"last_deep_scrub": "1057'53286",
"last_deep_scrub_stamp": "2019-07-01 03:18:26.190174",
"last_clean_scrub_stamp": "2019-07-01 03:18:26.190174"
},
"stats": {
"version": "1089'53466",
"reported_seq": "515750",
"reported_epoch": "1089",
"state": "active+clean",
"last_fresh": "2019-07-03 00:00:55.949673",
"last_change": "2019-07-02 18:12:25.973469",
"last_active": "2019-07-03 00:00:55.949673",
"last_peered": "2019-07-03 00:00:55.949673",
"last_clean": "2019-07-03 00:00:55.949673",
"last_became_active": "2019-07-02 18:12:25.973256",
"last_became_peered": "2019-07-02 18:12:25.973256",
"last_unstale": "2019-07-03 00:00:55.949673",
"last_undegraded": "2019-07-03 00:00:55.949673",
"last_fullsized": "2019-07-03 00:00:55.949673",
"mapping_epoch": 1745,
"log_start": "967'51759",
"ondisk_log_start": "967'51759",
"created": 748,
"last_epoch_clean": 1083,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "1057'53286",
"last_scrub_stamp": "2019-07-01 03:18:26.190174",
"last_deep_scrub": "1057'53286",
"last_deep_scrub_stamp": "2019-07-01 03:18:26.190174",
"last_clean_scrub_stamp": "2019-07-01 03:18:26.190174",
"log_size": 1707,
"ondisk_log_size": 1707,
"stats_invalid": true,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"manifest_stats_invalid": true,
"snaptrimq_len": 0,
"stat_sum": {
"num_bytes": 26809010770,
"num_objects": 6529,
"num_object_clones": 0,
"num_object_copies": 32648,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 0,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 6529,
"num_whiteouts": 0,
"num_read": 226620,
"num_read_kb": 148056701,
"num_write": 17095,
"num_write_kb": 30657414,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0,
"num_objects_manifest": 0
},
"up": [
4,
2,
3,
1,
2147483647
],
"acting": [
4,
2,
3,
1,
2147483647
],
"blocked_by": [],
"up_primary": 4,
"acting_primary": 4,
"purged_snaps": []
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 1300,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
{
"peer": "3(1)",
"pgid": "8.cs1",
"last_update": "1089'53466",
"last_complete": "1089'53466",
"log_tail": "967'51759",
"last_user_version": 53466,
"last_backfill": "MAX",
"last_backfill_bitwise": 0,
"purged_snaps": [],
"history": {
"epoch_created": 748,
"epoch_pool_created": 748,
"last_epoch_started": 1300,
"last_interval_started": 1299,
"last_epoch_clean": 1083,
"last_interval_clean": 1082,
"last_epoch_split": 1299,
"last_epoch_marked_full": 0,
"same_up_since": 1745,
"same_interval_since": 1745,
"same_primary_since": 1745,
"last_scrub": "1057'53286",
"last_scrub_stamp": "2019-07-01 03:18:26.190174",
"last_deep_scrub": "1057'53286",
"last_deep_scrub_stamp": "2019-07-01 03:18:26.190174",
"last_clean_scrub_stamp": "2019-07-01 03:18:26.190174"
},
"stats": {
"version": "1089'53466",
"reported_seq": "516131",
"reported_epoch": "1707",
"state": "peering",
"last_fresh": "2019-07-03 03:07:07.329932",
"last_change": "2019-07-03 03:07:07.329932",
"last_active": "2019-07-03 00:01:59.538782",
"last_peered": "2019-07-03 00:00:55.949673",
"last_clean": "2019-07-03 00:00:55.949673",
"last_became_active": "2019-07-02 18:12:25.973256",
"last_became_peered": "2019-07-02 18:12:25.973256",
"last_unstale": "2019-07-03 03:07:07.329932",
"last_undegraded": "2019-07-03 03:07:07.329932",
"last_fullsized": "2019-07-03 03:07:07.329932",
"mapping_epoch": 1745,
"log_start": "967'51759",
"ondisk_log_start": "967'51759",
"created": 748,
"last_epoch_clean": 1083,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "1057'53286",
"last_scrub_stamp": "2019-07-01 03:18:26.190174",
"last_deep_scrub": "1057'53286",
"last_deep_scrub_stamp": "2019-07-01 03:18:26.190174",
"last_clean_scrub_stamp": "2019-07-01 03:18:26.190174",
"log_size": 1707,
"ondisk_log_size": 1707,
"stats_invalid": true,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"manifest_stats_invalid": true,
"snaptrimq_len": 0,
"stat_sum": {
"num_bytes": 26809010770,
"num_objects": 6529,
"num_object_clones": 0,
"num_object_copies": 32645,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 0,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 6529,
"num_whiteouts": 0,
"num_read": 226620,
"num_read_kb": 148056701,
"num_write": 17095,
"num_write_kb": 30657414,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0,
"num_objects_manifest": 0
},
"up": [
4,
2,
3,
1,
2147483647
],
"acting": [
4,
2,
3,
1,
2147483647
],
"blocked_by": [
1,
2,
3,
4
],
"up_primary": 4,
"acting_primary": 4,
"purged_snaps": []
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 1300,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
{
"peer": "3(2)",
"pgid": "8.cs2",
"last_update": "1089'53466",
"last_complete": "1089'53466",
"log_tail": "967'51759",
"last_user_version": 0,
"last_backfill": "8:30020acc:::1000016a007.00000401:head",
"last_backfill_bitwise": 1,
"purged_snaps": [],
"history": {
"epoch_created": 748,
"epoch_pool_created": 748,
"last_epoch_started": 1300,
"last_interval_started": 1299,
"last_epoch_clean": 1083,
"last_interval_clean": 1082,
"last_epoch_split": 1299,
"last_epoch_marked_full": 0,
"same_up_since": 1745,
"same_interval_since": 1745,
"same_primary_since": 1745,
"last_scrub": "1057'53286",
"last_scrub_stamp": "2019-07-01 03:18:26.190174",
"last_deep_scrub": "1057'53286",
"last_deep_scrub_stamp": "2019-07-01 03:18:26.190174",
"last_clean_scrub_stamp": "2019-07-01 03:18:26.190174"
},
"stats": {
"version": "0'0",
"reported_seq": "0",
"reported_epoch": "0",
"state": "unknown",
"last_fresh": "0.000000",
"last_change": "0.000000",
"last_active": "0.000000",
"last_peered": "0.000000",
"last_clean": "0.000000",
"last_became_active": "0.000000",
"last_became_peered": "0.000000",
"last_unstale": "0.000000",
"last_undegraded": "0.000000",
"last_fullsized": "0.000000",
"mapping_epoch": 1745,
"log_start": "0'0",
"ondisk_log_start": "0'0",
"created": 0,
"last_epoch_clean": 0,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "0'0",
"last_scrub_stamp": "0.000000",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "0.000000",
"last_clean_scrub_stamp": "0.000000",
"log_size": 0,
"ondisk_log_size": 0,
"stats_invalid": true,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"manifest_stats_invalid": false,
"snaptrimq_len": 0,
"stat_sum": {
"num_bytes": 213909504,
"num_objects": 51,
"num_object_clones": 0,
"num_object_copies": 0,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 6528,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 51,
"num_whiteouts": 0,
"num_read": 0,
"num_read_kb": 0,
"num_write": 0,
"num_write_kb": 0,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0,
"num_objects_manifest": 0
},
"up": [
4,
2,
3,
1,
2147483647
],
"acting": [
4,
2,
3,
1,
2147483647
],
"blocked_by": [],
"up_primary": 4,
"acting_primary": 4,
"purged_snaps": []
},
"empty": 0,
"dne": 0,
"incomplete": 1,
"last_epoch_started": 1094,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
{
"peer": "4(3)",
"pgid": "8.cs3",
"last_update": "1089'53466",
"last_complete": "1089'53466",
"log_tail": "967'51759",
"last_user_version": 53466,
"last_backfill": "MAX",
"last_backfill_bitwise": 0,
"purged_snaps": [],
"history": {
"epoch_created": 748,
"epoch_pool_created": 748,
"last_epoch_started": 1300,
"last_interval_started": 1299,
"last_epoch_clean": 1083,
"last_interval_clean": 1082,
"last_epoch_split": 1299,
"last_epoch_marked_full": 0,
"same_up_since": 1745,
"same_interval_since": 1745,
"same_primary_since": 1745,
"last_scrub": "1057'53286",
"last_scrub_stamp": "2019-07-01 03:18:26.190174",
"last_deep_scrub": "1057'53286",
"last_deep_scrub_stamp": "2019-07-01 03:18:26.190174",
"last_clean_scrub_stamp": "2019-07-01 03:18:26.190174"
},
"stats": {
"version": "1089'53466",
"reported_seq": "515752",
"reported_epoch": "1710",
"state": "remapped",
"last_fresh": "2019-07-03 03:07:08.720389",
"last_change": "2019-07-03 03:07:08.720163",
"last_active": "2019-07-03 03:07:08.720163",
"last_peered": "2019-07-03 00:00:55.949673",
"last_clean": "2019-07-03 00:00:55.949673",
"last_became_active": "2019-07-02 18:12:25.973256",
"last_became_peered": "2019-07-02 18:12:25.973256",
"last_unstale": "2019-07-03 03:07:08.720389",
"last_undegraded": "2019-07-03 03:07:08.720389",
"last_fullsized": "2019-07-03 03:07:08.720389",
"mapping_epoch": 1745,
"log_start": "967'51759",
"ondisk_log_start": "967'51759",
"created": 748,
"last_epoch_clean": 1083,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "1057'53286",
"last_scrub_stamp": "2019-07-01 03:18:26.190174",
"last_deep_scrub": "1057'53286",
"last_deep_scrub_stamp": "2019-07-01 03:18:26.190174",
"last_clean_scrub_stamp": "2019-07-01 03:18:26.190174",
"log_size": 1707,
"ondisk_log_size": 1707,
"stats_invalid": true,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"manifest_stats_invalid": true,
"snaptrimq_len": 0,
"stat_sum": {
"num_bytes": 26809010770,
"num_objects": 6529,
"num_object_clones": 0,
"num_object_copies": 32645,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 0,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 6529,
"num_whiteouts": 0,
"num_read": 226620,
"num_read_kb": 148056701,
"num_write": 17095,
"num_write_kb": 30657414,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0,
"num_objects_manifest": 0
},
"up": [
4,
2,
3,
1,
2147483647
],
"acting": [
4,
2,
3,
1,
2147483647
],
"blocked_by": [],
"up_primary": 4,
"acting_primary": 4,
"purged_snaps": []
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 1300,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
}
],
"recovery_state": [
{
"name": "Started/Primary/Peering/Incomplete",
"enter_time": "2019-07-03 03:55:22.658428",
"comment": "not enough complete instances of this PG"
},
{
"name": "Started/Primary/Peering",
"enter_time": "2019-07-03 03:55:22.633592",
"past_intervals": [
{
"first": "1082",
"last": "1744",
"all_participants": [
{
"osd": 0,
"shard": 0
},
{
"osd": 0,
"shard": 3
},
{
"osd": 1,
"shard": 2
},
{
"osd": 1,
"shard": 3
},
{
"osd": 1,
"shard": 4
},
{
"osd": 2,
"shard": 1
},
{
"osd": 2,
"shard": 4
},
{
"osd": 3,
"shard": 1
},
{
"osd": 3,
"shard": 2
},
{
"osd": 4,
"shard": 0
},
{
"osd": 4,
"shard": 3
}
],
"intervals": [
{
"first": "1299",
"last": "1424",
"acting": "0(0),1(2),2(4),3(1),4(3)"
}
]
}
],
"probing_osds": [
"1(2)",
"1(3)",
"1(4)",
"2(1)",
"2(4)",
"3(1)",
"3(2)",
"4(0)",
"4(3)"
],
"down_osds_we_would_probe": [
0
],
"peering_blocked_by": [],
"peering_blocked_by_detail": [
{
"detail": "peering_blocked_by_history_les_bound"
}
]
},
{
"name": "Started",
"enter_time": "2019-07-03 03:55:22.633565"
}
],
"agent_state": {}
}
Raw
ceph-health-detail
HEALTH_WARN 1 MDSs report slow metadata IOs; Reduced data availability: 171 pgs inactive, 1 pg down, 149 pgs incomplete; too many PGs per OSD (319 > max 250)
MDS_SLOW_METADATA_IO 1 MDSs report slow metadata IOs
mdsarodd-linux(mds.0): 17 slow metadata IOs are blocked > 30 secs, oldest blocked for 32920 secs
PG_AVAILABILITY Reduced data availability: 171 pgs inactive, 1 pg down, 149 pgs incomplete
pg 1.93 is stuck inactive for 35600.731528, current state unknown, last acting []
pg 1.a0 is stuck inactive for 35600.731528, current state unknown, last acting []
pg 8.60 is incomplete, acting [1,4,3,2147483647,2] (reducing pool cephfs_ec42data min_size from 5 may help; search ceph.com/docs for 'incomplete')
pg 8.61 is incomplete, acting [1,3,2,2147483647,4] (reducing pool cephfs_ec42data min_size from 5 may help; search ceph.com/docs for 'incomplete')
pg 8.62 is incomplete, acting [2,4,2147483647,1,3] (reducing pool cephfs_ec42data min_size from 5 may help; search ceph.com/docs for 'incomplete')
pg 8.63 is incomplete, acting [4,3,1,2,2147483647] (reducing pool cephfs_ec42data min_size from 5 may help; search ceph.com/docs for 'incomplete')
pg 8.64 is incomplete, acting [4,2,1,2147483647,3] (reducing pool cephfs_ec42data min_size from 5 may help; search ceph.com/docs for 'incomplete')
pg 8.65 is incomplete, acting [3,1,2,2147483647,4] (reducing pool cephfs_ec42data min_size from 5 may help; search ceph.com/docs for 'incomplete')
pg 8.66 is incomplete, acting [2,1,4,3,2147483647] (reducing pool cephfs_ec42data min_size from 5 may help; search ceph.com/docs for 'incomplete')
pg 8.67 is incomplete, acting [2,4,3,2147483647,1] (reducing pool cephfs_ec42data min_size from 5 may help; search ceph.com/docs for 'incomplete')
pg 8.6c is stuck inactive for 39772.311807, current state incomplete, last acting [2,3,4,1,2147483647] (reducing pool cephfs_ec42data min_size from 5 may help; search ceph.com/docs for 'incomplete')
pg 8.6e is incomplete, acting [4,3,2,1,2147483647] (reducing pool cephfs_ec42data min_size from 5 may help; search ceph.com/docs for 'incomplete')
pg 8.6f is incomplete, acting [1,2,4,3,2147483647] (reducing pool cephfs_ec42data min_size from 5 may help; search ceph.com/docs for 'incomplete')
pg 8.70 is incomplete, acting [1,4,3,2,2147483647] (reducing pool cephfs_ec42data min_size from 5 may help; search ceph.com/docs for 'incomplete')
pg 8.71 is incomplete, acting [1,3,4,2,2147483647] (reducing pool cephfs_ec42data min_size from 5 may help; search ceph.com/docs for 'incomplete')
pg 8.72 is incomplete, acting [2,2147483647,3,4,1] (reducing pool cephfs_ec42data min_size from 5 may help; search ceph.com/docs for 'incomplete')
pg 8.73 is incomplete, acting [4,2,3,1,2147483647] (reducing pool cephfs_ec42data min_size from 5 may help; search ceph.com/docs for 'incomplete')
pg 8.74 is incomplete, acting [3,1,2,4,2147483647] (reducing pool cephfs_ec42data min_size from 5 may help; search ceph.com/docs for 'incomplete')
pg 8.75 is incomplete, acting [3,1,4,2147483647,2] (reducing pool cephfs_ec42data min_size from 5 may help; search ceph.com/docs for 'incomplete')
pg 8.76 is incomplete, acting [3,1,2,2147483647,4] (reducing pool cephfs_ec42data min_size from 5 may help; search ceph.com/docs for 'incomplete')
pg 8.77 is incomplete, acting [4,2147483647,2,1,3] (reducing pool cephfs_ec42data min_size from 5 may help; search ceph.com/docs for 'incomplete')
pg 8.78 is incomplete, acting [2,3,2147483647,1,4] (reducing pool cephfs_ec42data min_size from 5 may help; search ceph.com/docs for 'incomplete')
pg 8.79 is incomplete, acting [4,3,2,1,2147483647] (reducing pool cephfs_ec42data min_size from 5 may help; search ceph.com/docs for 'incomplete')
pg 8.7a is incomplete, acting [3,1,4,2,2147483647] (reducing pool cephfs_ec42data min_size from 5 may help; search ceph.com/docs for 'incomplete')
pg 8.7b is incomplete, acting [2,4,1,3,2147483647] (reducing pool cephfs_ec42data min_size from 5 may help; search ceph.com/docs for 'incomplete')
pg 8.7c is incomplete, acting [1,3,4,2,2147483647] (reducing pool cephfs_ec42data min_size from 5 may help; search ceph.com/docs for 'incomplete')
pg 8.7d is incomplete, acting [2,3,1,4,2147483647] (reducing pool cephfs_ec42data min_size from 5 may help; search ceph.com/docs for 'incomplete')
pg 8.7e is incomplete, acting [2,3,4,1,2147483647] (reducing pool cephfs_ec42data min_size from 5 may help; search ceph.com/docs for 'incomplete')
pg 8.7f is incomplete, acting [1,3,2147483647,4,2] (reducing pool cephfs_ec42data min_size from 5 may help; search ceph.com/docs for 'incomplete')
pg 8.80 is incomplete, acting [2,1,3,2147483647,4] (reducing pool cephfs_ec42data min_size from 5 may help; search ceph.com/docs for 'incomplete')
pg 8.81 is incomplete, acting [2,3,1,2147483647,4] (reducing pool cephfs_ec42data min_size from 5 may help; search ceph.com/docs for 'incomplete')
pg 8.82 is incomplete, acting [3,4,1,2,2147483647] (reducing pool cephfs_ec42data min_size from 5 may help; search ceph.com/docs for 'incomplete')
pg 8.83 is incomplete, acting [1,2147483647,4,3,2] (reducing pool cephfs_ec42data min_size from 5 may help; search ceph.com/docs for 'incomplete')
pg 8.84 is incomplete, acting [4,2,1,3,2147483647] (reducing pool cephfs_ec42data min_size from 5 may help; search ceph.com/docs for 'incomplete')
pg 8.85 is incomplete, acting [2147483647,3,4,2,1] (reducing pool cephfs_ec42data min_size from 5 may help; search ceph.com/docs for 'incomplete')
pg 8.86 is incomplete, acting [4,1,2,2147483647,3] (reducing pool cephfs_ec42data min_size from 5 may help; search ceph.com/docs for 'incomplete')
pg 8.87 is incomplete, acting [2,1,3,4,2147483647] (reducing pool cephfs_ec42data min_size from 5 may help; search ceph.com/docs for 'incomplete')
pg 8.88 is incomplete, acting [2,1,4,2147483647,3] (reducing pool cephfs_ec42data min_size from 5 may help; search ceph.com/docs for 'incomplete')
pg 8.89 is incomplete, acting [4,2,2147483647,1,3] (reducing pool cephfs_ec42data min_size from 5 may help; search ceph.com/docs for 'incomplete')
pg 8.8a is incomplete, acting [3,2147483647,1,4,2] (reducing pool cephfs_ec42data min_size from 5 may help; search ceph.com/docs for 'incomplete')
pg 8.8b is incomplete, acting [3,2,1,4,2147483647] (reducing pool cephfs_ec42data min_size from 5 may help; search ceph.com/docs for 'incomplete')
pg 8.8c is incomplete, acting [2147483647,2,3,4,1] (reducing pool cephfs_ec42data min_size from 5 may help; search ceph.com/docs for 'incomplete')
pg 8.8d is incomplete, acting [1,3,4,2,2147483647] (reducing pool cephfs_ec42data min_size from 5 may help; search ceph.com/docs for 'incomplete')
pg 8.8e is incomplete, acting [2147483647,1,4,3,2] (reducing pool cephfs_ec42data min_size from 5 may help; search ceph.com/docs for 'incomplete')
pg 8.8f is incomplete, acting [3,1,2147483647,2,4] (reducing pool cephfs_ec42data min_size from 5 may help; search ceph.com/docs for 'incomplete')
pg 8.90 is incomplete, acting [4,3,2,2147483647,1] (reducing pool cephfs_ec42data min_size from 5 may help; search ceph.com/docs for 'incomplete')
pg 8.91 is incomplete, acting [3,2,4,1,2147483647] (reducing pool cephfs_ec42data min_size from 5 may help; search ceph.com/docs for 'incomplete')
pg 8.92 is incomplete, acting [4,1,2,3,2147483647] (reducing pool cephfs_ec42data min_size from 5 may help; search ceph.com/docs for 'incomplete')
pg 8.93 is incomplete, acting [3,4,2147483647,2,1] (reducing pool cephfs_ec42data min_size from 5 may help; search ceph.com/docs for 'incomplete')
pg 8.94 is incomplete, acting [3,1,4,2,2147483647] (reducing pool cephfs_ec42data min_size from 5 may help; search ceph.com/docs for 'incomplete')
pg 8.95 is incomplete, acting [2147483647,1,4,3,2] (reducing pool cephfs_ec42data min_size from 5 may help; search ceph.com/docs for 'incomplete')
TOO_MANY_PGS too many PGs per OSD (319 > max 250)
Raw
ceph-osd-0-start
Uptime(secs): 0.7 total, 0.0 interval
Flush(GB): cumulative 0.000, interval 0.000
AddFile(GB): cumulative 0.000, interval 0.000
AddFile(Total Files): cumulative 0, interval 0
AddFile(L0 Files): cumulative 0, interval 0
AddFile(Keys): cumulative 0, interval 0
Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds
Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds
Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 1 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count
** File Read Latency Histogram By Level [default] **
-342> 2019-07-03 04:09:53.951 7f7d1d08e700 4 rocksdb: [/build/ceph-13.2.6/src/rocksdb/db/compaction_job.cc:1494] [default] [JOB 3] Compacting 36@0 + 1@1 files to L1, score 9.00
-342> 2019-07-03 04:09:53.951 7f7d1d08e700 4 rocksdb: [/build/ceph-13.2.6/src/rocksdb/db/compaction_job.cc:1498] [default] Compaction start summary: Base version 3 Base level 0, inputs: [9123(1141B) 9119(1141B) 9115(1141B) 9111(1141B) 9098(1141B) 9094(1141B) 9090(1141B) 9080(1141B) 9076(1141B) 9069(1141B) 9065(1141B) 9061(1141B) 9057(1141B) 9053(1141B) 9049(1141B) 9045(1141B) 9041(1141B) 9034(1141B) 9024(1141B) 9020(1141B) 9016(1141B) 9012(1141B) 9008(1141B) 9004(1141B) 9000(1141B) 8996(1141B) 8989(1141B) 8985(1141B) 8978(1141B) 8974(1141B) 8961(1141B) 8957(1141B) 8932(1141B) 8928(9619KB) 8926(136MB) 8920(226KB)], [8902(19MB)]
-342> 2019-07-03 04:09:53.951 7f7d1d08e700 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1562144993956692, "job": 3, "event": "compaction_started", "files_L0": [9123, 9119, 9115, 9111, 9098, 9094, 9090, 9080, 9076, 9069, 9065, 9061, 9057, 9053, 9049, 9045, 9041, 9034, 9024, 9020, 9016, 9012, 9008, 9004, 9000, 8996, 8989, 8985, 8978, 8974, 8961, 8957, 8932, 8928, 8926, 8920], "files_L1": [8902], "score": 9, "input_data_size": 173341352}
-342> 2019-07-03 04:09:54.207 7f7d34b5de00 1 freelist init
-342> 2019-07-03 04:09:54.247 7f7d34b5de00 1 bluestore(/var/lib/ceph/osd/ceph-0) _open_alloc opening allocation metadata
-342> 2019-07-03 04:09:55.695 7f7d34b5de00 1 bluestore(/var/lib/ceph/osd/ceph-0) _open_alloc loaded 2.0 TiB in 44113 extents
-342> 2019-07-03 04:09:55.719 7f7d34b5de00 2 osd.0 0 journal looks like hdd
-342> 2019-07-03 04:09:55.719 7f7d34b5de00 2 osd.0 0 boot
-342> 2019-07-03 04:09:55.719 7f7d1c08c700 5 bluestore.MempoolThread(0x23c4b08) _tune_cache_size target: 1963336226 heap: 171302912 unmapped: 294912 mapped: 171008000 old cache_size: 134217728 new cache size: 800005849
-342> 2019-07-03 04:09:55.719 7f7d1c08c700 5 bluestore.MempoolThread(0x23c4b08) _trim_shards cache_size: 800005849 kv_alloc: 134217728 kv_used: 35018071 meta_alloc: 0 meta_used: 43528 data_alloc: 0 data_used: 0
-342> 2019-07-03 04:09:55.915 7f7d34b5de00 0 _get_class not permitted to load lua
-342> 2019-07-03 04:09:55.915 7f7d34b5de00 1 <cls> /build/ceph-13.2.6/src/cls/replica_log/cls_replica_log.cc:135: Loaded replica log class!
-342> 2019-07-03 04:09:55.915 7f7d34b5de00 0 _get_class not permitted to load kvs
-342> 2019-07-03 04:09:55.915 7f7d34b5de00 1 <cls> /build/ceph-13.2.6/src/cls/rgw/cls_rgw.cc:3875: Loaded rgw class!
-342> 2019-07-03 04:09:55.915 7f7d34b5de00 1 <cls> /build/ceph-13.2.6/src/cls/user/cls_user.cc:419: Loaded user class!
-342> 2019-07-03 04:09:55.919 7f7d34b5de00 1 <cls> /build/ceph-13.2.6/src/cls/version/cls_version.cc:214: Loaded version class!
-342> 2019-07-03 04:09:55.919 7f7d34b5de00 0 _get_class not permitted to load sdk
-342> 2019-07-03 04:09:55.919 7f7d34b5de00 1 <cls> /build/ceph-13.2.6/src/cls/log/cls_log.cc:299: Loaded log class!
-342> 2019-07-03 04:09:55.919 7f7d34b5de00 1 <cls> /build/ceph-13.2.6/src/cls/timeindex/cls_timeindex.cc:242: Loaded timeindex class!
-342> 2019-07-03 04:09:55.923 7f7d34b5de00 1 <cls> /build/ceph-13.2.6/src/cls/refcount/cls_refcount.cc:399: Loaded refcount class!
-342> 2019-07-03 04:09:55.923 7f7d34b5de00 0 <cls> /build/ceph-13.2.6/src/cls/cephfs/cls_cephfs.cc:197: loading cephfs
-342> 2019-07-03 04:09:55.923 7f7d34b5de00 0 <cls> /build/ceph-13.2.6/src/cls/hello/cls_hello.cc:296: loading cls_hello
-342> 2019-07-03 04:09:55.923 7f7d34b5de00 1 <cls> /build/ceph-13.2.6/src/cls/statelog/cls_statelog.cc:279: Loaded log class!
-342> 2019-07-03 04:09:55.923 7f7d34b5de00 0 osd.0 1424 crush map has features 288514119978713088, adjusting msgr requires for clients
-342> 2019-07-03 04:09:55.923 7f7d34b5de00 0 osd.0 1424 crush map has features 288514119978713088 was 8705, adjusting msgr requires for mons
-342> 2019-07-03 04:09:55.923 7f7d34b5de00 0 osd.0 1424 crush map has features 1009090060358008832, adjusting msgr requires for osds
-342> 2019-07-03 04:09:59.899 7f7d1d08e700 4 rocksdb: [/build/ceph-13.2.6/src/rocksdb/db/compaction_job.cc:1166] [default] [JOB 3] Generated table #9233: 297927 keys, 68734140 bytes
-342> 2019-07-03 04:09:59.899 7f7d1d08e700 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1562144999903617, "cf_name": "default", "job": 3, "event": "table_file_creation", "file_number": 9233, "file_size": 68734140, "table_properties": {"data_size": 67114749, "index_size": 720430, "filter_size": 897909, "raw_key_size": 15339015, "raw_average_key_size": 51, "raw_value_size": 60576098, "raw_average_value_size": 203, "num_data_blocks": 15475, "num_entries": 297927, "filter_policy_name": "rocksdb.BuiltinBloomFilter", "kDeletedKeys": "125680", "kMergeOperands": "0"}}
-342> 2019-07-03 04:10:00.727 7f7d1c08c700 5 bluestore.MempoolThread(0x23c4b08) _tune_cache_size target: 1963336226 heap: 241983488 unmapped: 18128896 mapped: 223854592 old cache_size: 863521357 new cache size: 863528504
-342> 2019-07-03 04:10:00.727 7f7d1c08c700 5 bluestore.MempoolThread(0x23c4b08) _trim_shards cache_size: 863528504 kv_alloc: 466204498 kv_used: 83836773 meta_alloc: 264877906 meta_used: 43552 data_alloc: 132438952 data_used: 0
-342> 2019-07-03 04:10:02.651 7f7d34b5de00 0 osd.0 1424 load_pgs
-342> 2019-07-03 04:10:02.723 7f7d34b5de00 5 osd.0 pg_epoch: 1424 pg[8.fs4(unlocked)] enter Initial
-342> 2019-07-03 04:10:02.803 7f7d34b5de00 5 osd.0 pg_epoch: 1424 pg[8.fs4( v 1089'49973 (967'48279,1089'49973] local-lis/les=1299/1300 n=6510 ec=748/748 lis/c 1299/1299 les/c/f 1300/1300/0 1276/1299/1269) [3,2,4,1,0]p3(0) r=4 lpr=0 crt=1089'49973 lcod 0'0 unknown mbc={}] exit Initial 0.079029 0 0.000000
-342> 2019-07-03 04:10:02.803 7f7d34b5de00 5 osd.0 pg_epoch: 1424 pg[8.fs4( v 1089'49973 (967'48279,1089'49973] local-lis/les=1299/1300 n=6510 ec=748/748 lis/c 1299/1299 les/c/f 1300/1300/0 1276/1299/1269) [3,2,4,1,0]p3(0) r=4 lpr=0 crt=1089'49973 lcod 0'0 unknown mbc={}] enter Reset
-342> 2019-07-03 04:10:02.803 7f7d34b5de00 5 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0
-342> 2019-07-03 04:10:02.819 7f7d34b5de00 5 osd.0 pg_epoch: 1421 pg[8.4as2(unlocked)] enter Initial
-342> 2019-07-03 04:10:02.819 7f7d34b5de00 -1 /build/ceph-13.2.6/src/osd/ECUtil.h: In function 'ECUtil::stripe_info_t::stripe_info_t(uint64_t, uint64_t)' thread 7f7d34b5de00 time 2019-07-03 04:10:02.822097
/build/ceph-13.2.6/src/osd/ECUtil.h: 34: FAILED assert(stripe_width % stripe_size == 0)
ceph version 13.2.6 (7b695f835b03642f85998b2ae7b6dd093d9fbce4) mimic (stable)
1: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x14e) [0x7f7d2be3e97e]
2: (()+0x2fab07) [0x7f7d2be3eb07]
3: (ECBackend::ECBackend(PGBackend::Listener*, coll_t const&, boost::intrusive_ptr<ObjectStore::CollectionImpl>&, ObjectStore*, CephContext*, std::shared_ptr<ceph::ErasureCodeInterface>, unsigned long)+0x4de) [0xa513ce]
4: (PGBackend::build_pg_backend(pg_pool_t const&, std::map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > const&, PGBackend::Listener*, coll_t, boost::intrusive_ptr<ObjectStore::CollectionImpl>&, ObjectStore*, CephContext*)+0x2f9) [0x94b789]
5: (PrimaryLogPG::PrimaryLogPG(OSDService*, std::shared_ptr<OSDMap const>, PGPool const&, std::map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > const&, spg_t)+0x138) [0x8fd9f8]
6: (OSD::_make_pg(std::shared_ptr<OSDMap const>, spg_t)+0x11d3) [0x7546a3]
7: (OSD::load_pgs()+0x4a9) [0x759459]
8: (OSD::init()+0xcd3) [0x762a03]
9: (main()+0x367b) [0x64f23b]
10: (__libc_start_main()+0xf0) [0x7f7d299d9830]
11: (_start()+0x29) [0x718929]
-342> 2019-07-03 04:10:02.823 7f7d34b5de00 -1 *** Caught signal (Aborted) **
in thread 7f7d34b5de00 thread_name:ceph-osd
ceph version 13.2.6 (7b695f835b03642f85998b2ae7b6dd093d9fbce4) mimic (stable)
1: (()+0x11390) [0x7f7d2a4bb390]
2: (gsignal()+0x38) [0x7f7d299ee428]
3: (abort()+0x16a) [0x7f7d299f002a]
4: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x256) [0x7f7d2be3ea86]
5: (()+0x2fab07) [0x7f7d2be3eb07]
6: (ECBackend::ECBackend(PGBackend::Listener*, coll_t const&, boost::intrusive_ptr<ObjectStore::CollectionImpl>&, ObjectStore*, CephContext*, std::shared_ptr<ceph::ErasureCodeInterface>, unsigned long)+0x4de) [0xa513ce]
7: (PGBackend::build_pg_backend(pg_pool_t const&, std::map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > const&, PGBackend::Listener*, coll_t, boost::intrusive_ptr<ObjectStore::CollectionImpl>&, ObjectStore*, CephContext*)+0x2f9) [0x94b789]
8: (PrimaryLogPG::PrimaryLogPG(OSDService*, std::shared_ptr<OSDMap const>, PGPool const&, std::map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > const&, spg_t)+0x138) [0x8fd9f8]
9: (OSD::_make_pg(std::shared_ptr<OSDMap const>, spg_t)+0x11d3) [0x7546a3]
10: (OSD::load_pgs()+0x4a9) [0x759459]
11: (OSD::init()+0xcd3) [0x762a03]
12: (main()+0x367b) [0x64f23b]
13: (__libc_start_main()+0xf0) [0x7f7d299d9830]
14: (_start()+0x29) [0x718929]
NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
Raw
ceph-osd-5-start
-333> 2019-07-03 04:06:56.736 7f5ec40ece00 2 osd.5 0 journal looks like hdd
-333> 2019-07-03 04:06:56.736 7f5ec40ece00 2 osd.5 0 boot
-333> 2019-07-03 04:06:56.736 7f5eab61b700 5 bluestore.MempoolThread(0x3484b08) _tune_cache_size target: 1963336226 heap: 28368896 unmapped: 532480 mapped: 27836416 old cache_size: 134217728 new cache size: 853189155
-333> 2019-07-03 04:06:56.736 7f5eab61b700 5 bluestore.MempoolThread(0x3484b08) _trim_shards cache_size: 853189155 kv_alloc: 134217728 kv_used: 4710921 meta_alloc: 0 meta_used: 34504 data_alloc: 0 data_used: 0
-333> 2019-07-03 04:06:56.928 7f5ec40ece00 0 _get_class not permitted to load lua
-333> 2019-07-03 04:06:56.928 7f5ec40ece00 1 <cls> /build/ceph-13.2.6/src/cls/replica_log/cls_replica_log.cc:135: Loaded replica log class!
-333> 2019-07-03 04:06:56.928 7f5ec40ece00 0 _get_class not permitted to load kvs
-333> 2019-07-03 04:06:56.928 7f5ec40ece00 1 <cls> /build/ceph-13.2.6/src/cls/rgw/cls_rgw.cc:3875: Loaded rgw class!
-333> 2019-07-03 04:06:56.932 7f5ec40ece00 1 <cls> /build/ceph-13.2.6/src/cls/user/cls_user.cc:419: Loaded user class!
-333> 2019-07-03 04:06:56.932 7f5ec40ece00 1 <cls> /build/ceph-13.2.6/src/cls/version/cls_version.cc:214: Loaded version class!
-333> 2019-07-03 04:06:56.932 7f5ec40ece00 0 _get_class not permitted to load sdk
-333> 2019-07-03 04:06:56.932 7f5ec40ece00 1 <cls> /build/ceph-13.2.6/src/cls/log/cls_log.cc:299: Loaded log class!
-333> 2019-07-03 04:06:56.932 7f5ec40ece00 1 <cls> /build/ceph-13.2.6/src/cls/timeindex/cls_timeindex.cc:242: Loaded timeindex class!
-333> 2019-07-03 04:06:56.936 7f5ec40ece00 1 <cls> /build/ceph-13.2.6/src/cls/refcount/cls_refcount.cc:399: Loaded refcount class!
-333> 2019-07-03 04:06:56.936 7f5ec40ece00 0 <cls> /build/ceph-13.2.6/src/cls/cephfs/cls_cephfs.cc:197: loading cephfs
-333> 2019-07-03 04:06:56.936 7f5ec40ece00 0 <cls> /build/ceph-13.2.6/src/cls/hello/cls_hello.cc:296: loading cls_hello
-333> 2019-07-03 04:06:56.936 7f5ec40ece00 1 <cls> /build/ceph-13.2.6/src/cls/statelog/cls_statelog.cc:279: Loaded log class!
-333> 2019-07-03 04:06:56.936 7f5ec40ece00 0 osd.5 1430 crush map has features 288514119978713088, adjusting msgr requires for clients
-333> 2019-07-03 04:06:56.936 7f5ec40ece00 0 osd.5 1430 crush map has features 288514119978713088 was 8705, adjusting msgr requires for mons
-333> 2019-07-03 04:06:56.936 7f5ec40ece00 0 osd.5 1430 crush map has features 1009090060358008832, adjusting msgr requires for osds
-333> 2019-07-03 04:07:00.172 7f5ec40ece00 0 osd.5 1430 load_pgs
-333> 2019-07-03 04:07:00.216 7f5ec40ece00 5 osd.5 pg_epoch: 1430 pg[8.cs0(unlocked)] enter Initial
-333> 2019-07-03 04:07:00.216 7f5ec40ece00 -1 /build/ceph-13.2.6/src/osd/ECUtil.h: In function 'ECUtil::stripe_info_t::stripe_info_t(uint64_t, uint64_t)' thread 7f5ec40ece00 time 2019-07-03 04:07:00.218913
/build/ceph-13.2.6/src/osd/ECUtil.h: 34: FAILED assert(stripe_width % stripe_size == 0)
ceph version 13.2.6 (7b695f835b03642f85998b2ae7b6dd093d9fbce4) mimic (stable)
1: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x14e) [0x7f5ebb3cd97e]
2: (()+0x2fab07) [0x7f5ebb3cdb07]
3: (ECBackend::ECBackend(PGBackend::Listener*, coll_t const&, boost::intrusive_ptr<ObjectStore::CollectionImpl>&, ObjectStore*, CephContext*, std::shared_ptr<ceph::ErasureCodeInterface>, unsigned long)+0x4de) [0xa513ce]
4: (PGBackend::build_pg_backend(pg_pool_t const&, std::map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > const&, PGBackend::Listener*, coll_t, boost::intrusive_ptr<ObjectStore::CollectionImpl>&, ObjectStore*, CephContext*)+0x2f9) [0x94b789]
5: (PrimaryLogPG::PrimaryLogPG(OSDService*, std::shared_ptr<OSDMap const>, PGPool const&, std::map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > const&, spg_t)+0x138) [0x8fd9f8]
6: (OSD::_make_pg(std::shared_ptr<OSDMap const>, spg_t)+0x11d3) [0x7546a3]
7: (OSD::load_pgs()+0x4a9) [0x759459]
8: (OSD::init()+0xcd3) [0x762a03]
9: (main()+0x367b) [0x64f23b]
10: (__libc_start_main()+0xf0) [0x7f5eb8f68830]
11: (_start()+0x29) [0x718929]
-333> 2019-07-03 04:07:00.220 7f5ec40ece00 -1 *** Caught signal (Aborted) **
in thread 7f5ec40ece00 thread_name:ceph-osd
ceph version 13.2.6 (7b695f835b03642f85998b2ae7b6dd093d9fbce4) mimic (stable)
1: (()+0x11390) [0x7f5eb9a4a390]
2: (gsignal()+0x38) [0x7f5eb8f7d428]
3: (abort()+0x16a) [0x7f5eb8f7f02a]
4: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x256) [0x7f5ebb3cda86]
5: (()+0x2fab07) [0x7f5ebb3cdb07]
6: (ECBackend::ECBackend(PGBackend::Listener*, coll_t const&, boost::intrusive_ptr<ObjectStore::CollectionImpl>&, ObjectStore*, CephContext*, std::shared_ptr<ceph::ErasureCodeInterface>, unsigned long)+0x4de) [0xa513ce]
7: (PGBackend::build_pg_backend(pg_pool_t const&, std::map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > const&, PGBackend::Listener*, coll_t, boost::intrusive_ptr<ObjectStore::CollectionImpl>&, ObjectStore*, CephContext*)+0x2f9) [0x94b789]
8: (PrimaryLogPG::PrimaryLogPG(OSDService*, std::shared_ptr<OSDMap const>, PGPool const&, std::map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > const&, spg_t)+0x138) [0x8fd9f8]
9: (OSD::_make_pg(std::shared_ptr<OSDMap const>, spg_t)+0x11d3) [0x7546a3]
10: (OSD::load_pgs()+0x4a9) [0x759459]
11: (OSD::init()+0xcd3) [0x762a03]
12: (main()+0x367b) [0x64f23b]
13: (__libc_start_main()+0xf0) [0x7f5eb8f68830]
14: (_start()+0x29) [0x718929]
NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
Raw
ceph-osd-dump
epoch 1751
fsid 985f1a8d-e24c-4196-a5cc-4432d5597aef
created 2018-02-26 21:32:24.116991
modified 2019-07-03 05:01:22.769961
flags sortbitwise,recovery_deletes,purged_snapdirs
crush_version 19
full_ratio 0.95
backfillfull_ratio 0.9
nearfull_ratio 0.85
require_min_compat_client jewel
min_compat_client jewel
require_osd_release mimic
pool 1 'cephfs_data' replicated size 2 min_size 1 crush_rule 0 object_hash rjenkins pg_num 256 pgp_num 256 last_change 1072 lfor 0/76 flags hashpspool,selfmanaged_snaps stripe_width 0 application cephfs
removed_snaps [2~2]
pool 2 'cephfs_metadata' replicated size 3 min_size 1 crush_rule 0 object_hash rjenkins pg_num 8 pgp_num 8 last_change 1072 flags hashpspool stripe_width 0 application cephfs
pool 5 'rbd' replicated size 2 min_size 1 crush_rule 0 object_hash rjenkins pg_num 16 pgp_num 16 last_change 1072 flags hashpspool,selfmanaged_snaps stripe_width 0 application rbd
removed_snaps [1~3]
pool 8 'cephfs_ec42data' erasure size 5 min_size 5 crush_rule 3 object_hash rjenkins pg_num 150 pgp_num 150 last_change 1303 lfor 0/1299 flags hashpspool,ec_overwrites stripe_width 16384 application cephfs
max_osd 6
osd.0 down out weight 0 up_from 1252 up_thru 1415 down_at 1425 last_clean_interval [1245,1249) 10.13.0.8:6805/684092 10.13.0.8:6809/684092 10.13.0.8:6812/684092 10.13.0.8:6816/684092 autoout,exists 726e5802-1699-47df-ae90-0971bd51f408
osd.1 up in weight 1 up_from 1738 up_thru 1745 down_at 1737 last_clean_interval [1710,1735) 10.13.0.8:6801/4495 10.13.0.8:6804/4495 10.13.0.8:6806/4495 10.13.0.8:6809/4495 exists,up 9bcdc0a6-09ad-4e6c-924c-0504c2337457
osd.2 up in weight 1 up_from 1739 up_thru 1745 down_at 1738 last_clean_interval [1713,1735) 10.13.0.8:6816/4498 10.13.0.8:6819/4498 10.13.0.8:6822/4498 10.13.0.8:6823/4498 exists,up bcc2a6f7-6532-4497-9cd0-3b39320bfcfb
osd.3 up in weight 1 up_from 1742 up_thru 1745 down_at 1741 last_clean_interval [1717,1735) 10.13.0.8:6812/4496 10.13.0.8:6814/4496 10.13.0.8:6817/4496 10.13.0.8:6820/4496 exists,up 8585d1ec-c725-4017-bf2a-75915c202476
osd.4 up in weight 1 up_from 1745 up_thru 1745 down_at 1744 last_clean_interval [1718,1735) 10.13.0.8:6807/4497 10.13.0.8:6810/4497 10.13.0.8:6813/4497 10.13.0.8:6815/4497 exists,up cd36efed-c31f-48bd-bad9-b77554c87683
osd.5 down out weight 0 up_from 1282 up_thru 1430 down_at 1431 last_clean_interval [1227,1278) 10.13.0.8:6822/689246 10.13.0.8:6823/689246 10.13.0.8:6824/689246 10.13.0.8:6825/689246 autoout,exists b63f8df7-8914-4c79-ad9d-9a0ea7e5ea06
blacklist 10.13.0.8:6800/448336511 expires 2019-07-04 03:06:34.636141
blacklist 10.13.0.8:6800/3949592801 expires 2019-07-04 03:54:51.814401
blacklist 10.13.0.8:6804/2000849713 expires 2019-07-03 15:24:57.324632
Raw
ec-profile
original
crush-device-class=hdd
crush-failure-domain=osd
crush-root=default
jerasure-per-chunk-alignment=false
k=4
m=1
plugin=jerasure
technique=reed_sol_van
w=8
updated
crush-device-class=
crush-failure-domain=osd
crush-root=default
jerasure-per-chunk-alignment=false
k=5
m=1
plugin=jerasure
technique=reed_sol_van
w=8
fixed
crush-device-class=hdd
crush-failure-domain=osd
crush-root=default
jerasure-per-chunk-alignment=false
k=5
m=1
plugin=jerasure
technique=reed_sol_van
w=8
Reverted to original(to see if it would help)
crush-device-class=hdd
crush-failure-domain=osd
crush-root=default
jerasure-per-chunk-alignment=false
k=4
m=1
plugin=jerasure
technique=reed_sol_van
w=8
Raw
osd-semi-healthy
HEALTH_WARN 4181394/8653349 objects misplaced (48.321%); Degraded data redundancy: 1353837/8653349 objects degraded (15.645%), 340 pgs degraded, 340 pgs undersized
OBJECT_MISPLACED 4181394/8653349 objects misplaced (48.321%)
PG_DEGRADED Degraded data redundancy: 1353837/8653349 objects degraded (15.645%), 340 pgs degraded, 340 pgs undersized
pg 8.147 is active+undersized+degraded+remapped+backfill_wait, acting [1,3,2147483647,4,2]
pg 8.148 is stuck undersized for 1451.613487, current state active+undersized+degraded+remapped+backfill_wait, last acting [1,3,2,2147483647,4]
pg 8.149 is stuck undersized for 1451.589228, current state active+undersized+degraded+remapped+backfill_wait, last acting [2147483647,1,4,3,2]
pg 8.14a is stuck undersized for 1451.573763, current state active+undersized+degraded+remapped+backfill_wait, last acting [1,4,2147483647,2,3]
pg 8.14b is stuck undersized for 1451.576107, current state active+undersized+degraded+remapped+backfill_wait, last acting [4,2147483647,1,2,3]
pg 8.14c is stuck undersized for 1451.587996, current state active+undersized+degraded+remapped+backfill_wait, last acting [2147483647,3,1,4,2]
pg 8.14d is stuck undersized for 1451.599530, current state active+undersized+degraded+remapped+backfill_wait, last acting [2,3,4,1,2147483647]
pg 8.14e is stuck undersized for 1451.579089, current state active+undersized+degraded+remapped+backfill_wait, last acting [2147483647,4,1,3,2]
pg 8.14f is stuck undersized for 1451.603701, current state active+undersized+degraded+remapped+backfill_wait, last acting [3,2,4,1,2147483647]
pg 8.150 is stuck undersized for 1451.590575, current state active+undersized+degraded+remapped+backfill_wait, last acting [3,4,2,2147483647,1]
pg 8.151 is stuck undersized for 1451.603103, current state active+undersized+degraded+remapped+backfill_wait, last acting [3,1,2,2147483647,4]
pg 8.152 is stuck undersized for 1451.566661, current state active+undersized+degraded+remapped+backfill_wait, last acting [3,1,2147483647,2,4]
pg 8.153 is stuck undersized for 1451.606319, current state active+undersized+degraded+remapped+backfill_wait, last acting [3,2,1,2147483647,4]
pg 8.154 is stuck undersized for 1451.573541, current state active+undersized+degraded+remapped+backfill_wait, last acting [4,2,2147483647,1,3]
pg 8.155 is stuck undersized for 1451.584014, current state active+undersized+degraded+remapped+backfill_wait, last acting [3,2,1,4,2147483647]
pg 8.156 is stuck undersized for 1451.585858, current state active+undersized+degraded+remapped+backfill_wait, last acting [3,2,4,1,2147483647]
pg 8.157 is stuck undersized for 1451.613104, current state active+undersized+degraded+remapped+backfill_wait, last acting [1,2147483647,2,4,3]
pg 8.158 is stuck undersized for 1451.598424, current state active+undersized+degraded+remapped+backfill_wait, last acting [2,1,4,2147483647,3]
pg 8.15a is stuck undersized for 1451.600534, current state active+undersized+degraded+remapped+backfill_wait, last acting [2147483647,2,1,4,3]
pg 8.15c is stuck undersized for 1451.565935, current state active+undersized+degraded+remapped+backfill_wait, last acting [3,2,4,1,2147483647]
pg 8.15d is stuck undersized for 1451.583515, current state active+undersized+degraded+remapped+backfill_wait, last acting [2147483647,1,3,2,4]
pg 8.15e is stuck undersized for 1451.606102, current state active+undersized+degraded+remapped+backfill_wait, last acting [3,2147483647,1,4,2]
pg 8.15f is stuck undersized for 1451.595147, current state active+undersized+degraded+remapped+backfill_wait, last acting [4,1,2,3,2147483647]
pg 8.160 is stuck undersized for 1451.595757, current state active+undersized+degraded+remapped+backfill_wait, last acting [2,3,4,2147483647,1]
pg 8.161 is stuck undersized for 1451.575525, current state active+undersized+degraded+remapped+backfill_wait, last acting [4,3,1,2,2147483647]
pg 8.162 is stuck undersized for 1451.614154, current state active+undersized+degraded+remapped+backfill_wait, last acting [3,1,2147483647,4,2]
pg 8.163 is stuck undersized for 1451.583443, current state active+undersized+degraded+remapped+backfill_wait, last acting [2147483647,4,1,3,2]
pg 8.164 is stuck undersized for 1451.594661, current state active+undersized+degraded+remapped+backfill_wait, last acting [4,2,2147483647,3,1]
pg 8.165 is stuck undersized for 1451.600728, current state active+undersized+degraded+remapped+backfill_wait, last acting [1,2147483647,2,4,3]
pg 8.167 is stuck undersized for 1451.584528, current state active+undersized+degraded+remapped+backfill_wait, last acting [2147483647,3,4,1,2]
pg 8.168 is stuck undersized for 1451.593813, current state active+undersized+degraded+remapped+backfill_wait, last acting [2147483647,2,4,3,1]
pg 8.169 is stuck undersized for 1451.598141, current state active+undersized+degraded+remapped+backfill_wait, last acting [2,2147483647,4,3,1]
pg 8.16a is stuck undersized for 1451.605299, current state active+undersized+degraded+remapped+backfill_wait, last acting [3,4,2147483647,2,1]
pg 8.16b is stuck undersized for 1451.590011, current state active+undersized+degraded+remapped+backfill_wait, last acting [1,2,3,2147483647,4]
pg 8.16c is stuck undersized for 1451.594453, current state active+undersized+degraded+remapped+backfill_wait, last acting [2,4,1,3,2147483647]
pg 8.16d is stuck undersized for 1451.612758, current state active+undersized+degraded+remapped+backfill_wait, last acting [4,2147483647,2,3,1]
pg 8.16e is stuck undersized for 1451.612786, current state active+undersized+degraded+remapped+backfill_wait, last acting [1,2147483647,3,2,4]
pg 8.16f is stuck undersized for 1451.606797, current state active+undersized+degraded+remapped+backfill_wait, last acting [2147483647,2,1,3,4]
pg 8.170 is stuck undersized for 1451.576733, current state active+undersized+degraded+remapped+backfill_wait, last acting [1,4,2,3,2147483647]
pg 8.171 is stuck undersized for 1451.602517, current state active+undersized+degraded+remapped+backfill_wait, last acting [1,2,2147483647,4,3]
pg 8.172 is stuck undersized for 1450.601718, current state active+undersized+degraded+remapped+backfill_wait, last acting [2147483647,4,3,2,1]
pg 8.173 is stuck undersized for 1451.613286, current state active+undersized+degraded+remapped+backfill_wait, last acting [1,4,2147483647,2,3]
pg 8.174 is stuck undersized for 1451.601214, current state active+undersized+degraded+remapped+backfill_wait, last acting [3,1,2147483647,2,4]
pg 8.176 is stuck undersized for 1451.588587, current state active+undersized+degraded+remapped+backfill_wait, last acting [1,2,3,2147483647,4]
pg 8.177 is stuck undersized for 1451.612573, current state active+undersized+degraded+remapped+backfill_wait, last acting [3,2147483647,2,1,4]
pg 8.178 is stuck undersized for 1451.614556, current state active+undersized+degraded+remapped+backfill_wait, last acting [3,2147483647,2,4,1]
pg 8.179 is stuck undersized for 1451.605077, current state active+undersized+degraded+remapped+backfill_wait, last acting [2147483647,2,3,4,1]
pg 8.17b is stuck undersized for 1451.604478, current state active+undersized+degraded+remapped+backfill_wait, last acting [2,4,1,3,2147483647]
pg 8.17c is stuck undersized for 1451.612525, current state active+undersized+degraded+remapped+backfill_wait, last acting [1,2147483647,4,3,2]
pg 8.17d is stuck undersized for 1451.597766, current state active+undersized+degraded+remapped+backfill_wait, last acting [2147483647,2,4,1,3]
pg 8.17f is stuck undersized for 1451.611759, current state active+undersized+degraded+remapped+backfill_wait, last acting [1,4,3,2,2147483647]
osd: 6 osds: 5 up, 5 in; 384 remapped pgs
data:
pools: 4 pools, 664 pgs
objects: 1.97 M objects, 6.3 TiB
usage: 7.1 TiB used, 11 TiB / 18 TiB avail
pgs: 1424475/9234562 objects degraded (15.425%)
4397785/9234562 objects misplaced (47.623%)
338 active+undersized+degraded+remapped+backfill_wait
280 active+clean
45 active+remapped+backfill_wait
1 active+undersized+degraded+remapped+backfilling
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment