Reactor stall in shard 10

logs

-- Logs begin at Mon 2020-04-27 11:24:23 UTC, end at Wed 2020-04-29 05:59:01 UTC. --
Apr 29 02:44:31 scylla-league-001 scylla[1592]:  [shard 10] large_data - Writing large cell league_stats/champion_stats_aggregate_fixed: GOLD:10.8:76JUNGLE data (1112227 bytes)
Apr 29 02:44:31 scylla-league-001 scylla[1592]:  [shard 10] large_data - Writing large cell league_stats/champion_stats_aggregate_fixed: PLATINUM:10.8:86TOP data (3092144 bytes)
Apr 29 02:44:31 scylla-league-001 scylla[1592]:  [shard 10] large_data - Writing large cell league_stats/champion_stats_aggregate_fixed: GOLD:10.8:9JUNGLE data (2472416 bytes)
Apr 29 02:44:31 scylla-league-001 scylla[1592]:  [shard 10] large_data - Writing large cell league_stats/champion_stats_aggregate_fixed: GOLD:10.8:9SUPPORT data (1562444 bytes)
Apr 29 02:44:31 scylla-league-001 scylla[1592]:  [shard 10] large_data - Writing large cell league_stats/champion_stats_aggregate_fixed: SILVER:10.8:12SUPPORT data (3279738 bytes)
Apr 29 02:44:31 scylla-league-001 scylla[1592]:  [shard 10] large_data - Writing large cell league_stats/champion_stats_aggregate_fixed: PLATINUM:10.8:21ADC data (5487894 bytes)
Apr 29 02:44:31 scylla-league-001 scylla[1592]:  [shard 10] large_data - Writing large cell league_stats/champion_stats_aggregate_fixed: SILVER:10.8:26SUPPORT data (2570122 bytes)
Apr 29 02:44:31 scylla-league-001 scylla[1592]:  [shard 10] large_data - Writing large cell league_stats/champion_stats_aggregate_fixed: BRONZE:10.8:37SUPPORT data (3054366 bytes)
Apr 29 02:44:31 scylla-league-001 scylla[1592]:  [shard 10] large_data - Writing large cell league_stats/champion_stats_aggregate_fixed: PLATINUM_PLUS:10.8:245JUNGLE data (1440505 bytes)
Apr 29 02:44:31 scylla-league-001 scylla[1592]:  [shard 10] large_data - Writing large cell league_stats/champion_stats_aggregate_fixed: PLATINUM_PLUS:10.8:245MID data (1505392 bytes)
Apr 29 02:44:31 scylla-league-001 scylla[1592]:  [shard 10] large_data - Writing large cell league_stats/champion_stats_aggregate_fixed: BRONZE:10.8:45MID data (3836736 bytes)
Apr 29 02:44:31 scylla-league-001 scylla[1592]:  [shard 10] large_data - Writing large cell league_stats/champion_stats_aggregate_fixed: BRONZE:10.8:45SUPPORT data (1768880 bytes)
Apr 29 02:44:31 scylla-league-001 scylla[1592]:  [shard 10] large_data - Writing large cell league_stats/champion_stats_aggregate_fixed: PLATINUM_PLUS:10.8:55MID data (1960441 bytes)
Apr 29 02:44:31 scylla-league-001 scylla[1592]:  [shard 10] large_data - Writing large cell league_stats/champion_stats_aggregate_fixed: BRONZE:10.8:26SUPPORT data (1348798 bytes)
Apr 29 02:44:31 scylla-league-001 scylla[1592]:  [shard 10] large_data - Writing large cell league_stats/champion_stats_aggregate_fixed: BRONZE:10.8:80JUNGLE data (1064745 bytes)
Apr 29 02:44:31 scylla-league-001 scylla[1592]:  [shard 10] large_data - Writing large cell league_stats/champion_stats_aggregate_fixed: BRONZE:10.8:80MID data (1574659 bytes)
Apr 29 02:44:31 scylla-league-001 scylla[1592]:  [shard 10] large_data - Writing large cell league_stats/champion_stats_aggregate_fixed: BRONZE:10.8:80TOP data (1198392 bytes)
Apr 29 02:44:31 scylla-league-001 scylla[1592]:  [shard 10] large_data - Writing large cell league_stats/champion_stats_aggregate_fixed: GOLD:10.8:41TOP data (2263870 bytes)
Apr 29 02:44:31 scylla-league-001 scylla[1592]:  [shard 10] large_data - Writing large cell league_stats/champion_stats_aggregate_fixed: SILVER:10.8:104JUNGLE data (3881427 bytes)
Apr 29 02:44:31 scylla-league-001 scylla[1592]:  [shard 10] large_data - Writing large cell league_stats/champion_stats_aggregate_fixed: BRONZE:10.8:90MID data (1706653 bytes)
Apr 29 02:44:32 scylla-league-001 scylla[1592]:  [shard 10] large_data - Writing large cell league_stats/champion_stats_aggregate_fixed: SILVER:10.8:429ADC data (2143228 bytes)
Apr 29 02:44:32 scylla-league-001 scylla[1592]:  [shard 10] large_data - Writing large cell league_stats/champion_stats_aggregate_fixed: GOLD:10.8:44SUPPORT data (2928714 bytes)
Apr 29 02:44:32 scylla-league-001 scylla[1592]:  [shard 10] large_data - Writing large cell league_stats/champion_stats_aggregate_fixed: PLATINUM_PLUS:10.8:98TOP data (2264125 bytes)
Apr 29 02:44:32 scylla-league-001 scylla[1592]:  [shard 10] large_data - Writing large cell league_stats/champion_stats_aggregate_fixed: GOLD:10.8:90MID data (2334684 bytes)
Apr 29 02:44:32 scylla-league-001 scylla[1592]:  [shard 10] large_data - Writing large cell league_stats/champion_stats_aggregate_fixed: DIAMOND:10.8:236ADC data (2083794 bytes)
Apr 29 02:44:32 scylla-league-001 scylla[1592]:  [shard 10] large_data - Writing large cell league_stats/champion_stats_aggregate_fixed: PLATINUM_PLUS:10.8:238MID data (2047760 bytes)
Apr 29 02:44:32 scylla-league-001 scylla[1592]:  [shard 10] large_data - Writing large cell league_stats/champion_stats_aggregate_fixed: BRONZE:10.8:117SUPPORT data (1985462 bytes)
Apr 29 02:44:32 scylla-league-001 scylla[1592]:  [shard 10] compaction - Compacted 2 sstables to [/mnt/disks/data/db/league_stats/champion_ban_full-db6986200b9511ea92a600000000000a/mc-68778-big-Data.db:le
vel=0, ]. 3797724 bytes to 3669272 (~96% of original) in 1407ms = 2.49MB/s. ~896 total partitions merged to 676.
Apr 29 02:44:32 scylla-league-001 scylla[1592]:  [shard 10] large_data - Writing large cell league_stats/champion_stats_aggregate_fixed: DIAMOND:10.8:59JUNGLE data (1179071 bytes)
Apr 29 02:44:32 scylla-league-001 scylla[1592]:  [shard 10] large_data - Writing large cell league_stats/champion_stats_aggregate_fixed: BRONZE:10.8:134MID data (1505761 bytes)
Apr 29 02:44:32 scylla-league-001 scylla[1592]:  [shard 10] large_data - Writing large cell league_stats/champion_stats_aggregate_fixed: BRONZE:10.8:1MID data (1477116 bytes)
Apr 29 02:44:32 scylla-league-001 scylla[1592]:  [shard 10] large_data - Writing large cell league_stats/champion_stats_aggregate_fixed: PLATINUM_PLUS:10.8:102JUNGLE data (1479012 bytes)
Apr 29 02:44:32 scylla-league-001 scylla[1592]:  [shard 10] large_data - Writing large cell league_stats/champion_stats_aggregate_fixed: PLATINUM:10.8:61MID data (1537348 bytes)
Apr 29 02:44:32 scylla-league-001 scylla[1592]:  [shard 10] large_data - Writing large cell league_stats/champion_stats_aggregate_fixed: BRONZE:10.8:83TOP data (1730495 bytes)
Apr 29 02:44:32 scylla-league-001 scylla[1592]:  [shard 10] large_data - Writing large cell league_stats/champion_stats_aggregate_fixed: BRONZE:10.8:29ADC data (2422608 bytes)
Apr 29 02:44:32 scylla-league-001 scylla[1592]:  [shard 10] large_data - Writing large cell league_stats/champion_stats_aggregate_fixed: PLATINUM:10.8:15ADC data (1357925 bytes)
Apr 29 02:44:32 scylla-league-001 scylla[1592]:  [shard 10] large_data - Writing large cell league_stats/champion_stats_aggregate_fixed: GOLD:10.8:75TOP data (6077720 bytes)
Apr 29 02:44:32 scylla-league-001 scylla[1592]:  [shard 10] large_data - Writing large cell league_stats/champion_stats_aggregate_fixed: GOLD:10.8:103MID data (2886467 bytes)
Apr 29 02:44:32 scylla-league-001 scylla[1592]:  [shard 10] large_data - Writing large cell league_stats/champion_stats_aggregate_fixed: GOLD:10.8:53SUPPORT data (4885591 bytes)
Apr 29 02:44:32 scylla-league-001 scylla[1592]:  [shard 10] large_data - Writing large cell league_stats/champion_stats_aggregate_fixed: PLATINUM:10.8:29ADC data (1131354 bytes)
Apr 29 02:44:32 scylla-league-001 scylla[1592]:  [shard 10] large_data - Writing large cell league_stats/champion_stats_aggregate_fixed: GOLD:10.8:64JUNGLE data (6232469 bytes)
Apr 29 02:44:32 scylla-league-001 scylla[1592]:  [shard 10] large_data - Writing large cell league_stats/champion_stats_aggregate_fixed: PLATINUM_PLUS:10.8:56JUNGLE data (1579478 bytes)
Apr 29 02:44:32 scylla-league-001 scylla[1592]:  [shard 10] large_data - Writing large cell league_stats/champion_stats_aggregate_fixed: BRONZE:10.8:57SUPPORT data (1283949 bytes)
Apr 29 02:44:32 scylla-league-001 scylla[1592]:  [shard 10] large_data - Writing large cell league_stats/champion_stats_aggregate_fixed: BRONZE:10.8:57TOP data (1356293 bytes)
Apr 29 02:44:32 scylla-league-001 scylla[1592]:  [shard 10] large_data - Writing large cell league_stats/champion_stats_aggregate_fixed: SILVER:10.8:350SUPPORT data (3191459 bytes)
Apr 29 02:44:32 scylla-league-001 scylla[1592]:  [shard 10] large_data - Writing large cell league_stats/champion_stats_aggregate_fixed: GOLD:10.8:54JUNGLE data (1788611 bytes)
Apr 29 02:44:32 scylla-league-001 scylla[1592]:  [shard 10] large_data - Writing large cell league_stats/champion_stats_aggregate_fixed: GOLD:10.8:54MID data (2113245 bytes)
Apr 29 02:44:32 scylla-league-001 scylla[1592]:  [shard 10] large_data - Writing large cell league_stats/champion_stats_aggregate_fixed: GOLD:10.8:54SUPPORT data (2398548 bytes)
Apr 29 02:44:32 scylla-league-001 scylla[1592]:  [shard 10] large_data - Writing large cell league_stats/champion_stats_aggregate_fixed: GOLD:10.8:54TOP data (6238836 bytes)
Apr 29 02:44:32 scylla-league-001 scylla[1592]:  [shard 10] large_data - Writing large cell league_stats/champion_stats_aggregate_fixed: GOLD:10.8:5JUNGLE data (2716166 bytes)
Apr 29 02:44:32 scylla-league-001 scylla[1592]:  [shard 10] large_data - Writing large cell league_stats/champion_stats_aggregate_fixed: SILVER:10.8:101MID data (1616542 bytes)
Apr 29 02:44:32 scylla-league-001 scylla[1592]:  [shard 10] large_data - Writing large cell league_stats/champion_stats_aggregate_fixed: SILVER:10.8:101SUPPORT data (1598241 bytes)
Apr 29 02:44:35 scylla-league-001 scylla[1592]:  [shard 10] compaction - Compacting [/mnt/disks/data/db/league_stats/league_ranks-f4382cd037b211ea920b00000000000c/mc-157510-big-Data.db:level=0, /mnt/disks
/data/db/league_stats/league_ranks-f4382cd037b211ea920b00000000000c/mc-157496-big-Data.db:level=0, ]
Apr 29 02:44:35 scylla-league-001 scylla[1592]:  [shard 10] compaction - Compacted 2 sstables to [/mnt/disks/data/db/league_stats/duo_stat_full_aggregate-d5fcf9600b9511ea92a600000000000a/mc-65320-big-Data
.db:level=0, ]. 12743646 bytes to 11292030 (~88% of original) in 6498ms = 1.66MB/s. ~768 total partitions merged to 417.
Apr 29 02:44:35 scylla-league-001 scylla[1592]:  [shard 0] migration_manager - Requesting schema pull from 10.128.15.209:12
Apr 29 02:44:35 scylla-league-001 scylla[1592]:  [shard 0] migration_manager - Pulling schema from 10.128.15.209:0
Apr 29 02:44:35 scylla-league-001 scylla[1592]:  [shard 11] compaction - Compacting [/mnt/disks/data/db/system_schema/columns-24101c25a2ae3af787c1b40ee1aca33f/mc-4399539-big-Data.db:level=0, /mnt/disks/da
ta/db/system_schema/columns-24101c25a2ae3af787c1b40ee1aca33f/mc-4399525-big-Data.db:level=0, ]
Apr 29 02:44:36 scylla-league-001 scylla[1592]:  [shard 8] compaction - Compacting [/mnt/disks/data/db/system_schema/columns-24101c25a2ae3af787c1b40ee1aca33f/mc-4403148-big-Data.db:level=0, /mnt/disks/dat
a/db/system_schema/columns-24101c25a2ae3af787c1b40ee1aca33f/mc-4403134-big-Data.db:level=0, ]
Apr 29 02:44:36 scylla-league-001 scylla[1592]:  [shard 11] compaction - Compacting [/mnt/disks/data/db/system_schema/keyspaces-abac5682dea631c5b535b3d6cffd0fb6/mc-4399595-big-Data.db:level=0, /mnt/disks/
data/db/system_schema/keyspaces-abac5682dea631c5b535b3d6cffd0fb6/mc-4399581-big-Data.db:level=0, ]
Apr 29 02:44:36 scylla-league-001 scylla[1592]:  [shard 10] compaction - Compacting [/mnt/disks/data/db/system_schema/columns-24101c25a2ae3af787c1b40ee1aca33f/mc-4399538-big-Data.db:level=0, /mnt/disks/da
ta/db/system_schema/columns-24101c25a2ae3af787c1b40ee1aca33f/mc-4399524-big-Data.db:level=0, ]
Apr 29 02:44:36 scylla-league-001 scylla[1592]:  [shard 10] compaction - Compacting [/mnt/disks/data/db/system_schema/keyspaces-abac5682dea631c5b535b3d6cffd0fb6/mc-4399566-big-Data.db:level=0, /mnt/disks/
data/db/system_schema/keyspaces-abac5682dea631c5b535b3d6cffd0fb6/mc-4399552-big-Data.db:level=0, ]
Apr 29 02:44:36 scylla-league-001 scylla[1592]:  [shard 10] compaction - Compacting [/mnt/disks/data/db/system_schema/tables-afddfb9dbc1e30688056eed6c302ba09/mc-4399538-big-Data.db:level=0, /mnt/disks/dat
a/db/system_schema/tables-afddfb9dbc1e30688056eed6c302ba09/mc-4399524-big-Data.db:level=0, ]
Apr 29 02:44:36 scylla-league-001 scylla[1592]:  [shard 11] compaction - Compacting [/mnt/disks/data/db/system_schema/tables-afddfb9dbc1e30688056eed6c302ba09/mc-4399539-big-Data.db:level=0, /mnt/disks/dat
a/db/system_schema/tables-afddfb9dbc1e30688056eed6c302ba09/mc-4399525-big-Data.db:level=0, ]
Apr 29 02:44:36 scylla-league-001 scylla[1592]:  [shard 8] compaction - Compacting [/mnt/disks/data/db/system_schema/scylla_tables-5d912ff1f7593665b2c88042ab5103dd/mc-4403148-big-Data.db:level=0, /mnt/dis
ks/data/db/system_schema/scylla_tables-5d912ff1f7593665b2c88042ab5103dd/mc-4403134-big-Data.db:level=0, ]
Apr 29 02:44:36 scylla-league-001 scylla[1592]:  [shard 8] compaction - Compacting [/mnt/disks/data/db/system_schema/views-9786ac1cdd583201a7cdad556410c985/mc-4399732-big-Data.db:level=0, /mnt/disks/data/
db/system_schema/views-9786ac1cdd583201a7cdad556410c985/mc-4399718-big-Data.db:level=0, ]
Apr 29 02:44:36 scylla-league-001 scylla[1592]:  [shard 8] compaction - Compacting [/mnt/disks/data/db/system_schema/keyspaces-abac5682dea631c5b535b3d6cffd0fb6/mc-4403204-big-Data.db:level=0, /mnt/disks/d
ata/db/system_schema/keyspaces-abac5682dea631c5b535b3d6cffd0fb6/mc-4403190-big-Data.db:level=0, ]
Apr 29 02:44:36 scylla-league-001 scylla[1592]:  [shard 8] compaction - Compacting [/mnt/disks/data/db/system_schema/tables-afddfb9dbc1e30688056eed6c302ba09/mc-4403148-big-Data.db:level=0, /mnt/disks/data
/db/system_schema/tables-afddfb9dbc1e30688056eed6c302ba09/mc-4403134-big-Data.db:level=0, ]
Apr 29 02:44:36 scylla-league-001 scylla[1592]:  [shard 11] compaction - Compacted 2 sstables to [/mnt/disks/data/db/system_schema/columns-24101c25a2ae3af787c1b40ee1aca33f/mc-4399553-big-Data.db:level=0, 
]. 11484 bytes to 5896 (~51% of original) in 111ms = 0.05MB/s. ~256 total partitions merged to 1.
Apr 29 02:44:36 scylla-league-001 scylla[1592]:  [shard 11] compaction - Compacted 2 sstables to [/mnt/disks/data/db/system_schema/keyspaces-abac5682dea631c5b535b3d6cffd0fb6/mc-4399609-big-Data.db:level=0
, ]. 10974 bytes to 5641 (~51% of original) in 134ms = 0.04MB/s. ~256 total partitions merged to 1.
Apr 29 02:44:36 scylla-league-001 scylla[1592]:  [shard 0] database - Schema version changed to 0b0b9665-eb31-38e1-9254-97fa03e6aff3
Apr 29 02:44:36 scylla-league-001 scylla[1592]:  [shard 0] migration_manager - Schema merge with 10.128.15.209:0 completed
Apr 29 02:44:36 scylla-league-001 scylla[1592]:  [shard 8] compaction - Compacted 2 sstables to [/mnt/disks/data/db/system_schema/columns-24101c25a2ae3af787c1b40ee1aca33f/mc-4403162-big-Data.db:level=0, ]
. 180394 bytes to 90353 (~50% of original) in 169ms = 0.51MB/s. ~256 total partitions merged to 2.
Apr 29 02:44:36 scylla-league-001 scylla[1592]:  [shard 11] compaction - Compacted 2 sstables to [/mnt/disks/data/db/system_schema/tables-afddfb9dbc1e30688056eed6c302ba09/mc-4399553-big-Data.db:level=0, ]
. 14194 bytes to 7251 (~51% of original) in 141ms = 0.05MB/s. ~256 total partitions merged to 1.
Apr 29 02:44:36 scylla-league-001 scylla[1592]:  [shard 8] compaction - Compacted 2 sstables to [/mnt/disks/data/db/system_schema/scylla_tables-5d912ff1f7593665b2c88042ab5103dd/mc-4403162-big-Data.db:leve
l=0, ]. 11232 bytes to 5772 (~51% of original) in 207ms = 0.03MB/s. ~256 total partitions merged to 1.
Apr 29 02:44:36 scylla-league-001 scylla[1592]:  [shard 8] compaction - Compacted 2 sstables to [/mnt/disks/data/db/system_schema/views-9786ac1cdd583201a7cdad556410c985/mc-4399746-big-Data.db:level=0, ]. 
18922 bytes to 9615 (~50% of original) in 189ms = 0.05MB/s. ~256 total partitions merged to 1.
Apr 29 02:44:36 scylla-league-001 scylla[1592]:  [shard 8] compaction - Compacted 2 sstables to [/mnt/disks/data/db/system_schema/keyspaces-abac5682dea631c5b535b3d6cffd0fb6/mc-4403218-big-Data.db:level=0,
 ]. 142596 bytes to 71452 (~50% of original) in 183ms = 0.37MB/s. ~256 total partitions merged to 2.
Apr 29 02:44:36 scylla-league-001 scylla[1592]:  [shard 8] compaction - Compacted 2 sstables to [/mnt/disks/data/db/system_schema/tables-afddfb9dbc1e30688056eed6c302ba09/mc-4403162-big-Data.db:level=0, ].
 158782 bytes to 79547 (~50% of original) in 184ms = 0.41MB/s. ~256 total partitions merged to 2.
Apr 29 02:44:36 scylla-league-001 scylla[1592]:  [shard 10] compaction - Compacted 2 sstables to [/mnt/disks/data/db/system_schema/columns-24101c25a2ae3af787c1b40ee1aca33f/mc-4399552-big-Data.db:level=0, 
]. 11582 bytes to 5945 (~51% of original) in 418ms = 0.01MB/s. ~256 total partitions merged to 1.
Apr 29 02:44:36 scylla-league-001 scylla[1592]:  [shard 10] compaction - Compacted 2 sstables to [/mnt/disks/data/db/system_schema/keyspaces-abac5682dea631c5b535b3d6cffd0fb6/mc-4399580-big-Data.db:level=0
, ]. 10900 bytes to 5604 (~51% of original) in 415ms = 0.01MB/s. ~256 total partitions merged to 1.
Apr 29 02:44:36 scylla-league-001 scylla[1592]:  [shard 10] compaction - Compacted 2 sstables to [/mnt/disks/data/db/system_schema/tables-afddfb9dbc1e30688056eed6c302ba09/mc-4399552-big-Data.db:level=0, ]
. 14306 bytes to 7307 (~51% of original) in 417ms = 0.02MB/s. ~256 total partitions merged to 1.
Apr 29 02:44:37 scylla-league-001 scylla[1592]:  [shard 10] compaction - Compacting [/mnt/disks/data/db/league_stats/patches-ddc243300b9511ea92a600000000000a/mc-63682-big-Data.db:level=0, /mnt/disks/data/
db/league_stats/patches-ddc243300b9511ea92a600000000000a/mc-63668-big-Data.db:level=0, ]
Apr 29 02:44:37 scylla-league-001 scylla[1592]:  [shard 10] compaction - Compacting [/mnt/disks/data/db/league_stats/champion_ban_tier_blocks-96067f201a7711eab5a5000000000001/mc-17734-big-Data.db:level=0,
 /mnt/disks/data/db/league_stats/champion_ban_tier_blocks-96067f201a7711eab5a5000000000001/mc-17720-big-Data.db:level=0, ]
Apr 29 02:44:37 scylla-league-001 scylla[1592]:  [shard 10] compaction - Compacted 2 sstables to [/mnt/disks/data/db/league_stats/patches-ddc243300b9511ea92a600000000000a/mc-63696-big-Data.db:level=0, ]. 
79562 bytes to 40581 (~51% of original) in 266ms = 0.15MB/s. ~256 total partitions merged to 2.
Apr 29 02:44:37 scylla-league-001 scylla[1592]:  [shard 10] compaction - Compacted 2 sstables to [/mnt/disks/data/db/league_stats/champion_ban_tier_blocks-96067f201a7711eab5a5000000000001/mc-17748-big-Dat
a.db:level=0, ]. 218893 bytes to 211974 (~96% of original) in 320ms = 0.63MB/s. ~256 total partitions merged to 18.
Apr 29 02:44:41 scylla-league-001 scylla[1592]:  [shard 10] compaction - Compacting [/mnt/disks/data/db/league_stats/champion_stats_aggregate_fixed-974604001a7711eab5a5000000000001/mc-54148-big-Data.db:le
vel=0, /mnt/disks/data/db/league_stats/champion_stats_aggregate_fixed-974604001a7711eab5a5000000000001/mc-54134-big-Data.db:level=0, ]
Apr 29 02:44:41 scylla-league-001 scylla[1592]:  [shard 10] large_data - Writing large cell league_stats/champion_stats_aggregate_fixed: SILVER:10.8:157ADC data (2455888 bytes)
Apr 29 02:44:45 scylla-league-001 scylla[1592]: Reactor stalled for 5294 ms on shard 10.
Apr 29 02:44:45 scylla-league-001 scylla[1592]: Backtrace:
Apr 29 02:44:45 scylla-league-001 scylla[1592]:   0x00000000041df5e2
Apr 29 02:44:45 scylla-league-001 scylla[1592]:   0x00000000040cc8f5
Apr 29 02:44:45 scylla-league-001 scylla[1592]:   0x00000000040cce35
Apr 29 02:44:45 scylla-league-001 scylla[1592]:   0x00000000040cd177
Apr 29 02:44:45 scylla-league-001 scylla[1592]:   0x00007fe3c561702f
Apr 29 02:44:45 scylla-league-001 scylla[1592]:   0x0000000000ddb830
Apr 29 02:44:45 scylla-league-001 scylla[1592]:   0x0000000000eaf489
Apr 29 02:44:45 scylla-league-001 scylla[1592]:   0x0000000000eaf6c8
Apr 29 02:44:45 scylla-league-001 scylla[1592]:   0x0000000000ebc1ce
Apr 29 02:44:45 scylla-league-001 scylla[1592]:   0x000000000407d368
Apr 29 02:44:45 scylla-league-001 scylla[1592]:   0x000000000407dc0e
Apr 29 02:44:45 scylla-league-001 scylla[1592]:   0x000000000408151d
Apr 29 02:44:45 scylla-league-001 scylla[1592]:   0x00000000014fa1b7
Apr 29 02:44:45 scylla-league-001 scylla[1592]:   0x0000000001523d9e
Apr 29 02:44:45 scylla-league-001 scylla[1592]:   0x00000000015271ba
Apr 29 02:44:45 scylla-league-001 scylla[1592]:   0x0000000001529fb1
Apr 29 02:44:45 scylla-league-001 scylla[1592]:   0x000000000152b7c8
Apr 29 02:44:45 scylla-league-001 scylla[1592]:   0x00000000040c9dc1
Apr 29 02:44:45 scylla-league-001 scylla[1592]:   0x00000000040c9fde
Apr 29 02:44:45 scylla-league-001 scylla[1592]:   0x00000000041abd6d
Apr 29 02:44:45 scylla-league-001 scylla[1592]:   0x00000000041d63db
Apr 29 02:44:45 scylla-league-001 scylla[1592]:   0x000000000409809d
Apr 29 02:44:45 scylla-league-001 scylla[1592]:   /opt/scylladb/libreloc/libpthread.so.0+0x000000000000858d
Apr 29 02:44:45 scylla-league-001 scylla[1592]:   /opt/scylladb/libreloc/libc.so.6+0x00000000000fd6a2
Apr 29 02:44:50 scylla-league-001 scylla[1592]: Reactor stalled for 9962 ms on shard 10.
Apr 29 02:44:50 scylla-league-001 scylla[1592]: Backtrace:
Apr 29 02:44:50 scylla-league-001 scylla[1592]:   0x00000000041df5e2
Apr 29 02:44:50 scylla-league-001 scylla[1592]:   0x00000000040cc8f5
Apr 29 02:44:50 scylla-league-001 scylla[1592]:   0x00000000040cce35
Apr 29 02:44:50 scylla-league-001 scylla[1592]:   0x00000000040cd177
Apr 29 02:44:50 scylla-league-001 scylla[1592]:   0x00007fe3c561702f
Apr 29 02:44:50 scylla-league-001 scylla[1592]:   0x0000000000edbfd6
Apr 29 02:44:50 scylla-league-001 scylla[1592]:   0x0000000000d8b4c4
Apr 29 02:44:50 scylla-league-001 scylla[1592]:   0x0000000000ec7127
Apr 29 02:44:50 scylla-league-001 scylla[1592]:   0x0000000000eb6f57
Apr 29 02:44:50 scylla-league-001 scylla[1592]:   0x0000000000ebc1a9
Apr 29 02:44:50 scylla-league-001 scylla[1592]:   0x000000000407d368
Apr 29 02:44:50 scylla-league-001 scylla[1592]:   0x000000000407dc0e
Apr 29 02:44:50 scylla-league-001 scylla[1592]:   0x000000000408151d
Apr 29 02:44:50 scylla-league-001 scylla[1592]:   0x00000000014fa1b7
Apr 29 02:44:50 scylla-league-001 scylla[1592]:   0x0000000001523d9e
Apr 29 02:44:50 scylla-league-001 scylla[1592]:   0x00000000015271ba
Apr 29 02:44:50 scylla-league-001 scylla[1592]:   0x0000000001529fb1
Apr 29 02:44:50 scylla-league-001 scylla[1592]:   0x000000000152b7c8
Apr 29 02:44:50 scylla-league-001 scylla[1592]:   0x00000000040c9dc1
Apr 29 02:44:50 scylla-league-001 scylla[1592]:   0x00000000040c9fde
Apr 29 02:44:50 scylla-league-001 scylla[1592]:   0x00000000041abd6d
Apr 29 02:44:50 scylla-league-001 scylla[1592]:   0x00000000041d63db
Apr 29 02:44:50 scylla-league-001 scylla[1592]:   0x000000000409809d
Apr 29 02:44:50 scylla-league-001 scylla[1592]:   /opt/scylladb/libreloc/libpthread.so.0+0x000000000000858d

resolved_backtrace

void seastar::backtrace<seastar::backtrace_buffer::append_backtrace()::{lambda(seastar::frame)#1}>(seastar::backtrace_buffer::append_backtrace()::{lambda(seastar::frame)#1}&&) at /usr/include/boost/program_options/variables_map.hpp:146
seastar::backtrace_buffer::append_backtrace() at /usr/include/boost/program_options/variables_map.hpp:146
 (inlined by) print_with_backtrace at /jenkins/workspace/scylla-3.1/relocatable-pkg/scylla/seastar/build/release/../../src/core/reactor.cc:1104
seastar::internal::cpu_stall_detector::generate_trace() at /usr/include/boost/program_options/variables_map.hpp:146
seastar::internal::cpu_stall_detector::maybe_report() at /usr/include/boost/program_options/variables_map.hpp:146
 (inlined by) seastar::internal::cpu_stall_detector::on_signal() at /jenkins/workspace/scylla-3.1/relocatable-pkg/scylla/seastar/build/release/../../src/core/reactor.cc:1480
 (inlined by) seastar::reactor::block_notifier(int) at /jenkins/workspace/scylla-3.1/relocatable-pkg/scylla/seastar/build/release/../../src/core/reactor.cc:1603
?? ??:0
rows_entry::rows_entry(rows_entry&&) at crtstuff.c:?
 (inlined by) boost::intrusive::circular_list_algorithms<boost::intrusive::list_node_traits<void*> >::link_after(boost::intrusive::list_node<void*>* const&, boost::intrusive::list_node<void*>* const&) at /usr/include/boost/intrusive/circular_list_algorithms.hpp:201
 (inlined by) rows_entry::rows_entry(rows_entry&&) at /jenkins/workspace/scylla-3.1/relocatable-pkg/scylla/mutation_partition.cc:1499
standard_migrator<rows_entry>::migrate(void*, void*, unsigned long) const at /jenkins/workspace/scylla-3.1/relocatable-pkg/scylla/./utils/allocation_strategy.hh:75
logalloc::region_impl::compact(logalloc::segment*, logalloc::segment_descriptor&)::{lambda(logalloc::region_impl::object_descriptor const*, void*)#1}::operator()(logalloc::region_impl::object_descriptor const*, void*) const at /jenkins/workspace/scylla-3.1/relocatable-pkg/scylla/utils/logalloc.cc:1255
 (inlined by) void logalloc::region_impl::for_each_live<logalloc::region_impl::compact(logalloc::segment*, logalloc::segment_descriptor&)::{lambda(logalloc::region_impl::object_descriptor const*, void*)#1}>(logalloc::segment*, logalloc::region_impl::compact(logalloc::segment*, logalloc::segment_descriptor&)::{lambda(logalloc::region_impl::object_descriptor const*, void*)#1}&&) at /jenkins/workspace/scylla-3.1/relocatable-pkg/scylla/utils/logalloc.cc:1199
 (inlined by) logalloc::region_impl::compact(logalloc::segment*, logalloc::segment_descriptor&) at /jenkins/workspace/scylla-3.1/relocatable-pkg/scylla/utils/logalloc.cc:1251
 (inlined by) logalloc::region_impl::compact_single_segment_locked() at /jenkins/workspace/scylla-3.1/relocatable-pkg/scylla/utils/logalloc.cc:1539
logalloc::tracker::impl::compact_and_evict_locked(unsigned long, unsigned long) at crtstuff.c:?
 (inlined by) logalloc::tracker::impl::compact_and_evict_locked(unsigned long, unsigned long) at /jenkins/workspace/scylla-3.1/relocatable-pkg/scylla/utils/logalloc.cc:2009
std::_Function_handler<seastar::memory::reclaiming_result (), logalloc::tracker::tracker()::{lambda()#1}>::_M_invoke(std::_Any_data const&) at logalloc.cc:?
 (inlined by) logalloc::tracker::reclaim(unsigned long) at /jenkins/workspace/scylla-3.1/relocatable-pkg/scylla/utils/logalloc.cc:380
 (inlined by) logalloc::tracker::reclaim() at /jenkins/workspace/scylla-3.1/relocatable-pkg/scylla/utils/logalloc.cc:1680
 (inlined by) operator() at /jenkins/workspace/scylla-3.1/relocatable-pkg/scylla/utils/logalloc.cc:373
 (inlined by) _M_invoke at /usr/include/c++/8/bits/std_function.h:282
seastar::memory::cpu_pages::run_reclaimers(seastar::memory::reclaimer_scope, unsigned long) at crtstuff.c:?
 (inlined by) seastar::memory::reclaimer::do_reclaim(unsigned long) at /jenkins/workspace/scylla-3.1/relocatable-pkg/scylla/seastar/build/release/../../include/seastar/core/memory.hh:131
 (inlined by) seastar::memory::cpu_pages::run_reclaimers(seastar::memory::reclaimer_scope, unsigned long) at /jenkins/workspace/scylla-3.1/relocatable-pkg/scylla/seastar/build/release/../../src/core/memory.cc:1033
seastar::memory::cpu_pages::allocate_large_and_trim(unsigned int) at crtstuff.c:?
 (inlined by) seastar::memory::cpu_pages::allocate_large_and_trim(unsigned int) at /jenkins/workspace/scylla-3.1/relocatable-pkg/scylla/seastar/build/release/../../src/core/memory.cc:599
__libc_malloc at ??:?
 (inlined by) seastar::memory::allocate_large(unsigned long) at /jenkins/workspace/scylla-3.1/relocatable-pkg/scylla/seastar/build/release/../../src/core/memory.cc:1215
 (inlined by) seastar::memory::allocate(unsigned long) at /jenkins/workspace/scylla-3.1/relocatable-pkg/scylla/seastar/build/release/../../src/core/memory.cc:1265
 (inlined by) malloc at /jenkins/workspace/scylla-3.1/relocatable-pkg/scylla/seastar/build/release/../../src/core/memory.cc:1517
seastar::temporary_buffer<char>::temporary_buffer(unsigned long) at /usr/include/c++/8/variant:1334
 (inlined by) data_consumer::continuous_data_consumer<sstables::data_consume_rows_context_m>::read_bytes(seastar::temporary_buffer<char>&, unsigned int, seastar::temporary_buffer<char>&) at /jenkins/workspace/scylla-3.1/relocatable-pkg/scylla/sstables/consumer.hh:195
 (inlined by) data_consumer::continuous_data_consumer<sstables::data_consume_rows_context_m>::read_unsigned_vint_length_bytes(seastar::temporary_buffer<char>&, seastar::temporary_buffer<char>&) at /jenkins/workspace/scylla-3.1/relocatable-pkg/scylla/sstables/consumer.hh:246
sstables::data_consume_rows_context_m::do_process_state(seastar::temporary_buffer<char>&) at /usr/include/c++/8/variant:1334
sstables::data_consume_rows_context_m::process_state(seastar::temporary_buffer<char>&) at /usr/include/c++/8/variant:1334
 (inlined by) data_consumer::continuous_data_consumer<sstables::data_consume_rows_context_m>::process(seastar::temporary_buffer<char>&) at /jenkins/workspace/scylla-3.1/relocatable-pkg/scylla/sstables/consumer.hh:404
data_consumer::continuous_data_consumer<sstables::data_consume_rows_context_m>::operator()(seastar::temporary_buffer<char>) at /usr/include/c++/8/variant:1334
 (inlined by) seastar::future<seastar::consumption_result<char> > std::__invoke_impl<seastar::future<seastar::consumption_result<char> >, sstables::data_consume_rows_context_m&, seastar::temporary_buffer<char> >(std::__invoke_other, sstables::data_consume_rows_context_m&, seastar::temporary_buffer<char>&&) at /usr/include/c++/8/bits/invoke.h:60
 (inlined by) std::__invoke_result<sstables::data_consume_rows_context_m&, seastar::temporary_buffer<char> >::type std::__invoke<sstables::data_consume_rows_context_m&, seastar::temporary_buffer<char> >(sstables::data_consume_rows_context_m&, seastar::temporary_buffer<char>&&) at /usr/include/c++/8/bits/invoke.h:96
 (inlined by) std::result_of<sstables::data_consume_rows_context_m& (seastar::temporary_buffer<char>&&)>::type std::reference_wrapper<sstables::data_consume_rows_context_m>::operator()<seastar::temporary_buffer<char> >(seastar::temporary_buffer<char>&&) const at /usr/include/c++/8/bits/refwrap.h:319
 (inlined by) seastar::future<> seastar::input_stream<char>::consume<std::reference_wrapper<sstables::data_consume_rows_context_m> >(std::reference_wrapper<sstables::data_consume_rows_context_m>&&)::{lambda()#1}::operator()() at /jenkins/workspace/scylla-3.1/relocatable-pkg/scylla/seastar/include/seastar/core/iostream-impl.hh:227
 (inlined by) seastar::future<> seastar::repeat<seastar::future<> seastar::input_stream<char>::consume<std::reference_wrapper<sstables::data_consume_rows_context_m> >(std::reference_wrapper<sstables::data_consume_rows_context_m>&&)::{lambda()#1}>(seastar::future<> seastar::input_stream<char>::consume<std::reference_wrapper<sstables::data_consume_rows_context_m> >(std::reference_wrapper<sstables::data_consume_rows_context_m>&&)::{lambda()#1}) at /jenkins/workspace/scylla-3.1/relocatable-pkg/scylla/seastar/include/seastar/core/future-util.hh:285
seastar::future<> seastar::input_stream<char>::consume<std::reference_wrapper<sstables::data_consume_rows_context_m> >(std::reference_wrapper<sstables::data_consume_rows_context_m>&&) at /usr/include/c++/8/variant:1334
 (inlined by) seastar::future<> seastar::input_stream<char>::consume<sstables::data_consume_rows_context_m>(sstables::data_consume_rows_context_m&) at /jenkins/workspace/scylla-3.1/relocatable-pkg/scylla/seastar/include/seastar/core/iostream-impl.hh:236
 (inlined by) data_consumer::continuous_data_consumer<sstables::data_consume_rows_context_m>::consume_input() at /jenkins/workspace/scylla-3.1/relocatable-pkg/scylla/sstables/consumer.hh:377
 (inlined by) sstables::data_consume_context<sstables::data_consume_rows_context_m>::read() at /jenkins/workspace/scylla-3.1/relocatable-pkg/scylla/sstables/data_consume_context.hh:98
 (inlined by) sstables::sstable_mutation_reader<sstables::data_consume_rows_context_m, sstables::mp_row_consumer_m>::fill_buffer(std::chrono::time_point<seastar::lowres_clock, std::chrono::duration<long, std::ratio<1l, 1000l> > >)::{lambda()#3}::operator()() const::{lambda()#2}::operator()() const::{lambda()#1}::operator()() const at /jenkins/workspace/scylla-3.1/relocatable-pkg/scylla/sstables/partition.cc:479
 (inlined by) seastar::apply_helper<sstables::sstable_mutation_reader<sstables::data_consume_rows_context_m, sstables::mp_row_consumer_m>::fill_buffer(std::chrono::time_point<seastar::lowres_clock, std::chrono::duration<long, std::ratio<1l, 1000l> > >)::{lambda()#3}::operator()() const::{lambda()#2}::operator()() const::{lambda()#1}, std::tuple<>&&, std::integer_sequence<unsigned long> >::apply({lambda()#2}&&, sstables::sstable_mutation_reader<sstables::data_consume_rows_context_m, sstables::mp_row_consumer_m>::fill_buffer(std::chrono::time_point<seastar::lowres_clock, std::chrono::duration<long, std::ratio<1l, 1000l> > >)::{lambda()#3}::operator()() const::{lambda()#2}::operator()() const::{lambda()#1}) at /jenkins/workspace/scylla-3.1/relocatable-pkg/scylla/seastar/include/seastar/core/apply.hh:35
 (inlined by) auto seastar::apply<sstables::sstable_mutation_reader<sstables::data_consume_rows_context_m, sstables::mp_row_consumer_m>::fill_buffer(std::chrono::time_point<seastar::lowres_clock, std::chrono::duration<long, std::ratio<1l, 1000l> > >)::{lambda()#3}::operator()() const::{lambda()#2}::operator()() const::{lambda()#1}>(sstables::sstable_mutation_reader<sstables::data_consume_rows_context_m, sstables::mp_row_consumer_m>::fill_buffer(std::chrono::time_point<seastar::lowres_clock, std::chrono::duration<long, std::ratio<1l, 1000l> > >)::{lambda()#3}::operator()() const::{lambda()#2}::operator()() const::{lambda()#1}&&, std::tuple<>&&) at /jenkins/workspace/scylla-3.1/relocatable-pkg/scylla/seastar/include/seastar/core/apply.hh:43
 (inlined by) seastar::future<> seastar::futurize<seastar::future<> >::apply<sstables::sstable_mutation_reader<sstables::data_consume_rows_context_m, sstables::mp_row_consumer_m>::fill_buffer(std::chrono::time_point<seastar::lowres_clock, std::chrono::duration<long, std::ratio<1l, 1000l> > >)::{lambda()#3}::operator()() const::{lambda()#2}::operator()() const::{lambda()#1}>(sstables::sstable_mutation_reader<sstables::data_consume_rows_context_m, sstables::mp_row_consumer_m>::fill_buffer(std::chrono::time_point<seastar::lowres_clock, std::chrono::duration<long, std::ratio<1l, 1000l> > >)::{lambda()#3}::operator()() const::{lambda()#2}::operator()() const::{lambda()#1}&&, std::tuple<>&&) at /jenkins/workspace/scylla-3.1/relocatable-pkg/scylla/seastar/include/seastar/core/future.hh:1392
 (inlined by) seastar::future<> seastar::future<>::then_impl<sstables::sstable_mutation_reader<sstables::data_consume_rows_context_m, sstables::mp_row_consumer_m>::fill_buffer(std::chrono::time_point<seastar::lowres_clock, std::chrono::duration<long, std::ratio<1l, 1000l> > >)::{lambda()#3}::operator()() const::{lambda()#2}::operator()() const::{lambda()#1}, seastar::future<> >(sstables::sstable_mutation_reader<sstables::data_consume_rows_context_m, sstables::mp_row_consumer_m>::fill_buffer(std::chrono::time_point<seastar::lowres_clock, std::chrono::duration<long, std::ratio<1l, 1000l> > >)::{lambda()#3}::operator()() const::{lambda()#2}::operator()() const::{lambda()#1}&&)::{lambda()#1}::operator()() const::{lambda(sstables::sstable_mutation_reader<sstables::data_consume_rows_context_m, sstables::mp_row_consumer_m>::fill_buffer(std::chrono::time_point<seastar::lowres_clock, std::chrono::duration<long, std::ratio<1l, 1000l> > >)::{lambda()#3}::operator()() const::{lambda()#2}::operator()() const::{lambda()#1})#1}::operator()<seastar::future_state<> >(auto, sstables::sstable_mutation_reader<sstables::data_consume_rows_context_m, sstables::mp_row_consumer_m>::fill_buffer(std::chrono::time_point<seastar::lowres_clock, std::chrono::duration<long, std::ratio<1l, 1000l> > >)::{lambda()#3}::operator()() const::{lambda()#2}::operator()() const::{lambda()#1}) at /jenkins/workspace/scylla-3.1/relocatable-pkg/scylla/seastar/include/seastar/core/future.hh:950
 (inlined by) seastar::continuation<seastar::future<> seastar::future<>::then_impl<sstables::sstable_mutation_reader<sstables::data_consume_rows_context_m, sstables::mp_row_consumer_m>::fill_buffer(std::chrono::time_point<seastar::lowres_clock, std::chrono::duration<long, std::ratio<1l, 1000l> > >)::{lambda()#3}::operator()() const::{lambda()#2}::operator()() const::{lambda()#1}, seastar::future<> >(sstables::sstable_mutation_reader<sstables::data_consume_rows_context_m, sstables::mp_row_consumer_m>::fill_buffer(std::chrono::time_point<seastar::lowres_clock, std::chrono::duration<long, std::ratio<1l, 1000l> > >)::{lambda()#3}::operator()() const::{lambda()#2}::operator()() const::{lambda()#1}&&)::{lambda()#1}::operator()() const::{lambda(sstables::sstable_mutation_reader<sstables::data_consume_rows_context_m, sstables::mp_row_consumer_m>::fill_buffer(std::chrono::time_point<seastar::lowres_clock, std::chrono::duration<long, std::ratio<1l, 1000l> > >)::{lambda()#3}::operator()() const::{lambda()#2}::operator()() const::{lambda()#1})#1}>::run_and_dispose() at /jenkins/workspace/scylla-3.1/relocatable-pkg/scylla/seastar/include/seastar/core/future.hh:377
seastar::reactor::run_tasks(seastar::reactor::task_queue&) at /usr/include/boost/program_options/variables_map.hpp:146
seastar::reactor::run_some_tasks() at /usr/include/boost/program_options/variables_map.hpp:146
seastar::reactor::run_some_tasks() at /usr/include/boost/program_options/variables_map.hpp:146
 (inlined by) seastar::reactor::run() at /jenkins/workspace/scylla-3.1/relocatable-pkg/scylla/seastar/build/release/../../src/core/reactor.cc:4246
seastar::smp::configure(boost::program_options::variables_map, seastar::reactor_config)::{lambda()#3}::operator()() const at /usr/include/boost/program_options/variables_map.hpp:146
std::function<void ()>::operator()() const at /usr/include/c++/8/bits/std_function.h:687
 (inlined by) seastar::posix_thread::start_routine(void*) at /jenkins/workspace/scylla-3.1/relocatable-pkg/scylla/seastar/build/release/../../src/core/posix.cc:52