generate_test_file.sh

#!/bin/bash -xe

mkdir 1m_10000
mkdir 10m_1000
mkdir 100m_100

for i in {0000..9999}; do dd if=/dev/urandom of=1m_10000/1m_$i.dat count=1024 bs=1K; done
for i in {0000..999}; do dd if=/dev/urandom of=10m_1000/10m_$i.dat count=10240 bs=1K; done
for i in {000..99}; do dd if=/dev/urandom of=100m_100/100m_$i.dat count=102400 bs=1K; done

mkdir 1m_10000_4
mkdir 1m_10000_4/1
mkdir 1m_10000_4/2
mkdir 1m_10000_4/3
mkdir 1m_10000_4/4

for i in {000..2499}; do dd if=/dev/urandom of=1m_10000_4/1/1m_$i.dat count=1024 bs=1K; done
for i in {000..2499}; do dd if=/dev/urandom of=1m_10000_4/2/1m_$i.dat count=1024 bs=1K; done
for i in {000..2499}; do dd if=/dev/urandom of=1m_10000_4/3/1m_$i.dat count=1024 bs=1K; done
for i in {000..2499}; do dd if=/dev/urandom of=1m_10000_4/4/1m_$i.dat count=1024 bs=1K; done


mkdir 10m_1000_4
mkdir 10m_1000_4/1
mkdir 10m_1000_4/2
mkdir 10m_1000_4/3
mkdir 10m_1000_4/4

for i in {000..249}; do dd if=/dev/urandom of=10m_1000_4/1/10m_$i.dat count=10240 bs=1K; done
for i in {000..249}; do dd if=/dev/urandom of=10m_1000_4/2/10m_$i.dat count=10240 bs=1K; done
for i in {000..249}; do dd if=/dev/urandom of=10m_1000_4/3/10m_$i.dat count=10240 bs=1K; done
for i in {000..249}; do dd if=/dev/urandom of=10m_1000_4/4/10m_$i.dat count=10240 bs=1K; done


mkdir 100m_100_4
mkdir 100m_100_4/1
mkdir 100m_100_4/2
mkdir 100m_100_4/3
mkdir 100m_100_4/4

for i in {00..24}; do dd if=/dev/urandom of=100m_100_4/1/100m_$i.dat count=102400 bs=1K; done
for i in {00..24}; do dd if=/dev/urandom of=100m_100_4/2/100m_$i.dat count=102400 bs=1K; done
for i in {00..24}; do dd if=/dev/urandom of=100m_100_4/3/100m_$i.dat count=102400 bs=1K; done
for i in {00..24}; do dd if=/dev/urandom of=100m_100_4/4/100m_$i.dat count=102400 bs=1K; done

test.sh

#/bin/bash -xe

BUCKET=s3://hogehoge
S3_OPTIONS=--only-show-errors

time aws s3 sync ${S3_OPTIONS} 100m_100 s3://midaisuk-s3-test/test1/
time aws s3 sync ${S3_OPTIONS} 10m_1000 s3://midaisuk-s3-test/test1/
time aws s3 sync ${S3_OPTIONS} 1m_10000 s3://midaisuk-s3-test/test1/


CASE=1m_10000_4
TARGET=1m_10000_4
time ( \
       aws s3 sync ${S3_OPTIONS} ${CASE}/1 ${BUCKET}/${TARGET}/1 \
       & aws s3 sync ${S3_OPTIONS} ${CASE}/2 ${BUCKET}/${TARGET}/2 \
       & aws s3 sync ${S3_OPTIONS} ${CASE}/3 ${BUCKET}/${TARGET}/3 \
       & aws s3 sync ${S3_OPTIONS} ${CASE}/4 ${BUCKET}/${TARGET}/4 \
)

CASE=10m_1000_4
TARGET=10m_1000_4
time ( \
       aws s3 sync ${S3_OPTIONS} ${CASE}/1 ${BUCKET}/${TARGET}/1 \
       & aws s3 sync ${S3_OPTIONS} ${CASE}/2 ${BUCKET}/${TARGET}/2 \
       & aws s3 sync ${S3_OPTIONS} ${CASE}/3 ${BUCKET}/${TARGET}/3 \
       & aws s3 sync ${S3_OPTIONS} ${CASE}/4 ${BUCKET}/${TARGET}/4 \
)

CASE=100m_100_4
TARGET=100m_100_4
time ( \
       aws s3 sync ${S3_OPTIONS} ${CASE}/1 ${BUCKET}/${TARGET}/1 \
       & aws s3 sync ${S3_OPTIONS} ${CASE}/2 ${BUCKET}/${TARGET}/2 \
       & aws s3 sync ${S3_OPTIONS} ${CASE}/3 ${BUCKET}/${TARGET}/3 \
       & aws s3 sync ${S3_OPTIONS} ${CASE}/4 ${BUCKET}/${TARGET}/4 \
)