Skip to content

Instantly share code, notes, and snippets.

@Flamefire

Flamefire/createSubmoduleDeps.sh

Created June 17, 2020 14:31
Show Gist options
  • Save Flamefire/648a52a2784c0a21c172ad90545cb5eb to your computer and use it in GitHub Desktop.
Save Flamefire/648a52a2784c0a21c172ad90545cb5eb to your computer and use it in GitHub Desktop.
Download ZIP
Raw
createSubmoduleDeps.sh
#!/usr/bin/env bash
set -euo pipefail
git_repo="/home/alex/git/pytorch"
cmd_var="local_extract_cmd_pattern"
sub_cmd_var="local_mkdir_and_extract_cmd_pattern"
thirdparty_dir="third_party"
thirdparty_var="local_pytorchthirdpartydir"
# Repo names for dependencies that should be ignored
repo_ignore=(zstd nccl six enum34 ios-cmake ARM_NEON_2_x86_SSE clang-cindex-python3)
# Subdir of git_repo to ignore
subdir_ignore=("onnx-tensorrt/third_party/onnx" "protobuf/third_party/benchmark" "protobuf/third_party/googletest")
function printSubmoduleDeps(){
local dir="$1"
local submodules_dir="$2"
local extract_cmd="$3"
local submodules
mapfile -t submodules < <(cd "$dir" && git submodule status "$submodules_dir")
for submodule_data in "${submodules[@]}"; do
hash=$(echo "$submodule_data" | awk '{print $1;}')
submodule=$(echo "$submodule_data" | awk '{print $2;}')
if [[ "${hash::1}" == "-" || "${hash::1}" == "+" || "${hash::1}" == "U" ]]; then
echo "Submodule $submodule is dirty. Please reset"
exit 1
fi
submodule_dir="$dir/$submodule"
subfolder=${submodule_dir#*/$thirdparty_dir/}
if [[ " ${subdir_ignore[*]} " =~ " $subfolder " ]]; then
continue
fi
remote=$(cd "$submodule_dir" && git remote get-url origin)
remote="${remote%.git}"
source_url="${remote}/archive"
if [[ " ${repo_ignore[*]} " =~ " $(basename "$remote") " ]]; then
continue
fi
commit_date=$(cd "$submodule_dir" && git show -s --format='%cd' --date=format:'%Y%m%d')
filename="$(basename "$remote")-${commit_date}.tar.gz"
filename="${filename/-git-mirror/}"
echo "\
{
'source_urls': ['$source_url'],
'download_filename': '${hash}.tar.gz',
'filename': '$filename',
'extract_cmd': $extract_cmd % ($thirdparty_var, '$subfolder'),
},"
printSubmoduleDeps "$submodule_dir" "." "$cmd_var"
done
}
printSubmoduleDeps "$git_repo" "$thirdparty_dir" "$cmd_var"
Raw
generated.py
{
'source_urls': ['https://github.com/Maratyszcza/FP16/archive'],
'download_filename': 'febbb1c163726b5db24bed55cc9dc42529068997.tar.gz',
'filename': 'FP16-20181128.tar.gz',
'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'FP16'),
},
{
'source_urls': ['https://github.com/Maratyszcza/FXdiv/archive'],
'download_filename': 'b742d1143724d646cd0f914646f1240eacf5bd73.tar.gz',
'filename': 'FXdiv-20181116.tar.gz',
'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'FXdiv'),
},
{
'source_urls': ['https://github.com/Maratyszcza/NNPACK/archive'],
'download_filename': '24b55303f5cf65d75844714513a0d1b1409809bd.tar.gz',
'filename': 'NNPACK-20191007.tar.gz',
'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'NNPACK'),
},
{
'source_urls': ['https://github.com/pytorch/QNNPACK/archive'],
'download_filename': '7d2a4e9931a82adc3814275b6219a03e24e36b4c.tar.gz',
'filename': 'QNNPACK-20190828.tar.gz',
'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'QNNPACK'),
},
{
'source_urls': ['https://github.com/google/benchmark/archive'],
'download_filename': '505be96ab23056580a3a2315abba048f4428b04e.tar.gz',
'filename': 'benchmark-20180606.tar.gz',
'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'benchmark'),
},
{
'source_urls': ['https://github.com/pytorch/cpuinfo/archive'],
'download_filename': '89fe1695edf9ee14c22f815f24bac45577a4f135.tar.gz',
'filename': 'cpuinfo-20190117.tar.gz',
'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'cpuinfo'),
},
{
'source_urls': ['https://github.com/NVlabs/cub/archive'],
'download_filename': '285aeebaa34b0e8a7670867a2e66c1a52d998d6a.tar.gz',
'filename': 'cub-20170829.tar.gz',
'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'cub'),
},
{
'source_urls': ['https://github.com/eigenteam/eigen-git-mirror/archive'],
'download_filename': 'd41dc4dd74acce21fb210e7625d5d135751fa9e5.tar.gz',
'filename': 'eigen-20190125.tar.gz',
'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'eigen'),
},
{
'source_urls': ['https://github.com/pytorch/fbgemm/archive'],
'download_filename': '6aaaa4754fe2a7467c5f0584648ba47482caad7d.tar.gz',
'filename': 'fbgemm-20191120.tar.gz',
'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'fbgemm'),
},
{
'source_urls': ['https://github.com/asmjit/asmjit/archive'],
'download_filename': '17556b2d4984ab67bdc00e6b5a626efc75d95e07.tar.gz',
'filename': 'asmjit-20191001.tar.gz',
'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'fbgemm/third_party/asmjit'),
},
{
'source_urls': ['https://github.com/pytorch/cpuinfo/archive'],
'download_filename': 'd5e37adf1406cf899d7d9ec1d317c47506ccb970.tar.gz',
'filename': 'cpuinfo-20190201.tar.gz',
'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'fbgemm/third_party/cpuinfo'),
},
{
'source_urls': ['https://github.com/google/googletest/archive'],
'download_filename': '0fc5466dbb9e623029b1ada539717d10bd45e99e.tar.gz',
'filename': 'googletest-20180925.tar.gz',
'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'fbgemm/third_party/googletest'),
},
{
'source_urls': ['https://github.com/houseroad/foxi/archive'],
'download_filename': '97fe555430a857581b9b826ecd955e4f0a3653f0.tar.gz',
'filename': 'foxi-20191029.tar.gz',
'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'foxi'),
},
{
'source_urls': ['https://github.com/google/gemmlowp/archive'],
'download_filename': '3fb5c176c17c765a3492cd2f0321b0dab712f350.tar.gz',
'filename': 'gemmlowp-20181126.tar.gz',
'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'gemmlowp/gemmlowp'),
},
{
'source_urls': ['https://github.com/facebookincubator/gloo/archive'],
'download_filename': '7c541247a6fa49e5938e304ab93b6da661823d0f.tar.gz',
'filename': 'gloo-20191105.tar.gz',
'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'gloo'),
},
{
'source_urls': ['https://github.com/google/googletest/archive'],
'download_filename': '2fe3bd994b3189899d93f1d5a881e725e046fdc2.tar.gz',
'filename': 'googletest-20180831.tar.gz',
'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'googletest'),
},
{
'source_urls': ['https://github.com/intel/ideep/archive'],
'download_filename': '7def110361c05ee18569272e7b4e0b370998c6b1.tar.gz',
'filename': 'ideep-20191008.tar.gz',
'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'ideep'),
},
{
'source_urls': ['https://github.com/intel/mkl-dnn/archive'],
'download_filename': '7d2fd500bc78936d1d648ca713b901012f470dbc.tar.gz',
'filename': 'mkl-dnn-20190927.tar.gz',
'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'ideep/mkl-dnn'),
},
{
'source_urls': ['https://github.com/onnx/onnx/archive'],
'download_filename': 'fea8568cac61a482ed208748fdc0e1a8e47f62f5.tar.gz',
'filename': 'onnx-20191106.tar.gz',
'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'onnx'),
},
{
'source_urls': ['https://github.com/google/benchmark/archive'],
'download_filename': 'e776aa0275e293707b6a0901e0e8d8a8a3679508.tar.gz',
'filename': 'benchmark-20180525.tar.gz',
'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'onnx/third_party/benchmark'),
},
{
'source_urls': ['https://github.com/pybind/pybind11/archive'],
'download_filename': 'a1041190c8b8ff0cd9e2f0752248ad5e3789ea0c.tar.gz',
'filename': 'pybind11-20170828.tar.gz',
'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'onnx/third_party/pybind11'),
},
{
'source_urls': ['https://github.com/onnx/onnx-tensorrt/archive'],
'download_filename': 'c153211418a7c57ce071d9ce2a41f8d1c85a878f.tar.gz',
'filename': 'onnx-tensorrt-20190916.tar.gz',
'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'onnx-tensorrt'),
},
{
'source_urls': ['https://github.com/protocolbuffers/protobuf/archive'],
'download_filename': '48cb18e5c419ddd23d9badcfe4e9df7bde1979b2.tar.gz',
'filename': 'protobuf-20180727.tar.gz',
'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'protobuf'),
},
{
'source_urls': ['https://github.com/Maratyszcza/psimd/archive'],
'download_filename': '90a938f30ba414ada2f4b00674ee9631d7d85e19.tar.gz',
'filename': 'psimd-20180906.tar.gz',
'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'psimd'),
},
{
'source_urls': ['https://github.com/Maratyszcza/pthreadpool/archive'],
'download_filename': '13da0b4c21d17f94150713366420baaf1b5a46f4.tar.gz',
'filename': 'pthreadpool-20181008.tar.gz',
'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'pthreadpool'),
},
{
'source_urls': ['https://github.com/pybind/pybind11/archive'],
'download_filename': '25abf7efba0b2990f5a6dfb0a31bc65c0f2f4d17.tar.gz',
'filename': 'pybind11-20190204.tar.gz',
'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'pybind11'),
},
{
'source_urls': ['https://github.com/Maratyszcza/PeachPy/archive'],
'download_filename': '07d8fde8ac45d7705129475c0f94ed8925b93473.tar.gz',
'filename': 'PeachPy-20180218.tar.gz',
'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'python-peachpy'),
},
{
'source_urls': ['https://github.com/shibatch/sleef/archive'],
'download_filename': '7f523de651585fe25cade462efccca647dcc8d02.tar.gz',
'filename': 'sleef-20190730.tar.gz',
'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'sleef'),
},
{
'source_urls': ['https://github.com/01org/tbb/archive'],
'download_filename': 'a51a90bc609bb73db8ea13841b5cf7aa4344d4a9.tar.gz',
'filename': 'tbb-20181009.tar.gz',
'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'tbb'),
},
Raw
orig.py
{
'source_urls': ['https://github.com/Maratyszcza/FP16/archive'],
'download_filename': 'febbb1c163726b5db24bed55cc9dc42529068997.tar.gz',
'filename': 'FP16-20181128.tar.gz',
'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'FP16'),
},
{
'source_urls': ['https://github.com/Maratyszcza/FXdiv/archive'],
'download_filename': 'b742d1143724d646cd0f914646f1240eacf5bd73.tar.gz',
'filename': 'FXdiv-20181016.tar.gz',
'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'FXdiv'),
},
{
'source_urls': ['https://github.com/Maratyszcza/NNPACK/archive'],
'download_filename': '24b55303f5cf65d75844714513a0d1b1409809bd.tar.gz',
'filename': 'NNPACK-20191008.tar.gz',
'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'NNPACK'),
},
{
'source_urls': ['https://github.com/pytorch/QNNPACK/archive'],
'download_filename': '7d2a4e9931a82adc3814275b6219a03e24e36b4c.tar.gz',
'filename': 'QNNPACK-20190828.tar.gz',
'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'QNNPACK'),
},
{
'source_urls': ['https://github.com/google/benchmark/archive'],
'download_filename': '505be96ab23056580a3a2315abba048f4428b04e.tar.gz',
'filename': 'benchmark-20180606.tar.gz',
'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'benchmark'),
},
{
'source_urls': ['https://github.com/pytorch/cpuinfo/archive'],
'download_filename': '89fe1695edf9ee14c22f815f24bac45577a4f135.tar.gz',
'filename': 'cpuinfo-20190117.tar.gz',
'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'cpuinfo'),
},
{
'source_urls': ['https://github.com/NVlabs/cub/archive'],
'download_filename': '285aeebaa34b0e8a7670867a2e66c1a52d998d6a.tar.gz',
'filename': 'cub-20170829.tar.gz',
'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'cub'),
},
{
'source_urls': ['https://github.com/eigenteam/eigen-git-mirror/archive'],
'download_filename': 'd41dc4dd74acce21fb210e7625d5d135751fa9e5.tar.gz',
'filename': 'eigen-20190126.tar.gz',
'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'eigen'),
},
{
'source_urls': ['https://github.com/pytorch/fbgemm/archive'],
'download_filename': '6aaaa4754fe2a7467c5f0584648ba47482caad7d.tar.gz',
'filename': 'fbgemm-20191120.tar.gz',
'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'fbgemm'),
},
{
'source_urls': ['https://github.com/asmjit/asmjit/archive'],
'download_filename': '17556b2d4984ab67bdc00e6b5a626efc75d95e07.tar.gz',
'filename': 'asmjit-20190930.tar.gz',
'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'fbgemm/third_party/asmjit'),
},
{
'source_urls': ['https://github.com/pytorch/cpuinfo/archive'],
'download_filename': 'd5e37adf1406cf899d7d9ec1d317c47506ccb970.tar.gz',
'filename': 'cpuinfo-20190201.tar.gz',
'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'fbgemm/third_party/cpuinfo'),
},
{
'source_urls': ['https://github.com/google/googletest/archive'],
'download_filename': '0fc5466dbb9e623029b1ada539717d10bd45e99e.tar.gz',
'filename': 'googletest-20180920.tar.gz',
'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'fbgemm/third_party/googletest'),
},
{
'source_urls': ['https://github.com/houseroad/foxi/archive'],
'download_filename': '97fe555430a857581b9b826ecd955e4f0a3653f0.tar.gz',
'filename': 'foxi-20191029.tar.gz',
'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'foxi'),
},
{
'source_urls': ['https://github.com/google/gemmlowp/archive'],
'download_filename': '3fb5c176c17c765a3492cd2f0321b0dab712f350.tar.gz',
'filename': 'gemmlowp-20181127.tar.gz',
'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'gemmlowp/gemmlowp'),
},
{
'source_urls': ['https://github.com/facebookincubator/gloo/archive'],
'download_filename': '7c541247a6fa49e5938e304ab93b6da661823d0f.tar.gz',
'filename': 'gloo-20191105.tar.gz',
'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'gloo'),
},
{
'source_urls': ['https://github.com/google/googletest/archive'],
'download_filename': '2fe3bd994b3189899d93f1d5a881e725e046fdc2.tar.gz',
'filename': 'googletest-20180831.tar.gz',
'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'googletest'),
},
{
'source_urls': ['https://github.com/intel/ideep/archive'],
'download_filename': '7def110361c05ee18569272e7b4e0b370998c6b1.tar.gz',
'filename': 'ideep-20191008.tar.gz',
'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'ideep'),
},
{
'source_urls': ['https://github.com/intel/mkl-dnn/archive'],
'download_filename': '7d2fd500bc78936d1d648ca713b901012f470dbc.tar.gz',
'filename': 'mkl-dnn-20190928.tar.gz',
'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'ideep/mkl-dnn'),
},
{
'source_urls': ['https://github.com/onnx/onnx/archive'],
'download_filename': 'fea8568cac61a482ed208748fdc0e1a8e47f62f5.tar.gz',
'filename': 'onnx-20191107.tar.gz',
'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'onnx'),
},
{
'source_urls': ['https://github.com/google/benchmark/archive'],
'download_filename': 'e776aa0275e293707b6a0901e0e8d8a8a3679508.tar.gz',
'filename': 'benchmark-20180525.tar.gz',
'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'onnx/third_party/benchmark'),
},
{
'source_urls': ['https://github.com/pybind/pybind11/archive'],
'download_filename': 'a1041190c8b8ff0cd9e2f0752248ad5e3789ea0c.tar.gz',
'filename': 'pybind11-20170828.tar.gz',
'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'onnx/third_party/pybind11'),
},
{
'source_urls': ['https://github.com/onnx/onnx-tensorrt/archive'],
'download_filename': 'c153211418a7c57ce071d9ce2a41f8d1c85a878f.tar.gz',
'filename': 'onnx-tensorrt-20190916.tar.gz',
'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'onnx-tensorrt'),
},
{
'source_urls': ['https://github.com/google/protobuf/archive'],
'download_filename': '48cb18e5c419ddd23d9badcfe4e9df7bde1979b2.tar.gz',
'filename': 'protobuf-20180727.tar.gz',
'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'protobuf'),
},
{
'source_urls': ['https://github.com/Maratyszcza/psimd/archive'],
'download_filename': '90a938f30ba414ada2f4b00674ee9631d7d85e19.tar.gz',
'filename': 'psimd-20180906.tar.gz',
'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'psimd'),
},
{
'source_urls': ['https://github.com/Maratyszcza/pthreadpool/archive'],
'download_filename': '13da0b4c21d17f94150713366420baaf1b5a46f4.tar.gz',
'filename': 'pthreadpool-20181008.tar.gz',
'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'pthreadpool'),
},
{
'source_urls': ['https://github.com/pybind/pybind11/archive'],
'download_filename': '25abf7efba0b2990f5a6dfb0a31bc65c0f2f4d17.tar.gz',
'filename': 'pybind11-20190204.tar.gz',
'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'pybind11'),
},
{
'source_urls': ['https://github.com/wjakob/clang-cindex-python3/archive'],
'download_filename': '6a00cbc4a9b8e68b71caf7f774b3f9c753ae84d5.tar.gz',
'filename': 'clang-cindex-python3-20170330.tar.gz',
'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'pybind11/tools/clang'),
},
{
'source_urls': ['https://github.com/Maratyszcza/PeachPy/archive'],
'download_filename': '07d8fde8ac45d7705129475c0f94ed8925b93473.tar.gz',
'filename': 'PeachPy-20180219.tar.gz',
'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'python-peachpy'),
},
{
'source_urls': ['https://github.com/shibatch/sleef/archive'],
'download_filename': '7f523de651585fe25cade462efccca647dcc8d02.tar.gz',
'filename': 'sleef-20190730.tar.gz',
'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'sleef'),
},
{
'source_urls': ['https://github.com/intel/tbb/archive'],
'download_filename': 'a51a90bc609bb73db8ea13841b5cf7aa4344d4a9.tar.gz',
'filename': 'tbb-20181009.tar.gz',
'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'tbb'),
},
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment