This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# Parse the whole git history and show files larger than 1Mb (2^20 b) | |
git rev-list --objects --all | | |
git cat-file --batch-check='%(objecttype) %(objectname) %(objectsize) %(rest)' | | |
sed -n 's/^blob //p' | | |
awk '$2 >= 2^20' | | |
sort --numeric-sort --key=2 | | |
cut -c 1-12,41- | | |
$(command -v gnumfmt || echo numfmt) --field=2 --to=iec-i --suffix=B --padding=7 --round=nearest | |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# Store in ~/.config/direnv/direnvrc to run for all folders automatically | |
# check if VENV is loaded | |
if [[ -z "${VIRTUAL_ENV_PROMPT}" ]] ; then | |
if [ ! -d ".venv" ] ; then | |
echo "Installing virtualenv for $(python -V)" | |
python -m venv .venv | |
fi | |
echo "Activating $(python -V) virtualenv from .venv" | |
source .venv/bin/activate | |
fi |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# Test SMTP with STARTTLS | |
openssl s_client -showcerts -connect smtp.office365.com:587 -servername smtp.office365.com -starttls smtp | |
# Test IMAP with SSL/TLS | |
openssl s_client -showcerts -connect outlook.office365.com:993 -servername outlook.office365.com | |
# Test POP3 with SSL/TLS | |
openssl s_client -showcerts -connect outlook.office365.com:995 -servername outlook.office365.com | |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/env python3 | |
# -*- coding: utf-8 -*- | |
import argparse | |
import tarfile | |
import json | |
def get_tar_metadata(file_name): | |
tar = tarfile.open(file_name, encoding="iso8859-1") |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import os | |
import snowflake.connector | |
ctx = snowflake.connector.connect( | |
authenticator="snowflake", | |
user=os.getenv("SNOWSQL_USER"), | |
password=os.getenv("SNOWSQL_PWD"), | |
account=os.getenv("SNOWSQL_ACCOUNT"), | |
warehouse=os.getenv("SNOWSQL_WAREHOUSE") | |
) |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# Get the Hive Symlinks | |
aws s3 ls s3://<BUCKET>/hive/ --recursive | awk '{print "s3://<BUCKET>/"$4}' | |
# Delete all data files for the selected Hive partition | |
aws s3 cp s3://<BUCKET>/hive/dt=2019-07-24-00-00/symlink.txt - | xargs -I {} sh -c 'aws s3 rm {}' |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import pandas as pd | |
import pyarrow.orc as orc | |
file0 = open('/hive/warehouse/000000_0', 'rb') | |
data0 = orc.ORCFile(file0) | |
df0 = data0.read(columns=['_col10', '_col50']).to_pandas() | |
df0.describe() |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# Unzip in-place (i.e. in the folder containing the file and not the current one) | |
find . -type f -name "*.zip" | xargs -P4 -I fileName sh -c 'unzip -o -d "$(dirname "fileName")" "fileName" && rm "fileName"' | |
# Gzip all CSV extracted from the ZIP files | |
find . -type f -name *.csv -print0 | xargs -0 -n1 -P4 gzip |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# Concatenate the Root and SubCA certs from DigiCert | |
# to get the full certification chain | |
cat DigiCert_Global_Root_CA.pem DigiCertSHA2SecureServerCA.pem > DigiCertCA_Chain.pem | |
# Generate a new key store from the signed cert, the private key | |
openssl pkcs12 -export \ | |
-in my_cert_signed.crt | |
-inkey my_cert_key.pem | |
-chain -CAfile DigiCertCA_Chain.pem \ | |
-name "my_cert" -out my_cert.keystore.p12 |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# Get the requested resources for all pods by container | |
kubectl get pods -ao jsonpath='{range .items[*]}{@.metadata.name}{"\n"}{range @.spec.containers[*]}{"\t"}{@.name}{" cpu:"}{@.resources.requests.cpu}{" mem:"}{@.resources.requests.memory}{"\n"}{end}{end}' |
NewerOlder