jaimevalero · May 13, 2023 00:29 · jaimevalero · Feb 17, 2018 · jaimevalero · Feb 17, 2018
diff --git a/extract_awesome.sh b/extract_awesome.sh
 # Order an awesome list by number stars.
 #
 # Tis one liner scripts extracts the number of stars from each repo from a given awesome list, and order repos by the number of start
 # This one liner uses jq command, so you should have it installed in your machine

 # Parameter
 # Awesome List to extract, in raw
 AWESOME_LIST=https://raw.githubusercontent.com/pditommaso/awesome-pipeline/master/README.md
 #CREDENTIALS="jaimevalero:mysecret"
 CREDENTIALS="replace-for-your-github-user:replace-for-your-github-password"

 curl -L --user  "$CREDENTIALS" -s "$AWESOME_LIST" | \
  egrep -E  -o  'https://github.com/.*/.*\)'      | \
  cut -d\/ -f 4-5 | cut -d\) -f1 | \
  while read line ; do \
    echo "[$line](https://github.com/$line)" \|  \
    `curl --user  "$CREDENTIALS" -s  -L -k "https://api.github.com/repos/$line" | \
    jq -c '[ .stargazers_count  ,"º" ,  .description , "º"] ' | \
    tr -d '\[' | tr -d '\]' | tr -d ',' | tr -d '\"' |  tr -d '\|' | tr 'º' '|' `; \
  done |  sort -r -u -t \| -k2 -n | sed -e 's/^/\|/g'   | sed -e 's@||@|@g'
	# Order an awesome list by number stars.
	#
	# Tis one liner scripts extracts the number of stars from each repo from a given awesome list, and order repos by the number of start
	# This one liner uses jq command, so you should have it installed in your machine

	# Parameter
	# Awesome List to extract, in raw
	AWESOME_LIST=https://raw.githubusercontent.com/pditommaso/awesome-pipeline/master/README.md
	#CREDENTIALS="jaimevalero:mysecret"
	CREDENTIALS="replace-for-your-github-user:replace-for-your-github-password"

	curl -L --user "$CREDENTIALS" -s "$AWESOME_LIST" \| \
	egrep -E -o 'https://github.com/./.\)' \| \
	cut -d\/ -f 4-5 \| cut -d\) -f1 \| \
	while read line ; do \
	echo "[$line](https://github.com/$line)" \\| \
	`curl --user "$CREDENTIALS" -s -L -k "https://api.github.com/repos/$line" \| \
	jq -c '[ .stargazers_count ,"º" , .description , "º"] ' \| \
	tr -d '\[' \| tr -d '\]' \| tr -d ',' \| tr -d '\"' \| tr -d '\\|' \| tr 'º' '\|' `; \
	done \| sort -r -u -t \\| -k2 -n \| sed -e 's/^/\\|/g' \| sed -e 's@\|\|@\|@g'
Repo	Stars	Description
kahun/awesome-sysadmin	20723	A curated list of amazingly awesome open source sysadmin resources inspired by Awesome PHP.
airbnb/airflow	17868	Apache Airflow - A platform to programmatically author schedule and monitor workflows
spotify/luigi	13643	Luigi is a Python module that helps you build complex pipelines of batch jobs. It handles dependency resolution workflow management visualization etc. It also comes with Hadoop support built in.
ray-project/ray	12782	A fast and simple framework for building and running distributed applications. Ray is packaged with RLlib a scalable reinforcement learning library and Tune a scalable hyperparameter tuning library.
dask/dask	7085	Parallel computing with task scheduling
argoproj/argo	6248	Argo Workflows: Get stuff done with Kubernetes.
deis/deis	6116	Deis v1 the CoreOS and Docker PaaS: Your PaaS. Your Rules.
uber/cadence	4074	Cadence is a distributed scalable durable and highly available orchestration engine to execute asynchronous long-running business logic in a scalable and resilient way.
polynote/polynote	3920	A better notebook for Scala (and more)
ovh/cds	3030	Enterprise-Grade Continuous Delivery & DevOps Automation Open Source Platform
quantumblacklabs/kedro	2827	A Python library that implements software engineering best-practice for data and ML pipelines.
polyaxon/polyaxon	2539	Machine Learning Platform for Kubernetes
dagster-io/dagster	2032	A data orchestrator for machine learning analytics and ETL.
pawl/awesome-etl	2008	A curated list of awesome ETL frameworks libraries and software.
manuzhang/awesome-streaming	1549	a curated list of awesome streaming frameworks applications etc
meirwah/awesome-workflow-engines	1531	A curated list of awesome open source workflow engines
mara/data-integration	1530	A lightweight opinionated ETL framework halfway between plain scripts and Apache Airflow
Factual/drake	1432	Data workflow tool like a \Make for data\|
ropensci/drake	1191	An R-focused pipeline toolkit for reproducibility and high-performance computing
common-workflow-language/common-workflow-language	1154	Repository for the CWL standards. Use https://cwl.discourse.group/ for support 😊
pinterest/pinball	1043	Pinball is a scalable workflow manager
lyft/flyte	984	develop execute and monitor distributed workflows reliably at scale.
knipknap/SpiffWorkflow	898	A powerful workflow engine implemented in pure Python
pydoit/doit	876	task management & automation tool
Netflix/suro	747	Netflix's distributed Data Pipeline
BD2KGenomics/toil	696	A scalable efficient cross-platform (Linux/macOS) and easy-to-use workflow engine in pure Python.
thieman/dagobah	684	Simple DAG-based job scheduler in Python
grailbio/reflow	664	A language and runtime for distributed incremental data processing in the cloud
substantic/rain	639	Framework for large distributed pipelines
ndmitchell/shake	634	Shake build system
broadinstitute/cromwell	583	Scientific workflow engine designed for simplicity & scalability. Trivially transition between one off use cases to massive scale production environments
nipy/nipype	518	Workflows and interfaces for neuroimaging packages
soravux/scoop	460	SCOOP (Scalable COncurrent Operations in Python)
CenturyLinkLabs/dray	374	An engine for managing the execution of container-based workflows.
openwdl/wdl	372	Workflow Description Language - Specification and Implementations
calebwin/pipelines	342	Pipelines is a language for high-level data analysis.
temporalio/temporal	330	Temporal service and CLI
richfitz/remake	328	Make-like declarative workflows in R
fission/fission-workflows	283	Workflows for Fission: Fast reliable and lightweight function composition for serverless functions
pharmbio/sciluigi	280	A light-weight wrapper library around Spotify's Luigi workflow library to make writing scientific workflows more fluent flexible and modular
Parsl/parsl	236	Parsl - Parallel Scripting Library
dataform-co/dataform	234	Dataform is a framework for managing SQL based data operations in BigQuery Snowflake and Redshift
openstack/mistral	222	Workflow Service for OpenStack. Mirror of code maintained at opendev.org.
materialsproject/fireworks	193	The Fireworks Workflow Management Repo.
systemslab/popper	188	Container-native workflow execution and automation engine.
ssadedin/bpipe	177	Bpipe - a tool for running and managing bioinformatics pipelines
joergen7/cuneiform	160	Cuneiform distributed programming language
robdmc/consecution	158	🚎 A pipeline abstraction for Python
intentmedia/mario	137	Functional Typesafe Declarative Data Pipelines
minerva-ml/steppy	121	Lightweight Python library for fast and reproducible experimentation 🔬
sailthru/stolos	112	A Directed Acyclic Graph task dependency scheduler designed to simplify complex distributed pipelines
rabix/rabix	104	Historical Reproducible Analyses for Bioinformatics
Illumina/pyflow	101	A lightweight parallel task engine
gems-uff/noworkflow	95	Supporting infrastructure to run scientific experiments without a scientific workflow management system.
bloomreach/briefly	92	Briefly - A Python Meta-programming Library for Job Flow Control
googlegenomics/dockerflow	91	Dockerflow is a workflow runner that uses Dataflow to run a series of tasks in Docker with the Pipelines API
evoldoers/biomake	72	GNU-Make-like utility for managing builds and complex workflows
sahilseth/flowr	71	Robust and efficient workflows using a simple language agnostic approach
gilt/sundial	65	a jobsystem on AWS ECS AWS Batch or AWS EMR managing dependencies and scheduling
tweag/porcupine	63	Express parametrable composable and portable data pipelines
MG-RAST/AWE	62	workflow and resource management system for bioinformatics data analysis
LLNL/maestrowf	61	A tool to easily orchestrate general computational workflows both locally and on supercomputers
masa16/Pwrake	56	Parallel Workflow extension for Rake runs on multicores clusters clouds.
druths/flex	53	A framework (comand line tool + libraries) for creating flexible compute pipelines
fulcrumgenomics/dagr	51	A scala based DSL and framework for writing and executing bioinformatics pipelines as Directed Acyclic GRaphs
4dn-dcic/tibanna	48	Tibanna helps you run your genomic pipelines on Amazon cloud (AWS). It is used by the 4DN DCIC (4D Nucleome Data Coordination and Integration Center) to process data. Tibanna supports CWL/WDL (w/ docker) Snakemake (w/ conda) and custom Docker/shell command.
pwwang/pyppl	47	PyPPL - A Python PiPeLine framework.
papenfusslab/bioshake	44	Bioinformatics pipelines with Haskell and Shake
nipype/pydra	43	Pydra Dataflow Engine
Ensembl/ensembl-hive	41	EnsEMBL Hive - a system for creating and running pipelines on a distributed compute resource
pveber/bistro	40	A library to build and execute typed scientific workflows
uzh/gc3pie	39	Python libraries and tools for running applications on diverse Grids and clusters
diana-hep/adage	37	dynamic DAG workflows
baffelli/pyperator	35	Simple python workflow engine based on asyncio and a DAG structure.
kirillseva/ruigi	34	Ruigi is a pipeline specialist much like his python counterpart Luigi.
adaptivegenome/openge	27	An accelerated framework for manipulating and interpreting high-throughput sequencing data
StanfordBioinformatics/loom	25	A tool for running bioinformatics workflows locally or in the cloud.
liyao001/BioQueue	22	A novel pipeline framework to accelerate bioinformatics analysis
mailund/gwf	21	A flexible pragmatic workflow tool.
balsam-alcf/balsam	20	High throughput workflows and automation for HPC
fstrozzi/bioruby-pipengine	19	An ultra light YAML-based pipeline execution engine
jtaghiyar/kronos	18	A workflow assembler for cancer genome analytics and informatics
monajemi/clusterjob	16	ClusterJob: An automated system for painless and reproducible massive computational experiments
uzh/sushi	15	SUSHI: Supporting User for SHell script Integration
PacificBiosciences/pypeFLOW	11	a simple lightweight workflow engine for data analysis scripting
MD-Studio/MDStudio	10	A general framework for microservice based distributed applications
It4innovations/HyperLoom	9	Platform for defining and executing scientific pipelines in distributed environments
alpha-unito/streamflow	6	StreamFlow Workflow Manager
tburdett/Conan2	3	Conan2 lightweight workflow manager
Repo	Stars	Description
coreos/etcd	32443	Distributed reliable key-value store for the most critical data of a distributed system
drone/drone	21609	Drone is a Container-Native Continuous Delivery Platform
google/leveldb	21491	LevelDB is a fast key-value storage library written at Google that provides an ordered mapping from string keys to string values.
etsy/statsd	15268	Daemon for easy but powerful stats aggregation
textmate/textmate	13001	TextMate is a graphical text editor for macOS 10.12 or later
gollum/gollum	10913	A simple Git-powered wiki with a sweet API and local frontend.
restic/restic	10697	Fast secure efficient backup program
JetBrains/intellij-community	10393	IntelliJ IDEA Community Edition & IntelliJ Platform
jordansissel/fpm	9674	Effing package management! Build packages for multiple platforms (deb rpm etc) with great ease and sanity.
afaqurk/linux-dash	9295	A beautiful web dashboard for Linux
apenwarr/sshuttle	8976	Wrong project! You should head over to http://github.com/sshuttle/sshuttle
takezoe/gitbucket	8127	A Git platform powered by Scala with easy installation high extensibility & GitHub API compatibility
digitalocean/netbox	6823	IP address management (IPAM) and data center infrastructure management (DCIM) tool.
bup/bup	6314	Very efficient backup system based on the git packfile format providing fast incremental saves and global deduplication (among and within files including virtual machine images). Current release is 0.31 and the development branch is master. Please post problems or patches to the mailing list for discussion (see the end of the README below).
Freeboard/freeboard	5968	A damn-sexy open source real-time dashboard builder for IOT and other web mashups. A free open-source alternative to Geckoboard.
StackExchange/blackbox	5408	Safely store secrets in Git/Mercurial/Subversion
meskyanichi/backup	4619	Easy full stack backup operations on UNIX-like systems.
axkibe/lsyncd	4147	Lsyncd (Live Syncing Daemon) synchronizes local directories with remote targets
twitter/flockdb	3234	A distributed fault-tolerant graph database
ha/doozerd	3189	A consistent distributed data store.
unbit/uwsgi	2825	uWSGI application server container
Smashing/smashing	2361	The exceptionally handsome dashboard framework in Ruby and Coffeescript.
purpleidea/mgmt	2293	Next generation distributed event-driven parallel config management!
librenms/librenms	1963	Community-based GPL-licensed network monitoring system
guardian/alerta	1746	Alerta monitoring system
ytti/oxidized	1510	Oxidized is a network device configuration backup tool. It's a RANCID replacement!
allegro/ralph	1407	Ralph is the CMDB / Asset Management system for data center and back office hardware.
desaster/kippo	1303	Kippo - SSH Honeypot
BrightcoveOS/Diamond	1207	null
opscode/omnibus-ruby	1099	Easily create full-stack installers for your project across a variety of platforms.
scobal/seyren	871	An alerting dashboard for Graphite
m242/maildrop	739	MailDrop is an open-source scalable high-performance version of Mailinator.
trigger/trigger	505	Trigger is a robust network automation toolkit written in Python that was designed for interfacing with network devices.
dgoodwin/tito	303	A tool for managing rpm based git projects.
clusto/clusto	268	Clusto is a cluster management tool. It helps you keep track of your inventory where it is how it's connected and provides an abstracted interface for interacting with the elements of the infrastructure.
CloudBotIRC/CloudBot	262	CloudBot - The simple fast expandable open-source Python IRC Bot!
minad/olelo	242	Wiki with git backend
jdleesmiller/jotgit	207	Git-backed real time collaborative editor built with meteor.
djosephsen/lazlo	141	A chatops automation framework for Slack in Go
repleo/bounca	112	BounCA is a web tool to generate self-signed SSL certificates and setup a key infrastructure
dojo4/drebs	35	Disaster Recovery for Elastic Block Store
ddanier/serverstats	11	null
sicekit/sicekit	8	null
eleme/node-bell	null	null