Last active
May 19, 2022 00:11
Revisions
-
dmsimard revised this gist
Mar 14, 2021 . 2 changed files with 0 additions and 0 deletions.There are no files selected for viewing
File renamed without changes.File renamed without changes. -
Mar 14, 2021 . 3 changed files with 235 additions and 32 deletions.There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -0,0 +1,83 @@ [ "https://github.com/ansible-collections/amazon.aws", "https://github.com/ansible-collections/ansible.netcommon", "https://github.com/ansible-collections/ansible.posix", "https://github.com/ansible-collections/ansible.windows", "https://github.com/ansible-collections/arista.eos", "https://github.com/ansible/awx", "https://github.com/ansible-collections/azure", "https://github.com/CheckPointSW/CheckPointAnsibleMgmtCollection", "https://github.com/chocolatey/chocolatey-ansible", "https://github.com/CiscoDevNet/ansible-aci", "https://github.com/ansible-collections/cisco.asa", "https://github.com/CiscoDevNet/intersight-ansible", "https://github.com/ansible-collections/cisco.ios", "https://github.com/ansible-collections/cisco.iosxr", "https://github.com/CiscoDevNet/ansible-meraki", "https://github.com/CiscoDevNet/ansible-mso", "https://github.com/CiscoDevNet/ansible-nso", "https://github.com/ansible-collections/nxos", "https://github.com/CiscoDevNet/ansible-ucs", "https://github.com/cloudscale-ch/ansible-collection-cloudscale", "https://github.com/ansible-collections/community.aws", "https://github.com:ansible-collections/community.azure", "https://github.com/ansible-collections/community.crypto", "https://github.com/ansible-collections/community.digitalocean", "https://github.com/ansible-collections/community.docker", "https://github.com/ansible-collections/community.fortios", "https://github.com/ansible-collections/community.general", "https://github.com/ansible-collections/community.google", "https://github.com/ansible-collections/grafana.git", "https://github.com/ansible-collections/community.hashi_vault", "https://github.com/ansible-collections/community.hrobot", "https://github.com/ansible-collections/community.kubernetes", "https://github.com/ansible-collections/community.kubevirt", "https://github.com/ansible-collections/community.libvirt", "https://github.com/ansible-collections/community.mongodb", "https://github.com/ansible-collections/community.mysql", "https://github.com/ansible-collections/community.network", "https://github.com/ansible-collections/community.okd", "https://github.com/ansible-collections/community.postgresql", "https://github.com/ansible-collections/community.proxysql", "https://github.com/ansible-collections/community.rabbitmq", "https://github.com/ansible-collections/community.routeros", "https://github.com/ansible-collections/skydive", "https://github.com/ansible-collections/community.vmware.git", "https://github.com/ansible-collections/community.windows", "https://github.com/ansible-collections/community.zabbix.git", "https://github.com/containers/ansible-podman-collections.git", "https://github.com/cyberark/ansible-conjur-collection", "https://github.com/cyberark/ansible-security-automation-collection", "https://github.com/ansible-collections/dellemc.os10", "https://github.com/ansible-collections/dellemc.os6", "https://github.com/ansible-collections/dellemc.os9", "https://github.com/F5Networks/f5-ansible", "https://github.com/fortinet-ansible-dev/ansible-galaxy-fortimanager-collection/tree/galaxy/2.0.1", "https://github.com/fortinet-ansible-dev/ansible-galaxy-fortios-collection/tree/fos_v6.0.0/galaxy_1.1.9", "https://github.com/ansible-collections/frr.frr", "https://github.com/gluster/gluster-ansible-collection", "http://github.com/ansible/ansible_collections_google", "https://github.com/ansible-collections/hetzner.hcloud", "https://github.com/ansible-collections/ibm.qradar", "https://www.github.com/infinidat/ansible-infinidat-collection", "https://github.com/ansible-collections/junipernetworks.junos", "https://github.com/ansible-collections/mellanox.onyx", "https://github.com/ansible-collections/netapp", "https://github.com/ansible-collections/netapp", "https://www.github.com/netapp-eseries/santricity", "https://github.com/ansible-collections/netapp", "https://github.com/netbox-community/ansible_modules", "https://github.com/ngine-io/ansible-collection-cloudstack", "https://github.com/ngine-io/ansible-collection-exoscale", "https://github.com/ngine-io/ansible-collection-vultr", "https://opendev.org/openstack/ansible-collections-openstack", "https://github.com/ansible-collections/openvswitch.openvswitch", "https://github.com/ovirt/ovirt-ansible-collection", "https://github.com/Pure-Storage-Ansible/FlashArray-Collection", "https://github.com/Pure-Storage-Ansible/FlashBlade-Collection", "https://github.com/ServiceNowITOM/servicenow-ansible", "https://github.com/ansible-collections/splunk.es", "https://github.com/theforeman/foreman-ansible-modules", "https://github.com/ansible-collections/vyos.vyos", "https://github.com/wtinetworkgear/wti-collection" ] This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -0,0 +1,87 @@ [ "https://github.com/ansible-collections/amazon.aws", "https://github.com/ansible-collections/ansible.netcommon", "https://github.com/ansible-collections/ansible.posix", "https://github.com/ansible-collections/ansible.windows", "https://github.com/ansible-collections/arista.eos", "https://github.com/ansible/awx", "https://github.com/ansible-collections/azure", "https://github.com/CheckPointSW/CheckPointAnsibleMgmtCollection", "https://github.com/chocolatey/chocolatey-ansible", "https://github.com/CiscoDevNet/ansible-aci", "https://github.com/ansible-collections/cisco.asa", "https://github.com/CiscoDevNet/intersight-ansible", "https://github.com/ansible-collections/cisco.ios", "https://github.com/ansible-collections/cisco.iosxr", "https://github.com/CiscoDevNet/ansible-meraki", "https://github.com/CiscoDevNet/ansible-mso", "https://github.com/CiscoDevNet/ansible-nso", "https://github.com/ansible-collections/nxos", "https://github.com/CiscoDevNet/ansible-ucs", "https://github.com/cloudscale-ch/ansible-collection-cloudscale", "https://github.com/ansible-collections/community.aws", "https://github.com:ansible-collections/community.azure", "https://github.com/ansible-collections/community.crypto", "https://github.com/ansible-collections/community.digitalocean", "https://github.com/ansible-collections/community.docker", "https://github.com/ansible-collections/community.fortios", "https://github.com/ansible-collections/community.general", "https://github.com/ansible-collections/community.google", "https://github.com/ansible-collections/grafana.git", "https://github.com/ansible-collections/community.hashi_vault", "https://github.com/ansible-collections/community.hrobot", "https://github.com/ansible-collections/community.kubernetes", "https://github.com/ansible-collections/community.kubevirt", "https://github.com/ansible-collections/community.libvirt", "https://github.com/ansible-collections/community.mongodb", "https://github.com/ansible-collections/community.mysql", "https://github.com/ansible-collections/community.network", "https://github.com/ansible-collections/community.okd", "https://github.com/ansible-collections/community.postgresql", "https://github.com/ansible-collections/community.proxysql", "https://github.com/ansible-collections/community.rabbitmq", "https://github.com/ansible-collections/community.routeros", "https://github.com/ansible-collections/skydive", "https://github.com/ansible-collections/community.sops", "https://github.com/ansible-collections/community.vmware.git", "https://github.com/ansible-collections/community.windows", "https://github.com/ansible-collections/community.zabbix.git", "https://github.com/containers/ansible-podman-collections.git", "https://github.com/cyberark/ansible-conjur-collection", "https://github.com/cyberark/ansible-security-automation-collection", "https://github.com/dell/dellemc-openmanage-ansible-modules/tree/collections", "https://github.com/ansible-collections/dellemc.os10", "https://github.com/ansible-collections/dellemc.os6", "https://github.com/ansible-collections/dellemc.os9", "https://github.com/F5Networks/f5-ansible", "https://github.com/fortinet-ansible-dev/ansible-galaxy-fortimanager-collection/tree/galaxy/2.0.1", "https://github.com/fortinet-ansible-dev/ansible-galaxy-fortios-collection/tree/fos_v6.0.0/galaxy_1.1.9", "https://github.com/ansible-collections/frr.frr", "https://github.com/gluster/gluster-ansible-collection", "http://github.com/ansible/ansible_collections_google", "https://github.com/ansible-collections/hetzner.hcloud", "https://github.com/ansible-collections/ibm.qradar", "https://www.github.com/infinidat/ansible-infinidat-collection", "https://github.com/ansible-collections/junipernetworks.junos", "https://github.com/ansible-collections/mellanox.onyx", "https://github.com/ansible-collections/netapp", "https://github.com/ansible-collections/netapp", "https://www.github.com/netapp-eseries/santricity", "https://github.com/ansible-collections/netapp", "https://github.com/netbox-community/ansible_modules", "https://github.com/ngine-io/ansible-collection-cloudstack", "https://github.com/ngine-io/ansible-collection-exoscale", "https://github.com/ngine-io/ansible-collection-vultr", "https://opendev.org/openstack/ansible-collections-openstack", "https://github.com/ansible-collections/openvswitch.openvswitch", "https://github.com/ovirt/ovirt-ansible-collection", "https://github.com/Pure-Storage-Ansible/FlashArray-Collection", "https://github.com/Pure-Storage-Ansible/FlashBlade-Collection", "https://github.com/sensu/sensu-go-ansible", "https://github.com/ServiceNowITOM/servicenow-ansible", "https://github.com/ansible-collections/splunk.es", "https://github.com/theforeman/foreman-ansible-modules", "https://github.com/T-Systems-MMS/ansible-collection-icinga-director", "https://github.com/ansible-collections/vyos.vyos", "https://github.com/wtinetworkgear/wti-collection" ] This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -1,27 +1,71 @@ #!/usr/bin/env python3 # Query an ansible.in file to retrieve the list of collections included in the Ansible # package so we can query galaxy and retrieve their git repositories. # ansible.in files can be cloned from ansible-build-data: # - https://github.com/ansible-community/ansible-build-data/blob/main/2.10/ansible.in # - https://github.com/ansible-community/ansible-build-data/blob/main/3/ansible.in import argparse import json import logging import requests # from requests.adapters import HTTPAdapter import time GALAXY = "https://galaxy.ansible.com" NAMESPACES = f"{GALAXY}/api/v1/namespaces" REPOSITORIES = f"{GALAXY}/api/v1/repositories" COLLECTIONS_API = f"{GALAXY}/api/v2/collections" # Retry Galaxy API calls if they fail RETRY_COUNT = 15 def get_args(): parser = argparse.ArgumentParser() parser.add_argument("--infile", help="Path to an ansible.in file", required=True) parser.add_argument( "--ignore", help=( "Omit repositories belonging to specified git organizations (not collection namespaces), can be repeated.\n" "For example: --ignore ansible --ignore ansible-community --ignore ansible-collections\n" ), action="append", default=[], required=False, ) args = parser.parse_args() return args def query_galaxy(session, url): """ Queries galaxy and re-tries a few times on failures which can improve the odds of succeeding past rate-limiting or unreliability. """ logging.info(f"Querying galaxy: {url}") retries = 0 query = None while query is None: try: query = session.get(url).json() except Exception as e: retries += 1 if retries >= RETRY_COUNT: logging.error(f"giving up retrying on {url}: {e}") break logging.warning(f"retrying call to galaxy: {url}: {e}") time.sleep(2) query = None return query def main(): args = get_args() logging.basicConfig(level="INFO", format="%(asctime)s %(levelname)s %(name)s: %(message)s") session = requests.Session() # session.mount(GALAXY, HTTPAdapter(max_retries=5)) with open(args.infile) as f: collections = f.readlines() @@ -32,40 +76,29 @@ def main(): if not collection.startswith("#"): namespace, repository = collection.split(".") # Galaxy knows about the git repository for a collection and we can find it in the latest_version of a collection # ex: https://galaxy.ansible.com/api/v2/collections/community/general/ collection = query_galaxy(session, f"{COLLECTIONS_API}/{namespace}/{repository}/") if collection is None: logging.error("couldn't get the latest version for {collection}") continue # Now query the latest version so we can get the git repo # ex: https://galaxy.ansible.com/api/v2/collections/community/general/versions/2.2.0/ version = query_galaxy(session, collection["latest_version"]["href"]) if version is None: logging.error("couldn't get the git repository for {collection}") continue repository = version["metadata"]["repository"] # We might want to exclude organizations or namespaces for repositories we otherwise already know about ignored = False for ignored_ns in args.ignore: if f"/{ignored_ns}/" in repository: ignored = True if ignored: logging.info(f"ignoring {repository}, in ignored namespace") continue repositories.append(repository) @@ -74,4 +107,4 @@ def main(): if __name__ == "__main__": main() -
Jan 26, 2021 .There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -0,0 +1,77 @@ #!/usr/bin/env python3 import argparse import json import os import requests import time GALAXY = "https://galaxy.ansible.com" NAMESPACES = f"{GALAXY}/api/v1/namespaces" REPOSITORIES = f"{GALAXY}/api/v1/repositories" COLLECTIONS_API = f"{GALAXY}/api/v2/collections" IGNORED_NAMESPACES = ["/ansible/", "/ansible-community/", "/ansible-collections/"] def get_args(): parser = argparse.ArgumentParser() parser.add_argument('--infile', help='Path to the ansible.in file', required=True) args = parser.parse_args() return args def main(): args = get_args() session = requests.Session() with open(args.infile) as f: collections = f.readlines() collections = [x.strip() for x in collections] repositories = [] for collection in collections: if not collection.startswith("#"): namespace, repository = collection.split(".") # Find the repository with galaxy print(f"Querying {COLLECTIONS_API}/{namespace}/{repository}/") query = None while query is None: try: query = session.get(f"{COLLECTIONS_API}/{namespace}/{repository}/").json() except Exception as e: print("retrying: %s" % str(e)) time.sleep(1) query = None version = None while version is None: try: version = session.get(query["latest_version"]["href"]).json() except Exception as e: print("retrying: %s" % str(e)) time.sleep(1) version = None repository = version["metadata"]["repository"] # Only projects on github for now if "github.com" not in repository: print(f"ignoring {repository}, not on github") continue # Don't recover repositories we already know about ignored = False for ignored_ns in IGNORED_NAMESPACES: if ignored_ns in repository: ignored = True if ignored: print(f"ignoring {repository}, in ignored namespace") continue repositories.append(repository) print(json.dumps(repositories, indent=2)) if __name__ == "__main__": main()