Sometimes its helpful to edit lots of melange files quickly.
This is something that I used to do that.
| #!/usr/bin/python3.12 | |
| import os | |
| import sys | |
| import subprocess | |
| from ruamel.yaml import YAML | |
| syspath = sys.path | |
| sys.path = [path for path in sys.path if | |
| not (path == os.getcwd() or path == ".")] | |
| yaml = YAML(typ='safe') | |
| mytestdata = yaml.load("""\ | |
| test: | |
| environment: | |
| contents: | |
| packages: | |
| - busybox | |
| pipeline: | |
| - uses: py/pip-check | |
| """) | |
| for f in sys.argv[1:]: | |
| yaml2 = YAML() | |
| with open(f, "rb") as fp: | |
| content = fp.read() | |
| y = yaml2.load(content) | |
| #if "data" in y: | |
| # if "items" in y["data"][0]: | |
| # v = y["data"][0]["items"].get(3.1) | |
| # if v is not None: | |
| # y["data"][0]["items"]["3.10"] = v | |
| # del y["data"][0]["items"][3.1] | |
| is_subpkg = False | |
| pkg = y | |
| if "subpackages" in y and y["subpackages"][0].get("range") == "py-versions": | |
| pkg = y["subpackages"][0] | |
| is_subpkg = True | |
| if "test" not in pkg: | |
| pkg["test"] = mytestdata["test"] | |
| op = "add-test" | |
| elif "pipeline" not in pkg["test"]: | |
| pkg["test"]["pipeline"] = mytestdata["test"]["pipeline"] | |
| op = "add-pipeline" | |
| else: | |
| found = False | |
| for k in pkg["test"]["pipeline"]: | |
| if k.get("uses") == "py/pip-check": | |
| found = True | |
| if found: | |
| op = "done" | |
| else: | |
| pkg["test"]["pipeline"].extend(mytestdata["test"]["pipeline"]) | |
| op = "add-uses" | |
| print("%s: %s%s" % (f, op, "[sub]" if is_subpkg else "")) | |
| if op == "" or op == "done": | |
| continue | |
| with open(f, "w") as fp: | |
| yaml2.dump(y, fp) | |
| subprocess.check_output(["yam", f]) |
| #!/usr/bin/python3 | |
| import sys | |
| import subprocess | |
| from ruamel.yaml import YAML | |
| fails = set([ | |
| "openexr", # https://github.com/wolfi-dev/os/issues/47534 | |
| ]) | |
| # call like merge({"foo": "bar", "myarray": ["1"]}, {"myarray": ["2"]}) | |
| def merge(existing, newdata): | |
| if isinstance(existing, dict): | |
| if not isinstance(newdata, dict): | |
| raise RuntimeError(f"Can't put f{newdata} into f{existing}") | |
| for key, val in newdata.items(): | |
| if key in existing: | |
| merge(existing[key], val) | |
| else: | |
| existing[key] = val | |
| elif isinstance(existing, list): | |
| if not isinstance(newdata, list): | |
| raise RuntimeError(f"Can't put f{newdata} into list f{existing}") | |
| existing.extend(newdata) | |
| k = [k for k in existing] | |
| k.extend(newdata) | |
| k.sort() | |
| k = list(set(k)) | |
| print(f"k={k} existing={existing}") | |
| for i, v in enumerate(existing): | |
| if i >= len(k): | |
| existing.pop(i) | |
| else: | |
| existing[i] = k[i] | |
| # get a 'path' from a nested dictionary. path is '/' delimited. | |
| # get_dpath({"foo": "bar"}, "foo"} => "foo") | |
| # get_dpath({"test": "pipeline": [{"uses": "ldd-check"}]}, "test/pipeline") | |
| # returns ["uses"...} | |
| def find(element, data, default=None): | |
| keys = element.split('.') | |
| rv = data | |
| for key in keys: | |
| if key not in rv: | |
| return default | |
| rv = rv[key] | |
| return rv | |
| def update_uses_ldd(name, pipelines): | |
| if pipelines is None or len(pipelines) == 0: | |
| return False | |
| updated = False | |
| equivs = [ | |
| name, | |
| '${{package.name}}', | |
| '$(basename ${{targets.contextdir}})', | |
| '${{subpkg.name}}', | |
| '${{context.name}}'] | |
| for i, p in enumerate(pipelines): | |
| np = {"uses": "test/tw/ldd-check"} | |
| if 'runs' in p and 'uses' in p: | |
| continue | |
| uses = p.get("uses", {}) | |
| if uses not in ("test/tw/ldd-check", "test/ldd-check"): | |
| continue | |
| withv = p.get('with', {}) | |
| if not withv or len(withv) != 1: | |
| continue | |
| if withv['packages'] in equivs: | |
| pipelines[i] = np | |
| updated = True | |
| return updated | |
| # https://gist.github.com/justinvreeland/6c798760fb4b7709c6251bc30c0652b1 | |
| yml = YAML() | |
| # Otherwise yam gest confused about comments trailing multilined strings | |
| yml.indent(mapping=2, sequence=4, offset=2) | |
| # Reduce noise in the diff | |
| yml.preserve_quotes = True | |
| # Don't split single line strings | |
| yml.width = sys.maxsize | |
| for f in sys.argv[1:]: | |
| with open(f, "rb") as fp: | |
| content = fp.read() | |
| pkgdata = yml.load(content) | |
| updates = [] | |
| name = find("package.name", pkgdata) | |
| if name in fails: | |
| continue | |
| if update_uses_ldd(name, find("test.pipeline", pkgdata)): | |
| updates.append(name) | |
| for sp in pkgdata.get("subpackages", {}): | |
| spname = sp['name'] | |
| if update_uses_ldd(spname, find("test.pipeline", sp)): | |
| updates.append(spname) | |
| if len(updates) == 0: | |
| # print(f"{name}: n/a") | |
| continue | |
| with open(f, "w") as fp: | |
| yml.dump(pkgdata, fp) | |
| subprocess.check_output(["yam", f]) | |
| print(f"{name}: {updates}") | |
| #subprocess.check_output(["wolfictl", "bump", f]) |
| #!/usr/bin/python3.12 | |
| import os | |
| import re | |
| import sys | |
| import subprocess | |
| from ruamel.yaml import YAML | |
| syspath = sys.path | |
| sys.path = [path for path in sys.path if | |
| not (path == os.getcwd() or path == ".")] | |
| yaml = YAML(typ='safe') | |
| mytestdata = yaml.load("""\ | |
| test: | |
| environment: | |
| contents: | |
| packages: | |
| - busybox | |
| pipeline: | |
| - uses: py/pip-check | |
| """) | |
| for f in sys.argv[1:]: | |
| yaml2 = YAML() | |
| with open(f, "rb") as fp: | |
| content = fp.read() | |
| y = yaml2.load(content) | |
| #if "data" in y: | |
| # if "items" in y["data"][0]: | |
| # v = y["data"][0]["items"].get(3.1) | |
| # if v is not None: | |
| # y["data"][0]["items"]["3.10"] = v | |
| # del y["data"][0]["items"][3.1] | |
| ops = [] | |
| pyver = False | |
| if "data" in y: | |
| n = -1 | |
| for i, d in enumerate(y["data"]): | |
| if d["name"] == "py-versions": | |
| if "300" in d["items"].values(): | |
| break | |
| pyver = True | |
| n = i | |
| break | |
| if n >= 0: | |
| y["data"][n]["items"][3.13] = "300" | |
| ops.append("py-versions") | |
| if pyver and "subpackages" in y: | |
| n = -1 | |
| found = False | |
| bname = "${{vars.pypi-package}}" | |
| for i, sp in enumerate(y["subpackages"]): | |
| if sp.get("name").startswith("py3-supported-"): | |
| print("got it: %s" % (sp.get("name"))) | |
| if "dependencies" not in sp: | |
| sp["dependencies"] = {} | |
| if "runtime" in sp["dependencies"]: | |
| bnmatch = re.match("py3.[0-9]+-(.*)$", sp["dependencies"]["runtime"][0]) | |
| if bnmatch: | |
| bname = bnmatch.groups()[0] | |
| n = i | |
| break | |
| deps = ["py%s-%s" % (v, bname) for v in ["3.10", "3.11", "3.12", "3.13"]] | |
| supsub = { | |
| "name": "py3-supported-${{vars.pypi-package}}", | |
| "description" : "meta package providing ${{vars.pypi-package}} for supported python versions.", | |
| "dependencies": {"runtime": deps} | |
| } | |
| if n > 0: | |
| y["subpackages"][n]["dependencies"]["runtime"] = deps | |
| y["subpackages"][n]["dependencies"]["runtime"].sort() | |
| ops.append("update-supported") | |
| else: | |
| y["subpackages"].append(supsub) | |
| ops.append("new-supported") | |
| print("%s: %s" % (f, ops)) | |
| if len(ops): | |
| with open(f, "w") as fp: | |
| yaml2.dump(y, fp) | |
| subprocess.check_output(["yam", f]) | |
| subprocess.check_output(["wolfictl", "bump", f]) |
| #!/usr/bin/python3 | |
| import sys | |
| import subprocess | |
| from ruamel.yaml import YAML | |
| # 1. read file with 'origin binary' to figure out what binary packages | |
| # to update. | |
| # 2. edit yaml to | |
| # a. insert usrmerge repo from https://github.com/wolfi-dev/os/pull/40273 | |
| # b. insert merged-sbin dependency | |
| # c. bump epoch | |
| # https://github.com/wolfi-dev/os/pull/42522 | |
| add_repo = "https://apk.cgr.dev/wolfi-presubmit/4325ca278487afbfe85a38c1d32d81bb2d3de208" | |
| add_dep = "merged-sbin" | |
| epoch_bump = 1 | |
| # https://docs.google.com/spreadsheets/d/1lupbYGYLL3KKQJPORgXtJQCvV3uAWKGh40_8aV764L0/edit?gid=826790128#gid=826790128&fvid=19945842 | |
| # I just picked origin and binary-pkg column from a 'wolfi sbin' view. | |
| # copy and pasted here. | |
| pkgdata = """\ | |
| apk-tools apk-tools | |
| cifs-utils cifs-utils | |
| cilium-1.16 cilium-1.16-iptables | |
| cryptsetup cryptsetup | |
| dhclient dhclient | |
| e2fsprogs e2fsprogs | |
| efs-utils efs-utils | |
| fuse2 fuse2 | |
| glibc glibc | |
| glibc sln | |
| iproute2 iproute2 | |
| iptables ip6tables | |
| iptables iptables | |
| iptables iptables-xtables-privileged | |
| keyutils keyutils | |
| lvm2 device-mapper | |
| lvm2 lvm2 | |
| net-tools mii-tool | |
| net-tools net-tools | |
| nfs-utils nfs-utils | |
| ntfs-3g ntfs-3g | |
| openrc openrc | |
| runit runit | |
| shadow shadow-login | |
| su-exec su-exec | |
| systemd systemd-init | |
| tini tini | |
| tini tini-static | |
| util-linux agetty | |
| util-linux blkid | |
| util-linux blockdev | |
| util-linux cfdisk | |
| util-linux fstrim | |
| util-linux losetup | |
| util-linux runuser | |
| util-linux sfdisk | |
| util-linux util-linux-login | |
| util-linux util-linux-misc | |
| util-linux wipefs | |
| utmps utmps | |
| zfs zfs | |
| """ | |
| # util-linux has a vars/data/range packages that mess this up | |
| # so I did that one manually | |
| skips = ["glibc", "util-linux"] | |
| def add_test_repos(meldata, repos, binpkgs): | |
| found = [] | |
| rkeyname = "repositories" | |
| def _add_test_repos(t, repos): | |
| if "environment" not in t: | |
| t["environment"] = {} | |
| if "contents" not in t["environment"]: | |
| t["environment"]["contents"] = {} | |
| if rkeyname not in t["environment"]["contents"]: | |
| t["environment"]["contents"][rkeyname] = [] | |
| t["environment"]["contents"][rkeyname].extend(repos) | |
| mainpkg = meldata["package"]["name"] | |
| if "test" in meldata and mainpkg in binpkgs: | |
| _add_test_repos(meldata["test"], repos) | |
| found.append(mainpkg) | |
| if "subpackages" in meldata: | |
| for subp in meldata["subpackages"]: | |
| subpname = subp["name"].replace("${{package.name}}", mainpkg) | |
| if subpname not in binpkgs: | |
| continue | |
| if "test" not in subp: | |
| continue | |
| print("adding repos for %s (%s)" % (mainpkg, subpname)) | |
| _add_test_repos(subp["test"], repos) | |
| found.append(subpname) | |
| return found | |
| def add_pkg_runtime_deps(meldata, newdeps, binpkgs): | |
| mainpkg = meldata["package"]["name"] | |
| found = [] | |
| def adddeps(p, deps): | |
| if "dependencies" not in p: | |
| p["dependencies"] = {} | |
| if "runtime" not in p["dependencies"]: | |
| p["dependencies"]["runtime"] = [] | |
| d = list(p["dependencies"]["runtime"]) | |
| d.extend(newdeps) | |
| p["dependencies"]["runtime"] = [k for k in sorted(set(d))] | |
| return | |
| if mainpkg in binpkgs: | |
| adddeps(meldata["package"], newdeps) | |
| found.append(mainpkg) | |
| if "subpackages" in meldata: | |
| for subp in meldata["subpackages"]: | |
| subpname = subp["name"].replace("${{package.name}}", mainpkg) | |
| if subpname not in binpkgs: | |
| continue | |
| adddeps(subp, newdeps) | |
| found.append(subpname) | |
| if set(found) != set(binpkgs): | |
| print("[%s] found=%s" % (mainpkg, set(found))) | |
| print("[%s] binpkgs=%s" % (mainpkg, binpkgs)) | |
| raise RuntimeError("%s did not find packages %s" % | |
| (mainpkg, set(found).difference(set(binpkgs)))) | |
| def bump_epoch(meldata, count): | |
| meldata["package"]["epoch"] += count | |
| pkgs = {} | |
| for line in pkgdata.splitlines(): | |
| if not line: | |
| continue | |
| origin, binpkg = line.split() | |
| if origin not in pkgs: | |
| pkgs[origin] = [] | |
| pkgs[origin].append(binpkg) | |
| for origin, binpkgs in pkgs.items(): | |
| if origin in skips: | |
| print(f"skipping {origin}") | |
| continue | |
| yaml2 = YAML() | |
| yaml2.preserve_quotes = True | |
| fname = origin + ".yaml" | |
| with open(fname, "rb") as fp: | |
| content = fp.read() | |
| y = yaml2.load(content) | |
| print(f"opened {fname} . bins: {binpkgs}") | |
| add_pkg_runtime_deps(y, [add_dep], binpkgs) | |
| with open(fname, "w") as fp: | |
| yaml2.dump(y, fp) | |
| subprocess.check_output(["yam", fname]) | |
| subprocess.check_output( | |
| ["git", "commit", "-m", f"{origin} - add dep on {add_dep}", fname]) | |
| #subpkgs_updated = add_test_repos(y, [add_repo], binpkgs) | |
| #if len(subpkgs_updated) == 0: | |
| # print("%s did not have tests in any of %s" % (fname, binpkgs)) | |
| #else: | |
| # with open(fname, "w") as fp: | |
| # yaml2.dump(y, fp) | |
| # subprocess.check_output(["yam", fname]) | |
| # x = subprocess.check_output( | |
| # ["git", "commit", "-m", f"{origin} add build_dep repo", fname]) | |
| bump_epoch(y, epoch_bump) | |
| with open(fname, "w") as fp: | |
| yaml2.dump(y, fp) | |
| subprocess.check_output(["yam", fname]) | |
| subprocess.check_output( | |
| ["git", "commit", "-m", f"{origin} - bump epoch by {epoch_bump}", fname]) |
| #!/usr/bin/python3.12 | |
| import os | |
| import re | |
| import sys | |
| import subprocess | |
| from ruamel.yaml import YAML | |
| syspath = sys.path | |
| sys.path = [path for path in sys.path if | |
| not (path == os.getcwd() or path == ".")] | |
| yaml = YAML(typ='safe') | |
| mytestdata = yaml.load("""\ | |
| test: | |
| environment: | |
| contents: | |
| packages: | |
| - busybox | |
| pipeline: | |
| - uses: py/pip-check | |
| """) | |
| for f in sys.argv[1:]: | |
| yaml2 = YAML() | |
| with open(f, "rb") as fp: | |
| content = fp.read() | |
| y = yaml2.load(content) | |
| #if "data" in y: | |
| # if "items" in y["data"][0]: | |
| # v = y["data"][0]["items"].get(3.1) | |
| # if v is not None: | |
| # y["data"][0]["items"]["3.10"] = v | |
| # del y["data"][0]["items"][3.1] | |
| ops = [] | |
| pyver = False | |
| if "data" in y: | |
| n = -1 | |
| for i, d in enumerate(y["data"]): | |
| if d["name"] == "py-versions": | |
| if "313" in d["items"].values(): | |
| break | |
| pyver = True | |
| n = i | |
| break | |
| if n >= 0: | |
| y["data"][n]["items"][3.13] = "313" | |
| ops.append("py-versions") | |
| if pyver and "subpackages" in y: | |
| n = -1 | |
| found = False | |
| bname = "${{vars.pypi-package}}" | |
| for i, sp in enumerate(y["subpackages"]): | |
| if sp.get("name").startswith("py3-supported-"): | |
| print("got it: %s" % (sp.get("name"))) | |
| if "dependencies" not in sp: | |
| sp["dependencies"] = {} | |
| if "runtime" in sp["dependencies"]: | |
| bnmatch = re.match("py3.[0-9]+-(.*)$", sp["dependencies"]["runtime"][0]) | |
| if bnmatch: | |
| bname = bnmatch.groups()[0] | |
| n = i | |
| break | |
| deps = ["py%s-%s" % (v, bname) for v in ["3.10", "3.11", "3.12", "3.13"]] | |
| supsub = { | |
| "name": "py3-supported-${{vars.pypi-package}}", | |
| "description" : "meta package providing ${{vars.pypi-package}} for supported python versions.", | |
| "dependencies": {"runtime": deps} | |
| } | |
| if n > 0: | |
| y["subpackages"][n]["dependencies"]["runtime"] = deps | |
| y["subpackages"][n]["dependencies"]["runtime"].sort() | |
| ops.append("update-supported") | |
| else: | |
| y["subpackages"].append(supsub) | |
| ops.append("new-supported") | |
| print("%s: %s" % (f, ops)) | |
| if len(ops): | |
| with open(f, "w") as fp: | |
| yaml2.dump(y, fp) | |
| subprocess.check_output(["yam", f]) | |
| subprocess.check_output(["wolfictl", "bump", f]) |
Here's a simple class I use https://github.com/dannf/py-melange-yaml