Skip to content

Instantly share code, notes, and snippets.

@wu-lee
Last active May 31, 2023 12:48
Show Gist options
  • Save wu-lee/0e02d21503db95d534bfb12fcd1d20e3 to your computer and use it in GitHub Desktop.
Save wu-lee/0e02d21503db95d534bfb12fcd1d20e3 to your computer and use it in GitHub Desktop.
A multi-purpose script for SEA developers
# Cross-platform formatting config
# See https://EditorConfig.org
# top-most EditorConfig file
root = true
# Unix-style newlines with a newline ending every file
[*]
end_of_line = lf
insert_final_newline = true
# 2 space indentation
[{screenshot,*.{js,json,css,sh}}]
indent_style = space
indent_size = 2
charset = utf-8
/node_modules/
{
"name": "seodo-js",
"version": "1.0.0",
"lockfileVersion": 1,
"requires": true,
"dependencies": {
"@types/node": {
"version": "16.11.6",
"resolved": "https://registry.npmjs.org/@types/node/-/node-16.11.6.tgz",
"integrity": "sha512-ua7PgUoeQFjmWPcoo9khiPum3Pd60k4/2ZGXt18sm2Slk0W0xZTqt5Y0Ny1NyBiN1EVQ/+FaF9NcY4Qe6rwk5w==",
"optional": true
},
"@types/yauzl": {
"version": "2.9.2",
"resolved": "https://registry.npmjs.org/@types/yauzl/-/yauzl-2.9.2.tgz",
"integrity": "sha512-8uALY5LTvSuHgloDVUvWP3pIauILm+8/0pDMokuDYIoNsOkSwd5AiHBTSEJjKTDcZr5z8UpgOWZkxBF4iJftoA==",
"optional": true,
"requires": {
"@types/node": "*"
}
},
"agent-base": {
"version": "6.0.2",
"resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz",
"integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==",
"requires": {
"debug": "4"
}
},
"balanced-match": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz",
"integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw=="
},
"base64-js": {
"version": "1.5.1",
"resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz",
"integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA=="
},
"bl": {
"version": "4.1.0",
"resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz",
"integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==",
"requires": {
"buffer": "^5.5.0",
"inherits": "^2.0.4",
"readable-stream": "^3.4.0"
}
},
"brace-expansion": {
"version": "1.1.11",
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
"integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
"requires": {
"balanced-match": "^1.0.0",
"concat-map": "0.0.1"
}
},
"buffer": {
"version": "5.7.1",
"resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz",
"integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==",
"requires": {
"base64-js": "^1.3.1",
"ieee754": "^1.1.13"
}
},
"buffer-crc32": {
"version": "0.2.13",
"resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz",
"integrity": "sha1-DTM+PwDqxQqhRUq9MO+MKl2ackI="
},
"chownr": {
"version": "1.1.4",
"resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz",
"integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg=="
},
"concat-map": {
"version": "0.0.1",
"resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
"integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s="
},
"debug": {
"version": "4.3.2",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.3.2.tgz",
"integrity": "sha512-mOp8wKcvj7XxC78zLgw/ZA+6TSgkoE2C/ienthhRD298T7UNwAg9diBpLRxC0mOezLl4B0xV7M0cCO6P/O0Xhw==",
"requires": {
"ms": "2.1.2"
}
},
"devtools-protocol": {
"version": "0.0.901419",
"resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.901419.tgz",
"integrity": "sha512-4INMPwNm9XRpBukhNbF7OB6fNTTCaI8pzy/fXg0xQzAy5h3zL1P8xT3QazgKqBrb/hAYwIBizqDBZ7GtJE74QQ=="
},
"end-of-stream": {
"version": "1.4.4",
"resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz",
"integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==",
"requires": {
"once": "^1.4.0"
}
},
"extract-zip": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/extract-zip/-/extract-zip-2.0.1.tgz",
"integrity": "sha512-GDhU9ntwuKyGXdZBUgTIe+vXnWj0fppUEtMDL0+idd5Sta8TGpHssn/eusA9mrPr9qNDym6SxAYZjNvCn/9RBg==",
"requires": {
"@types/yauzl": "^2.9.1",
"debug": "^4.1.1",
"get-stream": "^5.1.0",
"yauzl": "^2.10.0"
}
},
"fd-slicer": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.1.0.tgz",
"integrity": "sha1-JcfInLH5B3+IkbvmHY85Dq4lbx4=",
"requires": {
"pend": "~1.2.0"
}
},
"find-up": {
"version": "4.1.0",
"resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz",
"integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==",
"requires": {
"locate-path": "^5.0.0",
"path-exists": "^4.0.0"
}
},
"fs-constants": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz",
"integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow=="
},
"fs.realpath": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
"integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8="
},
"get-stream": {
"version": "5.2.0",
"resolved": "https://registry.npmjs.org/get-stream/-/get-stream-5.2.0.tgz",
"integrity": "sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA==",
"requires": {
"pump": "^3.0.0"
}
},
"glob": {
"version": "7.2.0",
"resolved": "https://registry.npmjs.org/glob/-/glob-7.2.0.tgz",
"integrity": "sha512-lmLf6gtyrPq8tTjSmrO94wBeQbFR3HbLHbuyD69wuyQkImp2hWqMGB47OX65FBkPffO641IP9jWa1z4ivqG26Q==",
"requires": {
"fs.realpath": "^1.0.0",
"inflight": "^1.0.4",
"inherits": "2",
"minimatch": "^3.0.4",
"once": "^1.3.0",
"path-is-absolute": "^1.0.0"
}
},
"https-proxy-agent": {
"version": "5.0.0",
"resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.0.tgz",
"integrity": "sha512-EkYm5BcKUGiduxzSt3Eppko+PiNWNEpa4ySk9vTC6wDsQJW9rHSa+UhGNJoRYp7bz6Ht1eaRIa6QaJqO5rCFbA==",
"requires": {
"agent-base": "6",
"debug": "4"
}
},
"ieee754": {
"version": "1.2.1",
"resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz",
"integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA=="
},
"inflight": {
"version": "1.0.6",
"resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz",
"integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=",
"requires": {
"once": "^1.3.0",
"wrappy": "1"
}
},
"inherits": {
"version": "2.0.4",
"resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz",
"integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="
},
"locate-path": {
"version": "5.0.0",
"resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz",
"integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==",
"requires": {
"p-locate": "^4.1.0"
}
},
"minimatch": {
"version": "3.0.4",
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz",
"integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==",
"requires": {
"brace-expansion": "^1.1.7"
}
},
"mkdirp-classic": {
"version": "0.5.3",
"resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz",
"integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A=="
},
"ms": {
"version": "2.1.2",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
"integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
},
"node-fetch": {
"version": "2.6.5",
"resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.5.tgz",
"integrity": "sha512-mmlIVHJEu5rnIxgEgez6b9GgWXbkZj5YZ7fx+2r94a2E+Uirsp6HsPTPlomfdHtpt/B0cdKviwkoaM6pyvUOpQ==",
"requires": {
"whatwg-url": "^5.0.0"
}
},
"once": {
"version": "1.4.0",
"resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
"integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=",
"requires": {
"wrappy": "1"
}
},
"p-limit": {
"version": "2.3.0",
"resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz",
"integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==",
"requires": {
"p-try": "^2.0.0"
}
},
"p-locate": {
"version": "4.1.0",
"resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz",
"integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==",
"requires": {
"p-limit": "^2.2.0"
}
},
"p-try": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz",
"integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ=="
},
"path-exists": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz",
"integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w=="
},
"path-is-absolute": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz",
"integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18="
},
"pend": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz",
"integrity": "sha1-elfrVQpng/kRUzH89GY9XI4AelA="
},
"pkg-dir": {
"version": "4.2.0",
"resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-4.2.0.tgz",
"integrity": "sha512-HRDzbaKjC+AOWVXxAU/x54COGeIv9eb+6CkDSQoNTt4XyWoIJvuPsXizxu/Fr23EiekbtZwmh1IcIG/l/a10GQ==",
"requires": {
"find-up": "^4.0.0"
}
},
"progress": {
"version": "2.0.3",
"resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz",
"integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA=="
},
"proxy-from-env": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz",
"integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg=="
},
"pump": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz",
"integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==",
"requires": {
"end-of-stream": "^1.1.0",
"once": "^1.3.1"
}
},
"puppeteer": {
"version": "11.0.0",
"resolved": "https://registry.npmjs.org/puppeteer/-/puppeteer-11.0.0.tgz",
"integrity": "sha512-6rPFqN1ABjn4shgOICGDBITTRV09EjXVqhDERBDKwCLz0UyBxeeBH6Ay0vQUJ84VACmlxwzOIzVEJXThcF3aNg==",
"requires": {
"debug": "4.3.2",
"devtools-protocol": "0.0.901419",
"extract-zip": "2.0.1",
"https-proxy-agent": "5.0.0",
"node-fetch": "2.6.5",
"pkg-dir": "4.2.0",
"progress": "2.0.3",
"proxy-from-env": "1.1.0",
"rimraf": "3.0.2",
"tar-fs": "2.1.1",
"unbzip2-stream": "1.4.3",
"ws": "8.2.3"
}
},
"readable-stream": {
"version": "3.6.0",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz",
"integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==",
"requires": {
"inherits": "^2.0.3",
"string_decoder": "^1.1.1",
"util-deprecate": "^1.0.1"
}
},
"rimraf": {
"version": "3.0.2",
"resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz",
"integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==",
"requires": {
"glob": "^7.1.3"
}
},
"safe-buffer": {
"version": "5.2.1",
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz",
"integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ=="
},
"string_decoder": {
"version": "1.3.0",
"resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz",
"integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==",
"requires": {
"safe-buffer": "~5.2.0"
}
},
"tar-fs": {
"version": "2.1.1",
"resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.1.tgz",
"integrity": "sha512-V0r2Y9scmbDRLCNex/+hYzvp/zyYjvFbHPNgVTKfQvVrb6guiE/fxP+XblDNR011utopbkex2nM4dHNV6GDsng==",
"requires": {
"chownr": "^1.1.1",
"mkdirp-classic": "^0.5.2",
"pump": "^3.0.0",
"tar-stream": "^2.1.4"
}
},
"tar-stream": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz",
"integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==",
"requires": {
"bl": "^4.0.3",
"end-of-stream": "^1.4.1",
"fs-constants": "^1.0.0",
"inherits": "^2.0.3",
"readable-stream": "^3.1.1"
}
},
"through": {
"version": "2.3.8",
"resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz",
"integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU="
},
"tr46": {
"version": "0.0.3",
"resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz",
"integrity": "sha1-gYT9NH2snNwYWZLzpmIuFLnZq2o="
},
"unbzip2-stream": {
"version": "1.4.3",
"resolved": "https://registry.npmjs.org/unbzip2-stream/-/unbzip2-stream-1.4.3.tgz",
"integrity": "sha512-mlExGW4w71ebDJviH16lQLtZS32VKqsSfk80GCfUlwT/4/hNRFsoscrF/c++9xinkMzECL1uL9DDwXqFWkruPg==",
"requires": {
"buffer": "^5.2.1",
"through": "^2.3.8"
}
},
"util-deprecate": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
"integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8="
},
"webidl-conversions": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz",
"integrity": "sha1-JFNCdeKnvGvnvIZhHMFq4KVlSHE="
},
"whatwg-url": {
"version": "5.0.0",
"resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz",
"integrity": "sha1-lmRU6HZUYuN2RNNib2dCzotwll0=",
"requires": {
"tr46": "~0.0.3",
"webidl-conversions": "^3.0.0"
}
},
"wrappy": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
"integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8="
},
"ws": {
"version": "8.2.3",
"resolved": "https://registry.npmjs.org/ws/-/ws-8.2.3.tgz",
"integrity": "sha512-wBuoj1BDpC6ZQ1B7DWQBYVLphPWkm8i9Y0/3YdHjHKHiohOJ1ws+3OccDWtH+PoC9DZD5WOTrJvNbWvjS6JWaA=="
},
"yauzl": {
"version": "2.10.0",
"resolved": "https://registry.npmjs.org/yauzl/-/yauzl-2.10.0.tgz",
"integrity": "sha1-x+sXyT4RLLEIb6bY5R+wZnt5pfk=",
"requires": {
"buffer-crc32": "~0.2.3",
"fd-slicer": "~1.1.0"
}
}
}
}
{
"name": "seodo-js",
"version": "1.0.0",
"description": "",
"main": "index.js",
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1"
},
"repository": {
"type": "git",
"url": "https://github.com/0e02d21503db95d534bfb12fcd1d20e3.git"
},
"author": "",
"license": "ISC",
"dependencies": {
"puppeteer": "^11.0.0"
}
}
#!/bin/env node
const fs = require('fs');
const puppeteer = require('puppeteer');
const pageUrls = require('./seodo.config.json').urls;
async function snapshot(name, pageUrl) {
console.log(`start ${name}: with ${pageUrl}`);
// Launch the browser
const browser = await puppeteer.launch({
product: 'firefox',
defaultViewport: {
width: 810,
height: 1080,
isLandscape: false
}
});
try {
// Open a new page
const page = await browser.newPage();
// Log console messages
// page.on('console', msg => console.log(`${name}: ${msg.text()}`));
console.log(`${name}: goto ${pageUrl}`);
// Navigate to pageUrl
await page.goto(pageUrl);
// Wait for the loading animation to start, then finish
console.log(`${name}: wait for loading to begin`);
await page.waitForSelector('#loadingCirclespin', {visible: true});
console.log(`${name}: wait for loading to end`);
await page.waitForSelector('#loadingCirclespin', {hidden: true});
// Wait for all the map tiles to load
console.log(`${name}: wait for network to be idle`);
await page.waitForNetworkIdle();
// Take screenshot
await page.screenshot({path: `${name}-screenshot.png`, fullPage: true});
console.log(`${name}: screenshotted`);
}
finally {
await browser.close();
console.log(`${name}: finished`);
}
return name;
}
async function main() {
try {
for(var name in pageUrls) {
try {
await snapshot(name, pageUrls[name]);
}
catch(e) {
console.log(`${name}: error! ${e}`);
fs.writeFileSync(`${name}-snapshot.png`, ''); // write empty file
}
}
console.log(`Finished OK`);
}
catch(e) {
console.error(`Failure: ${e}`);
}
return true;
};
const result = main();
#!/bin/bash
script=$(readlink -f $0)
config="${script%/*}/seodo.config.json"
{ # Parse and validate the command options
while getopts "c:h" option; do
case $option in
c) config="$OPTARG" ; echo "using $config" ;;
h) action=help ;;
*)
echo "Invalid option: $OPTARG"
exit 1
;;
esac
done
shift $(( $OPTIND - 1 ))
}
[[ -z "$action" ]] && {
action=${1:-help}
shift 1
}
# Requirements
# - bash
# - curl
# - npm
# - git
# - ssh
# - jq
function die() {
printf "$*\n" >&2
exit 1
}
function warn() {
printf "$*\n" >&2
}
# Parse the JSON config into Bash assoc-array definitions
definitions=$(
jq -r -f /dev/stdin <<EOF $config
to_entries[] # for each top-level property
| "declare -A \( .key )=(\n" # declare an assoc array
+ (
[
.value # for each second-level property
| to_entries[] # convert into a key/value definition
| @sh " [\( .key )]=\( [.value] | flatten | join(" ") )"
]
| join("\n") # join these definitions on separate lines
)
+
"\n)" # terminate the assoc array defnition
EOF
) || die "jq failed to parse config: $config"
# Load the definitions
eval "$definitions"
function do_deploy {
local proj=${1:?No project name given. (Possible values: ${!projects[@]})}
local branch=${2:?No branch name given}
local server=${3:?No server name given. (Possible values: ${!endpoints[@]})}
local path=${paths["$proj@$server"]:?No project path defined for $proj@$server}
(
set -v
cd $proj
# check we are up to date on the correct branch
cur_branch=$(git symbolic-ref HEAD)
[[ "$cur_branch" == "refs/heads/$branch" ]] || {
warn "skipping project $proj: not checked out on $branch (it is on $cur_branch)"
return 1
}
git diff --quiet || {
warn "skipping project $proj: working directory has modifications"
return 1
}
# Check the branch has a remote branch (we assume remote is origin if no upstream set)
git fetch
remote_branch=$(git rev-parse --symbolic-full-name $branch@{u} 2>/dev/null ||
printf "refs/remotes/origin/$branch")
[[ $? == 0 ]] || {
warn "skipping project $proj: branch $branch has no remote branch"
return 1
}
# Check the branch is synced
remote_branch_id=$(git rev-parse "$remote_branch")
local_branch_id=$(git rev-parse refs/heads/$branch)
[[ "$remote_branch_id" == "$local_branch_id" ]] || {
warn "skipping project $proj: branch $branch is not in sync with $remote_branch"
return 1
}
# Now safe to build
rm -rf build node_modules
npm config set $proj:deploy_to=$server:$path
npm install
npm run build
npm run deploy
)
}
function doall_deploy_all {
do_deploy "$@"
}
function do_clone {
local proj=${1:?No project name given. (Possible values: ${!projects[@]})}
local url="[email protected]:SolidarityEconomyAssociation/$proj"
git clone "$url" "$proj"
}
function doall_clone_all {
local proj=$1
do_clone "$1"
}
function do_check_get_dataset {
local proj=${1:?No project name given. (Possible values: ${!projects[@]})}
local server=${2:?No server name given. (Possible values: ${!endpoints[@]})}
local url=${urls["$proj@$server"]:?No project url defined for $proj@$server. (Possible values: ${!urls[@]})}
local datasets=${datasets[$proj]:?No dataset name defined. (Possible values: ${!datasets[@]})}
for dataset in $datasets; do
printf "## Dataset $dataset\n"
curl -s $url/services/get_dataset.php?dataset=${dataset} |
jq '{status: .status, num_initiatives: .data | length}'
printf "\n"
done
}
function doall_check_get_datasets {
do_check_get_dataset "$@"
}
# Run the queries used by mapsites
function do_sparql_dump {
local proj=${1:?No project name given. (Possible values: ${!projects[@]})}
local server=${2:?No server name given. (Possible values: ${!endpoints[@]})}
local mime_type=${3:-text/csv}
local url=${urls["$proj@$server"]:?No project url defined for $proj@$server. (Possible values: ${!urls[@]})}
local datasets=${datasets[$proj]:?No dataset name defined. (Possible values: ${!datasets[@]})}
for dataset in $datasets; do
printf "## Dataset: $dataset\n" >&2
local endpoint=$(curl -s $url/configuration/${dataset}/endpoint.txt)
local dgu=$(curl -s $url/configuration/${dataset}/default-graph-uri.txt)
local query=$(curl -s $url/configuration/${dataset}/query.rq)
printf "sparql $endpoint\nDGU: $dgu\n" >&2
curl --no-progress-meter -G \
--header "Accept: $mime_type" \
--data-urlencode query="$query" \
--data-urlencode default-graph-uri="$dgu" \
"$endpoint" || printf "Bad dataset URL: $url/configuration/$dataset/endpoint.txt\n"
printf "\n"
done
}
function doall_sparql_dumps {
printf "sparql $endpoint\nDGU: $dgu\n"
do_sparql_dump "$@" | head -n 5 | sed 's/^/ csv: /'
}
function doall_listx {
local proj=${1:?No project name given. (Possible values: ${!projects[@]})}
local query="SELECT distinct ?g WHERE { GRAPH ?g { ?s ?p ?o } }"
local datasets=${datasets[$proj]:?No dataset name defined. (Possible values: ${!datasets[@]})}
for dataset in $datasets; do
local endpoint=$(< $proj/configuration/$dataset/endpoint.txt )
local mime_type=application/json
printf "## Dataset $dataset\n"
curl -s -G \
--header "Accept: $mime_type" \
--data-urlencode query="$query" \
$endpoint
printf "\n"
done
}
# list all the graphs
function do_list_graphs {
local server=${1:?No server name given. (Possible values: ${!endpoints[@]})}
local query="SELECT distinct ?graph WHERE { GRAPH ?graph { ?s ?p ?o } }"
local endpoint=${endpoints[$server]}
local mime_type=text/csv
printf "sparql $endpoint\n"
curl --no-progress-meter -G \
--header "Accept: $mime_type" \
--data-urlencode query="$query" \
$endpoint
}
# Attempts to count initiatives in graphs.
#
# Note, specific to one ESSGLOBAL version, and this might not work correctly, either.
function do_count_initiatives_by_graph {
local server=${1:?No server name given. (Possible values: ${!endpoints[@]})}
local dgu=${dgus[$server]}
local dgus query
# combine the object URIs. The leading comma will be stripped later
dgus=$(printf ",<%s>" {https://w3id.solidarityeconomy.coop,$dgu}/essglobal/{V2a,2.1}/vocab/SSEInitiative)
read -rd '' query <<HERE
SELECT distinct ?graph, COUNT(distinct ?s)
WHERE {
GRAPH ?graph { ?s ?p ?o }.
?s <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> ?o.
FILTER (?o IN (${dgus:1}))
}
HERE
local endpoint=${endpoints[$server]}
local mime_type=text/csv
printf "sparql $endpoint\n"
curl --no-progress-meter -G \
--header "Accept: $mime_type" \
--data-urlencode query="$query" \
$endpoint
}
# Attempts to create a table of vocab terms
#
# EXPERIMENTAL! Fragile too.
function do_vocab_table {
local server=${1:?No server name given. (Possible values: ${!endpoints[@]})}
local path="${2:?Please supply an LOD URI path}"
local dgu=${dgus[$server]}
local dgu="${dgus[$server]}/$path"
local lang="${3}"
local query formatter
# combine the object URIs. The leading comma will be stripped later
#dgus=$(printf ",<%s>" {https://w3id.solidarityeconomy.coop,$dgu}/essglobal/{V2a,2.1}/vocab/SSEInitiative)
[ -n "$lang" ] && local langFilter="FILTER (langMatches(lang(?label), \"$lang\"))"
read -rd '' query <<HERE
PREFIX skos: <http://www.w3.org/2004/02/skos/core#>
SELECT DISTINCT ?scheme ?term ?label
WHERE {
?term
a skos:Concept;
skos:inScheme ?scheme.
OPTIONAL { ?term skos:prefLabel ?label. }
$langFilter
}
HERE
read -rd '' formatter <<'HERE'
while(<>) {
chomp; @F = split "\t"; #s{".*/standard/}{"} for @F;
$h{$F[0]}{$F[1]}{$F[2]}++;
}
for $s (sort keys %h) {
print "$s\n";
for $t (sort keys %{$h{$s}}) {
print " $t\n";
for $l (sort keys %{$h{$s}{$t}}) {
print " $l (x$h{$s}{$t}{$l})\n";
}
}
}
HERE
local endpoint=${endpoints[$server]}
local mime_type=text/tab-separated-values
printf "sparql $endpoint\n"
printf "dgu: $dgu\n"
printf "lang: $lang\n"
curl --no-progress-meter -G \
--header "Accept: text/tab-separated-values" \
--data-urlencode query="$query" \
$endpoint | \
perl -ane "$formatter"
}
# Performs an arbitrary query on a SPARQL endpoint
function do_query {
server=${1:?Please supply a server name. (Possible values: ${!endpoints[@]})}
[[ -z "${endpoints[$server]}" ]] && die "No known endpoint for: $server"
endpoint="${endpoints[$server]}"
[[ -z "${dgus[$server]}" ]] && die "No default graph uri for: $server"
path="${2:?Please supply an LOD URI path}"
dgu="${dgus[$server]}/$path"
if [ -n "${3}" ]; then
query="$(<$3)"
else
query=$(cat)
fi
mime_type="text/csv"
printf "query: $endpoint\ndefault-graph: $dgu\n"
curl --no-progress-meter -G \
--header "Accept: $mime_type" \
--data-urlencode query="$query" \
--data-urlencode default-graph-uri="$dgu" \
$endpoint
printf "\n"
}
function do_show_query {
local proj=${1:?No project name given. (Possible values: ${!projects[@]})}
local server=${2:?No server name given. (Possible values: ${!endpoints[@]})}
local url=${urls["$proj@$server"]:?No project url defined for $proj@$server. (Possible values: ${!urls[@]})}
local datasets=${datasets[$proj]:?No dataset name defined. (Possible values: ${!datasets[@]})}
for dataset in $datasets; do
local endpoint=$(curl -s $url/configuration/${dataset}/endpoint.txt)
local dgu=$(curl -s $url/configuration/${dataset}/default-graph-uri.txt)
local query=$(curl -s $url/configuration/${dataset}/query.rq)
local mime_type=text/csv
printf "## Dataset $dataset\n$proj@$server: sparql\n${query}\n\n"
done
}
function doall_show_queries {
do_show_query "$@"
}
function do_show_config {
local proj=${1:?No project name given. (Possible values: ${!projects[@]})}
local server=${2:?No server name given. (Possible values: ${!endpoints[@]})}
local url=${urls["$proj@$server"]:?No project url defined for $proj@$server. (Possible values: ${!urls[@]})}
curl -s $url/configuration/config.json | jq
}
function doall_show_configs {
do_show_config "$@"
}
function do_show_version {
local proj=${1:?No project name given. (Possible values: ${!projects[@]})}
local server=${2:?No server name given. (Possible values: ${!endpoints[@]})}
local url=${urls[$proj@$server]:?No url defined for $proj@$server. (Possible values: ${!urls[@]})}
curl -s $url/configuration/version.json | jq
}
function doall_show_versions {
do_show_version "$@"
}
function do_show_lod_meta {
local proj=${1:?No project name given. (Possible values: ${!projects[@]})}
local server=${2:?No server name given. (Possible values: ${!endpoints[@]})}
local datasets=${datasets[$proj]:?No dataset name defined. (Possible values: ${!datasets[@]})}
local url=${urls[$proj@$server]:?No url defined for $proj@$server. (Possible values: ${!urls[@]})}
for dataset in $datasets; do
local endpoint="$url/configuration/${dataset}/default-graph-uri.txt"
local dgu=$(curl -sSf "$endpoint") ||
die "failed to get $endpoint"
local meta_json_url="$dgu/meta.json"
printf "## Dataset $dataset: $meta_json_url\n"
if meta_json=$(curl -LsSf "$meta_json_url"); then
printf "$meta_json" | jq
else
printf "Failed to get $meta_json_url:\n$meta_json\n";
fi
done
}
function doall_show_lod_metas {
do_show_lod_meta "$@"
}
function do_show_last_build_log {
# local proj=${1:?No project name given. (Possible values: ${!projects[@]})}
local server=${1:?No server hostname name given.}
printf "## Last se_open_data run on $server\n"
ssh $server 'journalctl _SYSTEMD_INVOCATION_ID=`systemctl show -p InvocationID --value se_open_data.service`'
}
function do_clear_data_cache {
local proj=${1:?No project name given. (Possible values: ${!projects[@]})}
local server=${2:?No server name given. (Possible values: ${!endpoints[@]})}
local path=${paths["$proj@$server"]:?No project path defined for $proj@$server. (Possible values: ${!paths[@]})}
local datasets=${datasets[$proj]:?No dataset name defined. (Possible values: ${!datasets[@]})}
ssh $server "set -x; rm $path/services/latest_timestamp.*"
ssh $server "set -x; for d in $datasets; do rm /home/seopendata/working/\$d/original-data/*.etag; done"
ssh $server "set -x; systemctl start se_open_data"
}
function doall_clear_data_caches {
do_clear_data_cache "$@"
}
function do_rcmd {
local proj=${1:?No project name given. (Possible values: ${!projects[@]})}
local server=${2:?No server name given. (Possible values: ${!endpoints[@]})}
local path=${paths["$proj@$server"]:?No project path defined for $proj@$server. (Possible values: ${!paths[@]})}
shift 2
printf "+ ssh $server \"cd $path; $*\n\""
(ssh $server "cd $path; $@")
}
function doall_rcmds {
local proj=${1:?No project name given. (Possible values: ${!projects[@]})}
local server=${2:?No server name given. (Possible values: ${!endpoints[@]})}
local path=${paths["$proj@$server"]:?No project path defined for $proj@$server}
shift 2
printf "+ ssh $server \"cd $path; $*\n\""
(ssh $server "cd $path; $@")
}
function doall_git {
local proj=${1:?No project name given. (Possible values: ${!projects[@]})}
shift 1
printf "+ git $*\n"
(cd $proj; git "$@")
}
function doall_git_branches {
local proj=${1:?No project name given. (Possible values: ${!projects[@]})}
( cd $proj; git branch -v )
}
function doall_cmds {
local proj=${1:?No project name given. (Possible values: ${!projects[@]})}
shift 1
printf "+ $*\n"
(cd $proj; "$@")
}
# Show help for this script
function do_help {
cat <<EOF
This command is a swiss-army knife for sea-map project deployment
and development. Usage:
${0##/} [options] <action> [options]
Options defined:
-h
Prints this help.
-c <config file>
Loads configuration from the given file. Otherwise,
a file ./seodo.conf is loaded. This file should be a Bash
script defining the servers, projects, urls, etc.
More details inline in the script.
Actions defined:
EOF
# Scans the script and strips out the `do_` and `doall_` function
# names, with the prefix trimmed, and then the preceeding
# comments, indented. A null delimiter follows each function
# description. The result is sorted by function name: a
# null-delimited sort which is then dead-headed and de-nulled.
sed -n -f - $0 <<EOF | sort -z | sed '1 d; s/\x00//'
1 s/.*/\x00/p # insert a blank entry, helps regularity
/^#/ { # store comment text, indented
s/^# */ /; H; d
}
/^function do/ { # print trimmed function name, then stored text
s/^function \(do_\|doall_\)\([^ ]*\).*/ \2:/
p; x; s/$/\n\x00/; p; d
}
/^#/! { # clear store on other lines
s/.*/ /; h; d
}
EOF
}
# Do the action
if [[ $(type -t "doall_$action") == "function" ]] ; then
for proj in "${!projects[@]}"; do
printf "\n== $action: $proj\n"
doall_$action $proj "$@"
printf "\n"
done
elif [[ $(type -t "do_$action") == "function" ]] ; then
do_$action "$@"
printf "\n"
else
die "stopping - don't know how to: $action"
fi
{
"projects": {
"coopsuk": "",
"covid-mutual-aid-project": "",
"dotcoop-project": "",
"ica-project": "",
"ica-youth-project": "",
"mutual-aid-project": "",
"mersey-green-project": "",
"newbridge-project": "",
"owned-by-oxford-project": "",
"oxford-project": ""
},
"paths": {
"coopsuk@dev": "/var/www/vhosts/coopsuk.solidarityeconomy.coop/www",
"covid-mutual-aid-project@dev": "/var/www/vhosts/covid-mutual-aid.solidarityeconomy.coop/www",
"dotcoop-project@dev": "/var/www/vhosts/dotcoop.solidarityeconomy.coop/www",
"ica-project@dev": "/var/www/vhosts/ica.solidarityeconomy.coop/www",
"ica-youth-project@dev": "/var/www/vhosts/ica-youth-network.solidarityeconomy.coop/www",
"mutual-aid-project@dev": "/var/www/vhosts/mutual-aid.solidarityeconomy.coop/www",
"mersey-green-project@dev": "/var/www/vhosts/mersey-green.solidarityeconomy.coop/www",
"newbridge-project@dev": "/var/www/vhosts/newbridge.solidarityeconomy.coop/www",
"owned-by-oxford-project@dev": "/var/www/vhosts/maps.solidarityeconomy.coop/www/owned-by-oxford",
"oxford-project@dev": "/var/www/vhosts/oxford.solidarityeconomy.coop/www",
"coopsuk@prod": "/var/www/vhosts/coopsuk.solidarityeconomy.coop/www",
"covid-mutual-aid-project@prod": "/var/www/vhosts/covid-mutual-aid.solidarityeconomy.coop/www",
"dotcoop-project@prod": "/var/www/vhosts/dotcoop.solidarityeconomy.coop/www",
"ica-project@prod": "/var/www/vhosts/ica.solidarityeconomy.coop/www",
"ica-youth-project@prod": "/var/www/vhosts/ica-youth-network.solidarityeconomy.coop/www",
"mutual-aid-project@prod": "/var/www/vhosts/mutual-aid.solidarityeconomy.coop/www",
"mersey-green-project@prod": "/var/www/vhosts/mersey-green.solidarityeconomy.coop/www",
"newbridge-project@prod": "/var/www/vhosts/newbridge.solidarityeconomy.coop/www",
"owned-by-oxford-project@prod": "/var/www/vhosts/maps.solidarityeconomy.coop/www/owned-by-oxford",
"oxford-project@prod": "/var/www/vhosts/oxford.solidarityeconomy.coop/www"
},
"urls": {
"coopsuk@dev": "https://dev.coopsuk.solidarityeconomy.coop",
"covid-mutual-aid-project@dev": "https://dev.covid-mutual-aid.solidarityeconomy.coop",
"dotcoop-project@dev": "https://dev.dotcoop.solidarityeconomy.coop",
"ica-project@dev": "https://dev.ica.solidarityeconomy.coop",
"ica-youth-project@dev": "https://dev.ica-youth-network.solidarityeconomy.coop",
"mersey-green-project@dev": "https://dev.mersey-green.solidarityeconomy.coop",
"mutual-aid-project@dev": "https://dev.mutual-aid.solidarityeconomy.coop",
"newbridge-project@dev": "https://dev.newbridge.solidarityeconomy.coop/map.html",
"owned-by-oxford-project@dev": "https://dev.maps.solidarityeconomy.coop/owned-by-oxford",
"oxford-project@dev": "https://dev.oxford.solidarityeconomy.coop/map.html",
"coopsuk@prod": "https://prod.coopsuk.solidarityeconomy.coop",
"covid-mutual-aid-project@prod": "https://prod.covid-mutual-aid.solidarityeconomy.coop",
"dotcoop-project@prod": "https://prod.dotcoop.solidarityeconomy.coop",
"ica-project@prod": "https://prod.ica.solidarityeconomy.coop",
"ica-youth-project@prod": "https://prod.ica-youth-network.solidarityeconomy.coop",
"mersey-green-project@prod": "https://prod.mersey-green.solidarityeconomy.coop",
"mutual-aid-project@prod": "https://prod.mutual-aid.solidarityeconomy.coop",
"newbridge-project@prod": "https://prod.newbridge.solidarityeconomy.coop/map.html",
"owned-by-oxford-project@prod": "https://prod.maps.solidarityeconomy.coop/owned-by-oxford",
"oxford-project@prod": "https://prod.oxford.solidarityeconomy.coop/map.html"
},
"endpoints": {
"dev": "http://dev-1.solidarityeconomy.coop:8890/sparql",
"prod": "http://prod-1.solidarityeconomy.coop:8890/sparql"
},
"dgus": {
"dev": "https://dev.lod.coop",
"prod": "https://lod.coop"
},
"datasets": {
"coopsuk": ["coops-uk"],
"covid-mutual-aid-project": ["covid-mutual-aid"],
"dotcoop-project": ["dotcoop"],
"ica-project": ["ica"],
"ica-youth-project": ["ica-youth-network"],
"mutual-aid-project": ["mutual-aid"],
"mersey-green-project": ["mersey-green"],
"newbridge-project": ["newbridge", "deep-adaptation"],
"owned-by-oxford-project": ["owned-by-oxford", "dotcoop", "coops-uk", "covid-mutual-aid", "good-food-oxford"],
"oxford-project": ["oxford"]
}
}
@wu-lee
Copy link
Author

wu-lee commented Sep 9, 2020

This script evolved out of the process of deploying data/map sites for 6 data sets on two servers at once, and it does help somewhat to reduce the amount of time and typing required. But it also has functions for checking the data is working, and could be the basis of some acceptance testing process.

It's poorly documented but the general idea is, put it in the directory below which the map-site repos are checked out, and then you can do things like:

  • run git status -s on all the sites: ./seodo git status -s
  • run git pull on all the sites: ./seodo git pull
  • deploy master branch of all the sites to prod-0: ./seodo deploy master prod-0 (use with care, check code first)
  • check the version.json of all the sites on dev-0: ./seodo remote_versions dev-0
  • check what get_dataset.php returns for all the sites on dev-0: ./seodo check_get_datasets dev-0
  • run the default SPARQL query for all the sites directly on dev-0: ./seodo check_queries dev-0

And so on. Just ./seodo alone will list the commands it knows.

It's rough and ready, and not especially user-friendly, but I think something like this would be invaluable to keep track of and verify the sites going forward. It could also do something like count the number of map pins using selenium (I was having problems with missing pins), or just take screenshots of all the sites with that or wkthmltopdf and display them for a visual check.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment