Last active
October 27, 2017 18:01
-
-
Save mariusae/0587ba225232b619aad6f8108f2f9920 to your computer and use it in GitHub Desktop.
How to manipulate and export file trees.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
/* | |
How to manipulate and export file trees. | |
The system module $/dirs contains a number of utilities for creating | |
and copying directories: | |
% reflow doc $/dirs | |
Declarations | |
val Groups func(dir dir, re string) [string:dir] | |
Groups assigns each path in a directory to a group according to the passed-in | |
regular expressio, which must have exactly one regexp group. Paths that do not | |
match the expression are filtered out. Group returns a map that maps each group | |
key to a directory of matched values. | |
val Make func(map [string:file]) dir | |
Make creates a new dir using the given map of paths to files. | |
val Pick func(dir dir, pattern string) (file, string) | |
Pick returns the first file in a directory matching a glob pattern. Pick fails | |
if no files match. | |
val Files func(dir dir) [file] | |
Files returns a sorted (by filename) list of files from a directory. | |
val Copy func(dir dir, url string) unit | |
Copy copies the directory to an extern location. | |
*/ | |
val dirs = make("$/dirs") | |
val image = "ubuntu" | |
// First, an exec can produce a directory directly: | |
val dirFromExec = exec(image, cpu := 1) (outdir dir) {" | |
echo 1 > {{outdir}}/1 | |
echo 2 > {{outdir}}/2 | |
"} | |
// We can turn a directory into a map, and vice-versa (through dirs.Make). | |
val files = map(dirFromExec) | |
// We can compose a new tree from both directories and new files. | |
val newFile = exec(image, cpu := 1) (out file) {" | |
echo foobar > {{out}} | |
"} | |
// This is the new file tree (types for exposition) | |
val tree [string:file] = [ | |
"foobar": newFile, | |
// As in Javascript, we can concatenate maps and lists with this | |
// "explosion" syntax. | |
...files, | |
] | |
val Main = { | |
newdir := dirs.Make(tree) | |
dirs.Copy(newdir, "s3://grail-marius/testdir") | |
} | |
/* | |
Example run: | |
% reflow run filetree.rf | |
2017/10/27 10:57:33 run name: [email protected]/69170181 | |
2017/10/27 10:57:57 ec2cluster: launched spot instance i-0e027d054752fb273: m3.medium: 3.6GiB 1 2.4TiB | |
2017/10/27 10:58:54 -> filetree.dirFromExec 1b2c17bb run exec ubuntu echo 1 > {{outdir}}/1.echo 2 > {{outdir}}/2 | |
2017/10/27 10:59:08 <- filetree.dirFromExec 1b2c17bb ok exec 0s 4B | |
2017/10/27 10:59:08 -> filetree.newFile 8c315f0f run exec ubuntu echo foobar > {{out}} | |
2017/10/27 10:59:09 <- filetree.newFile 8c315f0f ok exec 0s 7B | |
2017/10/27 10:59:09 -> filetree.Main 093e0702 run extern s3://grail-marius/testdir/ 11B | |
2017/10/27 10:59:22 <- filetree.Main 093e0702 ok extern 2s 0B | |
2017/10/27 10:59:23 total n=3 time=29s | |
ident n ncache runtime(m) cpu mem(GiB) disk(GiB) tmp(GiB) | |
filetree.Main 1 0 | |
filetree.newFile 1 0 | |
filetree.dirFromExec 1 0 | |
val<> | |
% aws s3 ls s3://grail-marius/testdir/ | |
2017-10-27 10:59:23 2 1 | |
2017-10-27 10:59:23 2 2 | |
2017-10-27 10:59:23 7 foobar | |
% | |
*/ |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment