Last active
March 8, 2019 20:37
-
-
Save da-moon/f9a6e74848464d17f9d1a4db7101cad2 to your computer and use it in GitHub Desktop.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
// func (c *Client) ListBuckets(s3Uris []string, recursive bool, delimiter string, searchDepth int, keyRegex string) (chan *BucketItem, error) { | |
func (c *Client) ListBucketsTMP(s3Uris string, recursive bool, delimiter string, searchDepth int, keyRegex string) { | |
fn := func(listOutput *BucketItem) { | |
// if listOutput.IsPrefix { | |
// fmt.Printf("%10s %s\n", "DIR", listOutput.FullKey) | |
// } else { | |
// var size string | |
// // if humanReadable { | |
// size = fmt.Sprintf("%10s", humanize.Bytes(uint64(listOutput.Size))) | |
// // } else { | |
// // size = fmt.Sprintf("%10d", listOutput.Size) | |
// // } | |
// date := "" | |
// // if includeDates { | |
// date = " " + (listOutput.LastModified).Format("2006-01-02T15:04:05") | |
// // } | |
// fmt.Printf("%s%s %s\n", size, date, listOutput.FullKey) | |
// } | |
} | |
outChan := make(chan *BucketItem, 10000) | |
slashRegex := regexp.MustCompile("/") | |
bucketExpandedS3Uris := make([]string, 0, 1000) | |
// object queries will always have atleast 3 slashes, e.g. s3://<bucket>/<object-dir> | |
if len(slashRegex.FindAllString(s3Uris, -1)) != 2 { | |
// If taget is not root | |
bucketExpandedS3Uris = append(bucketExpandedS3Uris, s3Uris) | |
} else { | |
// root queries will always have 2 slashes, e.g. s3://<bucket-prefix> | |
// Listing root buckets | |
buckets := c.ListAllBuckets("") | |
for _, bucket := range buckets { | |
// add the bucket back to the list of s3 uris in cases where | |
// we are searching beyond the bucket | |
if recursive || searchDepth > 0 { | |
_, resp := c.client.GetBucketLocationRequest(&s3.GetBucketLocationInput{Bucket: aws.String(bucket)}) | |
// Check and see if region is location constrained.If it is and it is not | |
// in the region defined in our config or the default region ( us-east-1) then don't list it | |
region, ok := c.config.Config(ConfigRegion) | |
if !ok { | |
region = "us-east-1" | |
} | |
if resp.LocationConstraint == nil || | |
*resp.LocationConstraint == region { | |
bucketExpandedS3Uris = append(bucketExpandedS3Uris, FormatS3Uri(bucket, "")) | |
} | |
} else { | |
key := "" | |
fullKey := FormatS3Uri(bucket, "") | |
outChan <- &BucketItem{ | |
IsPrefix: true, | |
Key: key, | |
FullKey: fullKey, | |
LastModified: time.Time{}, | |
Size: 0, | |
Bucket: bucket, | |
} | |
} | |
} | |
} | |
go func() { | |
defer close(outChan) | |
for i := 0; i < searchDepth; i++ { | |
newS3Uris := make([]string, 0) | |
// List all items | |
ch := make(chan *BucketItem, 10000) | |
var wg sync.WaitGroup | |
for _, uri := range bucketExpandedS3Uris { | |
wg.Add(1) | |
go func(uri string) { | |
defer wg.Done() | |
for itm := range c.GetBucketItem(uri, false, delimiter, keyRegex) { | |
ch <- itm | |
} | |
}(uri) | |
} | |
go func() { | |
wg.Wait() | |
close(ch) | |
}() | |
for itm := range ch { | |
if itm.IsPrefix { | |
newS3Uris = append(newS3Uris, strings.TrimRight(itm.FullKey, delimiter)+delimiter) | |
} else { | |
outChan <- itm | |
} | |
fmt.Printf("%10s %s\n", "ITM", itm.FullKey) | |
} | |
bucketExpandedS3Uris = newS3Uris | |
} | |
// for itm := range c.listAll(bucketExpandedS3Uris, recursive, delimiter, keyRegex) { | |
// outChan <- itm | |
// } | |
}() | |
for listOutput := range outChan { | |
fn(listOutput) | |
// fmt.Printf("%10s %s\n", "DIR", listOutput.FullKey) | |
} | |
} |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Data at rest encryption | |
------------------------------------------------------ | |
https://en.wikipedia.org/wiki/Data_at_rest | |
https://github.com/tristanls/data-at-rest | |
https://cloudonaut.io/encrypting-sensitive-data-stored-on-s3/ | |
https://github.com/widdix/s3-at-rest-encryption | |
https://pusher.com/sessions/meetup/viennajs/client-side-encryption-with-javascript | |
====================================================== | |
Two-level homomorphic encryption for Node.js by WebAssembly | |
------------------------------------------------------ | |
https://github.com/herumi/she-wasm | |
====================================================== | |
Cross-platform library for secure data storage, message exchange, socket connections, and authentication | |
------------------------------------------------------ | |
https://www.cossacklabs.com/themis/ | |
https://www.cossacklabs.com/zero-knowledge-protocols-without-magic.html | |
https://github.com/cossacklabs/webthemis | |
https://github.com/cossacklabs/themis | |
====================================================== | |
TS library with implementations of secp256k1, ripemd160, sha256, sha512, and sha1) in wasm | |
------------------------------------------------------ | |
https://github.com/bitjson/bitcoin-ts | |
------------------------------------------------------ | |
Benchmarks : | |
https://github.com/bitjson/bitcoin-ts/wiki/Benchmarks | |
====================================================== | |
WebAssembly implementation of Ed25519-based operations and more. | |
https://github.com/jedisct1/wasm-crypto | |
====================================================== |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
an api library by minio that simple provides APIs to access any Amazon S3 compatible object storage server | |
https://github.com/minio/minio-js | |
====================================================== |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
http://marcio.io/2015/07/handling-1-million-requests-per-minute-with-golang/ | |
https://medium.com/@odeke_et/tame-concurrency-in-go-with-a-simple-channel-fb872723cabc |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
https://learnxinyminutes.com/docs/go/ | |
https://www.goin5minutes.com/screencasts/ |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
go get -v github.com/minio/mc/... |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
golang tls for http/http2 examples | |
------------------------------------------------------ | |
https://github.com/smallnest/golang-tls | |
====================================================== |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Javascript to Typescript transpiler | |
------------------------------------------------------ | |
https://github.com/Lulubul/JSExperiment | |
https://github.com/ekoneko/es2ts | |
====================================================== |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment