Skip to content

Instantly share code, notes, and snippets.

@da-moon
Last active March 8, 2019 20:37
Show Gist options
  • Save da-moon/f9a6e74848464d17f9d1a4db7101cad2 to your computer and use it in GitHub Desktop.
Save da-moon/f9a6e74848464d17f9d1a4db7101cad2 to your computer and use it in GitHub Desktop.
// func (c *Client) ListBuckets(s3Uris []string, recursive bool, delimiter string, searchDepth int, keyRegex string) (chan *BucketItem, error) {
func (c *Client) ListBucketsTMP(s3Uris string, recursive bool, delimiter string, searchDepth int, keyRegex string) {
fn := func(listOutput *BucketItem) {
// if listOutput.IsPrefix {
// fmt.Printf("%10s %s\n", "DIR", listOutput.FullKey)
// } else {
// var size string
// // if humanReadable {
// size = fmt.Sprintf("%10s", humanize.Bytes(uint64(listOutput.Size)))
// // } else {
// // size = fmt.Sprintf("%10d", listOutput.Size)
// // }
// date := ""
// // if includeDates {
// date = " " + (listOutput.LastModified).Format("2006-01-02T15:04:05")
// // }
// fmt.Printf("%s%s %s\n", size, date, listOutput.FullKey)
// }
}
outChan := make(chan *BucketItem, 10000)
slashRegex := regexp.MustCompile("/")
bucketExpandedS3Uris := make([]string, 0, 1000)
// object queries will always have atleast 3 slashes, e.g. s3://<bucket>/<object-dir>
if len(slashRegex.FindAllString(s3Uris, -1)) != 2 {
// If taget is not root
bucketExpandedS3Uris = append(bucketExpandedS3Uris, s3Uris)
} else {
// root queries will always have 2 slashes, e.g. s3://<bucket-prefix>
// Listing root buckets
buckets := c.ListAllBuckets("")
for _, bucket := range buckets {
// add the bucket back to the list of s3 uris in cases where
// we are searching beyond the bucket
if recursive || searchDepth > 0 {
_, resp := c.client.GetBucketLocationRequest(&s3.GetBucketLocationInput{Bucket: aws.String(bucket)})
// Check and see if region is location constrained.If it is and it is not
// in the region defined in our config or the default region ( us-east-1) then don't list it
region, ok := c.config.Config(ConfigRegion)
if !ok {
region = "us-east-1"
}
if resp.LocationConstraint == nil ||
*resp.LocationConstraint == region {
bucketExpandedS3Uris = append(bucketExpandedS3Uris, FormatS3Uri(bucket, ""))
}
} else {
key := ""
fullKey := FormatS3Uri(bucket, "")
outChan <- &BucketItem{
IsPrefix: true,
Key: key,
FullKey: fullKey,
LastModified: time.Time{},
Size: 0,
Bucket: bucket,
}
}
}
}
go func() {
defer close(outChan)
for i := 0; i < searchDepth; i++ {
newS3Uris := make([]string, 0)
// List all items
ch := make(chan *BucketItem, 10000)
var wg sync.WaitGroup
for _, uri := range bucketExpandedS3Uris {
wg.Add(1)
go func(uri string) {
defer wg.Done()
for itm := range c.GetBucketItem(uri, false, delimiter, keyRegex) {
ch <- itm
}
}(uri)
}
go func() {
wg.Wait()
close(ch)
}()
for itm := range ch {
if itm.IsPrefix {
newS3Uris = append(newS3Uris, strings.TrimRight(itm.FullKey, delimiter)+delimiter)
} else {
outChan <- itm
}
fmt.Printf("%10s %s\n", "ITM", itm.FullKey)
}
bucketExpandedS3Uris = newS3Uris
}
// for itm := range c.listAll(bucketExpandedS3Uris, recursive, delimiter, keyRegex) {
// outChan <- itm
// }
}()
for listOutput := range outChan {
fn(listOutput)
// fmt.Printf("%10s %s\n", "DIR", listOutput.FullKey)
}
}
Data at rest encryption
------------------------------------------------------
https://en.wikipedia.org/wiki/Data_at_rest
https://github.com/tristanls/data-at-rest
https://cloudonaut.io/encrypting-sensitive-data-stored-on-s3/
https://github.com/widdix/s3-at-rest-encryption
https://pusher.com/sessions/meetup/viennajs/client-side-encryption-with-javascript
======================================================
Two-level homomorphic encryption for Node.js by WebAssembly
------------------------------------------------------
https://github.com/herumi/she-wasm
======================================================
Cross-platform library for secure data storage, message exchange, socket connections, and authentication
------------------------------------------------------
https://www.cossacklabs.com/themis/
https://www.cossacklabs.com/zero-knowledge-protocols-without-magic.html
https://github.com/cossacklabs/webthemis
https://github.com/cossacklabs/themis
======================================================
TS library with implementations of secp256k1, ripemd160, sha256, sha512, and sha1) in wasm
------------------------------------------------------
https://github.com/bitjson/bitcoin-ts
------------------------------------------------------
Benchmarks :
https://github.com/bitjson/bitcoin-ts/wiki/Benchmarks
======================================================
WebAssembly implementation of Ed25519-based operations and more.
https://github.com/jedisct1/wasm-crypto
======================================================
an api library by minio that simple provides APIs to access any Amazon S3 compatible object storage server
https://github.com/minio/minio-js
======================================================
http://marcio.io/2015/07/handling-1-million-requests-per-minute-with-golang/
https://medium.com/@odeke_et/tame-concurrency-in-go-with-a-simple-channel-fb872723cabc
https://learnxinyminutes.com/docs/go/
https://www.goin5minutes.com/screencasts/
go get -v github.com/minio/mc/...
golang tls for http/http2 examples
------------------------------------------------------
https://github.com/smallnest/golang-tls
======================================================
Javascript to Typescript transpiler
------------------------------------------------------
https://github.com/Lulubul/JSExperiment
https://github.com/ekoneko/es2ts
======================================================
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment