read_enc_data

readf.go

package cmd

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"io"
	"log"

	"github.com/fatih/structs"
)

//ID		Name	Key     Mode 	IV  AuthTag	KDA 	        Sig
// 03 78 	AES 	256 	GCM 	12 	16 	HKDF with SHA-384 	ECDSA with P-384 and SHA-384
// 03 46 	AES 	192 	GCM 	12 	16 	HKDF with SHA-384 	ECDSA with P-384 and SHA-384
// 02 14 	AES 	128 	GCM 	12 	16 	HKDF with SHA-256 	ECDSA with P-256 and SHA-256
// 01 78 	AES 	256 	GCM 	12 	16 	HKDF with SHA-256 	Not applicable
// 01 46 	AES 	192 	GCM 	12 	16 	HKDF with SHA-256 	Not applicable
// 01 14 	AES 	128 	GCM 	12 	16 	HKDF with SHA-256 	Not applicable
// 00 78 	AES 	256 	GCM 	12 	16 	Not applicable 	Not applicable
// 00 46 	AES 	192 	GCM 	12 	16 	Not applicable 	Not applicable
// 00 14 	AES 	128 	GCM 	12 	16 	Not applicable 	Not applicable

//EncryptedFileHeaders represents the header for the ciphertext blob
//This is common for both framed and non-framed files
type EncryptedFileHeaders struct {
	Version [1]byte //current version is 1.0 encoded in hex as 01

	// Field 	Length, in bytes
	// Version 	1
	// Type 	1
	// Algorithm ID 	2
	// Message ID 	16
	// AAD Length 	2
	// AAD 	Variable. Equal to the value specified in the previous 2 bytes (AAD Length).
	// Encrypted Data Key Count 	2
	// Encrypted Data Key(s) 	Variable. Determined by the number of encrypted data keys and the length of each.
	// Content Type 	1
	// Reserved 	4
	// IV Length 	1
	// Frame Length 	4
	// Header Authentication 	Variable. Determined by the algorithm that generated the message.
	//Current crypto type is customer authenticated encrypted data, type value is 128 encoded as 80 in hex
	CryptoType [1]byte

	// Algorithm ID (in 2-byte hex) 	Algorithm Name 	Data Key Length (in bits) 	Algorithm Mode 	IV Length (in bytes) 	Authentication Tag Length (in bytes) 	Key Derivation Algorithm 	Signature Algorithm
	// 03 78 	AES 	256 	GCM 	12 	16 	HKDF with SHA-384 	ECDSA with P-384 and SHA-384
	// 03 46 	AES 	192 	GCM 	12 	16 	HKDF with SHA-384 	ECDSA with P-384 and SHA-384
	// 02 14 	AES 	128 	GCM 	12 	16 	HKDF with SHA-256 	ECDSA with P-256 and SHA-256
	// 01 78 	AES 	256 	GCM 	12 	16 	HKDF with SHA-256 	Not applicable
	// 01 46 	AES 	192 	GCM 	12 	16 	HKDF with SHA-256 	Not applicable
	// 01 14 	AES 	128 	GCM 	12 	16 	HKDF with SHA-256 	Not applicable
	// 00 78 	AES 	256 	GCM 	12 	16 	Not applicable 	Not applicable
	// 00 46 	AES 	192 	GCM 	12 	16 	Not applicable 	Not applicable
	// 00 14 	AES 	128 	GCM 	12 	16 	Not applicable 	Not applicable
	AlgoID [2]byte

	//A randomly generated 128-bit value that identifies the message. The Message ID:

	// Uniquely identifies the encrypted message.

	// Weakly binds the message header to the message body.

	// Provides a mechanism to securely reuse a data key with multiple encrypted messages.

	// Protects against accidental reuse of a data key or the wearing out of keys in the AWS Encryption SDK
	MessageID [16]byte
	//AAD length?
	// The length of the additional authenticated data (AAD). It is a 2-byte value interpreted as a 16-bit unsigned integer that specifies the number of bytes that contain the AAD.
	LengthAAD [2]byte

	AdditionalAuthenticatedData []byte
	//How many KV pairs?
	// KvPairCount [2]byte
	// KeyLength   [2]byte
	// Key         []byte
	// ValLength   []byte
	// Val         []byte
	//Encrypted data key count
	EncryptedDataKeyCount [2]byte
	//EncryptedDataKey
	EncryptedDataKeys []byte
	// KeyProviderIdLength    [2]byte //usually KMS?
	// KeyProviderId          []byte  //has length as above
	// KeyProviderInfoLength  [2]byte
	// KeyProviderInfo        []byte
	// EncryptedDataKeyLength [2]byte //how long is the data key?
	// EncryptedDataKey       []byte

	ContentType          [1]byte
	Reserved             [4]byte
	IvLength             [1]byte
	FrameLength          [4]byte
	IV                   []byte
	HeaderAuthentication []byte
}

//required to write bytes in order
var sortOrderHeader = []string{
	"Version",
	"CryptoType",
	"AlgoID",
	"MessageID",
	"LengthAAD",
	"AdditionalAuthenticatedData",
	"EncryptedDataKeyCount",
	"EncryptedDataKeys",
	"ContentType",
	"Reserved",
	"IvLength",
	"FrameLength",
	"IV",
	"HeaderAuthentication",
}

//EncryptedNonFramedData represents a single blob of authenticated ciphertext that has not
//been broken down into frames
type EncryptedNonFramedData struct {
	Iv                     []byte
	EncryptedContentLength [8]byte
	EncryptedContent       []byte
	//Auth tag for the message body
	AuthenticationTag []byte
}

var sortOrderEncryptedNonFramedData = []string{
	"Iv",
	"EncryptedContentLength",
	"EncryptedContent",
	"AuthenticationTag",
}

//EncryptedRegularFrame represents the non-final frame in an encrypted blob (framed file)
type EncryptedRegularFrame struct {

	// The frame sequence number. It is an incremental counter number for the frame. It is a 4-byte value interpreted as a 32-bit unsigned integer.

	// Framed data must start at sequence number 1. Subsequent frames must be in order and must contain an increment of 1 of the previous frame. Otherwise, the decryption process stops and reports an error.

	SequenceNum [4]byte
	// The initialization vector (IV) for the frame. The SDK uses a deterministic method to construct a different IV for each frame in the message. Its length is specified by the algorithm suite used.
	Iv                []byte
	EncryptedContent  []byte
	AuthenticationTag []byte
}

var sortOrderEncryptedRegularFrame = []string{
	"SequenceNum",
	"Iv",
	"EncryptedContent",
	"AuthenticationTag",
}

//EncryptedFinalFrame represents the last frame in a framed encrypted file
//All framed files have this frame
type EncryptedFinalFrame struct {
	// Length, in bytes
	// Sequence Number End 	4
	// Sequence Number 	4
	// IV 	Variable. Equal to the value specified in the IV Length byte of the header.
	// Encrypted Content Length 	4
	// Encrypted Content 	Variable. Equal to the value specified in the previous 4 bytes (Encrypted Content Length).
	// Authentication Tag 	Variable. Determined by the algorithm used, as specified in the Algorithm ID of the header.
	SequenceNumberEnd []byte
	//Framed data must start at sequence number 1. Subsequent frames must be in order and must contain an increment of 1 of the previous frame. Otherwise, the decryption process stops and reports an error.
	SequenceNumber [4]byte
	// The initialization vector (IV) for the frame. The SDK uses a deterministic method to construct a different IV for each frame in the message. The length of the IV length is specified by the algorithm suite.
	Iv                     []byte
	EncryptedContentLength []byte
	EncryptedContent       []byte
	AuthenticationTag      []byte
}

var sortOrderEncryptedFinalFrame = []string{
	"SequenceNumberEnd",
	"SequenceNumber",
	"Iv",
	"EncryptedContentLength",
	"EncryptedContent",
	"AuthenticationTag",
}

//EncryptedFramedData represents one or more EncryptedRegularFrames and one final frame that form
//body of th ciphertext
type EncryptedFramedData struct {
	EncryptedRegularFrames []EncryptedRegularFrame
	EncryptedFinalFrame    EncryptedFinalFrame
}

var sortOrderEncryptedFramedData = []string{
	"EncryptedRegularFrame",
	"EncryptedFinalFrame",
}

//EncryptedFooter represents the footer of the encrypted file, common for both framed and non-framed files
type EncryptedFooter struct {
	// When the algorithms with signing are used, the message format contains a footer. The message footer contains a signature calculated over the message header and body. The following table describes the fields that form the footer. The bytes are appended in the order shown.

	SignatureLength [2]byte
	Signature       []byte
}

var sortOrderFooter = []string{
	"SignatureLength",
	"Signature",
}

//EncryptedNonFramedFile represents an encrypted file without frames
type EncryptedNonFramedFile struct {
	Header *EncryptedFileHeaders
	Body   *EncryptedNonFramedData
	Footer *EncryptedFooter
}

//EncryptedFramedFile represents an encrypted file that is framed
type EncryptedFramedFile struct {
	Header *EncryptedFileHeaders
	Body   *EncryptedFramedData
	Footer *EncryptedFooter
}

//AAD represents the additional authetnicated data in the header
type AAD struct {
	// The additional authenticated data. The AAD is an encoding of the encryption context, an array of key-value pairs where each key and value is a string of UTF-8 encoded characters. The encryption context is converted to a sequence of bytes and used for the AAD value.

	// When the algorithms with signing are used, the encryption context must contain the key-value pair {'aws-crypto-public-key', Qtxt}. Qtxt represents the elliptic curve point Q compressed according to SEC 1 version 2.0 and then base64-encoded. The encryption context can contain additional values, but the maximum length of the constructed AAD is 2^16 - 1 bytes.

	// The following table describes the fields that form the AAD. Key-value pairs are sorted, by key, in ascending order according to UTF-8 character code. The bytes are appended in the order shown.

	//How many KV pairs?
	KvPairCount [2]byte
	KeyLength   [2]byte
	Key         []byte
	ValLength   []byte
	Val         []byte
}

//EncryptedDataKeys represents one or more data keys used to encrypt the file
type EncryptedDataKeys struct {

	// Encrypted Data Key Structure
	// Field 	Length, in bytes
	// Key Provider ID Length 	2
	// Key Provider ID 	Variable. Equal to the value specified in the previous 2 bytes (Key Provider ID Length).
	// Key Provider Information Length 	2
	// Key Provider Information 	Variable. Equal to the value specified in the previous 2 bytes (Key Provider Information Length).
	// Encrypted Data Key Length 	2
	// Encrypted Data Key 	Variable. Equal to the value specified in the previous 2 bytes (Encrypted Data Key Length).

	KeyProviderIDLength    [2]byte //usually KMS?
	KeyProviderID          []byte  //has length as above
	KeyProviderInfoLength  [2]byte
	KeyProviderInfo        []byte
	EncryptedDataKeyLength [2]byte //how long is the data key?
	EncryptedDataKey       []byte
}

//MarshalBinary returns a binary representation of an encrypted, non framed file
func (e *EncryptedNonFramedFile) MarshalBinary() ([]byte, error) {
	b := new(bytes.Buffer)

	//Header first
	m := structs.Map(&e.Header)
	for _, v := range sortOrderHeader {
		err := binary.Write(b, binary.LittleEndian, m[v])
		if err != nil {
			return nil, err
		}
	}
	//Body next
	m = structs.Map(&e.Body)
	for _, v := range sortOrderEncryptedNonFramedData {
		err := binary.Write(b, binary.LittleEndian, m[v])
		if err != nil {
			return nil, err
		}
	}

	//Footer last
	m = structs.Map(&e.Footer)
	for _, v := range sortOrderFooter {
		err := binary.Write(b, binary.LittleEndian, m[v])
		if err != nil {
			return nil, err
		}
	}

	return b.Bytes(), nil

}

//MarshalBinary returns a binary representation of an encrypted, framed file
func (e *EncryptedFramedFile) MarshalBinary() ([]byte, error) {
	b := new(bytes.Buffer)

	//Header first
	m := structs.Map(&e.Header)
	for _, v := range sortOrderHeader {
		err := binary.Write(b, binary.LittleEndian, m[v])
		if err != nil {
			return nil, err
		}
	}
	//Body Next
	encRegularFrame := e.Body.EncryptedRegularFrames
	encFinalFrame := e.Body.EncryptedFinalFrame

	fmt.Printf("%v, %v", encRegularFrame, encFinalFrame)

	if len(e.Body.EncryptedRegularFrames) > 0 {
		for _, v := range e.Body.EncryptedRegularFrames {
			m = structs.Map(v)
			for _, x := range sortOrderEncryptedRegularFrame {
				err := binary.Write(b, binary.LittleEndian, m[x])
				if err != nil {
					log.Fatalf("err %v", err)
				}
			}
		}
	}

	m = structs.Map(encFinalFrame)
	for _, v := range sortOrderEncryptedFinalFrame {
		err := binary.Write(b, binary.LittleEndian, m[v])
		if err != nil {
			log.Fatalf("err %v", err)
		}
	}

	m = structs.Map(&e.Footer)
	for _, v := range sortOrderFooter {
		err := binary.Write(b, binary.LittleEndian, m[v])
		if err != nil {
			return nil, err
		}
	}

	return b.Bytes(), nil

}

func readEncryptedData(f io.Reader) ([]byte, error) {
	var encBytes []byte
	//Begin parsing header
	efh := &EncryptedFileHeaders{}
	n, err := f.Read(efh.Version[:])
	checkErr(n, err, len(efh.Version))
	//	fmt.Printf("e.version is %x\n", efh.Version)

	n, err = f.Read(efh.CryptoType[:])
	checkErr(n, err, len(efh.CryptoType))
	//	fmt.Printf("e.cryptotype is %x\n", efh.CryptoType)

	n, err = f.Read(efh.AlgoID[:])
	checkErr(n, err, len(efh.AlgoID))
	//	fmt.Printf("e.algoId is %x\n", efh.AlgoID)

	n, err = f.Read(efh.MessageID[:])
	checkErr(n, err, len(efh.MessageID))
	//	fmt.Printf("e.message id is %x\n", efh.MessageID)

	n, err = f.Read(efh.LengthAAD[:])
	checkErr(n, err, len(efh.LengthAAD))
	//	fmt.Printf("e.lengthAAD is %v\n", efh.LengthAAD)

	aadlength := binary.BigEndian.Uint16(efh.LengthAAD[:])
	//	fmt.Printf("aad is %x bytes\n", aadlength)

	efh.AdditionalAuthenticatedData = make([]byte, aadlength)
	_, err = f.Read(efh.AdditionalAuthenticatedData[:])
	if err != nil {
		log.Fatalf("couldn't read any aad")
	}

	n, err = f.Read(efh.EncryptedDataKeyCount[:])
	checkErr(n, err, len(efh.EncryptedDataKeyCount))

	numEncryptedDataKeys := binary.BigEndian.Uint16(efh.EncryptedDataKeyCount[:])
	//	fmt.Printf("total num of keys is %x\n", numEncryptedDataKeys)

	encryptedDataKeyStructure := new(bytes.Buffer)

	//Read the key data
	for x := uint16(0); x < numEncryptedDataKeys; x++ {
		keyProviderIdLengthByte := make([]byte, 2)
		n, err := f.Read(keyProviderIdLengthByte)

		//		fmt.Printf("keyProviderIdlengthbyte %v", keyProviderIdLengthByte)
		checkErr(n, err, len(keyProviderIdLengthByte))

		err = binary.Write(encryptedDataKeyStructure, binary.BigEndian, keyProviderIdLengthByte)
		//	//Figure out the length
		keyProviderIdLength := binary.BigEndian.Uint16(keyProviderIdLengthByte[:])
		//		fmt.Printf("keyprovideridlength is %v", keyProviderIdLength)

		t := make([]byte, keyProviderIdLength)
		n, err = f.Read(t)
		checkErr(n, err, len(t))

		err = binary.Write(encryptedDataKeyStructure, binary.BigEndian, t)
		//		fmt.Printf("the buffer is now %x\n", encryptedDataKeyStructure.Bytes())

		keyProviderInfoLengthByte := make([]byte, 2)
		n, err = f.Read(keyProviderInfoLengthByte)

		//		fmt.Printf("keyProviderInfolengthbyte %v", keyProviderInfoLengthByte)
		checkErr(n, err, len(keyProviderInfoLengthByte))

		err = binary.Write(encryptedDataKeyStructure, binary.BigEndian, keyProviderInfoLengthByte)

		//	//Figure out the length

		keyProviderInfoLength := binary.BigEndian.Uint16(keyProviderInfoLengthByte[:])
		//		fmt.Printf("keyprovider info length is %v\n", keyProviderInfoLength)

		t = make([]byte, keyProviderInfoLength)
		n, err = f.Read(t)
		checkErr(n, err, len(t))

		err = binary.Write(encryptedDataKeyStructure, binary.BigEndian, t)
		//		fmt.Printf("the buffer is now %x\n", encryptedDataKeyStructure.Bytes())

		//Encrypteddata keys
		dataKeyLengthByte := make([]byte, 2)
		n, err = f.Read(dataKeyLengthByte)

		//		fmt.Printf("data keylength %v\n", dataKeyLengthByte)
		checkErr(n, err, len(dataKeyLengthByte))

		err = binary.Write(encryptedDataKeyStructure, binary.BigEndian, dataKeyLengthByte)

		//	//Figure out the length

		dataKeyLength := binary.BigEndian.Uint16(dataKeyLengthByte[:])
		//		fmt.Printf("datakey length is %v\n", dataKeyLength)

		t = make([]byte, dataKeyLength)
		n, err = f.Read(t)
		checkErr(n, err, len(t))

		err = binary.Write(encryptedDataKeyStructure, binary.BigEndian, t)
		//		fmt.Printf("the buffer is now %x\n", encryptedDataKeyStructure.Bytes())
	}

	efh.EncryptedDataKeys = encryptedDataKeyStructure.Bytes()

	//Figure out if this is a framed or un-framed file
	//Usually framed
	n, err = f.Read(efh.ContentType[:])
	checkErr(n, err, len(efh.ContentType))
	//	fmt.Printf("\ncontent type is%x", efh.ContentType)

	var framed bool
	if bytes.Equal([]byte{0x01}, efh.ContentType[:]) {
		framed = false
	} else if bytes.Equal([]byte{0x02}, efh.ContentType[:]) {
		framed = true
	}

	n, err = f.Read(efh.Reserved[:])
	checkErr(n, err, len(efh.Reserved))
	//	fmt.Printf("\nreserved is%x", efh.Reserved)

	n, err = f.Read(efh.IvLength[:])
	checkErr(n, err, len(efh.IvLength))
	//	fmt.Printf("\nivlength is%x", efh.IvLength)

	//TODO: either a bug in the python SDK or a bug in how I'm reading the framelength
	//This is
	n, err = f.Read(efh.FrameLength[:])
	checkErr(n, err, len(efh.FrameLength))
	//	fmt.Printf("\nframe length is%x", efh.FrameLength)

	efh.IV = make([]byte, efh.IvLength[0])
	n, err = f.Read(efh.IV[:])
	checkErr(n, err, len(efh.IV))
	//	fmt.Printf("\nIV is%x", efh.IV)

	//TODO: auth tag lenght may not always be 16!
	efh.HeaderAuthentication = make([]byte, 16)
	n, err = f.Read(efh.HeaderAuthentication[:])
	checkErr(n, err, len(efh.HeaderAuthentication))
	//	fmt.Printf("\nauth tag is%x", efh.HeaderAuthentication)
	//End of header

	var encFramedFile *EncryptedFramedFile
	var efd *EncryptedFramedData
	var footer *EncryptedFooter

	switch framed {

	case framed:
		efd = &EncryptedFramedData{}
		footer = &EncryptedFooter{}
		encFramedFile = &EncryptedFramedFile{}
		seq := make([]byte, 4)
		n := 1
		for {
			//Run until we see the final frame
			//			fmt.Printf("\running loop %d\n", n)
			n, err = f.Read(seq)
			checkErr(n, err, len(seq))
			//			fmt.Printf("\nseq num is %x", seq)
			if bytes.Equal([]byte{0xff, 0xff, 0xff, 0xff}, seq) {
				break
			}

			regFrame := &EncryptedRegularFrame{}
			if n := copy(regFrame.SequenceNum[:], seq); n != len(seq) {
				log.Fatalf("wanted to copy %v, but could only copy %v", len(seq), n)
			}

			// n, err = f.Read(regFrame.SequenceNum[:])
			// checkErr(n, err, len(regFrame.SequenceNum))

			regFrame.Iv = make([]byte, efh.IvLength[0])
			n, err = f.Read(regFrame.Iv)
			checkErr(n, err, len(regFrame.Iv))

			encryptedContentLength := binary.BigEndian.Uint32(efh.FrameLength[:])

			regFrame.EncryptedContent = make([]byte, encryptedContentLength)
			n, err = f.Read(regFrame.EncryptedContent)
			checkErr(n, err, len(regFrame.EncryptedContent))

			regFrame.AuthenticationTag = make([]byte, 16)
			n, err = f.Read(regFrame.AuthenticationTag)
			checkErr(n, err, len(regFrame.AuthenticationTag))
			//			fmt.Printf("\nRead auth data for header\n")

			//Probably better way to do this
			// encFramedFile.Body.EncryptedRegularFrame = append(encFramedFile.Body.EncryptedRegularFrame, *regFrame)
			efd.EncryptedRegularFrames = append(efd.EncryptedRegularFrames, *regFrame)
			//			fmt.Printf("read body")
			n++
		}

		efd.EncryptedFinalFrame.SequenceNumberEnd = seq
		n, err = f.Read(efd.EncryptedFinalFrame.SequenceNumber[:])
		checkErr(n, err, len(efd.EncryptedFinalFrame.SequenceNumber))
		//log.Printf("getting seq number")
		efd.EncryptedFinalFrame.Iv = make([]byte, efh.IvLength[0])
		n, err = f.Read(efd.EncryptedFinalFrame.Iv)
		checkErr(n, err, len(efd.EncryptedFinalFrame.Iv))
		//		log.Printf("read iv")

		encFrameContentLength := make([]byte, 4)
		n, err = f.Read(encFrameContentLength)
		checkErr(n, err, len(encFrameContentLength))
		//		log.Printf("encFrameContentlength")

		efd.EncryptedFinalFrame.EncryptedContentLength = encFrameContentLength
		//How much to read
		encFrameContentLengthLen := binary.BigEndian.Uint32(encFrameContentLength[:])
		efd.EncryptedFinalFrame.EncryptedContent = make([]byte, encFrameContentLengthLen)
		n, err = f.Read(efd.EncryptedFinalFrame.EncryptedContent)
		checkErr(n, err, len(efd.EncryptedFinalFrame.EncryptedContent))
		//		log.Printf("Read enc final frame")

		//Read the auth tag
		efd.EncryptedFinalFrame.AuthenticationTag = make([]byte, 16)
		n, err = f.Read(efd.EncryptedFinalFrame.AuthenticationTag)
		checkErr(n, err, len(efd.EncryptedFinalFrame.AuthenticationTag))
		//At final frame so next is footer
		n, err = f.Read(footer.SignatureLength[:])
		checkErr(n, err, len(footer.SignatureLength))
		//		log.Printf("Read footer sig")

		sigLength := binary.BigEndian.Uint16(footer.SignatureLength[:])
		footer.Signature = make([]byte, sigLength)
		//		log.Printf("Read footer sig2 %v bytes", sigLength)

		n, err = f.Read(footer.Signature[:])
		checkErr(n, err, len(footer.Signature))

		//Write out a test file
		encFramedFile.Header = efh
		encFramedFile.Body = efd
		encFramedFile.Footer = footer
		encBytes, err = encFramedFile.MarshalBinary()
		if err != nil {
			return nil, err
		}
	default:
		log.Fatal(" not sure what file this is, not framed or un-framed ErrWhat")
	}

	return encBytes, nil

}
func checkErr(n int, err error, bytesRead int) {
	if n != bytesRead {
		log.Fatalf("wanted to read %v, read only %v", bytesRead, n)
	}
	if err != nil {
		log.Fatalf("error! %v", err)
	}
}