Skip to content

Instantly share code, notes, and snippets.

@steinybot
Last active December 15, 2020 06:22
Show Gist options
  • Select an option

  • Save steinybot/068d5e50c5f11824dcf776c380b7e881 to your computer and use it in GitHub Desktop.

Select an option

Save steinybot/068d5e50c5f11824dcf776c380b7e881 to your computer and use it in GitHub Desktop.
import pulumi_aws as aws
import ssh
temporary_key = ssh.GenerateSSHKey(
'GeneratedTemporaryPlatformSSHKey'
)
temporary_key_pair = aws.ec2.KeyPair(
'TemporaryPlatformSSHKey',
opts=opts,
key_name='Temporary Platform SSH Key',
public_key=temporary_key.public_key
)
import json
import traceback
from typing import Any
def __json_diff__(old: Any, new: Any) -> bool:
try:
olds_json_value = json.dumps(old, sort_keys=True, indent=2)
news_json_value = json.dumps(new, sort_keys=True, indent=2)
except TypeError as error:
if str(error) == 'Object of type Unknown is not JSON serializable':
return True
else:
print(f"Failed to calculate diff. {error}")
traceback.print_stack()
raise error
else:
return olds_json_value != news_json_value
def diff_inputs(olds: dict, news: dict) -> [str]:
diffs = []
for key in olds:
if key not in news:
diffs.append(key)
else:
olds_value = olds[key]
news_value = news[key]
if __json_diff__(olds_value, news_value):
diffs.append(key)
for key in news:
if key not in olds:
diffs.append(key)
return diffs
import hashlib
import io
from typing import Optional, Sequence
import paramiko
import time
from pulumi import Input, Output, ResourceOptions
from pulumi.dynamic import UpdateResult, DiffResult, Resource
from provider import diff_inputs
from .key import *
class ConnectionInputs(TypedDict, total=False):
host: Input[str]
"""The host to SSH into."""
port: Input[int]
"""The port to SSH into (default 22)."""
username: Input[str]
"""The username for the SSH login."""
password: Input[str]
"""The optional password for the SSH login (private key is recommended instead)."""
private_key: Input[str]
"""The private key, as an ASCII string, to use for the SSH connection."""
private_key_passphrase: Input[str]
"""The private key passphrase, if any, to use for the SSH private key."""
def connect(connection: ConnectionInputs) -> paramiko.SSHClient:
ssh_client = paramiko.SSHClient()
ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
skey = io.StringIO(connection['private_key'])
pkey = paramiko.RSAKey.from_private_key(skey, password=connection.get('private_key_passphrase'))
# Retry the connection until the endpoint is available (up to 2 minutes).
retries = 0
while True:
try:
ssh_client.connect(
hostname=connection['host'],
port=connection.get('port') or 22,
username=connection.get('username'),
password=connection.get('password'),
pkey=pkey,
)
return ssh_client
except paramiko.ssh_exception.NoValidConnectionsError:
if retries == 24:
raise
time.sleep(5)
retries = retries + 1
pass
def remote_file_hash(connection: ConnectionInputs, path: str) -> Optional[str]:
try:
ssh_client = connect(connection)
except TimeoutError:
print(f"WARNING: Timeout connecting to {connection['host']}. Assuming that the remote file does not exist.")
return None
try:
stdin, stdout, stderr = ssh_client.exec_command(f"sha256sum '{path}' | cut -d ' ' -f 1")
errors = ''.join(stderr.readlines())
if len(errors) != 0:
raise IOError(f"Failed to calculate SHA256 of remote file '{path}':\n{errors}")
lines = stdout.readlines()
if len(lines) != 1:
raise IOError(f"Unexpected output from sha256sum: {stdout}")
return lines[0].strip()
finally:
ssh_client.close()
def copy_file(connection: ConnectionInputs, src: str, dest: str):
ssh_client = connect(connection)
scp = ssh_client.open_sftp()
try:
scp.put(src, dest)
finally:
scp.close()
ssh_client.close()
def write_file(connection: ConnectionInputs, contents: str, dest: str):
ssh_client = connect(connection)
try:
stdin, stdout, stderr = ssh_client.exec_command(f"cat > '{dest}'")
stdin.write(contents)
stdin.flush()
stdin.close()
errors = ''.join(stderr.readlines())
if len(errors) != 0:
raise IOError(f"Failed to write to remote file '{dest}':\n{errors}")
return ''.join(stdout.readlines())
finally:
ssh_client.close()
def delete_file(connection: ConnectionInputs, path: str):
ssh_client = connect(connection)
try:
stdin, stdout, stderr = ssh_client.exec_command(f"rm -rf '{path}'")
errors = ''.join(stderr.readlines())
if len(errors) != 0:
raise IOError(f"Failed to delete to remote file '{path}':\n{errors}")
return ''.join(stdout.readlines())
finally:
ssh_client.close()
def run_commands(connection: ConnectionInputs, commands: [str]):
ssh_client = connect(connection)
try:
lines = []
for command in commands:
stdin, stdout, stderr = ssh_client.exec_command(command)
errors = ''.join(stderr.readlines())
if len(errors) != 0:
raise IOError(f"Failed to run command '{command}':\n{errors}")
lines + stdout.readlines()
return ''.join(lines)
finally:
ssh_client.close()
def contents_hash(contents: bytes) -> str:
hasher = hashlib.sha256()
hasher.update(contents)
return hasher.hexdigest()
def file_hash(path: str) -> str:
with open(path, 'rb') as file:
contents = file.read()
return contents_hash(contents)
class GenerateSSHKey(Resource):
private_key: Output[str]
public_key: Output[str]
passphrase: Output[str]
def __init__(self,
resource_name: str,
key_type: str = 'rsa',
bits: int = 4096,
opts: Optional[ResourceOptions] = None):
self.key_type = key_type
"""
key_type specifies the type of key to create.
"""
self.bits = bits
"""
bits specifies the number of bits in the key to create.
"""
super().__init__(
GenerateSSHKeyProvider(),
resource_name,
{
'key_type': key_type,
'bits': bits,
'private_key': None,
'public_key': None,
'passphrase': None,
},
ResourceOptions.merge(opts, ResourceOptions(additional_secret_outputs=['private_key', 'passphrase'])),
)
class _CopyFileProviderInputs(TypedDict):
connection: ConnectionInputs
src: str
dest: str
# CopyFile is a provisioner step that can copy a file over an SSH connection.
class CopyFile(Resource):
def __init__(self,
resource_name: str,
connection: Input[ConnectionInputs],
src: Optional[Input[str]],
dest: Input[str],
opts: Optional[ResourceOptions] = None):
self.connection = connection
"""conn contains information on how to connect to the destination, in addition to dependency information."""
self.src = src
"""
src is the source of the file or directory to copy. It can be specified as relative to the current
working directory or as an absolute path.
"""
self.dest = dest
"""dest is required and specifies the absolute path on the target where the file will be copied to."""
super().__init__(
CopyFileProvider(),
resource_name,
{
'connection': connection,
'src': src,
'dest': dest,
},
opts,
)
class CopyFileProvider(ResourceProvider):
def create(self, inputs: _CopyFileProviderInputs):
copy_file(inputs['connection'], inputs['src'], inputs['dest'])
return CreateResult(id_=uuid4().hex, outs=inputs)
# TODO: What are stables?
def diff(self, _id, _olds: _CopyFileProviderInputs, _news: _CopyFileProviderInputs):
inputs_diff = diff_inputs(_olds, _news)
if len(inputs_diff) > 0:
diff_dest = _olds['dest'] != _news['dest']
return DiffResult(changes=True, replaces=['dest'] if diff_dest else None, delete_before_replace=False)
dest_hash = remote_file_hash(_news['connection'], _news['dest'])
src_hash = file_hash(_news['src'])
if dest_hash != src_hash:
return DiffResult(changes=True, replaces=['src'])
else:
return DiffResult(changes=False)
def update(self, _id: str, _olds: _CopyFileProviderInputs, _news: _CopyFileProviderInputs) -> UpdateResult:
copy_file(_news['connection'], _news['src'], _news['dest'])
return UpdateResult(outs=_news)
def delete(self, _id: str, _props: _CopyFileProviderInputs) -> None:
delete_file(connection=_props['connection'], path=_props['dest'])
class _WriteFileProviderInputs(TypedDict):
connection: ConnectionInputs
contents: str
dest: str
class WriteFileProvider(ResourceProvider):
def create(self, inputs: _WriteFileProviderInputs):
write_file(inputs['connection'], inputs['contents'], inputs['dest'])
return CreateResult(id_=uuid4().hex, outs=inputs)
# TODO: What are stables?
def diff(self, _id, _olds: _WriteFileProviderInputs, _news: _WriteFileProviderInputs):
inputs_diff = diff_inputs(_olds, _news)
if len(inputs_diff) > 0:
diff_dest = _olds.get('dest') != _news.get('dest')
return DiffResult(changes=True, replaces=['dest'] if diff_dest else None, delete_before_replace=False)
dest_hash = remote_file_hash(_news['connection'], _news['dest'])
content_hash = contents_hash(_news['contents'].encode('utf-8'))
if dest_hash != content_hash:
return DiffResult(changes=True)
else:
return DiffResult(changes=False)
def update(self, _id: str, _olds: _WriteFileProviderInputs, _news: _WriteFileProviderInputs) -> UpdateResult:
write_file(_news['connection'], _news['contents'], _news['dest'])
return UpdateResult(outs=_news)
def delete(self, _id: str, _props: _WriteFileProviderInputs) -> None:
delete_file(connection=_props['connection'], path=_props['dest'])
class WriteFile(Resource):
def __init__(self,
resource_name: str,
connection: Input[ConnectionInputs],
contents: Optional[Input[str]],
dest: Input[str],
opts: Optional[ResourceOptions] = None):
self.connection = connection
"""conn contains information on how to connect to the destination, in addition to dependency information."""
self.contents = contents
"""
contents are the bytes to be written to the file.
"""
self.dest = dest
"""dest is required and specifies the absolute path on the target where the file will be written to."""
super().__init__(
WriteFileProvider(),
resource_name,
{
'connection': connection,
'contents': contents,
'dest': dest,
},
opts,
)
class _ExecuteCommandsProviderInputs(TypedDict):
connection: ConnectionInputs
commands: Sequence[str]
class ExecuteCommandsProvider(ResourceProvider):
def create(self, inputs: _ExecuteCommandsProviderInputs):
run_commands(inputs['connection'], inputs['commands'])
return CreateResult(id_=uuid4().hex, outs=inputs)
# TODO: What are stables?
def diff(self, _id, _olds: _ExecuteCommandsProviderInputs, _news: _ExecuteCommandsProviderInputs):
inputs_diff = diff_inputs(_olds, _news)
if len(inputs_diff) > 0:
return DiffResult(changes=True, replaces=inputs_diff)
else:
return DiffResult(changes=False)
def update(self,
_id: str,
_olds: _ExecuteCommandsProviderInputs,
_news: _ExecuteCommandsProviderInputs) -> UpdateResult:
run_commands(_news['connection'], _news['commands'])
return UpdateResult()
class ExecuteCommands(Resource):
def __init__(self,
resource_name: str,
connection: Input[ConnectionInputs],
commands: Input[Sequence[str]],
opts: Optional[ResourceOptions] = None):
self.connection = connection
"""conn contains information on how to connect to the destination, in addition to dependency information."""
self.commands = commands
"""
commands are the commands to run on the remote machine.
"""
super().__init__(
ExecuteCommandsProvider(),
resource_name,
{
'connection': connection,
'commands': commands
},
opts,
)
import getpass
import secrets
import socket
import string
import subprocess
import tempfile
from typing import TypedDict
from uuid import uuid4
import sys
from pulumi.dynamic import ResourceProvider, CreateResult
def get_public_key() -> str:
ssh_add = subprocess.run(['ssh-add', '-L'], capture_output=True, text=True)
if ssh_add.stderr != '':
print(ssh_add.stderr, file=sys.stderr)
if ssh_add.returncode != 0:
raise IOError(f"Failed to get public key from your SSH Agent. ssh-add returned with code {ssh_add.returncode}.")
lines = ssh_add.stdout.splitlines()
if len(lines) == 0:
raise ValueError('SSH Agent did not list any public keys.')
if len(lines) > 1:
raise ValueError('SSH Agent returned multiple public keys.')
public_key = lines[0]
if '(none)' in public_key:
user = getpass.getuser()
host = socket.gethostname()
public_key = public_key.replace('(none)', f"{user}@{host}")
return public_key
class KeyPair(TypedDict):
private_key: str
public_key: str
passphrase: str
def generate_rsa_key_pair() -> KeyPair:
chars = string.ascii_letters + string.digits + string.punctuation
passphrase = ''.join(secrets.choice(chars) for i in range(50))
with tempfile.TemporaryDirectory() as temp_dir:
args = ['ssh-keygen', '-t', 'rsa', '-P', passphrase, '-f', f"{temp_dir}/key", '-b', '4096']
ssh_keygen = subprocess.run(args, capture_output=True)
if ssh_keygen.returncode != 0:
print(ssh_keygen.stdout)
print(ssh_keygen.stderr)
raise IOError(f"Failed to generate RSA key. ssh-keygen returned with code {ssh_keygen.returncode}.")
with open(f"{temp_dir}/key") as private_key_file:
private_key = private_key_file.read()
with open(f"{temp_dir}/key.pub") as public_key_file:
public_key = public_key_file.read()
return KeyPair(
private_key=private_key,
public_key=public_key,
passphrase=passphrase
)
class GenerateSSHKeyInputs(TypedDict):
key_type: str
bits: int
class GenerateSSHKeyProvider(ResourceProvider):
def create(self, inputs: GenerateSSHKeyInputs):
key_pair = generate_rsa_key_pair()
outs = {
'private_key': key_pair['private_key'],
'public_key': key_pair['public_key'],
'passphrase': key_pair['passphrase'],
**inputs
}
return CreateResult(id_=uuid4().hex, outs=outs)
# TODO: Replace if type or size changes.
@steinybot
Copy link
Author

Move ssh-__init__.py and ssh-key.py to ssh/__init__.py and ssh/key.py.

@steinybot
Copy link
Author

steinybot commented Dec 10, 2020

Revision 1 works.

Revision 2 fails with:

❯ pulumi up
Previewing update (prod)

View Live: https://app.pulumi.com/steinybot/tlayen-infrastructure/prod/previews/d1b907d3-2c60-4d25-a162-139f457940b4

     Type                               Name                              Plan       Info
     pulumi:pulumi:Stack                tlayen-infrastructure-prod                   48 messages
 ~   ├─ pulumi-python:dynamic:Resource  GeneratedTemporaryPlatformSSHKey  update     [diff: ~__provider]
     └─ aws:ec2:KeyPair                 TemporaryPlatformSSHKey                      1 error

Diagnostics:
  aws:ec2:KeyPair (TemporaryPlatformSSHKey):
    error: transport is closing

  pulumi:pulumi:Stack (tlayen-infrastructure-prod):
    panic: interface conversion: interface {} is nil, not map[string]interface {}
    goroutine 299 [running]:
    github.com/terraform-providers/terraform-provider-aws/aws.resourceAwsCodeDeployTagSetHash(0x0, 0x0, 0xc0014afcf0)
    	/home/runner/go/pkg/mod/github.com/pulumi/[email protected]/aws/resource_aws_codedeploy_deployment_group.go:1414 +0x144
    github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema.(*Set).hash(0xc0014afce0, 0x0, 0x0, 0xc00215a780, 0x3)
    	/home/runner/go/pkg/mod/github.com/pulumi/terraform-plugin-sdk/[email protected]/helper/schema/set.go:251 +0x3d
    github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema.(*Set).add(0xc0014afce0, 0x0, 0x0, 0xc00086e500, 0x0, 0xc001580700)
    	/home/runner/go/pkg/mod/github.com/pulumi/terraform-plugin-sdk/[email protected]/helper/schema/set.go:231 +0x83
    github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema.(*ConfigFieldReader).readSet(0xc00195b170, 0xc0019733f0, 0x1, 0x1, 0xc00086e500, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
    	/home/runner/go/pkg/mod/github.com/pulumi/terraform-plugin-sdk/[email protected]/helper/schema/field_reader_config.go:284 +0x329
    github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema.(*ConfigFieldReader).readField(0xc00195b170, 0xc0019733f0, 0x1, 0x1, 0xc00197f400, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
    	/home/runner/go/pkg/mod/github.com/pulumi/terraform-plugin-sdk/[email protected]/helper/schema/field_reader_config.go:107 +0x7c9
    github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema.(*ConfigFieldReader).ReadField(0xc00195b170, 0xc0019733f0, 0x1, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc0019566c0, ...)
    	/home/runner/go/pkg/mod/github.com/pulumi/terraform-plugin-sdk/[email protected]/helper/schema/field_reader_config.go:29 +0xae
    github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema.(*MultiLevelFieldReader).ReadFieldExact(0xc0014af8c0, 0xc0019733f0, 0x1, 0x1, 0x6d18fdf, 0x6, 0x0, 0x0, 0x0, 0x0, ...)
    	/home/runner/go/pkg/mod/github.com/pulumi/terraform-plugin-sdk/[email protected]/helper/schema/field_reader_multi.go:31 +0xd0
    github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema.(*ResourceData).get(0xc0017b1f80, 0xc0019733f0, 0x1, 0x1, 0x12, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
    	/home/runner/go/pkg/mod/github.com/pulumi/terraform-plugin-sdk/[email protected]/helper/schema/resource_data.go:506 +0xfa
    github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema.(*ResourceData).getChange(0xc0017b1f80, 0x6d27c62, 0xb, 0xc000861201, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
    	/home/runner/go/pkg/mod/github.com/pulumi/terraform-plugin-sdk/[email protected]/helper/schema/resource_data.go:482 +0x122
    github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema.(*ResourceData).diffChange(0xc0017b1f80, 0x6d27c62, 0xb, 0x10, 0x0, 0x0, 0x0, 0xba797d8)
    	/home/runner/go/pkg/mod/github.com/pulumi/terraform-plugin-sdk/[email protected]/helper/schema/resource_data.go:459 +0x97
    github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema.schemaMap.diffSet(0xc000856870, 0x6d27c62, 0xb, 0xc00086e500, 0xc0006271c0, 0x7912b20, 0xc0017b1f80, 0xc000627100, 0x0, 0x0)
    	/home/runner/go/pkg/mod/github.com/pulumi/terraform-plugin-sdk/[email protected]/helper/schema/schema.go:1223 +0x66
    github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema.schemaMap.diff(0xc000856870, 0x6d27c62, 0xb, 0xc00086e500, 0xc0014af7a0, 0x7912b20, 0xc0017b1f80, 0x6da1c00, 0x0, 0x0)
    	/home/runner/go/pkg/mod/github.com/pulumi/terraform-plugin-sdk/[email protected]/helper/schema/schema.go:964 +0x557
    github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema.schemaMap.Diff(0xc000856870, 0x78f81a0, 0xc00018c010, 0xc00035c1c0, 0xc001909710, 0x0, 0x60f1b40, 0xc001806500, 0x0, 0xc00195a720, ...)
    	/home/runner/go/pkg/mod/github.com/pulumi/terraform-plugin-sdk/[email protected]/helper/schema/schema.go:523 +0x215
    github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema.(*Resource).SimpleDiff(0xc00086c790, 0x78f81a0, 0xc00018c010, 0xc00035c1c0, 0xc001909710, 0x60f1b40, 0xc001806500, 0x0, 0xc001909650, 0x0)
    	/home/runner/go/pkg/mod/github.com/pulumi/terraform-plugin-sdk/[email protected]/helper/schema/resource.go:437 +0x9f
    github.com/pulumi/pulumi-terraform-bridge/v2/pkg/tfshim/sdk-v2.v2Provider.Diff(0xc0005de0a0, 0x6da0525, 0x1f, 0x78f9360, 0xc00035c070, 0x7846d60, 0xc001909710, 0xc001909620, 0x0, 0x0, ...)
    	/home/runner/go/pkg/mod/github.com/pulumi/pulumi-terraform-bridge/[email protected]/pkg/tfshim/sdk-v2/provider.go:99 +0x1b0
    github.com/pulumi/pulumi-terraform-bridge/v2/pkg/tfbridge.(*Provider).Diff(0xc0000dbdc0, 0x78f8220, 0xc001908cf0, 0xc00035c000, 0xc0000dbdc0, 0x6066901, 0xc001914fc0)
    	/home/runner/go/pkg/mod/github.com/pulumi/pulumi-terraform-bridge/[email protected]/pkg/tfbridge/provider.go:683 +0x6cf
    github.com/pulumi/pulumi/sdk/v2/proto/go._ResourceProvider_Diff_Handler.func1(0x78f8220, 0xc001908cf0, 0x6aec880, 0xc00035c000, 0x6b2f7a0, 0xba797d8, 0x78f8220, 0xc001908cf0)
    	/home/runner/go/pkg/mod/github.com/pulumi/pulumi/sdk/[email protected]/proto/go/provider.pb.go:2199 +0x86
    github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x78f8220, 0xc001908ae0, 0x6aec880, 0xc00035c000, 0xc0014c6420, 0xc0014c6440, 0x0, 0x0, 0x7833f80, 0xc000090d90)
    	/home/runner/go/pkg/mod/github.com/grpc-ecosystem/[email protected]/go/otgrpc/server.go:57 +0x2eb
    github.com/pulumi/pulumi/sdk/v2/proto/go._ResourceProvider_Diff_Handler(0x6c1d620, 0xc0000dbdc0, 0x78f8220, 0xc001908ae0, 0xc0010cf920, 0xc0010fd180, 0x78f8220, 0xc001908ae0, 0xc0015e0500, 0x489)
    	/home/runner/go/pkg/mod/github.com/pulumi/pulumi/sdk/[email protected]/proto/go/provider.pb.go:2201 +0x14b
    google.golang.org/grpc.(*Server).processUnaryRPC(0xc0000dbc00, 0x791de60, 0xc001159080, 0xc001b82200, 0xc0010b3ce0, 0xba3b870, 0x0, 0x0, 0x0)
    	/home/runner/go/pkg/mod/google.golang.org/[email protected]/server.go:1171 +0x50a
    google.golang.org/grpc.(*Server).handleStream(0xc0000dbc00, 0x791de60, 0xc001159080, 0xc001b82200, 0x0)
    	/home/runner/go/pkg/mod/google.golang.org/[email protected]/server.go:1494 +0xccd
    google.golang.org/grpc.(*Server).serveStreams.func1.2(0xc00058cd20, 0xc0000dbc00, 0x791de60, 0xc001159080, 0xc001b82200)
    	/home/runner/go/pkg/mod/google.golang.org/[email protected]/server.go:834 +0xa1
    created by google.golang.org/grpc.(*Server).serveStreams.func1
    	/home/runner/go/pkg/mod/google.golang.org/[email protected]/server.go:832 +0x204

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment