| from boto3.session import Session | |
| from botocore.client import Config | |
| from botocore.handlers import set_list_objects_encoding_type_url | |
| import boto3 | |
| ACCESS_KEY = "xx" | |
| SECRET_KEY = "yy" | |
| boto3.set_stream_logger('') |
| ### prerequisites | |
| sudo yum groupinstall "Development Tools" | |
| git --version | |
| gcc --version | |
| bash --version | |
| python --version # (system) | |
| sudo yum install -y openssl-devel readline-devel zlib-devel | |
| sudo yum update | |
| ### install `pyenv` |
UPDATE (March 2020, thanks @ic): I don't know the exact AMI version but yum install docker now works on the latest Amazon Linux 2. The instructions below may still be relevant depending on the vintage AMI you are using.
Amazon changed the install in Linux 2. One no-longer using 'yum' See: https://aws.amazon.com/amazon-linux-2/release-notes/
sudo amazon-linux-extras install docker
sudo service docker start
| import pyspark.sql.functions as F | |
| from pyspark.sql import Window | |
| df = spark.createDataFrame([ | |
| ('d1',None), | |
| ('d2',10), | |
| ('d3',None), | |
| ('d4',30), | |
| ('d5',None), | |
| ('d6',None), |
| # Make sure the source index is actually open | |
| POST /source_index/_open | |
| # Put the source index in read-only mode | |
| PUT /source_index/_settings | |
| { | |
| "settings": { | |
| "index.blocks.write": "true" | |
| } | |
| } |
