developer-sdk

hadoop fs \
-Dfs.s3.impl=org.apache.hadoop.fs.s3a.S3AFileSystem \
-Dfs.s3a.aws.credentials.provider=org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider \
-Dfs.s3a.access.key=access_key \
-Dfs.s3a.secret.key=secret_key \
-Dfs.s3a.buckets.create.region=ap-northeast-1 \
-Dfs.s3a.endpoint=s3.ap-northeast-1.io \
-Ds3service.s3-endpoint=3.ap-northeast-1.io \
-Dfs.s3a.path.style.access=true \
-ls \

developer-sdk

import functools

def retry(retry_num, retry_sleep_sec):
    """
    retry help decorator.
    :param retry_num: the retry num
    :param retry_sleep_sec: retry sleep sec
    :return: decorator
    """
    def decorator(func):

developer-sdk

import requests
from requests.adapters import HTTPAdapter, Retry


def requests_retry_session(
        retries=3,
        backoff_factor=0.3,
        status_forcelist=(500, 502, 504),
        session=None):

developer-sdk

# s3 라이브러리가 있는 위치 추가 
export HADOOP_ROOT_LOGGER=INFO,console
export HADOOP_CLASSPATH="/opt/hadoop/share/hadoop/tools/lib/*"

# 일반 설정 
hadoop fs \
-Dfs.s3a.aws.credentials.provider=org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider \
-Dfs.s3a.access.key=xxx \
-Dfs.s3a.secret.key=sss \
-Dfs.s3a.buckets.create.region=ap-northeast-2 \

developer-sdk

# 에어플로우 실행 로그 삭제 
find {AIRFLOW_HOME}/logs/ -maxdepth 1 -name "airflow*" -type f -mtime +5 -delete
# DAG 실행 로그 삭제
find {AIRFLOW_HOME}/logs/ -name "20??-??-??*" -type d -mtime +5 -prune -exec rm -rf "{}" \;

developer-sdk

# 라이브러리 링크 
ln -s /opt/hive/lib/hive-exec-3.1.2.jar /ope/metastore/lib/hie-exec-3.1.2.jar

developer-sdk

______                             
| ___ \                            
| |_/ / __ _ _ __  _ __   ___ _ __ 
| ___ \/ _` | '_ \| '_ \ / _ \ '__|
| |_/ / (_| | | | | | | |  __/ |   
\____/ \__,_|_| |_|_| |_|\___|_|   
                                   
                                   

developer-sdk

# 환경변수 설정
export AIRFLOW__CORE__SQL_ALCHEMY_CONN="mysql://airflow_user:airflow_user@database.com:3306/airflow_db?charset=utf8"
export AIRFLOW__CELERY__RESULT_BACKEND="db+mysql://airflow_user:airflow_user@database.com:3306/airflow_db?charset=utf8"

# 웹서버 실행 
airflow webserver -D

developer-sdk

# 인증서 정보에서 확인할 수 있어야 함 
$ openssl x509 -in CertCA.pem -text
Certificate:
    Data:
        Version: 3 (0x2)
 ...
             X509v3 Basic Constraints: critical
                CA:TRUE, pathlen:0
 ...

developer-sdk

broker_url = rediss://redis-host.com:6379/0?ssl_cert_reqs=CERT_OPTIONAL&ssl_certfile=ca.crt&ssl_keyfile=ca.key&ssl_ca_certs=ca.crt