python3.9.9 소스설치 pyspark, airflow 설치

 

 

python bash_profile 설정

$ vi ~/.bash_profile
export PYTHON_HOME=$HOME/usr/local/Python3.9.9/
export LD_LIBRARY_PATH=${PYTHON_HOME}/lib:$LD_LIBRARY_PATH

 

python 다운로드

https://www.python.org/downloads/release/python-399/

https://www.python.org/ftp/python/3.9.9/Python-3.9.9.tgz

yum install gcc openssl openssl-devel zlib zlib-devel libffi-devel sqlite-devel

./configure --prefix=/home/eva/usr/local/Python3.9.9
make
make install



./python3 -m pip3 install pyspark
./python3 -m pip3 install psycopg2 (배포 시 libpq.so.5 파일 필요)

 

 

# postgres에 airflow계정 생성

create user airflow with password 'airflow';
create database airflow_meta owner=airflow;
create schema authorization airflow;
grant all privileges on all tables in schema airflow to airflow;
grant all privileges on database airflow_meta to airflow;

 

# airflow 설치

./pipe install --upgrade pip
./pip3 install pysqlite3
./pip3 install "apache-airflow==$2.2.2" --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.2.2/constraints-3.9.txt"

 

airflow-db 연동 (디렉토리 변경 방법 확인 필요 | 한번은 airflow명령어 사용해야 cfg 파일 생길 수 있음)

$ vi ~/airflow/airflow.cfg
sql_alchemy_conn = postgresql+psycopg2://airflow:airflow@hadoop01/airflow_meta

https://dhznsdl.tistory.com/19

https://progressivecoder.com/how-to-setup-an-airflow-postgres-connection/

 

airflow 실행

https://dydwnsekd.tistory.com/32

./airflow config get-value core sql_alchemy_connv
./airflow db init
./airflow version
./airflow webserver --port 9090
./airflow users create --username admin --firstname jh --lastname j --role Admin --email jjh@email.com

 

airflow 용어 개념

https://dydwnsekd.tistory.com/28

airflow 사용법

https://blog.naver.com/PostView.nhn?blogId=wideeyed&logNo=221565240108 

https://yogae.tistory.com/32

 

airflow sensor 사용법

https://dydwnsekd.tistory.com/76?category=897626 

 

airflow connections (db 접속 보안)

https://mentha2.tistory.com/217

 

airflow sparkoperator 사용법

https://louisdev.tistory.com/8

 

s3 cli 설치방법

https://blog.naver.com/PostView.naver?blogId=hl0geb&logNo=220665116617&redirect=Dlog&widgetTypeCall=true&directAccess=false