airflow-安装部署

Aconda安装（略）

MySQL安装（略）

(MariaDB >= 5.7)

(MariaDB >= 10.1.8)

pip更新

pip install --upgrade pip
pip install --upgrade setuptools
           

依赖库安装

yum install python3-devel
yum install libevent-devel
yum install mysql-devel
yum install -y mysql-devel
pip3 install pymysql
pip3 install mysql
pip3 install cryptography

           

创建mysql数据库

create database apache_airflow;
create user 'airflow'@'%' identified by '123456';
GRANT all privileges on apache_airflow.* TO 'airflow'@'%' IDENTIFIED BY '123456';
FLUSH PRIVILEGES;
set explicit_defaults_for_timestamp=1; 
           

airflow

export AIRFLOW_HOME=~/airflow
pip3 install apache-airflow
AIRFLOW_VERSION=2.0.1
PYTHON_VERSION="$(python --version | cut -d " " -f 2 | cut -d "." -f 1-2)"

CONSTRAINT_URL="https://raw.githubusercontent.com/apache/airflow/constraints-${AIRFLOW_VERSION}/constraints-${PYTHON_VERSION}.txt"

           

修改配置 ~/airflow/airflow.cfg

sql_alchemy_conn = mysql://airflow:[email protected]:3306/apache_airflow
           

初始化

airflow db init

airflow users create \
    --username admin \
    --firstname Peter \
    --lastname Parker \
    --role Admin \
    --email [email protected]


           

启动

airflow webserver --port 8086
airflow scheduler

           

用户

airflow 
admin 
admin
           

任务运行

# run your first task instance
airflow tasks run example_bash_operator runme_0 2015-01-01
# run a backfill over 2 days
airflow dags backfill example_bash_operator \
    --start-date 2015-01-01 \
    --end-date 2015-01-02
           

参考资料

https://airflow.apache.org/docs/apache-airflow/stable/tutorial.html#it-s-a-dag-definition-file