[GCP]Docker-Compose로 Airflow 설치 Bitnami/Airflow

sudo su

apt-get install apt-transport-https ca-certificates curl gnupg-agent software-properties-common

curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -

apt-key fingerprint 0EBFCD88

add-apt-repository "deb [arch=arm64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"

apt-get update

apt-get install docker-ce docker-ce-cli containerd.io
#여기까지는 Docker설치편에서 한번 다루었다.
 

# 버전 확인

docker version

 

# docker-compose

sudo su

curl -L "https://github.com/docker/compose/releases/download/1.27.0/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose

chmod +x /usr/local/bin/docker-compose

ln -s /usr/local/bin/docker-compose /usr/bin/docker-compose

 

# 버전 확인

docker-compose -v


컴포저를 설치 후 아래의 명령어를 하나씩 날려줌

여기까지 한다음 bitnamit airflow github에서 
https://github.com/bitnami/bitnami-docker-airflow

bitnami/bitnami-docker-airflow

docker-compose.yml을 복사하여 text에디터에서 수정

version: '2'

services:
  jupyter:
    image: 'jupyter/base-notebook:latest'
    volumes:
        - ./dags:/usr/local/airflow/dags
    ports:
        - "8888:8888"
    environment:
    	- JUPYTER_ENABLE_LAB=1
    command: "start-notebook.sh \
              --NotebookApp.password='sha1:b70a055565b1:854e3f96f91c51cc0c91da371ea5dfbf7c1a61a1' \
              --sudo mkdir /usr/local/airflow/dags"
  postgresql:
    image: docker.io/bitnami/postgresql:10
    volumes:
      - 'postgresql_data:/bitnami/postgresql'
    environment:
      - POSTGRESQL_DATABASE=bitnami_airflow
      - POSTGRESQL_USERNAME=bn_airflow
      - POSTGRESQL_PASSWORD=bitnami1
      - ALLOW_EMPTY_PASSWORD=yes
  redis:
    image: docker.io/bitnami/redis:6.0
    volumes:
      - 'redis_data:/bitnami'
    environment:
      - ALLOW_EMPTY_PASSWORD=yes
  airflow-scheduler:
    # TODO: to be reverted to use proper registry/distro on T39132
    # image: docker.io/bitnami/airflow-scheduler:2
    image: docker.io/bitnami/airflow-scheduler:2
    environment:
      - AIRFLOW_DATABASE_NAME=bitnami_airflow
      - AIRFLOW_DATABASE_USERNAME=bn_airflow
      - AIRFLOW_DATABASE_PASSWORD=bitnami1
      - AIRFLOW_EXECUTOR=CeleryExecutor
      - AIRFLOW_WEBSERVER_HOST=airflow
    volumes:
      - airflow_scheduler_data:/bitnami
  airflow-worker:
    # TODO: to be reverted to use proper registry/distro on T39132
    # image: docker.io/bitnami/airflow-worker:2
    image: docker.io/bitnami/airflow-worker:2
    environment:
      - AIRFLOW_DATABASE_NAME=bitnami_airflow
      - AIRFLOW_DATABASE_USERNAME=bn_airflow
      - AIRFLOW_DATABASE_PASSWORD=bitnami1
      - AIRFLOW_EXECUTOR=CeleryExecutor
      - AIRFLOW_WEBSERVER_HOST=airflow
    volumes:
      - airflow_worker_data:/bitnami
  airflow:
    image: docker.io/bitnami/airflow:2
    environment:
      - AIRFLOW_DATABASE_NAME=bitnami_airflow
      - AIRFLOW_DATABASE_USERNAME=bn_airflow
      - AIRFLOW_DATABASE_PASSWORD=bitnami1
      - AIRFLOW_EXECUTOR=CeleryExecutor
    ports:
      - '8080:8080'
    volumes:
      - airflow_data:/bitnami
volumes:
  airflow_scheduler_data:
    driver: local
  airflow_worker_data:
    driver: local
  airflow_data:
    driver: local
  postgresql_data:
    driver: local
  redis_data:
    driver: local

$mkdir /usr/local/airflow
$vi docker-compose.yml

#위의 코드 복사

$docker-compose -f docker-compose.yml up -d
#작석한 yaml파일로 올리기

#끝.

#서버의 ip주소:8080 => airflow
#서버의 ip주소:8888 => jupyterLab

#주피터랩에서 권한 어쩌구저쩌구 나올시

chmod 777 /usr/local/airflow/dags

#끝.