https://cloud.google.com/composer/docs/composer-2/save-load-snapshots?hl=ja
Airflow バージョン: 2.6.3
Composer バージョン: 2.4.6
-- 1. 前作業
gcloud init
gcloud auth list
gcloud --version
gcloud projects create project01-9999999 \
--name="project01"
gcloud config list
gcloud config set project project01-9999999
gcloud config set compute/region asia-northeast1 --quiet
gcloud config set compute/zone asia-northeast1-a --quiet
gcloud beta billing accounts list
gcloud beta billing projects link project01-9999999 --billing-account=111111-111111-111111
gcloud services enable compute.googleapis.com --project project01-9999999
gcloud components update
-- 2. バケットの作成
gcloud storage buckets create gs://bucket123 \
--default-storage-class=Standard \
--no-enable-autoclass \
--location=us \
--public-access-prevention \
--uniform-bucket-level-access
gcloud storage ls
-- 3. Cloud Composer の有効化
gcloud services list --enabled
gcloud services enable composer.googleapis.com --project project01-9999999
-- 4. サービス アカウントへ権限付与
gcloud projects get-iam-policy project01-9999999
service-00000000000@cloudcomposer-accounts.iam.gserviceaccount.com
へ権限付与
gcloud projects add-iam-policy-binding project01-9999999 \
--member="serviceAccount:service-00000000000@cloudcomposer-accounts.iam.gserviceaccount.com" \
--role="roles/owner"
-- 5. 環境の作成
gcloud composer environments create env01 \
--location asia-northeast1 \
--image-version composer-2.4.6-airflow-2.6.3
GKEやバケットが作成される
30分程度かかる
gcloud composer environments list \
--locations asia-northeast1
gcloud composer environments describe env01 \
--location asia-northeast1
-- 6. DAGファイルの作成
mkdir -p test
vim ./test/dag01.py
import datetime
import airflow
from airflow.operators import bash_operator
# If you are running Airflow in more than one time zone
# see https://airflow.apache.org/docs/apache-airflow/stable/timezone.html
# for best practices
YESTERDAY = datetime.datetime.now() - datetime.timedelta(days=1)
default_args = {
'owner': 'Composer Example',
'depends_on_past': False,
'email': [''],
'email_on_failure': False,
'email_on_retry': False,
'retries': 1,
'retry_delay': datetime.timedelta(minutes=5),
'start_date': YESTERDAY,
}
with airflow.DAG(
'dag01',
catchup=False,
default_args=default_args,
schedule_interval=datetime.timedelta(days=1)) as dag:
# Print the dag_run id from the Airflow logs
task01 = bash_operator.BashOperator(
task_id='task01', bash_command='echo {{ dag_run.id }}')
-- 7. DAGファイルをGCSにアップロードする
gcloud composer environments storage dags list \
--environment=env01 \
--location=asia-northeast1
gcloud composer environments storage dags import \
--environment=env01 \
--location=asia-northeast1 \
--source=./test
-- 8. 環境のスナップショットを保存する
gcloud beta composer environments snapshots save \
env01 \
--location asia-northeast1 \
--snapshot-location "gs://bucket123/snapshots"
gcloud storage ls gs://bucket123/snapshots/*
-- 9. DAGファイルを削除する
gcloud composer environments storage dags list \
--environment=env01 \
--location=asia-northeast1
gcloud composer environments storage dags delete dags/test/dag01.py \
--environment=env01 \
--location=asia-northeast1
→ なぜか削除できないので画面から実行
-- 10. 環境のスナップショットを読み込む
gcloud beta composer environments snapshots load \
env01 \
--location asia-northeast1 \
--snapshot-path "gs://bucket123/snapshots/project01-9999999_asia-northeast1_env01_2023-12-09T13-01-30"
→ DAG復旧を確認
-- 11. 環境のスナップショット スケジュールを有効にする
gcloud beta composer environments update env01 \
--location asia-northeast1 \
--enable-scheduled-snapshot-creation \
--snapshot-location "gs://bucket123/snapshots" \
--snapshot-creation-schedule "0 4 * * *" \
--snapshot-schedule-timezone "UTC+09"
-- 12. クリーンアップ
gcloud storage rm gs://bucket123 --recursive
gcloud storage ls
gcloud composer environments delete env01 \
--location asia-northeast1 \
--quiet
gcloud composer environments list \
--locations asia-northeast1
gcloud projects list
gcloud projects delete project01-9999999 \
--quiet
