You can refresh your dag directly from a git source
For that you just have to make a shell script like this :
clone_dags.sh
#!/bin/bash
set -e
GIT_REPO_URL="[email protected]:votre-utilisateur/votre-repo-dags.git"
DAGS_FOLDER="/opt/airflow/dags/repo"
# Clone or pull
if [ ! -d "$DAGS_FOLDER/.git" ]; then
echo "Cloning repository..."
export GIT_SSH_COMMAND="ssh -o StrictHostKeyChecking=no"
git clone "$GIT_REPO_URL" "$DAGS_FOLDER"
else
echo "Repository already cloned. Pulling latest changes..."
git config --global --add safe.directory {CLONE_DIR} && cd "$DAGS_FOLDER" && git fetch && git reset --hard origin/main
fi
exec "$@"
Then you can manage the refresh directly in Airflow with a dag :
from airflow import DAG
from airflow.operators.bash import BashOperator
from datetime import datetime
default_args = {
'owner': 'airflow',
'depends_on_past': False,
'start_date': datetime(2025, 1, 15),
'retries': 0,
'schedule_interval': '@daily'
}
test_dag = DAG(
'Refresh_Dags',
default_args=default_args,
)
# Define the BashOperator task
bash_task = BashOperator(
task_id='refresh_dag',
bash_command='bash -i /opt/airflow/dags/clone_dags.sh ',
dag=test_dag
)
# Set task dependencies
bash_task
Content type
Image
Digest
sha256:4f017ce9c…
Size
663.3 MB
Last updated
4 months ago
docker pull pafplouf/airflow-git