tyurnin/airflow repository overview

version: '3.8'

volumes:
  postgresql_data:
  logs:

services:
  postgres:
    image: postgres:13
    platform: linux/amd64
    environment:
      POSTGRES_USER: ${POSTGRES_USER}
      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
      POSTGRES_DB: ${POSTGRES_DB}
    volumes:
      - ./data/postgres:/var/lib/postgresql/data
    restart: always
    networks:
      - my_network
    ports:
      - "5432:5432"

  webserver:
    image: tyurnin/airflow:latest
    platform: linux/amd64
    depends_on:
      - postgres
    environment:
      AIRFLOW__CORE__EXECUTOR: ${AIRFLOW__CORE__EXECUTOR}
      AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB}
      AIRFLOW__CORE__LOAD_EXAMPLES: ${AIRFLOW__CORE__LOAD_EXAMPLES}
      AIRFLOW__WEBSERVER__DEFAULT_UI_TIMEZONE: ${AIRFLOW__WEBSERVER__DEFAULT_UI_TIMEZONE}
      AIRFLOW__WEBSERVER__RBAC: ${AIRFLOW__WEBSERVER__RBAC}
      AIRFLOW__WEBSERVER__BASE_URL: ${AIRFLOW__WEBSERVER__BASE_URL}
      AIRFLOW__CORE__FERNET_KEY: ${AIRFLOW__CORE__FERNET_KEY}
    volumes:
      - ./data/dags:/opt/airflow/dags
      - ./data/plugins:/opt/airflow/plugins
      - ./logs/airflow:/opt/airflow/logs
    command: >
      bash -c "mkdir -p /opt/airflow/logs && chmod -R 777 /opt/airflow/logs &&
               airflow db upgrade &&
               (airflow users list | grep -q 'admin' || airflow users create --username admin --password admin --firstname Admin --lastname User --role Admin --email [email protected]) &&
               airflow webserver"
    ports:
      - "8080:8080"
    restart: always
    networks:
      - my_network

  scheduler:
    image: tyurnin/airflow:latest
    platform: linux/amd64
    depends_on:
      - webserver
      - postgres
    environment:
      AIRFLOW__CORE__EXECUTOR: ${AIRFLOW__CORE__EXECUTOR}
      AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB}
    volumes:
      - ./data/dags:/opt/airflow/dags
      - ./data/plugins:/opt/airflow/plugins
      - ./logs/airflow:/opt/airflow/logs
    command: scheduler
    restart: always
    networks:
      - my_network

networks:
  my_network:
    driver: bridge

# requirements.txt

jira
keyring
openpyxl
pandas
pyzabbix
requests_ntlm
shareplum

FROM apache/airflow:2.7.0

USER airflow

# Копируем файл requirements.txt
COPY --chown=airflow:airflow requirements.txt /requirements.txt

# Устанавливаем зависимости с использованием пользователя airflow
RUN pip install --no-cache-dir -r /requirements.txt

# Переключаемся на пользователя root для изменения прав
USER root

# Создаем директории, если они не существуют, и устанавливаем права
RUN mkdir -p /opt/airflow/dags /opt/airflow/plugins && chmod -R 777 /opt/airflow/dags /opt/airflow/plugins

# Возвращаемся к пользователю airflow
USER airflow

version: '3.8'

services:
  airflow:
    build: .
    image: custom_airflow:latest
    platform: linux/amd64

  postgres:
    image: postgres:13
    platform: linux/amd64
    environment:
      POSTGRES_USER: airflow
      POSTGRES_PASSWORD: airflow
      POSTGRES_DB: airflow
    volumes:
      - postgresql_data:/var/lib/postgresql/data
    restart: always

volumes:
  postgresql_data: