ekoerner/heritrix repository overview

# tag it (-t) as "heritrix"
# specify version (if not latest default, specified in image) with: --build-arg version=3.4.0-20210923
docker build --build-arg version=3.4.0-20210923 -t heritrix .

# run it in foreground
# -it required for clean stopping with Ctrl+C
# --rm for cleaning up afterwards of volumes etc.
docker run --rm -it heritrix

# run it in background
# --name is optional but easier to find for stopping
docker run -d --name heritrix_container heritrix
# logs
docker logs heritrix_container
# stop it
docker stop heritrix_container

# --init : use tini init wrapper
# --rm : remove container after exit
# -it : runs docker interactively (pseudo TTY)
# -d : detach, run container in background
# -p : map public api port of 8443 (host) to 8443 (container port)
# -e : set environment variables for user/pass of REST API
#      JAVA_OPTS=-Xmx1024M (to restrict heritrix memory usage)
# -v : mount local folder into container (to persist job results)
#      on windows/WSL[2] volume mounts might not work (container files are not in local folder?)
#      heritrix is install at /opt/heritrix
#      heritrix jobs are at   /opt/heritrix/jobs
docker run --init --rm -d \
    --name heritrix_container \
    -p 8443:8443 \
    -e "USERNAME=admin" -e "PASSWORD=admin" \
    -v $(pwd)/jobs:/opt/heritrix/jobs \
    heritrix

# or mount a credentials file into the container (docker secrets?)
echo "admin:admin" > $(pwd)/creds
docker run --init --rm -d \
    --name heritrix_container \
    -p 8443:8443 \
    -e "CREDSFILE=/opt/heritrix/creds.txt" \
    -v $(pwd)/creds:/opt/heritrix/creds.txt \
    -v $(pwd)/jobs:/opt/heritrix/jobs \
    heritrix

# docker options the same as above
# * specify with -e "JOBNAME=<jobname> the job that should be run
# * mount the job folder with the crawler-beans.cxml to the <jobname>
#   folder within the container
# * the crawl will start immediately
# 
docker run --init --rm -d \
    --name heritrix_container \
    -p 8443:8443 \
    -e "USERNAME=admin" -e "PASSWORD=admin" -e "JOBNAME=myjob" \
    -v $(pwd)/myjob:/opt/heritrix/jobs/myjob \
    heritrix

# the last two lines are the relative path to the job lob
# (in the container) as well as the URI_PREFIX
docker run -it --rm \
    -v "$(pwd)/myjob:/opt/heritrix/jobs/myjob" \
    --entrypoint bin/hoppath.pl \
    heritrix \
    jobs/myjob/latest/logs/crawl.log \
    https://

PORT=8443
USERNAME=admin
PASSWORD=admin
#JOBNAME=myjob

version: "3.7"
services:

  heritrix:
    build: .
    container_name: "heritrix"
    # TEST: keeps the container running without doing anything (for inspections)
    # entrypoint: bash -c 'while :; do :; done & kill -STOP $$! && wait $$!'
    # env_file: .env
    environment:
      - USERNAME=admin
      - PASSWORD=admin
      # optional jobname to run (will only run this single job and exit!)
      # - JOBNAME=myjob
      # - JAVA_OPTS=-Xmx1024M
    init: true
    ports:
      # if you want to use a .env file with `PORT=8443` for example
      # - ${PORT}:8443
      - 8443:8443
    restart: unless-stopped
    volumes:
      # where jobs will be stored
      - job-files:/opt/heritrix/jobs
      # or if JOBNAME envvar is used (mount just the single job folder)
      # jobfolder in the container needs to have the same name as in JOBNAME
      # - $(pwd)/host_myjob:/opt/heritrix/jobs/myjob

volumes:
  job-files: