peihsinsu/fluentd repository overview

<source>
  type tail
  format apache
  path /logs/access.log
  tag nginx.access
#  pos_file /logs/nginx.pos
</source>

<match nginx.access>
  type bigquery

  method insert    # default

  auth_method private_key   # default
  email [email protected] # replace yours
  private_key_path /data/mykey.p12 # replace yours
  # private_key_passphrase notasecret # default

  project my-project-id # replace yours
  dataset bq_test # replace yours
  auto_create_table true
  table   access_log

  time_format %s
  time_field  time

  field_integer time,status,bytes
  field_string  rhost,vhost,path,method,protocol,agent,referer
  field_float   requesttime
  field_boolean bot_access,loginsession
</match>

docker run -v ~/data:/data -v ~/logs:/logs \
  -v /home/simonsu/dockerws/fluentd/td-agent:/etc/td-agent \
  -it peihsinsu/fluentd bash

# /etc/init.d/td-agent start

# tail -f /var/log/td-agent/td-agent.log