To run the harvester create a local folder, for example: /storage/download/
Create a file named config.json inside of it, that looks like:
{
"printinfo": false,
"overwrite_existing": false,
"outputfolder": "/storage/download/",
"harvesters": [
{
"hub": "mistral",
"ignore": true,
"datasets":[
{
"dataset": "COSMO_5M",
"variables": [
"t_g",
"w_snow"
],
"cloudcover": {
"variable": "clct",
"threshold_perc": 40,
"valid_map_perc": 70
}
}
],
"credentials": {
"username": "user",
"password": "pwd"
},
"schedule_hours": 12
},{
"hub": "dpc-radar",
"ignore": false,
"datasets":[
{
"dataset": "SRT1"
}
],
"schedule_hours": 1
},{
"hub": "infoterre_brgm_fr",
"hub_services": "https://infoterre.brgm.fr/page/geoservices-ogc",
"ignore": true,
"do_cleanup": true,
"datasets":[
{
"dataset": "ms:LITHO_1M_SIMPLIFIEE",
"dbtable": "litho_1m_simplifiee",
"entrypoint": "http://geoservices.brgm.fr/geologie"
},{
"dataset": "ms:MVT_COMMUNE",
"dbtable": "mvt_commune",
"entrypoint": "http://geoservices.brgm.fr/risques"
},{
"dataset": "ms:EXPLOIT_ACTIVE_L",
"dbtable": "exploit_active_l",
"entrypoint": "http://geoservices.brgm.fr/odmgm"
}
],
"schedule_hours": 48
},{
"hub": "imerg",
"ignore": true,
"do_cleanup": false,
"datasets":[
{
"dataset": "https://jsimpsonhttps.pps.eosdis.nasa.gov/imerg/gis/early/"
}
],
"credentials": {
"username": "user",
"password": "pwd"
},
"schedule_hours": 24
},{
"hub": "sensors-arpa",
"ignore": true,
"do_cleanup": true,
"datasets":[
{
"dataset": "idro"
},{
"dataset": "pluv"
},{
"dataset": "terma"
},{
"dataset": "nivo"
}
],
"schedule_hours": 1
},{
"hub": "eaufrance_hydrometrie",
"ignore": true,
"do_cleanup": true,
"datasets":[
{
"dataset": "H"
},{
"dataset": "Q"
}
],
"schedule_hours": 24
},{
"hub": "osm",
"ignore": true,
"do_cleanup": true,
"datasets":[
{
"dataset": "area_of_interest",
"type": "pbf",
"only_if_table_not_exists": false
}
],
"pbf_urls": [
{
"url": "https://download.geofabrik.de/europe/italy/nord-ovest-latest.osm.pbf"
},{
"url": "https://download.geofabrik.de/europe/france/provence-alpes-cote-d-azur-latest.osm.pbf"
},{
"url": "https://download.geofabrik.de/europe/france/rhone-alpes-latest.osm.pbf"
}
],
"schedule_hours": 48
},{
"hub": "idrogeo",
"ignore": true,
"do_cleanup": true,
"datasets":[
{
"dataset": "landslides",
"type": "geojson"
}
],
"geojson_urls": [
{
"url": "https://idrogeo.isprambiente.it/opendata/frane/frane_poly_piemonte_opendata.json.zip"
},{
"url": "https://idrogeo.isprambiente.it/opendata/frane/frane_poly_valle-d-aosta_opendata.json.zip"
},{
"url": "https://idrogeo.isprambiente.it/opendata/frane/frane_poly_liguria_opendata.json.zip"
}
],
"schedule_hours": 24
},{
"hub": "sentinel",
"ignore": false,
"do_cleanup": false,
"datasets":[
{
"dataset": "S2_MSI_L1C",
"description": "Sentinel-2 MultiSpectral Instrument Level 1C"
}
],
"credentials": {
"username": "user",
"password": "pwd",
},
"schedule_hours": 24
}
],
"options":{
"rows": 600,
"cols": 600,
"timesteps":[],
"remove_tmp": true,
"force_epsg": 4326
},
"database":{
"dbname": "dbname",
"user": "user",
"password": "pwd",
"host": "host",
"port": 5432
},
"area_of_interest": {
"geometry": {
"type": "Polygon",
"coordinates": [ [ [ 6.202484568571906, 42.987127928378349 ], ... ] ]
}
}
}
Make sure you change the parameters you need, ex:
Note that the outputfolder needs to stay as it is, as it is the container folder. You can set that folder when the container mounting a volume to it (See below).
Then just run the docker image like:
docker run -it --rm -v /storage/download/:/download --name pyenvdataharvester moovida/pyenvdataharvester:0.4.0
changing /storage/download/ to the local folder you choose.
Content type
Image
Digest
sha256:9620fe095…
Size
1.3 GB
Last updated
over 1 year ago
docker pull moovida/pyenvdataharvester:0.9.1