synchronicityiot/estimator-data-manager repository overview

{
  data_sources: {
    Santander: {
      key:  "header",
      token:  "token"
    }
}

{
  /**
   * Data Manager API configuration
   */
  api: {
    /**
     * Server listening port (to configure Express)
     */
    port: 3000,

    /**   
     * Root path of the API
     * EXAMPLE: http://<api_host>/<root_path>  
     */
    root_path: 'parkingareas',
  },
  /**
   * Underlying data sources. Information about the underlying IoT Infrastructures where the Data Manager will 
   * harvest the information from. Supported framworks:
   * - FIWARE and Synchronicity
   *  - Data context brokers: Orion CB (NGSI-v2) - 'ngsiv2'
   *  - Historical APIs: Synchronicity (~ STH Commet) - 'synchronicity'
   * 
   * NOTE: The compatibility list will grow as long as we gather new opportunities and partnerships
   */
  data_sources: [
    /**
     * Official name of the data source.     
     */
    {
      /**
       * ID pattern (i.e. substring) used for the device identification. This will be used when it comes
       *  to trigger historical data queries. To point to the corresponding historical API endpoint,
       *  we need to link between ID and endpoint, so we must have a connector.
       * 
       * EXAMPLE: Synchronicity's naming system (in this case, 'santander' will work)
       * urn	Context entity	Site	Service	Group	EntityName
       * urn:entity:santander:traffic:TrafficFlowObserved:1001
       */
      data_source_auth_id: 'Santander',

      /**
       *Label through we the data source can be detected (e.g. subscription-manager)
       */
      data_source_label: 'Santander-parking',
      /**
       * Type of data context broker endpoint: Current list of types
       *  - ngsiv2 --> Orion Context Broker (NGSI-v2 interface)
       * ToDo:
       *  - scorpio --> Scorpio Context Broker (NGSI-LD)
       */
      data_type: 'ngsiv2',
      /**
       * Data Context Broker endpoint
       */
      data_endpoint: '<endpoint>',

      /**
      * Enable/disable subscriptions 
      */
      subscriptions_enabled: true,

      /**
        * Enable/disable the delivery of all historic values from all assets to the AI prediction module
        * The dump will be sliced in chunks of "batch_size" observations
        */
      historical_dump_enabled: true,

      /**
       * Type of historical data API: Current list of types
       *  - synchronicity --> Synchronicity (~ STH Commet)
       * ToDo: 
       *  - csv --> Dump from a CSV file
       */
      historical_api_type: 'synchronicity',
      /**
       * Historical data API endpoint
       */
      historical_api_endpoint: '<endpoint>',
      /**
       * OAUth2 enabled/disabled
       * NOTE: if this value is set to true, the file 'config/auth.js' will be checked in order to set the proper
       * headers in the HTTP messages (the search will address the name of the data source)
       */
      auth: true,
      /**
       * (Optional) Data entity types to discover. During the discovery process, we can target a subset of entity
       * types in the search query
       */
      entity_types: [
        '<type_1>',
        '<type_2>'
      ],
      /**
       * (Optional) Attributes that will be dumped to the persistence system
       * 
       * NOTE: If nothing (empty array or commented variable), the data manager will
       *  proceed to dump everything to the persistence system
       */
      historic_attrs: [{
          name: '<hist_attr_1>',
          type: 'string/integer/float/double'
        },
        {
          name: '<hist_attr_2>',
          type: 'string/integer/float/double'
        },
      ],

      /**
       *  List of attributes that will be saved (if the data aggregation parameters - see below 
       *  are not enabled)
       */
      saved_attributes: [{
          name: '<saved_attr_1>',
          type: 'string/integer/float/double'
        },
        {
          name: '<saved_attr_2>',
          type: 'string/integer/float/double'
        },        
      ],

      /**
       * Index of the entities information
       */
      elastic_entities_index: 'entities-parking',

      /**
       * Index of the data
       */
      elastic_data_index: 'data-parking',

      /**
       * Raw dataset transformation before forwarding to the persistence system.
       * This is invoked from two different points: 
       *  1- Historical data query
       *  2- Event notification
       *
       * NOTE: If this field is enabled, the 'saved_attributes' parameter will be overriden
       */
      data_aggregation: {

        /**
         * Flag to enable/disable the aggregation function
         */
        enabled: false,

        /**
         * [Advanced] Callback to the function that will be called to aggregate the data (notification events/historical 
         * methods)
         */
        aggregation_function: callback_function_1
      },

      /**
       * IoT infrastructure specific params.
       * Current options:
       *  - [FIWARE] fiware_service --> Comment/Uncomment in case of need (tenant)
       *  - [FIWARE] fiware_servicepath --> Comment/Uncomment in case of need (hierarchical scope - NOTE: 
       *                                    must begin with a slash '/' character)
       */
      infrastructure_specific: {
        // fiware_service: '',
        // fiware_servicepath: ''
      }
    },
    // {
    //  More data sources (if necessary)  
    // }
    // ...
  ],

  /**
   * If data source subscriptions are enabled (see data_source.subscriptions_enabled flag), we must specify the
   * an endpoint from which the Data Manager will be listening to forthcoming events
   */
  subscriptions: {

    /**
     * Sink to send the asynchronous publications (from data sources)     
     *
     * NOTE: By default (i.e. set in the code), in 'ngsiv2' mode,  we set the endpoint as <endpoint>/notify as the
     * notificiation URL in the Subscribe to context request sent to Orion
     */    
    endpoint: '<subscription_endpoint>',    
  },

  /**
   * Parameters related with the historical data dump
   * At booting time, if the data source has the historical data dump enabled (see 
   * data_source.historical_dump_enabled flag), the Data Manager will proceed to harvest
   * and dump all the devices' history
   */
  history: {

    /**
     * Number of observations that will be sent in a single batch (the operation will be repeated till 
     * the query to the historical API returns an empty resultset)
     */
    batch_size: 10000,

  },

  /**
   * Datastore and search engine for the context element information and historical data
   *
   * NOTE: The use of Elastic clusters may force us to revamp this configuration
   */
  elasticsearch: {
    /**
     * Elasticsearch endpoint 
     *
     * EXAMPLE: 'http://127.0.0.1:9200' 
     */
    endpoint: '<elasticsearch_endpoint>',
  },

  /**
   * Data Manager Logging handler
   */
  logging: {
    /**
     * Configures the log level. Appropriate values are: 
     * levels: {
     *   emerg: 0,
     *   alert: 1,
     *   crit: 2,
     *   error: 3,
     *   warning: 4,
     *   notice: 5,
     *   perf: 6,
     *   info: 7,
     *   debug: 8
     * },
     */
    logLevel: 'debug',

    /**
     * Name of the module to be include in the different log files
     * (they will start by the %DATE% - see utils/log.js for more info)
     */
    moduleName: 'DATA-MANAGER',

  }
};


/**
 * (Advanced) Functions that will be responsible for shaping the output data, that is, the datasets that will feed the 
 * persistence system
 */
function callback_function_1() {
  //Aggregation function code goes here
};

docker run -p HOST_PORT:6000  -v "$(pwd)/config":/usr/src/app/config synchronicityiot/estimator-data-manager