- configurable sample granularity (WIP)

- improvements to inventory retrieval (threading)
- billing processing performance improvements
- refactored to trigger less API calls and use less memory
- make default IOs configurable
- dynamic inventory ingestion

Gemfile

@@ -2,6 +2,7 @@ source 'https://rubygems.org'
 
 gem 'activesupport'
 gem 'aws-sdk'
+gem 'concurrent-ruby', require: 'concurrent'
 gem 'fastest-csv'
 gem 'hash_dot'
 gem 'httparty'

Gemfile.lock

@@ -19,6 +19,7 @@ GEM
     bson (4.1.1)
     builder (3.2.2)
     coderay (1.1.1)
+    concurrent-ruby (1.0.5)
     diff-lcs (1.2.5)
     fastest-csv (0.0.4)
     hash_dot (2.0.2)
@@ -68,6 +69,7 @@ PLATFORMS
 DEPENDENCIES
   activesupport
   aws-sdk
+  concurrent-ruby
   fastest-csv
   hash_dot
   httparty

Rakefile

@@ -4,9 +4,9 @@ Bundler.require(:default)
 
 $:.unshift File.expand_path('lib/collectors/modules'), File.expand_path('lib/collectors'), File.expand_path('lib/helpers')
 
-STDOUT.sync = true
-$logger = Logger.new(STDOUT)
-$logger.level = ENV['LOG_LEVEL'] || Logger::INFO
+$stdout.sync = true
+$logger = Logger.new($stdout)
+$logger.level = Logger::DEBUG #ENV['LOG_LEVEL'] || Logger::INFO
 
 Dir.glob(File.join('./lib/helpers/**/*.rb'), &method(:require))
 Dir.glob(File.join('./lib/models/**/*.rb'), &method(:require))
@@ -16,12 +16,18 @@ Dir.glob(File.join('./lib/connector/**/*.rb'), &method(:require))
 Dir.glob(File.join('./lib/tasks/**/*.rb'), &method(:require))
 Dir.glob(File.join('./lib/*.rb'), &method(:require))
 
-CONFIG = YAML::load_file(File.expand_path('../config/application.yml', __FILE__)).to_dot
+yaml_erb = File.read(File.expand_path('../config/application.yml', __FILE__))
+renderer = ERB.new(yaml_erb, nil, '-')
+CONFIG = YAML::load(renderer.result(binding)).to_dot
 
 Mongoid.load_configuration(clients: {
     default: {
         database: 'metrics',
-        hosts: ["#{PropertyHelper.mongo_host}:#{PropertyHelper.mongo_port}"]
+        hosts: ["#{PropertyHelper.mongo_host}:#{PropertyHelper.mongo_port}"],
+        options: {
+          max_pool_size: 40,
+          min_pool_size: 2 }
+
     }
 })
 

config/application.yml

@@ -2,6 +2,7 @@ server:
   port: 8000
 
 startup:
+  - fetch_instance_types
   - bootstrap_inventory
   - fetch_price
   - fetch_detailed_report
@@ -14,13 +15,13 @@ scheduler:
 
   collect_samples:
     rake_command: collect_samples
-    interval: 5m
-    first_in: 0
+    interval: <%= PropertyHelper.collection_interval -%>m
+    first_in: 0m
 
   send_samples:
     rake_command: send_samples
-    interval: 5m
-    first_in: 5m
+    interval: 15m
+    first_in: 0
 
   fetch_detailed_report:
     rake_command: fetch_detailed_report
@@ -31,3 +32,9 @@ scheduler:
     rake_command: fetch_price
     interval: 30d
     first_in: 30d
+
+  fetch_instance_types:
+    rake_command: fetch_instance_types
+    interval: 1d
+    first_in: 1d
+