Various updates to make a multiplatform client side collector

* Updated stingcell.yaml.sample
* Added a salty stingcell.yaml.sample
* Added a "light" minion file (Need to remove my username in the future)
* Added saltcell.py a smaller driver for the clientcontroller.py objects
* clientcontroller.Host is an object to control collections as a unit.
More pythonic calls all the salty things.

stingcell/etc/stingcell/collector2.yaml.sample

@@ -0,0 +1,145 @@
+manowar_version: 3
+
+minion_config: minion
+
+collections:
+  packages:
+    salt: true
+    multi: true
+    saltfactor: "pkg.list_pkgs"
+    saltargs: []
+    saltkwargs: {}
+    jq_parse: "."
+  release:
+    salt: true
+    multi: false
+    saltfactor: "cmd.run"
+    saltargs: ["lsb_release -sc"]
+    saltkwargs: {}
+    jq_parse: "."
+  rkernel:
+    salt: true
+    multi: false
+    saltfactor: "grains.get"
+    saltargs: ["kernelrelease"]
+    saltkwargs: {}
+    jq_parse: "."
+  lsmod:
+    salt: true
+    multi: true
+    saltfactor: "kmod.lsmod"
+    saltargs: []
+    saltkwargs: {}
+    jq_parse: '[.[] | { (.module) : (.deps | join(",")) }] | add'
+  users:
+    salt: true
+    multi: true
+    saltfactor: "user.getent"
+    saltargs: []
+    saltkwargs: {}
+    jq_parse: "[.[] | { (.name) : (.shell) }] | add"
+  services:
+    salt: true
+    multi: true
+    saltfactor: "service.get_enabled"
+    saltargs: []
+    saltkwargs: {}
+    jq_parse: '[ .[] | { (.) : "ENABLED" } ] | add'
+  cpu-info:
+    salt: true
+    multi: true
+    saltfactor: "status.cpuinfo"
+    saltargs: []
+    saltkwargs: {}
+    jq_parse: '[ to_entries | .[] | { (.key) : (.value| tostring ) }] | add'
+  interfaces:
+    salt: true
+    multi: true
+    saltfactor: "network.interfaces"
+    saltargs: []
+    saltkwargs: {}
+    jq_parse: '[ to_entries | .[] | { "\(.key)-mac" : .value.hwaddr|tostring, "\(.key)-up" : .value.up|tostring, "\(.key)-ipv4" : .value.inet[0].address|tostring, "\(.key)-ipv6" : .value.inet6[0].address|tostring } ] | add'
+  mounts:
+    salt: true
+    multi: true
+    saltfactor: "mount.fstab"
+    saltargs: []
+    saltkwargs: {}
+    jq_parse: '[to_entries | .[] | { "\(.key)-device" : .value.device, "\(.key)-fstype" : .value.fstype }] | add'
+  listen:
+    salt: true
+    multi: true
+    saltfactor: "network.netstat"
+    saltargs: []
+    saltkwargs: {}
+    jq_parse: '[ .[] | if .state == "LISTEN" then {"\(.proto)_\(."local-address"|split(":")[-1])" : ."remote-address"} else {} end ] | add'
+  local-hosts:
+    salt: true
+    multi: true
+    saltfactor: "hosts.list_hosts"
+    saltargs: []
+    saltkwargs: {}
+    jq_parse: '[ to_entries | .[] | .  as $u | .value[] | { (.) : $u.key } ] | add'
+  os_family:
+    salt: true
+    multi: false
+    saltfactor: "grains.get"
+    saltargs: ["os_family"]
+    saltkwargs: {}
+    jq_parse: "."
+  os:
+    salt: true
+    multi: false
+    saltfactor: "grains.get"
+    saltargs: ["os"]
+    saltkwargs: {}
+    jq_parse: "."
+  os_version:
+    salt: true
+    multi: false
+    saltfactor: "grains.get"
+    saltargs: ["osversion"]
+    saltkwargs: {}
+    jq_parse: "."
+  kernelrelease:
+    salt: true
+    multi: false
+    saltfactor: "grains.get"
+    saltargs: ["kernelrelease"]
+    saltkwargs: {}
+    jq_parse: "."
+  cpuarch:
+    salt: true
+    multi: false
+    saltfactor: "grains.get"
+    saltargs: ["cpuarch"]
+    saltkwargs: {}
+    jq_parse: "."
+  locale:
+    salt: true
+    multi: true
+    saltfactor: "grains.get"
+    saltargs: ["locale_info"]
+    saltkwargs: {}
+    jq_parse: '[ to_entries | .[] | { (.key) : (.value| tostring ) }] | add'
+  virtual:
+    salt: true
+    multi: false
+    saltfactor: "grains.get"
+    saltargs: ["virtual"]
+    saltkwargs: {}
+    jq_parse: "."
+  ipv4_addr:
+    salt: true
+    multi: true
+    saltfactor: "network.ip_addrs"
+    saltargs: []
+    saltkwargs: {}
+    jq_parse: '[ .[] | { (.) : "IPV4" } ] | add'
+  ipv6_addr:
+    salt: true
+    multi: true
+    saltfactor: "network.ip_addrs6"
+    saltargs: []
+    saltkwargs: {}
+    jq_parse: '[ .[] | { (.) : "IPV6" } ] | add'

stingcell/etc/stingcell/minion

@@ -0,0 +1,8 @@
+file_client: local
+local: true
+master_type: disable
+root_dir: ./
+file_roots:
+  base:
+    - ./salt/
+user: chalbersma

stingcell/etc/stingcell/stingcell.yaml.sample

@@ -1,7 +1,7 @@
 ---
 hostinfo:
   externalid: 10001
-  pop: pop
+  pop: location
   srvtype: srvtype
   status: status
 sapi:
@@ -13,9 +13,11 @@ stingcell:
   #collection_use_api: true
   collection_use_api: false
   default_collection_timeout: 60
-  local_collections: true
+  local_collections: false
   local_collections_location: /etc/stingcell/collections.d/
-  collection_config_file: etc/stingcell/collector.yaml.sample
+  collection_config_file: etc/stingcell/collector2.yaml.sample
 ipintel:
   dointel: true
-version: 2
+version: 3
+salt:
+  minion_file: etc/stingcell/minion

stingcell/saltcell.py

@@ -0,0 +1,165 @@
+#!/usr/bin/env python3
+
+# Salt Cell
+
+import yaml
+import jq
+import requests
+import sys
+import argparse
+import logging
+import json
+
+#import salt.config
+#import salt.client
+
+from saltcell.clientcollector import Host
+
+#
+# Process
+# 1. Grab Configs
+# 1. Grab Collection Configuration
+# 1. Do Collection
+# 1. Submit Results
+#
+#
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-c", "--config", help="Stingcell Config File (Default /etc/stingcell/stingcell.ini)", default="/etc/stingcell/stingcell.ini")
+    parser.add_argument("-p", "--print", help="Print stingcell json to stdout in addition to sending it along.", action='store_true')
+    parser.add_argument("-n", "--noupload", help="Do not upload results to endoint.", action='store_true')
+    parser.add_argument("-v", "--verbose", action='append_const', help="Turn on Verbosity", const=1, default=[])
+    parser._optionals.title = "DESCRIPTION "
+
+    # Parser Args
+    args = parser.parse_args()
+
+    VERBOSE = len(args.verbose)
+
+    if VERBOSE == 0 :
+        logging.basicConfig(level=logging.ERROR)
+
+    elif VERBOSE == 1 :
+        logging.basicConfig(level=logging.WARNING)
+
+    elif VERBOSE == 2 :
+        logging.basicConfig(level=logging.INFO)
+
+    else:
+        logging.basicConfig(level=logging.DEBUG)
+
+    logger = logging.getLogger("saltcell.py")
+
+    logger.info("Welcome to Saltcell")
+
+    with open(args.config, "r") as config_file:
+        try:
+            configs = yaml.load(config_file)
+        except yaml.YAMLError as parse_error:
+            print("Unable to parse file {} with error : \n {}".format(args.config, parse_error))
+            sys.exit(1)
+
+    if args.print:
+        PRINT=True
+    else:
+        PRINT=False
+
+    if args.noupload:
+        NOUPLOAD = True
+    else:
+        NOUPLOAD = False
+
+
+def docoll(config_items=False, noupload=False) :
+
+    logger = logging.getLogger("saltcell.docoll")
+
+
+    '''
+    Do the Collection
+    '''
+
+    # Step 2 Grab Collection Configuration
+    collection_configuration_file = config_items["stingcell"]["collection_config_file"]
+
+    this_host = Host(minion_file=config_items["salt"].get("minion_file", "minion"), \
+                     base_config_file=config_items["stingcell"]["collection_config_file"], \
+                     local_cols=(config_items["stingcell"].get("local_collections", False), \
+                                 config_items["stingcell"].get("local_collections_location", "/etc/stingcell/collections.d")),
+                     host_configs=config_items["hostinfo"],
+                     ipintel_configs=config_items["ipintel"])
+
+    results = this_host.todict()
+
+    print(json.dumps(results))
+
+
+    # Collect Host Stuff
+    '''
+
+    multi_hostname = config_items["hostinfo"].get("hostname", socket_hostname)
+    multi_pop = config_items["hostinfo"].get("pop", found_pop)
+    multi_srvtype = config_items["hostinfo"].get("srvtype", found_srvtype)
+    multi_status = config_items["hostinfo"].get("status", found_status)
+    # No way to find uber id on host (yet)
+    multi_uberid = config_items["hostinfo"].get("uberid", "N/A")
+
+    collection_status = "STINGCELL"
+    connection_string = "Via StingCell"
+
+    # Add host_host data
+    results_dictionary["collection_data"]["host_host"] = { "HOSTNAME" : multi_hostname , "POP" : multi_pop , "SRVTYPE" : multi_srvtype, "UBERID" : multi_uberid, "STATUS" : multi_status }
+
+    results_dictionary["ip_intel"] = local_ipintel(config_items=config_items, verbose=verbose, hostname=multi_hostname, collections=results_dictionary["collection_data"] )
+
+    # Add Base Data
+    results_dictionary["collection_hostname"] = multi_hostname
+    results_dictionary["collection_status"] = collection_status
+    results_dictionary["collection_timestamp"] = int(time.time())
+    results_dictionary["connection_string"] = connection_string
+    results_dictionary["pop"] = multi_pop
+    results_dictionary["srvtype"] = multi_srvtype
+    results_dictionary["status"] = multi_status
+    results_dictionary["uber_id"] = multi_uberid
+
+    # Step 4 Stick that Shit in the API
+    request_headers = { "Authorization" : config_items["sapi"].get("sapi_username", "nobody") + ":" + config_items["sapi"].get("sapi_token", "nothing") ,
+                                                "Content-Type" : "application/json" }
+
+    collection_data_json = json.dumps(results_dictionary)
+
+    url = config_items["sapi"]["sapi_endpoint"] + "sapi/puthostjson/"
+
+    if ignoreupload == True:
+        # Don't do the upload just return the data
+        pass
+    else:
+        # The default, do the needful baby!
+        if verbose == True :
+            LOGGER.debug("Collections Finished, Uploading Data to : {} as user {} ".format(url, config_items["sapi"].get("sapi_username", "nobody") ) )
+
+        try :
+            this_request = requests.post(url, data=collection_data_json, headers=request_headers)
+        except Exception as e :
+            print("Error posting data back to SAPI {}".format(e))
+        else :
+            response_code = this_request.status_code
+            if response_code == 200 :
+                if verbose == True :
+                    LOGGER.debug("VERBOSE: Data Successfully Posted to : {}".format(url))
+            else :
+                LOGGER.warning("Data posted to : {} but returned status code {} ".format(url, response_code))
+
+    return collection_data_json
+    '''
+
+    return
+
+
+
+if __name__ == "__main__":
+    collection_data = docoll(config_items=configs)
+
+    if PRINT == True :
+        sys.stdout.write(collection_data)