diff --git a/stingcell/etc/stingcell/collector2.yaml.sample b/stingcell/etc/stingcell/collector2.yaml.sample new file mode 100644 index 0000000..9606ea0 --- /dev/null +++ b/stingcell/etc/stingcell/collector2.yaml.sample @@ -0,0 +1,145 @@ +manowar_version: 3 + +minion_config: minion + +collections: + packages: + salt: true + multi: true + saltfactor: "pkg.list_pkgs" + saltargs: [] + saltkwargs: {} + jq_parse: "." + release: + salt: true + multi: false + saltfactor: "cmd.run" + saltargs: ["lsb_release -sc"] + saltkwargs: {} + jq_parse: "." + rkernel: + salt: true + multi: false + saltfactor: "grains.get" + saltargs: ["kernelrelease"] + saltkwargs: {} + jq_parse: "." + lsmod: + salt: true + multi: true + saltfactor: "kmod.lsmod" + saltargs: [] + saltkwargs: {} + jq_parse: '[.[] | { (.module) : (.deps | join(",")) }] | add' + users: + salt: true + multi: true + saltfactor: "user.getent" + saltargs: [] + saltkwargs: {} + jq_parse: "[.[] | { (.name) : (.shell) }] | add" + services: + salt: true + multi: true + saltfactor: "service.get_enabled" + saltargs: [] + saltkwargs: {} + jq_parse: '[ .[] | { (.) : "ENABLED" } ] | add' + cpu-info: + salt: true + multi: true + saltfactor: "status.cpuinfo" + saltargs: [] + saltkwargs: {} + jq_parse: '[ to_entries | .[] | { (.key) : (.value| tostring ) }] | add' + interfaces: + salt: true + multi: true + saltfactor: "network.interfaces" + saltargs: [] + saltkwargs: {} + jq_parse: '[ to_entries | .[] | { "\(.key)-mac" : .value.hwaddr|tostring, "\(.key)-up" : .value.up|tostring, "\(.key)-ipv4" : .value.inet[0].address|tostring, "\(.key)-ipv6" : .value.inet6[0].address|tostring } ] | add' + mounts: + salt: true + multi: true + saltfactor: "mount.fstab" + saltargs: [] + saltkwargs: {} + jq_parse: '[to_entries | .[] | { "\(.key)-device" : .value.device, "\(.key)-fstype" : .value.fstype }] | add' + listen: + salt: true + multi: true + saltfactor: "network.netstat" + saltargs: [] + saltkwargs: {} + jq_parse: '[ .[] | if .state == "LISTEN" then {"\(.proto)_\(."local-address"|split(":")[-1])" : ."remote-address"} else {} end ] | add' + local-hosts: + salt: true + multi: true + saltfactor: "hosts.list_hosts" + saltargs: [] + saltkwargs: {} + jq_parse: '[ to_entries | .[] | . as $u | .value[] | { (.) : $u.key } ] | add' + os_family: + salt: true + multi: false + saltfactor: "grains.get" + saltargs: ["os_family"] + saltkwargs: {} + jq_parse: "." + os: + salt: true + multi: false + saltfactor: "grains.get" + saltargs: ["os"] + saltkwargs: {} + jq_parse: "." + os_version: + salt: true + multi: false + saltfactor: "grains.get" + saltargs: ["osversion"] + saltkwargs: {} + jq_parse: "." + kernelrelease: + salt: true + multi: false + saltfactor: "grains.get" + saltargs: ["kernelrelease"] + saltkwargs: {} + jq_parse: "." + cpuarch: + salt: true + multi: false + saltfactor: "grains.get" + saltargs: ["cpuarch"] + saltkwargs: {} + jq_parse: "." + locale: + salt: true + multi: true + saltfactor: "grains.get" + saltargs: ["locale_info"] + saltkwargs: {} + jq_parse: '[ to_entries | .[] | { (.key) : (.value| tostring ) }] | add' + virtual: + salt: true + multi: false + saltfactor: "grains.get" + saltargs: ["virtual"] + saltkwargs: {} + jq_parse: "." + ipv4_addr: + salt: true + multi: true + saltfactor: "network.ip_addrs" + saltargs: [] + saltkwargs: {} + jq_parse: '[ .[] | { (.) : "IPV4" } ] | add' + ipv6_addr: + salt: true + multi: true + saltfactor: "network.ip_addrs6" + saltargs: [] + saltkwargs: {} + jq_parse: '[ .[] | { (.) : "IPV6" } ] | add' diff --git a/stingcell/etc/stingcell/minion b/stingcell/etc/stingcell/minion new file mode 100644 index 0000000..ef794bd --- /dev/null +++ b/stingcell/etc/stingcell/minion @@ -0,0 +1,8 @@ +file_client: local +local: true +master_type: disable +root_dir: ./ +file_roots: + base: + - ./salt/ +user: chalbersma diff --git a/stingcell/etc/stingcell/stingcell.yaml.sample b/stingcell/etc/stingcell/stingcell.yaml.sample index 88f8044..8814f51 100644 --- a/stingcell/etc/stingcell/stingcell.yaml.sample +++ b/stingcell/etc/stingcell/stingcell.yaml.sample @@ -1,7 +1,7 @@ --- hostinfo: externalid: 10001 - pop: pop + pop: location srvtype: srvtype status: status sapi: @@ -13,9 +13,11 @@ stingcell: #collection_use_api: true collection_use_api: false default_collection_timeout: 60 - local_collections: true + local_collections: false local_collections_location: /etc/stingcell/collections.d/ - collection_config_file: etc/stingcell/collector.yaml.sample + collection_config_file: etc/stingcell/collector2.yaml.sample ipintel: dointel: true -version: 2 +version: 3 +salt: + minion_file: etc/stingcell/minion diff --git a/stingcell/saltcell.py b/stingcell/saltcell.py new file mode 100755 index 0000000..ce28630 --- /dev/null +++ b/stingcell/saltcell.py @@ -0,0 +1,165 @@ +#!/usr/bin/env python3 + +# Salt Cell + +import yaml +import jq +import requests +import sys +import argparse +import logging +import json + +#import salt.config +#import salt.client + +from saltcell.clientcollector import Host + +# +# Process +# 1. Grab Configs +# 1. Grab Collection Configuration +# 1. Do Collection +# 1. Submit Results +# +# + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("-c", "--config", help="Stingcell Config File (Default /etc/stingcell/stingcell.ini)", default="/etc/stingcell/stingcell.ini") + parser.add_argument("-p", "--print", help="Print stingcell json to stdout in addition to sending it along.", action='store_true') + parser.add_argument("-n", "--noupload", help="Do not upload results to endoint.", action='store_true') + parser.add_argument("-v", "--verbose", action='append_const', help="Turn on Verbosity", const=1, default=[]) + parser._optionals.title = "DESCRIPTION " + + # Parser Args + args = parser.parse_args() + + VERBOSE = len(args.verbose) + + if VERBOSE == 0 : + logging.basicConfig(level=logging.ERROR) + + elif VERBOSE == 1 : + logging.basicConfig(level=logging.WARNING) + + elif VERBOSE == 2 : + logging.basicConfig(level=logging.INFO) + + else: + logging.basicConfig(level=logging.DEBUG) + + logger = logging.getLogger("saltcell.py") + + logger.info("Welcome to Saltcell") + + with open(args.config, "r") as config_file: + try: + configs = yaml.load(config_file) + except yaml.YAMLError as parse_error: + print("Unable to parse file {} with error : \n {}".format(args.config, parse_error)) + sys.exit(1) + + if args.print: + PRINT=True + else: + PRINT=False + + if args.noupload: + NOUPLOAD = True + else: + NOUPLOAD = False + + +def docoll(config_items=False, noupload=False) : + + logger = logging.getLogger("saltcell.docoll") + + + ''' + Do the Collection + ''' + + # Step 2 Grab Collection Configuration + collection_configuration_file = config_items["stingcell"]["collection_config_file"] + + this_host = Host(minion_file=config_items["salt"].get("minion_file", "minion"), \ + base_config_file=config_items["stingcell"]["collection_config_file"], \ + local_cols=(config_items["stingcell"].get("local_collections", False), \ + config_items["stingcell"].get("local_collections_location", "/etc/stingcell/collections.d")), + host_configs=config_items["hostinfo"], + ipintel_configs=config_items["ipintel"]) + + results = this_host.todict() + + print(json.dumps(results)) + + + # Collect Host Stuff + ''' + + multi_hostname = config_items["hostinfo"].get("hostname", socket_hostname) + multi_pop = config_items["hostinfo"].get("pop", found_pop) + multi_srvtype = config_items["hostinfo"].get("srvtype", found_srvtype) + multi_status = config_items["hostinfo"].get("status", found_status) + # No way to find uber id on host (yet) + multi_uberid = config_items["hostinfo"].get("uberid", "N/A") + + collection_status = "STINGCELL" + connection_string = "Via StingCell" + + # Add host_host data + results_dictionary["collection_data"]["host_host"] = { "HOSTNAME" : multi_hostname , "POP" : multi_pop , "SRVTYPE" : multi_srvtype, "UBERID" : multi_uberid, "STATUS" : multi_status } + + results_dictionary["ip_intel"] = local_ipintel(config_items=config_items, verbose=verbose, hostname=multi_hostname, collections=results_dictionary["collection_data"] ) + + # Add Base Data + results_dictionary["collection_hostname"] = multi_hostname + results_dictionary["collection_status"] = collection_status + results_dictionary["collection_timestamp"] = int(time.time()) + results_dictionary["connection_string"] = connection_string + results_dictionary["pop"] = multi_pop + results_dictionary["srvtype"] = multi_srvtype + results_dictionary["status"] = multi_status + results_dictionary["uber_id"] = multi_uberid + + # Step 4 Stick that Shit in the API + request_headers = { "Authorization" : config_items["sapi"].get("sapi_username", "nobody") + ":" + config_items["sapi"].get("sapi_token", "nothing") , + "Content-Type" : "application/json" } + + collection_data_json = json.dumps(results_dictionary) + + url = config_items["sapi"]["sapi_endpoint"] + "sapi/puthostjson/" + + if ignoreupload == True: + # Don't do the upload just return the data + pass + else: + # The default, do the needful baby! + if verbose == True : + LOGGER.debug("Collections Finished, Uploading Data to : {} as user {} ".format(url, config_items["sapi"].get("sapi_username", "nobody") ) ) + + try : + this_request = requests.post(url, data=collection_data_json, headers=request_headers) + except Exception as e : + print("Error posting data back to SAPI {}".format(e)) + else : + response_code = this_request.status_code + if response_code == 200 : + if verbose == True : + LOGGER.debug("VERBOSE: Data Successfully Posted to : {}".format(url)) + else : + LOGGER.warning("Data posted to : {} but returned status code {} ".format(url, response_code)) + + return collection_data_json + ''' + + return + + + +if __name__ == "__main__": + collection_data = docoll(config_items=configs) + + if PRINT == True : + sys.stdout.write(collection_data) diff --git a/stingcell/saltcell/__init__.py b/stingcell/saltcell/__init__.py new file mode 100755 index 0000000..e69de29 diff --git a/stingcell/saltcell/clientcollector.py b/stingcell/saltcell/clientcollector.py new file mode 100644 index 0000000..317663b --- /dev/null +++ b/stingcell/saltcell/clientcollector.py @@ -0,0 +1,278 @@ +#!/usr/bin/env python3 + +''' +A ClientSide Collector +''' + +import socket +import time + +import json + +import ipaddress +import logging +import jq +import yaml + +class Host: + def __init__(self, minion_file="minion", base_config_file=False, local_cols=[False, None], + host_configs=False, ipintel_configs=False): + + self.logger = logging.getLogger("saltcell.clientcollector.Host") + + self.minion_file = minion_file + + self.salt_caller = self.start_minion() + + self.base_config_file = self.get_configs(base_config_file, local_cols) + + self.host_configs = host_configs + + self.logger.info("Read Information :\n{}".format(self.base_config_file)) + + self.collection_info = self.getall_collections() + + self.basedata = self.getbasedata() + + self.ipintel_configs = ipintel_configs + + if self.ipintel_configs.get("dointel", False) is True: + self.myipintel = self.ipintel() + else: + # Empty + self.myipintel = list() + + def todict(self): + + return_dict = {"collection_data" : self.collection_info, + "ip_intel" : self.myipintel, + **self.basedata} + + return return_dict + + + def get_configs(self, base_config_file, local_cols): + + if isinstance(base_config_file, dict): + # I've been given the configuration + to_collect_items = base_config_file + elif isinstance(base_config_file, str): + # I've been given a filename parse it + with open(base_config_file, "r") as base_config_file: + try: + to_collect_items = yaml.safe_load(base_config_file) + except yaml.YAMLError as yaml_error: + self.logger.error("Unable to read collection configuration file {} with error : \n{}".format(base_config_file, str(yaml_error))) + to_collect_items = dict() + + if local_cols[0] is True: + # Do Local Cols + collection_d_dir = local_cols[1] + collections_files = list() + for (dirpath, dirnames, filenames) in os.walk(collections_d_dir) : + for singlefile in filenames : + onefile = dirpath + "/" + singlefile + #print(singlefile.find(".ini", -4)) + if singlefile.find(".yaml", -4) > 0 : + # File ends with .ini Last 4 chars + collections_files.append(onefile) + + for collection_file in collections_files : + try: + # Read Our INI with our data collection rules + this_local_coll = yaml.safe_load(collection_file) + except Exception as e: # pylint: disable=broad-except, invalid-name + sys.stderr.write("Bad collection configuration file {} cannot parse: {}".format(collection_file, str(e))) + else: + # I've read and parsed this file let's add the things + for this_new_coll_key in this_local_coll.get("collections", {}).keys(): + to_collect_items["collections"][this_new_coll_key]=this_local_coll[this_new_coll_key] + + return to_collect_items + + def start_minion(self): + + # Any Earlier and I'll fubar the logger + import salt.config + import salt.client + + minion_opts = salt.config.minion_config(self.minion_file) + + salt_caller = salt.client.Caller(c_path=".", mopts=minion_opts) + + return salt_caller + + + def getone(self, cname, collection): + + results_dictionary = dict() + results_dictionary[cname] = dict() + + is_multi = collection.get("multi", False) + + if collection.get("salt", False) is True: + + this_find = self.salt_caller.function(collection["saltfactor"], \ + *collection["saltargs"], \ + **collection["saltkwargs"]) + + if is_multi: + # Multi so do the JQ bits + parsed_result = jq.jq(collection["jq_parse"]).transform(this_find) + + results_dictionary[cname] = parsed_result + else: + # Not Multi the whole thing goes + results_dictionary[cname]["default"] = str(this_find) + + else: + results_dictionary = {"type" : {"subtype", "value"}} + + return results_dictionary + + def getall_collections(self): + + myresults = dict() + + for this_collection in self.base_config_file["collections"].keys(): + self.logger.info("Collection {} Processing".format(this_collection)) + + this_result = self.getone(this_collection, self.base_config_file["collections"][this_collection]) + + myresults[this_collection] = this_result[this_collection] + + myresults["host_host"] = self.gethostmeta() + + return myresults + + def gethostmeta(self): + + ''' + Takes the host metadata given and stores it puts defaults for nothing. + ''' + + hostmeta = dict() + + hostconfig=self.host_configs + + hostmeta["hostname"] = socket.gethostname() + + if hostconfig.get("pop", False) is not False: + hostmeta["pop"] = str(hostconfig["pop"]) + else: + hostmeta["pop"] = "none" + + if hostconfig.get("srvtype", False) is not False: + hostmeta["srvtype"] = str(hostconfig["srvtype"]) + else: + hostmeta["srvtype"] = "none" + + if hostconfig.get("status", False) is not False: + hostmeta["status"] = str(hostconfig["status"]) + else: + hostmeta["status"] = "none" + + if hostconfig.get("externalid", False) is not False: + hostmeta["externalid"] = str(hostconfig["externalid"]) + + self.logger.debug("Host Meta Information: {}".format(hostmeta)) + + return hostmeta + + def getbasedata(self): + + ''' + Get the Basic Data like Collection Timestamp, + A copy of the host data + And any other meta data that shouldn't be stored as a collection + ''' + + basedata = self.gethostmeta() + + basedata["collection_timestamp"] = int(time.time()) + + return basedata + + def ipintel(self): + + ''' + Get's IPs from the IPV6 and IPV4 collection + + Future work, make configuralbe parsing + ''' + + PRIVATE_NETWORKS = [ipaddress.ip_network("127.0.0.0/8"), + ipaddress.ip_network("10.0.0.0/8"), + ipaddress.ip_network("172.16.0.0/12"), + ipaddress.ip_network("192.168.0.0/16"), + ipaddress.ip_network("fd00::/8")] + + found_intel = list() + + # Get Local Addresses + ipa_object = list() + ipa_object.extend(list(self.collection_info["ipv4_addr"].keys())) + ipa_object.extend(list(self.collection_info["ipv6_addr"].keys())) + + + self.logger.info("Raw Intel Object for this Host : \n{}".format(ipa_object)) + + ipv4 = list() + ipv6 = list() + + for this_unvalidated_ip in ipa_object : + + self.logger.info("Doing the needful for IP : \n{}".format(this_unvalidated_ip)) + + isipv4=False + isipv6=False + + try: + validated_ipv4 = socket.inet_pton(socket.AF_INET, this_unvalidated_ip) + isipv4 = True + except OSError : + # IPV4 Validation Failed, Try IPV6 + try: + validated_ipv6 = socket.inet_pton(socket.AF_INET6, this_unvalidated_ip) + isipv6 = True + except OSError : + pass + finally: + # After this checks let's see what showed up + if isipv4 or isipv6 : + # On or the other was true let's see if it's a private address + this_ip = ipaddress.ip_address(this_unvalidated_ip) + + is_private = False + for priv_net in PRIVATE_NETWORKS : + if this_ip in priv_net : + # This is a private ip + self.logger.debug("{} is in Private network {}".format(this_ip, priv_net)) + is_private = True + break + else : + # Check the other Private Networks + continue + + if is_private == False : + # It's not a private IP so add it to the intel report + if isipv4 : + ipv4.append(this_unvalidated_ip) + elif isipv6 : + ipv6.append(this_unvalidated_ip) + + unduped_ipv4 = list(set(ipv4)) + unduped_ipv6 = list(set(ipv6)) + + this_intel = dict() + this_intel["host4"] = unduped_ipv4 + this_intel["host6"] = unduped_ipv6 + + for thing in ["host4", "host6"] : + for ip in this_intel[thing] : + this_report = { "iptype" : thing , \ + "ip" : ip } + + found_intel.append(this_report) + + return found_intel