Support for connection to remote mongodb

install.sh

@@ -42,6 +42,20 @@ if [ -f /etc/redhat-release ]; then
     ./install_supervisord.sh
 fi
 
+echo "[`date`] Geting configuration"
+
+echo "[`date`] ========= MongoDB configuration ========="
+# run get config in context of current shell and export environment variables related to mongodb to be used in other install scripts
+. ./get_config.sh
+export REMOTE_MONGO
+export MONGO_AUTH
+export MONGO_HOST
+export MONGO_PORT
+export MONGO_USER
+export MONGO_PASSWORD
+export MONGO_AUTH_MECHANISM
+
+
 echo "[`date`] Starting Installation of all MHN packages"
 
 echo "[`date`] ========= Installing hpfeeds ========="

scripts/get_config.sh

@@ -0,0 +1,72 @@
+#!/bin/bash
+
+set -e
+
+echo "==========================================================="
+echo "  Configuration - MongoDB"
+echo "==========================================================="
+
+while true;
+do
+    echo -n "Would you like to use remote mongodb (must be installed and configured before installing mhn)? (y/n) "
+    read MONGO
+    if [ "$MONGO" == "y" -o "$MONGO" == "Y" ]
+    then
+        REMOTE_MONGO=true
+        echo -n "MongoDB Host: "
+        read MONGO_HOST
+        echo -n "MongoDB Port: "
+        read MONGO_PORT
+        echo "Using mongodb server $MONGO_HOST:$MONGO_PORT"
+        break
+    elif [ "$MONGO" == "n" -o "$MONGO" == "N" ]
+    then
+        REMOTE_MONGO=false
+        MONGO_HOST='localhost'
+        MONGO_PORT=27017
+        echo "Using default configuration:"
+        echo "    MongoDB Host: localhost"
+        echo "    MongoDB Port: 27017"
+        break
+    fi
+done
+
+# Remote Mongo -> ask for authentication
+if [ "$MONGO" == "y" -o "$MONGO" == "Y" ]
+then
+    while true;
+    do
+        echo -n "Would you like to use authentication for mongodb? (y/n) "
+        read MONGO_AUTH
+        if [ "$MONGO_AUTH" == "y" -o "$MONGO_AUTH" == "Y" ]
+        then
+            MONGO_AUTH='true'
+            echo -n "MongoDB user: "
+            read MONGO_USER
+            echo -n "MongoDB password: "
+            read MONGO_PASSWORD
+#            TODO add new authentication method to evnet
+#            echo -n "MongoDB authentication mechanism < SCRAM-SHA-1 | MONGODB-CR >:"
+#            read MONGO_AUTH_MECHANISM
+            MONGO_AUTH_MECHANISM="MONGODB-CR"
+            echo "The mongo will use username: $MONGO_USER and authentication mechanism $MONGO_AUTH_MECHANISM"
+            break
+        elif [ "$MONGO_AUTH" == "n" -o "$MONGO_AUTH" == "N" ]
+        then
+            MONGO_AUTH='false'
+            MONGO_USER='null'
+            MONGO_PASSWORD='null'
+            MONGO_AUTH_MECHANISM='null'
+            break
+        fi
+    done
+else
+    MONGO_AUTH='false'
+    MONGO_USER='null'
+    MONGO_PASSWORD='null'
+    MONGO_AUTH_MECHANISM='null'
+fi
+
+# set environment variable in supevisord.conf
+sed -i "s/\(^\[supervisord\]\)$/\1\nenvironment=REMOTE_MONGO=\"$REMOTE_MONGO\",MONGO_HOST=\"$MONGO_HOST\",MONGO_PORT=$MONGO_PORT,MONGO_AUTH=$MONGO_AUTH,MONGO_USER=$MONGO_USER,MONGO_PASSWORD=$MONGO_PASSWORD,MONGO_AUTH_MECHANISM=$MONGO_AUTH_MECHANISM/" /etc/supervisor/supervisord.conf
+

scripts/install_hpfeeds.sh

@@ -43,7 +43,7 @@ fi
 
 ldconfig /usr/local/lib/
 
-bash install_mongo.sh
+#bash install_mongo.sh
 
 $PIP install virtualenv
 
@@ -75,7 +75,6 @@ mkdir -p /var/log/mhn
 mkdir -p /etc/supervisor/
 mkdir -p /etc/supervisor/conf.d
 
-
 cat >> /etc/supervisor/conf.d/hpfeeds-broker.conf <<EOF 
 [program:hpfeeds-broker]
 command=/opt/hpfeeds/env/bin/python /opt/hpfeeds/broker/feedbroker.py

scripts/install_mhnserver.sh

@@ -152,6 +152,7 @@ MHN_UUID=`python -c 'import uuid;print str(uuid.uuid4())'`
 SECRET=`python -c 'import uuid;print str(uuid.uuid4()).replace("-","")'`
 /opt/hpfeeds/env/bin/python /opt/hpfeeds/broker/add_user.py "collector" "$SECRET" "" "geoloc.events"
 
+
 cat > $MHN_HOME/server/collector.json <<EOF
 {
   "IDENT": "collector",

scripts/install_mnemosyne.sh

@@ -39,7 +39,9 @@ else
 fi
 
 
-bash $SCRIPTS/install_mongo.sh
+if ! [ $REMOTE_MONGO == "true" ]; then 
+    bash $SCRIPTS/install_mongo.sh
+fi
 
 mkdir -p /opt
 cd /opt/

server/collector_v2.py

@@ -5,6 +5,7 @@
 from hpfeedslogger import processors
 import pymongo
 import requests
+import os
 
 ch = logging.StreamHandler(sys.stdout)
 ch.setLevel(logging.DEBUG)
@@ -30,15 +31,17 @@
     "wordpot.events",
 ]
 
-def ensure_user_permissions(ident, secret, publish, subscribe):
+def ensure_user_permissions(ident, secret, publish, subscribe, mongo_host, mongo_port, mongo_auth, mongo_user, mongo_password, mongo_auth_mechanism):
     rec = {
         "identifier": ident,
         "secret": secret,
         "publish": publish,
         "subscribe":subscribe
     }
 
-    client = pymongo.MongoClient()
+    client = pymongo.MongoClient(host=mongo_host, port=mongo_port)
+    if mongo_auth == 'true':
+        auth_res = client.hpfeeds.authenticate(mongo_user, mongo_password, mechanism=mongo_auth_mechanism)
     res = client.hpfeeds.auth_key.update({"identifier": ident}, {"$set": rec}, upsert=True)
     client.fsync()
     client.close()
@@ -88,7 +91,7 @@ def main():
         ip = None
     mhn_uuid = cfg['MHN_UUID']
 
-    ensure_user_permissions(cfg['IDENT'], cfg['SECRET'], [], cfg['CHANNELS'])
+    ensure_user_permissions(cfg['IDENT'], cfg['SECRET'], [], cfg['CHANNELS'], os.getenv('MONGO_HOST'), int(os.getenv('MONGO_PORT')), os.getenv('MONGO_AUTH'), os.getenv('MONGO_USER'), os.getenv('MONGO_PASSWORD'), os.getenv('MONGO_AUTH_MECHANISM'))
     subscriber = hpfeeds_connect(cfg['HOST'], cfg['PORT'], cfg['IDENT'], cfg['SECRET'])
     publisher = hpfeeds_connect(cfg['RHOST'], cfg['RPORT'], cfg['RIDENT'], cfg['RSECRET'])
     processor = processors.HpfeedsMessageProcessor(cfg['IP_GEO_DB'], cfg['IP_ASN_DB'])

server/mhn/__init__.py

@@ -71,6 +71,17 @@
     console.setFormatter(formatter)
     mhn.logger.addHandler(console)
 
+def new_clio_connection():
+    from mhn.common.clio import Clio
+    import os
+    return Clio(
+        os.getenv('MONGO_HOST'),
+        int(os.getenv('MONGO_PORT')),
+        True if os.getenv('MONGO_AUTH') == 'true' else False,
+        os.getenv('MONGO_USER'),
+        os.getenv('MONGO_PASSWORD'),
+        os.getenv('MONGO_AUTH_MECHANISM')
+    )
 
 @mhn.route('/feed.json')
 def json_feed():
@@ -89,14 +100,13 @@ def makeurl(uri):
 
 
 def get_feed():
-    from mhn.common.clio import Clio
     from mhn.auth import current_user
     authfeed = mhn.config['FEED_AUTH_REQUIRED']
     if authfeed and not current_user.is_authenticated():
         abort(404)
     feed = AtomFeed('MHN HpFeeds Report', feed_url=request.url,
                     url=request.url_root)
-    sessions = Clio().session.get(options={'limit': 1000})
+    sessions = new_clio_connection().session.get(options={'limit': 1000})
     for s in sessions:
         feedtext = u'Sensor "{identifier}" '
         feedtext += '{source_ip}:{source_port} on sensorip:{destination_port}.'

server/mhn/api/models.py

@@ -4,10 +4,9 @@
 
 from sqlalchemy import UniqueConstraint, func
 
-from mhn import db
+from mhn import db, mhn
 from mhn.api import APIModel
 from mhn.auth.models import User
-from mhn.common.clio import Clio
 
 
 class Sensor(db.Model, APIModel):
@@ -68,11 +67,13 @@ def new_auth_dict(self):
 
     @property
     def attacks_count(self):
-        return Clio().counts.get_count(identifier=self.uuid)
+        from mhn import new_clio_connection
+        return new_clio_connection().counts.get_count(identifier=self.uuid)
 
     @property
     def authkey(self):
-        return Clio().authkey.get(identifier=self.uuid)
+        from mhn import new_clio_connection
+        return new_clio_connection().authkey.get(identifier=self.uuid)
 
     @staticmethod
     def get_channels(honeypot):