Pickle (#1134)

* load pickle from scratch, don't deepcopy
* allow output module `import` without side effects
* set up intersphinx for py3 and twisted

docs/conf.py

@@ -189,3 +189,8 @@ epub_exclude_files = ['search.html']
 
 # If true, `todo` and `todoList` produce output, else they produce nothing.
 todo_include_todos = True
+
+# -- Options for intersphinx extension ---------------------------------------
+
+intersphinx_mapping = {'python': ('https://docs.python.org/3', None),
+                       'twisted': ('https://twistedmatrix.com/documents/current/api/', None)}

src/cowrie/core/config.py

@@ -2,7 +2,7 @@
 # See the COPYRIGHT file for more information
 
 """
-This module contains ...
+This module contains code to deal with Cowrie's configuration
 """
 
 from __future__ import absolute_import, division
@@ -31,7 +31,9 @@ class CowrieConfig(object):
 
 
 class EnvironmentConfigParser(configparser.ConfigParser):
-
+    """
+    ConfigParser with additional option to read from environment variables
+    """
     def has_option(self, section, option):
         if to_environ_key('_'.join((section, option))) in environ:
             return True
@@ -41,8 +43,7 @@ class EnvironmentConfigParser(configparser.ConfigParser):
         key = to_environ_key('_'.join((section, option)))
         if key in environ:
             return environ[key]
-        return super(EnvironmentConfigParser, self).get(
-            section, option, **kwargs)
+        return super(EnvironmentConfigParser, self).get(section, option, **kwargs)
 
 
 def readConfigFile(cfgfile):

src/cowrie/core/realm.py

@@ -28,9 +28,9 @@
 
 from __future__ import absolute_import, division
 
-import twisted
 from twisted.conch import interfaces as conchinterfaces
 from twisted.conch.telnet import ITelnetProtocol
+from twisted.cred.portal import IRealm
 from twisted.python import log
 
 from zope.interface import implementer
@@ -43,7 +43,7 @@ from cowrie.shell import server as shellserver
 from cowrie.telnet import session
 
 
-@implementer(twisted.cred.portal.IRealm)
+@implementer(IRealm)
 class HoneyPotRealm(object):
 
     def __init__(self):

src/cowrie/output/csirtg.py

@@ -21,12 +21,12 @@ class Output(cowrie.core.output.Output):
     """
     csirtg output
     """
-    user = CowrieConfig().get('output_csirtg', 'username') or USERNAME
-    feed = CowrieConfig().get('output_csirtg', 'feed') or FEED
-    token = CowrieConfig().get('output_csirtg', 'token') or TOKEN
-    description = CowrieConfig().get('output_csirtg', 'description', fallback=DESCRIPTION)
 
     def start(self, ):
+        self.user = CowrieConfig().get('output_csirtg', 'username') or USERNAME
+        self.feed = CowrieConfig().get('output_csirtg', 'feed') or FEED
+        self.token = CowrieConfig().get('output_csirtg', 'token') or TOKEN
+        self.description = CowrieConfig().get('output_csirtg', 'description', fallback=DESCRIPTION)
         self.context = {}
         self.client = Client(token=self.token)
 

src/cowrie/output/cuckoo.py

@@ -50,16 +50,15 @@ class Output(cowrie.core.output.Output):
     """
     cuckoo output
     """
-    url_base = CowrieConfig().get('output_cuckoo', 'url_base').encode('utf-8')
-    api_user = CowrieConfig().get('output_cuckoo', 'user')
-    api_passwd = CowrieConfig().get('output_cuckoo', 'passwd', raw=True)
-    cuckoo_force = int(CowrieConfig().getboolean('output_cuckoo', 'force'))
 
     def start(self):
         """
         Start output plugin
         """
-        pass
+        self.url_base = CowrieConfig().get('output_cuckoo', 'url_base').encode('utf-8')
+        self.api_user = CowrieConfig().get('output_cuckoo', 'user')
+        self.api_passwd = CowrieConfig().get('output_cuckoo', 'passwd', raw=True)
+        self.cuckoo_force = int(CowrieConfig().getboolean('output_cuckoo', 'force'))
 
     def stop(self):
         """

src/cowrie/output/dshield.py

@@ -26,12 +26,12 @@ class Output(cowrie.core.output.Output):
     """
     dshield output
     """
-    auth_key = CowrieConfig().get('output_dshield', 'auth_key')
-    userid = CowrieConfig().get('output_dshield', 'userid')
-    batch_size = CowrieConfig().getint('output_dshield', 'batch_size')
-    debug = CowrieConfig().getboolean('output_dshield', 'debug', fallback=False)
 
     def start(self):
+        self.auth_key = CowrieConfig().get('output_dshield', 'auth_key')
+        self.userid = CowrieConfig().get('output_dshield', 'userid')
+        self.batch_size = CowrieConfig().getint('output_dshield', 'batch_size')
+        self.debug = CowrieConfig().getboolean('output_dshield', 'debug', fallback=False)
         self.batch = []  # This is used to store login attempts in batches
 
     def stop(self):

src/cowrie/output/elasticsearch.py

@@ -12,13 +12,13 @@ class Output(cowrie.core.output.Output):
     """
     elasticsearch output
     """
-    host = CowrieConfig().get('output_elasticsearch', 'host')
-    port = CowrieConfig().get('output_elasticsearch', 'port')
-    index = CowrieConfig().get('output_elasticsearch', 'index')
-    type = CowrieConfig().get('output_elasticsearch', 'type')
-    pipeline = CowrieConfig().get('output_elasticsearch', 'pipeline')
 
     def start(self):
+        self.host = CowrieConfig().get('output_elasticsearch', 'host')
+        self.port = CowrieConfig().get('output_elasticsearch', 'port')
+        self.index = CowrieConfig().get('output_elasticsearch', 'index')
+        self.type = CowrieConfig().get('output_elasticsearch', 'type')
+        self.pipeline = CowrieConfig().get('output_elasticsearch', 'pipeline')
         self.es = Elasticsearch('{0}:{1}'.format(self.host, self.port))
 
     def stop(self):

src/cowrie/output/greynoise.py

@@ -20,14 +20,14 @@ class Output(cowrie.core.output.Output):
     """
     greynoise output
     """
-    apiKey = CowrieConfig().get('output_greynoise', 'api_key', fallback=None)
-    tags = CowrieConfig().get('output_greynoise', 'tags', fallback="all").split(",")
-    debug = CowrieConfig().getboolean('output_greynoise', 'debug', fallback=False)
 
     def start(self):
         """
         Start output plugin
         """
+        self.apiKey = CowrieConfig().get('output_greynoise', 'api_key', fallback=None)
+        self.tags = CowrieConfig().get('output_greynoise', 'tags', fallback="all").split(",")
+        self.debug = CowrieConfig().getboolean('output_greynoise', 'debug', fallback=False)
 
     def stop(self):
         """

src/cowrie/output/jsonlog.py

@@ -40,9 +40,9 @@ class Output(cowrie.core.output.Output):
     """
     jsonlog output
     """
-    epoch_timestamp = CowrieConfig().getboolean('output_jsonlog', 'epoch_timestamp', fallback=False)
 
     def start(self):
+        self.epoch_timestamp = CowrieConfig().getboolean('output_jsonlog', 'epoch_timestamp', fallback=False)
         fn = CowrieConfig().get('output_jsonlog', 'logfile')
         dirs = os.path.dirname(fn)
         base = os.path.basename(fn)

src/cowrie/output/localsyslog.py

@@ -41,9 +41,9 @@ class Output(cowrie.core.output.Output):
     """
     localsyslog output
     """
-    format = CowrieConfig().get('output_localsyslog', 'format')
 
     def start(self):
+        self.format = CowrieConfig().get('output_localsyslog', 'format')
         facilityString = CowrieConfig().get('output_localsyslog', 'facility')
         self.facility = vars(syslog)['LOG_' + facilityString]
         self.syslog = twisted.python.syslog.SyslogObserver(prefix='cowrie', facility=self.facility)

src/cowrie/output/mysql.py

@@ -47,9 +47,9 @@ class Output(cowrie.core.output.Output):
     mysql output
     """
     db = None
-    debug = CowrieConfig().getboolean('output_mysql', 'debug', fallback=False)
 
     def start(self):
+        self.debug = CowrieConfig().getboolean('output_mysql', 'debug', fallback=False)
         port = CowrieConfig().getint('output_mysql', 'port', fallback=3306)
         try:
             self.db = ReconnectingConnectionPool(

src/cowrie/output/redis.py

@@ -1,7 +1,7 @@
 from __future__ import absolute_import, division
 
 import json
-from ConfigParser import NoOptionError
+from configparser import NoOptionError
 
 import redis
 

src/cowrie/output/rethinkdblog.py

@@ -17,14 +17,14 @@ RETHINK_DB_SEGMENT = 'output_rethinkdblog'
 
 
 class Output(cowrie.core.output.Output):
-    host = CowrieConfig().get(RETHINK_DB_SEGMENT, 'host')
-    port = CowrieConfig().getint(RETHINK_DB_SEGMENT, 'port')
-    db = CowrieConfig().get(RETHINK_DB_SEGMENT, 'db')
-    table = CowrieConfig().get(RETHINK_DB_SEGMENT, 'table')
-    password = CowrieConfig().get(RETHINK_DB_SEGMENT, 'password', raw=True)
 
     # noinspection PyAttributeOutsideInit
     def start(self):
+        self.host = CowrieConfig().get(RETHINK_DB_SEGMENT, 'host')
+        self.port = CowrieConfig().getint(RETHINK_DB_SEGMENT, 'port')
+        self.db = CowrieConfig().get(RETHINK_DB_SEGMENT, 'db')
+        self.table = CowrieConfig().get(RETHINK_DB_SEGMENT, 'table')
+        self.password = CowrieConfig().get(RETHINK_DB_SEGMENT, 'password', raw=True)
         self.connection = r.connect(
             host=self.host,
             port=self.port,

src/cowrie/output/reversedns.py

@@ -15,14 +15,13 @@ class Output(cowrie.core.output.Output):
     """
     Output plugin used for reverse DNS lookup
     """
-    timeout = [CowrieConfig().getint(
-            'output_reversedns', 'timeout', fallback=3)]
 
     def start(self):
         """
         Start Output Plugin
         """
-        pass
+        self.timeout = [CowrieConfig().getint(
+            'output_reversedns', 'timeout', fallback=3)]
 
     def stop(self):
         """

src/cowrie/output/s3.py

@@ -20,9 +20,9 @@ class Output(cowrie.core.output.Output):
     """
     s3 output
     """
-    bucket = CowrieConfig().get("output_s3", "bucket")
 
     def start(self):
+        self.bucket = CowrieConfig().get("output_s3", "bucket")
         self.seen = set()
         self.session = get_session()
 

src/cowrie/output/slack.py

@@ -41,10 +41,10 @@ class Output(cowrie.core.output.Output):
     """
     slack output
     """
-    slack_channel = CowrieConfig().get('output_slack', 'channel')
-    slack_token = CowrieConfig().get('output_slack', 'token')
 
     def start(self):
+        self.slack_channel = CowrieConfig().get('output_slack', 'channel')
+        self.slack_token = CowrieConfig().get('output_slack', 'token')
         pass
 
     def stop(self):

src/cowrie/output/socketlog.py

@@ -13,9 +13,9 @@ class Output(cowrie.core.output.Output):
     """
     socketlog output
     """
-    timeout = CowrieConfig().getint('output_socketlog', 'timeout')
 
     def start(self):
+        self.timeout = CowrieConfig().getint('output_socketlog', 'timeout')
         addr = CowrieConfig().get('output_socketlog', 'address')
         self.host = addr.split(':')[0]
         self.port = int(addr.split(':')[1])

src/cowrie/output/splunk.py

@@ -29,14 +29,14 @@ class Output(cowrie.core.output.Output):
     """
     Splunk HEC output
     """
-    token = CowrieConfig().get('output_splunk', 'token')
-    url = CowrieConfig().get('output_splunk', 'url').encode('utf8')
-    index = CowrieConfig().get('output_splunk', 'index', fallback=None)
-    source = CowrieConfig().get('output_splunk', 'source', fallback=None)
-    sourcetype = CowrieConfig().get('output_splunk', 'sourcetype', fallback=None)
-    host = CowrieConfig().get('output_splunk', 'host', fallback=None)
 
     def start(self):
+        self.token = CowrieConfig().get('output_splunk', 'token')
+        self.url = CowrieConfig().get('output_splunk', 'url').encode('utf8')
+        self.index = CowrieConfig().get('output_splunk', 'index', fallback=None)
+        self.source = CowrieConfig().get('output_splunk', 'source', fallback=None)
+        self.sourcetype = CowrieConfig().get('output_splunk', 'sourcetype', fallback=None)
+        self.host = CowrieConfig().get('output_splunk', 'host', fallback=None)
         contextFactory = WebClientContextFactory()
         # contextFactory.method = TLSv1_METHOD
         self.agent = client.Agent(reactor, contextFactory)

src/cowrie/output/textlog.py

@@ -37,10 +37,10 @@ class Output(cowrie.core.output.Output):
     """
     textlog output
     """
-    format = CowrieConfig().get('output_textlog', 'format')
-    outfile = open(CowrieConfig().get('output_textlog', 'logfile'), 'a')
 
     def start(self):
+        self.format = CowrieConfig().get('output_textlog', 'format')
+        self.outfile = open(CowrieConfig().get('output_textlog', 'logfile'), 'a')
         pass
 
     def stop(self):

src/cowrie/output/virustotal.py

@@ -63,18 +63,18 @@ class Output(cowrie.core.output.Output):
     """
     virustotal output
     """
-    apiKey = CowrieConfig().get('output_virustotal', 'api_key')
-    debug = CowrieConfig().getboolean('output_virustotal', 'debug', fallback=False)
-    upload = CowrieConfig().getboolean('output_virustotal', 'upload', fallback=True)
-    comment = CowrieConfig().getboolean('output_virustotal', 'comment', fallback=True)
-    scan_file = CowrieConfig().getboolean('output_virustotal', 'scan_file', fallback=True)
-    scan_url = CowrieConfig().getboolean('output_virustotal', 'scan_url', fallback=False)
-    commenttext = CowrieConfig().get('output_virustotal', 'commenttext', fallback=COMMENT)
 
     def start(self):
         """
         Start output plugin
         """
+        self.apiKey = CowrieConfig().get('output_virustotal', 'api_key')
+        self.debug = CowrieConfig().getboolean('output_virustotal', 'debug', fallback=False)
+        self.upload = CowrieConfig().getboolean('output_virustotal', 'upload', fallback=True)
+        self.comment = CowrieConfig().getboolean('output_virustotal', 'comment', fallback=True)
+        self.scan_file = CowrieConfig().getboolean('output_virustotal', 'scan_file', fallback=True)
+        self.scan_url = CowrieConfig().getboolean('output_virustotal', 'scan_url', fallback=False)
+        self.commenttext = CowrieConfig().get('output_virustotal', 'commenttext', fallback=COMMENT)
         self.agent = client.Agent(reactor, WebClientContextFactory())
 
     def stop(self):

src/cowrie/shell/fs.py

@@ -20,18 +20,6 @@ from twisted.python import log
 
 from cowrie.core.config import CowrieConfig
 
-# At the moment this is the first place a config file is used
-# Put extra help here in case it goes wrong
-try:
-    with open(CowrieConfig().get('shell', 'filesystem'), 'rb') as f:
-        PICKLE = pickle.load(f)
-except UnicodeDecodeError:
-    with open(CowrieConfig().get('shell', 'filesystem'), 'rb') as f:
-        PICKLE = pickle.load(f, encoding='utf8')
-except Exception as e:
-    log.err(e, "ERROR: Failed to load filesystem")
-    exit(2)
-
 A_NAME, A_TYPE, A_UID, A_GID, A_SIZE, A_MODE, A_CTIME, A_CONTENTS, A_TARGET, A_REALFILE = list(range(0, 10))
 T_LINK, T_DIR, T_FILE, T_BLK, T_CHR, T_SOCK, T_FIFO = list(range(0, 7))
 
@@ -85,7 +73,16 @@ class PermissionDenied(Exception):
 class HoneyPotFilesystem(object):
 
     def __init__(self, fs, arch):
-        self.fs = fs
+
+        try:
+            with open(CowrieConfig().get('shell', 'filesystem'), 'rb') as f:
+                self.fs = pickle.load(f)
+        except UnicodeDecodeError:
+            with open(CowrieConfig().get('shell', 'filesystem'), 'rb') as f:
+                self.fs = pickle.load(f, encoding='utf8')
+        except Exception as e:
+            log.err(e, "ERROR: Failed to load filesystem")
+            exit(2)
 
         # Keep track of arch so we can return appropriate binary
         self.arch = arch

src/cowrie/shell/server.py

@@ -28,7 +28,6 @@
 
 from __future__ import absolute_import, division
 
-import copy
 import json
 import random
 from configparser import NoOptionError
@@ -74,7 +73,7 @@ class CowrieServer(object):
         """
         Do this so we can trigger it later. Not all sessions need file system
         """
-        self.fs = fs.HoneyPotFilesystem(copy.deepcopy(fs.PICKLE), self.arch)
+        self.fs = fs.HoneyPotFilesystem(None, self.arch)
 
         try:
             self.process = self.getCommandOutput(CowrieConfig().get('shell', 'processes'))['command']['ps']

src/cowrie/ssh/transport.py

@@ -231,7 +231,7 @@ class HoneyPotSSHTransport(transport.SSHServerTransport, TimeoutMixin):
         @param reason: the reason for the disconnect.  Should be one of the
                        DISCONNECT_* values.
         @type reason: C{int}
-        @param desc: a descrption of the reason for the disconnection.
+        @param desc: a description of the reason for the disconnection.
         @type desc: C{str}
         """
         if b'bad packet length' not in desc:
@@ -243,8 +243,7 @@ class HoneyPotSSHTransport(transport.SSHServerTransport, TimeoutMixin):
 
     def receiveError(self, reasonCode, description):
         """
-        Called when we receive a disconnect error message from the other
-        side.
+        Called when we receive a disconnect error message from the other side.
 
         @param reasonCode: the reason for the disconnect, one of the
                            DISCONNECT_ values.

src/cowrie/test/fake_server.py

@@ -5,10 +5,6 @@
 
 from __future__ import absolute_import, division
 
-import copy
-import pickle
-
-from cowrie.core.config import CowrieConfig
 from cowrie.shell import fs
 
 
@@ -22,9 +18,7 @@ class FakeServer:
         self.arch = 'linux-x64-lsb'
         self.hostname = "unitTest"
 
-        self.pckl = pickle.load(
-            open(CowrieConfig().get('honeypot', 'filesystem_file'), 'rb'))
-        self.fs = fs.HoneyPotFilesystem(copy.deepcopy(self.pckl), 'arch')
+        self.fs = fs.HoneyPotFilesystem(None, 'arch')
         self.process = None
 
 

src/cowrie/test/test_base_commands.py

@@ -13,7 +13,7 @@ from cowrie.shell import protocol
 from cowrie.test import fake_server, fake_transport
 
 os.environ["HONEYPOT_DATA_PATH"] = "../data"
-os.environ["HONEYPOT_FILESYSTEM_FILE"] = "../share/cowrie/fs.pickle"
+os.environ["SHELL_FILESYSTEM"] = "../share/cowrie/fs.pickle"
 
 PROMPT = b"root@unitTest:~# "
 

src/cowrie/test/test_echo.py

@@ -18,7 +18,7 @@ from cowrie.test import fake_server, fake_transport
 
 os.environ["HONEYPOT_DATA_PATH"] = "../data"
 os.environ["HONEYPOT_DOWNLOAD_PATH"] = "/tmp"
-os.environ["HONEYPOT_FILESYSTEM_FILE"] = "../share/cowrie/fs.pickle"
+os.environ["SHELL_FILESYSTEM"] = "../share/cowrie/fs.pickle"
 
 PROMPT = b"root@unitTest:~# "
 

src/cowrie/test/test_ftpget.py

@@ -15,7 +15,7 @@ from cowrie.test import fake_server, fake_transport
 
 os.environ["HONEYPOT_DATA_PATH"] = "../data"
 os.environ["HONEYPOT_DOWNLOAD_PATH"] = "/tmp"
-os.environ["HONEYPOT_FILESYSTEM_FILE"] = "../share/cowrie/fs.pickle"
+os.environ["SHELL_FILESYSTEM"] = "../share/cowrie/fs.pickle"
 
 PROMPT = b"root@unitTest:~# "
 

src/cowrie/test/test_tftp.py

@@ -18,7 +18,7 @@ from cowrie.test import fake_server, fake_transport
 
 os.environ["HONEYPOT_DATA_PATH"] = "../data"
 os.environ["HONEYPOT_DOWNLOAD_PATH"] = "/tmp"
-os.environ["HONEYPOT_FILESYSTEM_FILE"] = "../share/cowrie/fs.pickle"
+os.environ["SHELL_FILESYSTEM"] = "../share/cowrie/fs.pickle"
 
 PROMPT = b"root@unitTest:~# "