diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..bb560fb --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +qcow/* +db/* +iso/* +*.pyc diff --git a/conf/malrec.config b/conf/malrec.config index 747fd7c..e7f996e 100644 --- a/conf/malrec.config +++ b/conf/malrec.config @@ -1,7 +1,8 @@ [Main] -basedir = /home/brendan/malrec -panda = /home/brendan/git/panda/qemu -db = /home/brendan/malrec/db/panda.db +basedir = /home/users/grimm/panda-malrec +panda = /home/users/grimm/panda1/qemu +db = /home/users/grimm/panda-malrec/db/panda.db +volatility = /home/users/grimm/volatility/vol.py [VM] mem = 1G diff --git a/run.sh b/run.sh new file mode 100755 index 0000000..29a7cc1 --- /dev/null +++ b/run.sh @@ -0,0 +1,2 @@ +#!/bin/bash +ls queue/pending/$2/ | parallel -j 40 python scripts/runmal.py conf/malrec.config {/} {%} $1 ; sleep 600 diff --git a/scripts/dumpmem.py b/scripts/dumpmem.py new file mode 100644 index 0000000..7aaa4a0 --- /dev/null +++ b/scripts/dumpmem.py @@ -0,0 +1,83 @@ +import os +import tarfile +import subprocess +import errno +import re +import time +from Queue import Queue, Empty +from threading import Thread +import signal + +def dumpmem(panda_path, mem, sample_name, dumppath): + mkdir_p(dumppath) + path = os.path.join(dumppath, "memdump") + panda_args = [panda_path, + '-m', mem, + '-replay', sample_name, + '-panda', "n_instruct_mem:file={}".format(path) + ] + tarname = "{}.tar.gz".format(dumppath) + + rr_path = "/".join(dumppath.split("/")[0:-1]) + + proc = subprocess.Popen(panda_args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=rr_path) + + stdout_q = Queue() + stderr_q = Queue() + out_t = Thread(target=enqueue_output, args=(proc.stdout, stdout_q)) + err_t = Thread(target=enqueue_output, args=(proc.stderr, stderr_q)) + out_t.daemon = True + err_t.daemon = True + out_t.start() + err_t.start() + + finished = False + success = False + count = 0 + + while not finished: + stdout = get_nowait_q(stdout_q) + stderr = get_nowait_q(stderr_q) + + print(stdout) + + if stdout and re.search("Replay completed successfully", stdout): + print("Success case made") + finished = True + success = True + os.kill(proc.pid, signal.SIGKILL) + else: + time.sleep(2) + + out_t.join() + err_t.join() + + if success: + with tarfile.open(tarname, "w:gz") as tar: + print("Tart") + tar.add(dumppath, arcname=sample_name) + print("Tarted") + else: + return (False, "Failed to create memdumps for {}".format(sample_name), None) + + return (True, "Success for {}".format(sample_name), tarname) + +def mkdir_p(path): + try: + os.makedirs(path) + except OSError as exc: + if exc.errno == errno.EEXIST and os.path.isdir(path): + pass + else: + raise + +def enqueue_output(out, queue): + for line in iter(out.readline, b''): + queue.put(line) + out.close() + +def get_nowait_q(q): + try: + return q.get_nowait() + except Empty: + pass \ No newline at end of file diff --git a/scripts/runmal.py b/scripts/runmal.py index dbe9bf0..eacb354 100755 --- a/scripts/runmal.py +++ b/scripts/runmal.py @@ -17,9 +17,12 @@ import hashlib import tempfile import atexit +from dumpmem import dumpmem from mon_util import mon_cmd, guest_type import click_buttons +global sample_name, sample_file + def md5_for_file(fname, block_size=2**20): f = open(fname, 'rb') md5 = hashlib.md5() @@ -52,7 +55,7 @@ def cleanup(): c = conn.cursor() while True: try: - c.execute('INSERT INTO samples VALUES(?,?,?)', (run_id, sample_name, sample_md5)) + c.execute('INSERT INTO samples VALUES(?,?,?)', (sample_name, sample_name, sample_md5)) break except sqlite3.OperationalError: pass @@ -61,7 +64,7 @@ def cleanup(): conn.close() # All done, write the stamp - stampfile = os.path.join(logdir, 'stamps', run_id) + stampfile = os.path.join(logdir, 'stamps', sample_name) open(stampfile, 'w').close() atexit.register(cleanup) @@ -71,7 +74,14 @@ def cleanup(): sample_name = sys.argv[2] instance = int(sys.argv[3]) -run_id = str(uuid.uuid4()) +malicious = bool(int(sys.argv[4])) +run_subpath = None + +if malicious: + run_subpath = os.path.join("malicious", sample_name) +else: + run_subpath = os.path.join("benign", sample_name) + # Setup from config monitor_port = 1234 + instance @@ -81,8 +91,8 @@ def cleanup(): queuedir = os.path.join(basedir, 'queue') logdir = os.path.join(basedir, 'logs') rr_logdir = os.path.join(logdir, 'rr') -rr_logname = os.path.join(rr_logdir, run_id) -pcap_name = os.path.join(logdir, 'pcap', run_id + '.pcap') +rr_logname = os.path.join(rr_logdir, run_subpath) +pcap_name = os.path.join(logdir, 'pcap', sample_name+ '.pcap') logfile = os.path.join(logdir, 'text', time.strftime('%Y%m%d.%H.%M.%S.{0}.log').format(instance)) database = conf.get('Main', 'db') @@ -91,14 +101,20 @@ def cleanup(): # Startup msgs logging.info("Config file: {0}".format(sys.argv[1])) -logging.info("UUID: {0}".format(run_id)) logging.info("Sample: {0}".format(sample_name)) # Claim ownership of this file logging.info("Moving sample into 'running' queue.") sample_file = os.path.join(queuedir, 'running', sample_name) +pending_path = None + +if malicious: + pending_path = "pending/malicious" +else: + pending_path = "pending/benign" + shutil.move( - os.path.join(queuedir, 'pending', sample_name), + os.path.join(queuedir, pending_path, sample_name), sample_file ) @@ -107,7 +123,7 @@ def cleanup(): logging.info("MD5: {0}".format(sample_md5)) # Make the CD image -iso_file = os.path.join(basedir, 'iso', run_id + '.iso') +iso_file = os.path.join(basedir, 'iso', sample_name + '.iso') logging.info("Creating CD image {0}".format(iso_file)) genisoimage = ['/usr/bin/genisoimage', '-iso-level', '4', '-l', '-R', '-J', '-o', iso_file, sample_file] logging.info(str(genisoimage)) @@ -149,8 +165,8 @@ def cleanup(): ] # Start the QEMU process -panda_stdout = open(os.path.join(logdir, 'text', run_id + '.stdout'), 'w') -panda_stderr = open(os.path.join(logdir, 'text', run_id + '.stderr'), 'w') +panda_stdout = open(os.path.join(logdir, 'text', sample_name + '.stdout'), 'w') +panda_stderr = open(os.path.join(logdir, 'text', sample_name + '.stderr'), 'w') panda = subprocess.Popen(panda_args, stdin=subprocess.PIPE, stdout=panda_stdout, stderr=panda_stderr) # Connect to the monitor @@ -194,12 +210,11 @@ def cleanup(): # Create our memory access socket qemu_socket = tempfile.mktemp() +print("Qemu socket") logging.info("Creating memory access socket: {0}".format(qemu_socket)) mon_cmd("pmemaccess {0}\n".format(qemu_socket), mon) # Warm up the Volatility part -click_buttons.setup("Win7SP1x64" if is_64bit else "Win7SP1x86", "qemu://" + qemu_socket) - # Run the sample logging.info("Starting sample.") # Handle 3 cases: driver, exe, dll @@ -220,7 +235,6 @@ def cleanup(): period = 10 for _ in range(exec_time / period): time.sleep(period) - click_buttons.click_buttons(mon) # End the record logging.info("Ending record.") @@ -228,3 +242,23 @@ def cleanup(): logging.info("Quitting PANDA.") mon.write("q\n") +success, message, filepath = dumpmem(panda_exe, conf.get('VM', 'mem'), sample_name, rr_logname+"_memdump") + +if success: + logging.info(message) + + try: + shutil.rmtree(rr_logname+"_memdump") + os.remove(rr_logname+"-rr.cmd") + os.remove(rr_logname+"-rr-nondet.log") + os.remove(rr_logname+"-rr-snp") + except e: + logging.error(e) + raise e + + logging.info("Success. Exiting.") + sys.exit(0) +else: + logging.error(message) + + raise diff --git a/scripts/targetvaddump.py b/scripts/targetvaddump.py new file mode 100644 index 0000000..b8aa05e --- /dev/null +++ b/scripts/targetvaddump.py @@ -0,0 +1,132 @@ +import os +import sys +import tarfile +import glob +import ConfigParser +import subprocess +import time +import errno +from collections import Counter +from pprint import pprint +from functools import partial +from multiprocessing import Pool + +def find_possible_targets(folder_name): + proc_counts = Counter() + + for memfile in sorted(glob.glob(folder_name + "/*"), key=lambda name: name.split("_")[-1]): + stdout = vol_pslist(memfile) + count = len(stdout.split("\n")[2:]) + proc_key = memfile.split("_")[-1] + proc_counts[proc_key] = count + + count_by_dump = proc_counts.most_common() + + _max = count_by_dump[0] + _min = count_by_dump[-1] + + if _max[1] == _min[1]: + raise ValueError("It looks like the EXE was not run in the recording.") + + potential_targets = None + + max_index = int(_max[0].split("_")[-1]) + min_index = int(_max[0].split("_")[-1]) + + if max_index > len(count_by_dump) // 2: + potential_targets = test_for_targets(_min[1], range(max_index, len(count_by_dump)), proc_counts) + else: + potential_targets = test_for_targets(_min[1], range(min_index, (len(count_by_dump) // 2) + 1 ), proc_counts) + + return (_min, potential_targets) + +def test_for_targets(baseline, _range, counts): + passing = [] + + if len(counts.keys()) == 1: + return [0] + + for index in _range: + if int(counts[str(index)]) - baseline > 0: + passing.append(index) + + return passing + +def dump_vads(folder_name, _type, os_procs, dump_index): + vad_path = "/tmp/vads/{}".format(folder_name) + + mkdir_p(vad_path) + + filename = "/tmp/{0}/{1}/memdump_{2}".format(_type, folder_name, dump_index) + + vol_args = ["python", + conf.get("Main", 'volatility'), + '-f', filename, + '--profile', "Win7SP0x86", + 'vaddump', '-D', vad_path + ] + + subprocess.check_output(vol_args, stderr=subprocess.PIPE) + + for proc_name in os_procs: + for f in glob.glob(vad_path + "/*{}*".format(proc_name)): + os.remove(f) + +def mkdir_p(path): + try: + os.makedirs(path) + except OSError as exc: + if exc.errno == errno.EEXIST and os.path.isdir(path): + pass + else: + raise + +def proc_list_from_dump_index(folder_name, _type, dump_index): + memfile = "/tmp/{0}/{1}/memdump_{2}".format(_type, folder_name, dump_index) + stdout = filter(None, vol_pslist(memfile).split("\n")) + proc_names = map(lambda ln: ln.split(" ")[1], stdout[2:]) + + return proc_names + +def vol_pslist(memfile): + vol_args = ["python", + conf.get("Main", 'volatility'), + '-f', memfile, + '--profile', "Win7SP0x86", + 'pslist', + ] + + return subprocess.check_output(vol_args, stderr=subprocess.PIPE) + +def execute(_type, file): + with tarfile.open(file) as memdumps: + try: + memdumps.extractall(path="/tmp/{}/".format(_type)) + except: + print("Incomplete targz") + + #not the clearest but basically grabbing the untarred folder name from the tarred folder name + folder_name = "_".join((".".join(file.split("/")[-1].split(".")[0:-2])).split("_")[0:-1]) + baseline, potential_targets = find_possible_targets("/tmp/{0}/{1}".format(_type, folder_name)) + os_procs = proc_list_from_dump_index(folder_name, _type, baseline[0]) + + map(partial(dump_vads, folder_name, _type, os_procs), potential_targets) + +def main(): + folder = sys.argv[1] + _type = "benign" + + global conf + + conf = ConfigParser.ConfigParser() + conf.read("conf/malrec.config") + + if bool(int(sys.argv[2])): + _type = "malicious" + + p = Pool(10) + files = glob.glob(folder + "/*") + p.map(partial(execute, _type), files) + +if __name__ == "__main__": + main() \ No newline at end of file