Add parser for output

.gitignore

@@ -0,0 +1,109 @@
+__pycache__
+
+# Created by https://www.toptal.com/developers/gitignore/api/pycharm
+# Edit at https://www.toptal.com/developers/gitignore?templates=pycharm
+
+### PyCharm ###
+# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider
+# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
+
+# User-specific stuff
+.idea/**/workspace.xml
+.idea/**/tasks.xml
+.idea/**/usage.statistics.xml
+.idea/**/dictionaries
+.idea/**/shelf
+
+# Generated files
+.idea/**/contentModel.xml
+
+# Sensitive or high-churn files
+.idea/**/dataSources/
+.idea/**/dataSources.ids
+.idea/**/dataSources.local.xml
+.idea/**/sqlDataSources.xml
+.idea/**/dynamic.xml
+.idea/**/uiDesigner.xml
+.idea/**/dbnavigator.xml
+
+# Gradle
+.idea/**/gradle.xml
+.idea/**/libraries
+
+# Gradle and Maven with auto-import
+# When using Gradle or Maven with auto-import, you should exclude module files,
+# since they will be recreated, and may cause churn.  Uncomment if using
+# auto-import.
+# .idea/artifacts
+# .idea/compiler.xml
+# .idea/jarRepositories.xml
+# .idea/modules.xml
+# .idea/*.iml
+# .idea/modules
+# *.iml
+# *.ipr
+
+# CMake
+cmake-build-*/
+
+# Mongo Explorer plugin
+.idea/**/mongoSettings.xml
+
+# File-based project format
+*.iws
+
+# IntelliJ
+out/
+
+# mpeltonen/sbt-idea plugin
+.idea_modules/
+
+# JIRA plugin
+atlassian-ide-plugin.xml
+
+# Cursive Clojure plugin
+.idea/replstate.xml
+
+# Crashlytics plugin (for Android Studio and IntelliJ)
+com_crashlytics_export_strings.xml
+crashlytics.properties
+crashlytics-build.properties
+fabric.properties
+
+# Editor-based Rest Client
+.idea/httpRequests
+
+# Android studio 3.1+ serialized cache file
+.idea/caches/build_file_checksums.ser
+
+### PyCharm Patch ###
+# Comment Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-215987721
+
+# *.iml
+# modules.xml
+# .idea/misc.xml
+# *.ipr
+
+# Sonarlint plugin
+# https://plugins.jetbrains.com/plugin/7973-sonarlint
+.idea/**/sonarlint/
+
+# SonarQube Plugin
+# https://plugins.jetbrains.com/plugin/7238-sonarqube-community-plugin
+.idea/**/sonarIssues.xml
+
+# Markdown Navigator plugin
+# https://plugins.jetbrains.com/plugin/7896-markdown-navigator-enhanced
+.idea/**/markdown-navigator.xml
+.idea/**/markdown-navigator-enh.xml
+.idea/**/markdown-navigator/
+
+# Cache file creation bug
+# See https://youtrack.jetbrains.com/issue/JBR-2257
+.idea/$CACHE_FILE$
+
+# CodeStream plugin
+# https://plugins.jetbrains.com/plugin/12206-codestream
+.idea/codestream.xml
+
+# End of https://www.toptal.com/developers/gitignore/api/pycharm

scan_output_parser/.idea/.gitignore

@@ -0,0 +1,8 @@
+# Default ignored files
+/shelf/
+/workspace.xml
+# Datasource local storage ignored files
+/../../../../../:\Git-Repos\it-security-2\scan_output_parser\.idea/dataSources/
+/dataSources.local.xml
+# Editor-based HTTP Client requests
+/httpRequests/

scan_output_parser/.idea/inspectionProfiles/profiles_settings.xml

@@ -0,0 +1,6 @@
+<component name="InspectionProjectProfileManager">
+  <settings>
+    <option name="USE_PROJECT_PROFILE" value="false" />
+    <version value="1.0" />
+  </settings>
+</component>

scan_output_parser/.idea/misc.xml

@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.9" project-jdk-type="Python SDK" />
+</project>

scan_output_parser/.idea/modules.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/.idea/scan_output_parser.iml" filepath="$PROJECT_DIR$/.idea/scan_output_parser.iml" />
+    </modules>
+  </component>
+</project>

scan_output_parser/.idea/scan_output_parser.iml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="PYTHON_MODULE" version="4">
+  <component name="NewModuleRootManager">
+    <content url="file://$MODULE_DIR$" />
+    <orderEntry type="inheritedJdk" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+</module>

scan_output_parser/.idea/vcs.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="VcsDirectoryMappings">
+    <mapping directory="$PROJECT_DIR$/.." vcs="Git" />
+  </component>
+</project>

scan_output_parser/lynis.py

@@ -0,0 +1,108 @@
+import re
+
+LYNIS_REGEX = {
+    "green": r"\[1;32m",  # more advanced "green": r"\[ .*\[1;32m([\w\d ]*).*",
+    "yellow": r"\[1;33m",
+    "red": r"\[1;31m",
+    "white": r"\[1;37m",
+    "heading": r".*\[1;33m([\w\d ,:-]*).*"
+}
+LYNIS_BLOCKS = {
+    "Boot and services",
+    "Kernel",
+    "Memory and Processes",
+    "Users, Groups and Authentication",
+    "Shells",
+    "File systems",
+    "USB Devices",
+    "Storage",
+    "NFS",
+    "Name services",
+    "Ports and packages",
+    "Networking",
+    "Printers and Spools",
+    "Software: e-mail and messaging",
+    "Software: firewalls",
+    "Software: webserver",
+    "SSH Support",
+    "Databases",
+    "PHP",
+    "Logging and files",
+    "Insecure services",
+    "Scheduled tasks",
+    "Accounting",
+    "Time and Synchronization",
+    "Cryptography",
+    "Security frameworks",
+    "Software: Malware",
+    "File Permissions",
+    "Home directories",
+    "Kernel Hardening",
+    "Hardening"
+}
+
+
+def lynis_get_base(path_to_log):
+    with open(path_to_log) as handle:
+        text = handle.read()
+    blocks = text.split("[+]")
+    interesting_blocks = {}
+    for block in blocks:
+        heading = re.findall(LYNIS_REGEX["heading"], block.splitlines()[0])
+        if heading and heading[0] in LYNIS_BLOCKS:
+            block_text = "".join(block.splitlines()[2:])
+            interesting_blocks[heading[0]] = {
+                "text": block_text,
+                "counts": {
+                    "green": len(re.findall(LYNIS_REGEX['green'], block_text)),
+                    "yellow": len(re.findall(LYNIS_REGEX['yellow'], block_text)),
+                    "red": len(re.findall(LYNIS_REGEX['red'], block_text)),
+                    "white": len(re.findall(LYNIS_REGEX['white'], block_text)),
+                }
+            }
+    interesting_blocks["GENERAL"] = {}
+    if warning_count := re.findall(r".*Warnings.* \((\d+)\)", text):
+        interesting_blocks["GENERAL"]["warningCount"] = int(warning_count[0])
+
+    if suggestion_count := re.findall(r".*Suggestions.* \((\d+)\)", text):
+        interesting_blocks["GENERAL"]["suggestionCount"] = int(suggestion_count[0])
+
+    if hardening_index := re.findall(r".*Hardening index.*m(\d+)", text):
+        interesting_blocks["GENERAL"]["hardeningIndex"] = int(hardening_index[0])
+
+    if tests_performed := re.findall(r".*Tests performed.*m(\d+)", text):
+        interesting_blocks["GENERAL"]["testsPerformed"] = int(tests_performed[0])
+
+    return interesting_blocks
+
+
+def lynis_add_special_properties(lynis_base):
+    def find_prop(category, key, regex, as_int=False):
+        if category in lynis_base:
+            if match := re.findall(regex, lynis_base[category]["text"]):
+                lynis_base[category][key] = int(match[0]) if as_int else match[0]
+
+    find_prop("Boot and services", "runningServices", r"found (\d+) running services", as_int=True)
+    find_prop("Boot and services", "enabledServices", r"found (\d+) enabled services", as_int=True)
+    find_prop("Kernel", "defaultRunLevel", r"RUNLEVEL (\d+)", as_int=True)
+    find_prop("Kernel", "activeModules", r"Found (\d+) active modules", as_int=True)
+    find_prop("Kernel", "kernelUpdate", r"Checking for available kernel update.*? \[ .*?m([\w ]*).*? \]")
+    find_prop("Shells", "numShells", r"Result: found (\d*) shells", as_int=True)
+    find_prop("Networking", "ipv6Enabled", r"Checking IPv6 configuration.*? \[ .*?m([\w ]*).*? \]")
+    find_prop("Time and Synchronization", "lastSync", r"Last time synchronization.*? \[ .*?m([\w ]*).*? \]")
+    find_prop("Security frameworks", "unconfinedProcesses", r"Found (\d+) unconfined processes")
+
+
+def lynis_remove_text_from_blocks(lynis_base):
+    for block in lynis_base:
+        try:
+            del lynis_base[block]["text"]
+        except KeyError:
+            pass
+
+
+def lynis_parse(path):
+    lynis_base = lynis_get_base(path)
+    lynis_add_special_properties(lynis_base)
+    lynis_remove_text_from_blocks(lynis_base)
+    return lynis_base

scan_output_parser/main.py

@@ -0,0 +1,57 @@
+import glob
+import os.path
+import re
+from dataclasses import dataclass
+from typing import List
+
+from lynis import lynis_parse
+from otseca import otseca_parse
+from testssl import testssl_parse
+
+BASE_SCAN_PATH = os.path.join("..", "raw_scans")
+
+
+@dataclass
+class Result:
+    path: str
+    run_nr: int
+    result: dict
+
+
+@dataclass
+class Run:
+    id: int
+    platform: str
+    system: str
+    version: str
+    path: str
+    otseca_results: List[Result]
+    lynis_results: List[Result]
+    testssl_results: List[Result]
+
+
+def main():
+    list_of_all = []
+    all_scans = glob.glob(os.path.join(BASE_SCAN_PATH, "*", ""))
+    for scan in all_scans:
+        findings = re.findall(r"(\d+)_(.*)_(.*)_(.*)", os.path.dirname(scan))
+        findings = findings[0]
+        list_of_all.append(
+            Run(findings[0], findings[1], findings[2], findings[3], scan, [], [], [])
+        )
+    for run in list_of_all:
+        for otseca_path in glob.glob(os.path.join(run.path, "otseca*", "report*")):
+            nr = re.findall(r"otseca-(\d+)", otseca_path)[0]
+            run.otseca_results.append(Result(otseca_path, nr, otseca_parse(otseca_path)))
+        for log_file in os.listdir(run.path):
+            path = os.path.join(run.path, log_file)
+            nr = re.findall(r"(\d+)", log_file)[0]
+            if "lynis-console-" in log_file:
+                run.lynis_results.append(Result(path, nr, lynis_parse(path)))
+            if "testssl-" in log_file:
+                run.testssl_results.append(Result(path, nr, testssl_parse(path)))
+    [print(run) for run in list_of_all]
+
+
+if __name__ == '__main__':
+    main()

scan_output_parser/otseca.py

@@ -0,0 +1,49 @@
+import os.path
+import re
+
+OTSECA_FILES = {
+    "distro": "distro.all.log.html",
+    "kernel": "kernel.all.log.html",
+    "network": "network.all.log.html",
+    "permission": "permissions.all.log.html",
+    "services": "services.all.log.html",
+    "system": "system.all.log.html"
+}
+
+
+def otseca_box_counts(path_to_report):
+    counts = {}
+    for (name, curr_file) in OTSECA_FILES.items():
+        curr_path = os.path.join(path_to_report, curr_file)
+        with open(curr_path, encoding="UTF-8") as handle:
+            text = handle.read()
+        counts[name] = {
+            "green": (len(re.findall("background-color: #1F9D55", text))),
+            "yellow": (len(re.findall("background-color: #F2D024", text))),
+            "red": (len(re.findall("background-color: #CC1F1A", text))),
+            "total": (len(re.findall("background-color:", text)))
+        }
+    return counts
+
+
+def otseca_distro_info(path_to_report):
+    with open(os.path.join(path_to_report, OTSECA_FILES["distro"])) as handle:
+        text = handle.read()
+
+    pkg_count = len(re.findall("ii {2}", text))
+    upgrades_count = re.findall(r"(\d+) .*? (\d+) .*? (\d+) .*? (\d+) .*", text).pop()
+
+    return {
+        "pkgCount": pkg_count,
+        "upgraded": upgrades_count[0],
+        "newlyInstalled": upgrades_count[1],
+        "remove": upgrades_count[2],
+        "notUpgraded": upgrades_count[3]
+    }
+
+
+def otseca_parse(path):
+    return {
+        "boxes": otseca_box_counts(path),
+        "general": otseca_distro_info(path)
+    }

scan_output_parser/testssl.py

@@ -0,0 +1,32 @@
+import re
+
+
+def testssl_parse(path):
+    with open(path) as handle:
+        text = handle.read()
+
+    testssl_overall = {
+        "443": {"open": False, "ssl": False},
+        "21": {"open": False, "ssl": False},
+        "465": {"open": False, "ssl": False},
+        "587": {"open": False, "ssl": False},
+        "110": {"open": False, "ssl": False},
+        "995": {"open": False, "ssl": False},
+        "993": {"open": False, "ssl": False},
+        "5432": {"open": False, "ssl": False},
+        "3306": {"open": False, "ssl": False}
+    }
+
+    tests = text.split("## Scan started as: ")
+    for test in tests:
+        port = re.findall("Start .*? -->> 127\.0\.0\.1:(\d*) \(localhost\) <<--", test)
+        if not port:
+            continue
+        port = port[0]
+        if "Overall Grade" in test:
+            testssl_overall[port]["ssl"] = True
+            testssl_overall[port]["open"] = True
+        if "firewall" not in test:
+            testssl_overall[port]["open"] = True
+            continue
+    return testssl_overall