idaholab#290, add 'all files except common plain text files' to file …

…carving option for Malcolm

docs/file-scanning.md

@@ -6,6 +6,7 @@ To specify which files should be extracted, the following values are acceptable
 
 * `none`: no file extraction
 * `interesting`: extraction of files with mime types of common attack vectors
+* `notcommtxt`: extraction of all files except common plain text files
 * `mapped`: extraction of files with recognized mime types
 * `known`: extraction of files for which any mime type can be determined
 * `all`: extract all files

docs/kubernetes.md

@@ -401,6 +401,7 @@ Enable file extraction with Zeek? (y / N): y
 3: mapped
 4: all
 5: interesting
+6: notcommtxt
 
 Select file extraction behavior (none): 5
 1: quarantined

docs/malcolm-hedgehog-e2e-iso-install.md

@@ -228,6 +228,7 @@ The [configuration and tuning](malcolm-config.md#ConfigAndTuning) wizard's quest
         + `mapped`: extraction of files with recognized mime types
         + `known`: extraction of files for which any mime type can be determined
         + `all`: extract all files
+        + `notcommtxt`: extract all files except common plain text files
 * **Select file preservation behavior**
     - This determines the behavior for preservation of Zeek-extracted files:
         +  `quarantined`: preserve only flagged files in `./zeek-logs/extract_files/quarantine`

docs/ubuntu-install-example.md

@@ -160,6 +160,7 @@ Enable file extraction with Zeek? (y / N): y
 3: mapped
 4: all
 5: interesting
+6: notcommtxt
 Select file extraction behavior (none): 5
 
 1: quarantined

scripts/install.py

@@ -1184,7 +1184,14 @@ def tweak_malcolm_runtime(self, malcolm_install_path):
         )
 
         # input file extraction parameters
-        allowedFileCarveModes = ('none', 'known', 'mapped', 'all', 'interesting')
+        allowedFileCarveModes = {
+            'none': 'No file extraction',
+            'known': 'Extract recognized MIME types',
+            'mapped': 'Extract MIME types for which file extensions are known',
+            'all': 'Extract all files',
+            'interesting': 'Extract MIME types of common attack vectors',
+            'notcommtxt': 'Extract all except common plain text files',
+        }
         allowedFilePreserveModes = ('quarantined', 'all', 'none')
 
         fileCarveMode = None
@@ -1202,12 +1209,16 @@ def tweak_malcolm_runtime(self, malcolm_install_path):
 
         if InstallerYesOrNo('Enable file extraction with Zeek?', default=bool(fileCarveModeDefault)):
             loopBreaker = CountUntilException(MaxAskForValueCount, 'Invalid file extraction behavior')
-            while fileCarveMode not in allowedFileCarveModes and loopBreaker.increment():
+            while fileCarveMode not in allowedFileCarveModes.keys() and loopBreaker.increment():
                 fileCarveMode = InstallerChooseOne(
                     'Select file extraction behavior',
                     choices=[
-                        (x, '', x == fileCarveModeDefault if fileCarveModeDefault else allowedFileCarveModes[0])
-                        for x in allowedFileCarveModes
+                        (
+                            x,
+                            allowedFileCarveModes[x],
+                            x == fileCarveModeDefault if fileCarveModeDefault else 'none',
+                        )
+                        for x in allowedFileCarveModes.keys()
                     ],
                 )
             if fileCarveMode and (fileCarveMode != 'none'):
@@ -1256,9 +1267,9 @@ def tweak_malcolm_runtime(self, malcolm_install_path):
                         'Download updated file scanner signatures periodically?', default=args.fileScanRuleUpdate
                     )
 
-        if fileCarveMode not in allowedFileCarveModes:
-            fileCarveMode = allowedFileCarveModes[0]
-        if filePreserveMode not in allowedFileCarveModes:
+        if fileCarveMode not in allowedFileCarveModes.keys():
+            fileCarveMode = 'none'
+        if filePreserveMode not in allowedFilePreserveModes:
             filePreserveMode = allowedFilePreserveModes[0]
         if (vtotApiKey is None) or (len(vtotApiKey) <= 1):
             vtotApiKey = '0'
@@ -3685,7 +3696,7 @@ def main():
         '--file-extraction',
         dest='fileCarveMode',
         required=False,
-        metavar='<none|known|mapped|all|interesting>',
+        metavar='<none|known|mapped|all|interesting|notcommtxt>',
         type=str,
         default='none',
         help='Zeek file extraction behavior',

sensor-iso/config/includes.chroot/opt/zeek/share/zeek/site/extractor.zeek

@@ -17,7 +17,9 @@ event file_sniff(f: fa_file, meta: fa_metadata) {
       # we only want to extract knowns and we know the mime type OR
       ((extractor_extract_mode == extractor_extract_known) && meta?$mime_type) ||
       # we only want to extract mime->extension mapped files, we know the mimetype, and the mime type is mapped
-      ((extractor_extract_mode == extractor_extract_mapped) && meta?$mime_type && (meta$mime_type in extractor_mime_to_ext_map))) {
+      ((extractor_extract_mode == extractor_extract_mapped) && meta?$mime_type && (meta$mime_type in extractor_mime_to_ext_map)) ||
+      # we want to extract everything except common plain-text mimes, and either there's no mime type or the mime type isn't one of those
+      ((extractor_extract_mode == extractor_extract_notcommtxt) && ((! meta?$mime_type) || (meta$mime_type !in plain_text_mimes)))) {
 
     local ext: string = "";
     if (! meta?$mime_type)