Unknown members

In dealing with sophisticated MS Word files, it turns out that there are many member elements of the pkzip archive that mat2 doesn't currently know what to do with.

This series is an attempt to make the handling of those unknown members a bit more robust, so that it's easier to get reports about missing handlers, and so that people have a clear choice about what to do with them.

libmat2/office.py

@@ -40,6 +40,10 @@ class ArchiveBasedAbstractParser(abstract.AbstractParser):
     # no matter if they are supported or not.
     files_to_omit = set() # type: Set[Pattern]
 
+    # what should the parser do if it encounters an unknown file in
+    # the archive?  valid policies are 'abort', 'omit', 'keep'
+    unknown_member_policy = 'abort' # type: str
+
     def __init__(self, filename):
         super().__init__(filename)
         try:  # better fail here than later
@@ -79,10 +83,12 @@ class ArchiveBasedAbstractParser(abstract.AbstractParser):
         return metadata
 
     def remove_all(self) -> bool:
+        # pylint: disable=too-many-branches
         with zipfile.ZipFile(self.filename) as zin,\
              zipfile.ZipFile(self.output_filename, 'w') as zout:
 
             temp_folder = tempfile.mkdtemp()
+            abort = False
 
             for item in zin.infolist():
                 if item.filename[-1] == '/':  # `is_dir` is added in Python3.6
@@ -92,11 +98,10 @@ class ArchiveBasedAbstractParser(abstract.AbstractParser):
                 full_path = os.path.join(temp_folder, item.filename)
 
                 if self._specific_cleanup(full_path) is False:
-                    shutil.rmtree(temp_folder)
-                    os.remove(self.output_filename)
                     logging.warning("Something went wrong during deep cleaning of %s",
                                     item.filename)
-                    return False
+                    abort = True
+                    continue
 
                 if item.filename in self.files_to_keep:
                     # those files aren't supported, but we want to add them anyway
@@ -107,14 +112,25 @@ class ArchiveBasedAbstractParser(abstract.AbstractParser):
                     # supported files that we want to clean then add
                     tmp_parser, mtype = parser_factory.get_parser(full_path)  # type: ignore
                     if not tmp_parser:
-                        shutil.rmtree(temp_folder)
-                        os.remove(self.output_filename)
-                        logging.error("In file %s, element %s's format (%s) " +
-                                      "isn't supported",
-                                      self.filename, item.filename, mtype)
-                        return False
-                    tmp_parser.remove_all()
-                    os.rename(tmp_parser.output_filename, full_path)
+                        if self.unknown_member_policy == 'omit':
+                            logging.warning("In file %s, omitting unknown element %s (format: %s)",
+                                            self.filename, item.filename, mtype)
+                            continue
+                        elif self.unknown_member_policy == 'keep':
+                            logging.warning("In file %s, keeping unknown element %s (format: %s)",
+                                            self.filename, item.filename, mtype)
+                        else:
+                            if self.unknown_member_policy != 'abort':
+                                logging.warning("Invalid unknown_member_policy %s, " +
+                                                "treating as 'abort'", self.unknown_member_policy)
+                            logging.error("In file %s, element %s's format (%s) " +
+                                          "isn't supported",
+                                          self.filename, item.filename, mtype)
+                            abort = True
+                            continue
+                    if tmp_parser:
+                        tmp_parser.remove_all()
+                        os.rename(tmp_parser.output_filename, full_path)
 
                 zinfo = zipfile.ZipInfo(item.filename)  # type: ignore
                 clean_zinfo = self._clean_zipinfo(zinfo)
@@ -122,6 +138,9 @@ class ArchiveBasedAbstractParser(abstract.AbstractParser):
                     zout.writestr(clean_zinfo, f.read())
 
         shutil.rmtree(temp_folder)
+        if abort:
+            os.remove(self.output_filename)
+            return False
         return True