Skip to content
GitLab
Explore
Sign in
Register
Primary navigation
Search or go to…
Project
mat2
Manage
Activity
Members
Labels
Plan
Issues
Issue boards
Milestones
Wiki
Code
Merge requests
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Build
Pipelines
Jobs
Pipeline schedules
Artifacts
Deploy
Releases
Model registry
Operate
Environments
Monitor
Incidents
Service Desk
Analyze
Value stream analytics
Contributor analytics
CI/CD analytics
Repository analytics
Model experiments
Help
Help
Support
GitLab documentation
Compare GitLab plans
Community forum
Contribute to GitLab
Provide feedback
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
Simon Magnin
mat2
Commits
25230107
Commit
25230107
authored
6 years ago
by
Julien (jvoisin) Voisin
Browse files
Options
Downloads
Patches
Plain Diff
Yay, it's working!
parent
30e567de
No related branches found
No related tags found
No related merge requests found
Changes
1
Hide whitespace changes
Inline
Side-by-side
Showing
1 changed file
libmat2/office.py
+31
-10
31 additions, 10 deletions
libmat2/office.py
with
31 additions
and
10 deletions
libmat2/office.py
+
31
−
10
View file @
25230107
import
logging
import
os
import
os
import
re
import
re
import
zipfile
import
zipfile
...
@@ -14,8 +15,15 @@ assert Pattern
...
@@ -14,8 +15,15 @@ assert Pattern
def
_parse_xml
(
full_path
:
str
):
def
_parse_xml
(
full_path
:
str
):
"""
This function parses XML, with namespace support.
"""
"""
This function parses XML, with namespace support.
"""
cpt
=
0
namespace_map
=
dict
()
namespace_map
=
dict
()
for
_
,
(
key
,
value
)
in
ET
.
iterparse
(
full_path
,
(
"
start-ns
"
,
)):
for
_
,
(
key
,
value
)
in
ET
.
iterparse
(
full_path
,
(
"
start-ns
"
,
)):
# The ns[0-9]+ namespaces are reserved for interal usage, so
# we have to use an other nomenclature.
if
re
.
match
(
'
^ns[0-9]+$
'
,
key
):
key
=
'
mat%d
'
%
cpt
cpt
+=
1
namespace_map
[
key
]
=
value
namespace_map
[
key
]
=
value
ET
.
register_namespace
(
key
,
value
)
ET
.
register_namespace
(
key
,
value
)
...
@@ -23,16 +31,16 @@ def _parse_xml(full_path: str):
...
@@ -23,16 +31,16 @@ def _parse_xml(full_path: str):
def
_sort_xml_attributes
(
full_path
:
str
)
->
bool
:
def
_sort_xml_attributes
(
full_path
:
str
)
->
bool
:
"""
Sort xml attributes lexicographically,
because it
'
s possible to fingerprint producers (MS Office, Libreoffice, …)
since they are all using different orders.
"""
tree
=
ET
.
parse
(
full_path
)
tree
=
ET
.
parse
(
full_path
)
root
=
tree
.
getroot
()
root
=
tree
.
getroot
()
for
c
in
root
:
for
c
in
root
:
c
[:]
=
sorted
(
c
,
key
=
lambda
child
:
(
child
.
tag
,
child
.
get
(
'
desc
'
)))
c
[:]
=
sorted
(
c
,
key
=
lambda
child
:
(
child
.
tag
,
child
.
get
(
'
desc
'
)))
print
(
'
CLENAING %s
'
%
full_path
)
xmlstr
=
ET
.
tostring
(
root
,
encoding
=
"
utf-8
"
,
method
=
"
xml
"
)
print
(
xmlstr
.
decode
(
"
utf-8
"
))
tree
.
write
(
full_path
,
xml_declaration
=
True
)
tree
.
write
(
full_path
,
xml_declaration
=
True
)
return
True
return
True
...
@@ -64,7 +72,8 @@ class MSOfficeParser(ArchiveBasedAbstractParser):
...
@@ -64,7 +72,8 @@ class MSOfficeParser(ArchiveBasedAbstractParser):
"""
"""
try
:
try
:
tree
,
namespace
=
_parse_xml
(
full_path
)
tree
,
namespace
=
_parse_xml
(
full_path
)
except
ET
.
ParseError
:
except
ET
.
ParseError
as
e
:
logging
.
error
(
"
Unable to parse %s: %s
"
,
full_path
,
e
)
return
False
return
False
# Revisions are either deletions (`w:del`) or
# Revisions are either deletions (`w:del`) or
...
@@ -98,6 +107,9 @@ class MSOfficeParser(ArchiveBasedAbstractParser):
...
@@ -98,6 +107,9 @@ class MSOfficeParser(ArchiveBasedAbstractParser):
return
True
return
True
def
_specific_cleanup
(
self
,
full_path
:
str
)
->
bool
:
def
_specific_cleanup
(
self
,
full_path
:
str
)
->
bool
:
if
os
.
stat
(
full_path
).
st_size
==
0
:
# Don't process empty files
return
True
if
full_path
.
endswith
(
'
/word/document.xml
'
):
if
full_path
.
endswith
(
'
/word/document.xml
'
):
# this file contains the revisions
# this file contains the revisions
return
self
.
__remove_revisions
(
full_path
)
return
self
.
__remove_revisions
(
full_path
)
...
@@ -154,7 +166,8 @@ class LibreOfficeParser(ArchiveBasedAbstractParser):
...
@@ -154,7 +166,8 @@ class LibreOfficeParser(ArchiveBasedAbstractParser):
def
__remove_revisions
(
full_path
:
str
)
->
bool
:
def
__remove_revisions
(
full_path
:
str
)
->
bool
:
try
:
try
:
tree
,
namespace
=
_parse_xml
(
full_path
)
tree
,
namespace
=
_parse_xml
(
full_path
)
except
ET
.
ParseError
:
except
ET
.
ParseError
as
e
:
logging
.
error
(
"
Unable to parse %s: %s
"
,
full_path
,
e
)
return
False
return
False
if
'
office
'
not
in
namespace
.
keys
():
# no revisions in the current file
if
'
office
'
not
in
namespace
.
keys
():
# no revisions in the current file
...
@@ -169,11 +182,19 @@ class LibreOfficeParser(ArchiveBasedAbstractParser):
...
@@ -169,11 +182,19 @@ class LibreOfficeParser(ArchiveBasedAbstractParser):
return
True
return
True
def
_specific_cleanup
(
self
,
full_path
:
str
)
->
bool
:
def
_specific_cleanup
(
self
,
full_path
:
str
)
->
bool
:
if
os
.
path
.
basename
(
full_path
).
endswith
(
'
.xml
'
):
if
os
.
stat
(
full_path
).
st_size
==
0
:
# Don't process empty files
_sort_xml_attributes
(
full_path
)
return
True
if
os
.
path
.
basename
(
full_path
)
==
'
content.xml
'
:
if
os
.
path
.
basename
(
full_path
).
endswith
(
'
.xml
'
):
return
self
.
__remove_revisions
(
full_path
)
if
os
.
path
.
basename
(
full_path
)
==
'
content.xml
'
:
if
self
.
__remove_revisions
(
full_path
)
is
False
:
return
False
try
:
_sort_xml_attributes
(
full_path
)
except
ET
.
ParseError
as
e
:
logging
.
error
(
"
Unable to parse %s: %s
"
,
full_path
,
e
)
return
False
return
True
return
True
def
get_meta
(
self
)
->
Dict
[
str
,
str
]:
def
get_meta
(
self
)
->
Dict
[
str
,
str
]:
...
...
This diff is collapsed.
Click to expand it.
Preview
0%
Loading
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment