1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
|
"""Various miscellaneous utility functions."""
# based on https://github.com/audreyr/binaryornot
#
# Copyright (c) 2013, Audrey Roy
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# * Redistributions of source code must retain the above copyright notice, this
# list of conditions and the following disclaimer.
#
# * Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation
# and/or other materials provided with the distribution.
#
# * Neither the name of BinaryOrNot nor the names of its contributors may be used
# to endorse or promote products derived from this software without specific
# prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
_control_chars = b"\n\r\t\f\b"
_printable_ascii = _control_chars + bytes(range(32, 127))
_printable_high_ascii = bytes(range(127, 256))
def is_binary(path, blocksize=1024):
"""Check if a given file is binary or not.
Uses a simplified version of the Perl detection algorithm, based roughly on
Eli Bendersky's translation to Python:
http://eli.thegreenplace.net/2011/10/19/perls-guess-if-file-is-text-or-binary-implemented-in-python/
This is biased slightly more in favour of deeming files as text files than
the Perl algorithm, since all ASCII compatible character sets are accepted as
text, not just utf-8.
:param path: Path to a file to check.
:param blocksize: Amount of bytes to read for determination.
:returns: True if appears to be a binary, otherwise False.
"""
try:
with open(path, "rb") as f:
byte_str = f.read(blocksize)
except IOError:
return False
# empty files are considered text
if not byte_str:
return False
# Now check for a high percentage of ASCII control characters
# Binary if control chars are > 30% of the string
low_chars = byte_str.translate(None, _printable_ascii)
nontext_ratio1 = len(low_chars) / len(byte_str)
# and check for a low percentage of high ASCII characters:
# Binary if high ASCII chars are < 5% of the string
# From: https://en.wikipedia.org/wiki/UTF-8
# If the bytes are random, the chances of a byte with the high bit set
# starting a valid UTF-8 character is only 6.64%. The chances of finding 7
# of these without finding an invalid sequence is actually lower than the
# chance of the first three bytes randomly being the UTF-8 BOM.
high_chars = byte_str.translate(None, _printable_high_ascii)
nontext_ratio2 = len(high_chars) / len(byte_str)
is_likely_binary = (nontext_ratio1 > 0.3 and nontext_ratio2 < 0.05) or (
nontext_ratio1 > 0.8 and nontext_ratio2 > 0.8
)
decodable = False
try:
byte_str.decode()
decodable = True
except UnicodeDecodeError:
# Delay import to hide during wheel/sdist builds that iterate over and
# import most modules to generate check/keyword/reporter lists.
import chardet
# guess character encoding using chardet
detected_encoding = chardet.detect(byte_str)
if detected_encoding["confidence"] > 0.8:
try:
byte_str.decode(encoding=detected_encoding["encoding"])
decodable = True
except (UnicodeDecodeError, LookupError):
pass
# finally use all the checks to decide binary or text
if decodable:
return False
if is_likely_binary or b"\x00" in byte_str:
return True
return False
|
GitWeb