Understanding test_binary

change.diff

diff --git a/src/util.c b/src/util.c
index 3045bdf..bf0dad6 100644
--- a/src/util.c
+++ b/src/util.c
@@ -324,6 +324,7 @@ int is_binary(const void *buf, const size_t buf_len) {
                 }
             }
             suspicious_bytes++;
+            log_debug("found suspicious byte %#x at pos %i", buf_c[i], i);
             /* Disk IO is so slow that it's worthwhile to do this calculation after every suspicious byte. */
             /* This is true even on a 1.6Ghz Atom with an Intel 320 SSD. */
             /* Read at least 32 bytes before making a decision */

file.sh-session

Hesperos:testcase jschpp$ file test.lua 
test.lua: ERROR: line 22: regexec error 17, (illegal byte sequence)

hexdump

00000000  0a 20 20 20 20 c2 ed b5  c2 c0 ef ce f7 b1 b4 c0  |.    ...........|
00000010  b3 cb b9 b9 e3 b3 a1 a3  ac d3 ce c8 cb c8 e7 d6  |................|
00000020  af a3 ac ce bb d3 da c0  d7 bf c6 c0 b3 cd d0 cb  |................|
00000030  b9 b4 f3 b5 c0 ba cd c6  d5 c0 ad b6 e0 b4 f3 b5  |................|
00000040  c0 bd bb bd e7 b4 a6 b5  c4 ce f7 b1 b4 c0 b3 cb  |................|
00000050  b9 b9 e3 b3 a1 ca c7 c2  ed b5 c2 c0 ef ca d0 b5  |................|
00000060  c4 b1 ea d6 be a3 ac b8  a5 c0 ef d1 c7 bc aa b7  |................|
00000070  e1 ca d5 c5 ae c9 f1 ce  f7 b1 b4 c0 b3 cb b9 d7  |................|
00000080  f8 d4 da d3 c9 c1 bd d6  bb ca a8 d7 d3 c0 ad d7  |................|
00000090  c5 b5 c4 d5 bd b3 b5 c9  cf a3 ac d3 ba c8 dd bb  |................|
000000a0  aa b9 f3 a3 ac c9 ed b1  df b5 c4 c5 e7 c8 aa b8  |................|
000000b0  fc ca c7 b3 c4 cd d0 d7  c5 d5 e2 ce bb b4 ab cb  |................|
000000c0  b5 d6 d0 cf f3 d5 f7 d7  c5 b7 e1 ca d5 b5 c4 c5  |................|
000000d0  ae c9 f1 a3 ac c3 c0 d1  de b2 bb bf c9 b7 bd ce  |................|
000000e0  ef a1 a3 0a 20 20 20 20  b6 f8 d5 e2 b4 a6 b9 e3  |....    ........|
000000f0  b3 a1 a3 ac d2 b2 ca c7  bb ca bc d2 c2 ed b5 c2  |................|
00000100  c0 ef be e3 c0 d6 b2 bf  b4 ab cd b3 b5 c4 c7 ec  |................|
00000110  d7 a3 ca a4 c0 fb b5 c4  b5 d8 b7 bd a3 ac c3 bf  |................|
00000120  b5 b1 d5 e2 bc e4 ca c0  bc cd d7 ee bc d1 be e3  |................|
00000130  c0 d6 b2 bf bb f1 b5 c3  d3 d6 d2 bb b4 ce ca a4  |................|
00000140  c0 fb d6 ae ba f3 a3 ac  bb ca c2 ed c8 ab b6 d3  |................|
00000150  be cd bb e1 c0 b4 b5 bd  d5 e2 c0 ef a3 ac ba cd  |................|
00000160  b7 e1 ca d5 c5 ae c9 f1  d2 bb cd ac c7 ec d7 a3  |................|
00000170  bb ca c2 ed be e3 c0 d6  b2 bf b5 c4 d3 d6 d2 bb  |................|
00000180  b4 ce ca a4 c0 fb a1 a3  0a 20 20 20 20 cb e4 c8  |.........    ...|
00000190  bb cf d6 d4 da d6 bb ca  c7 d2 bb b8 f6 c6 d5 cd  |................|
000001a0  a8 b5 c4 ca ae d4 c2 b5  c4 cf c2 ce e7 a3 ac c8  |................|
000001b0  bb b6 f8 d5 e2 c0 ef d2  c0 be c9 d3 ce c8 cb c8  |................|
000001c0  e7 d6 af a3 ac ce de ca  fd d3 ce bf cd b6 bc d2  |................|
000001d0  f2 ce aa d1 f6 c4 bd d5  e2 d7 f9 b9 e3 b3 a1 b5  |................|
000001e0  c4 c3 fb c6 f8 b6 f8 c0  b4 a3 ac c6 e4 d6 d0 c5  |................|
000001f0  c4 c9 e3 d7 c5 ce f7 b1  b4 c0 b3 cb b9 c5 ae c9  |................|
00000200  f1 b5 f1 cf f1 b5 c4 d3  ce c8 cb d4 f2 ca c7 b8  |................|
00000210  fc b6 e0 a1 a3 0a 20 20  20 20 c8 bb b6 f8 d4 da  |......    ......|
00000220  d5 e2 d2 bb c4 bb d6 d0  a3 ac c8 b4 d3 d0 d7 c5  |................|
00000230  b2 bb ba cd d0 b3 b5 c4  be b0 cf f3 a3 ac b4 d3  |................|
00000240  d6 d0 ce e7 bf aa ca bc  a3 ac be cd d2 bb d6 b1  |................|
00000250  d3 d0 d2 bb b8 f6 d2 c2  d7 c5 c6 d5 cd a8 b5 c4  |................|
00000260  c4 d0 d7 d3 d5 be d4 da  c5 e7 c8 aa c5 d4 b1 df  |................|
00000270  a3 ac b3 c9 ce aa c1 cb  cb f9 d3 d0 cf eb d2 aa  |................|
00000280  c5 c4 c9 e3 c5 ae c9 f1  cf f1 b5 c4 d3 ce bf cd  |................|
00000290  d1 db d6 d0 d7 ee b2 bb  ba cd d0 b3 b5 c4 d2 bb  |................|
000002a0  b8 f6 b5 e3 a1 a3 0a 20  20 20 20 bf c9 ca c7 cb  |.......    .....|
000002b0  fb c3 c7 d3 d6 b2 bb c4  dc cb b5 ca b2 c3 b4 a3  |................|
000002c0  ac b1 cf be b9 d5 e2 c0  ef ca c7 b9 ab b9 b2 b3  |................|
000002d0  a1 ba cf a3 ac c8 cb bc  d2 d4 b8 d2 e2 d5 be d4  |................|
000002e0  da c4 c7 c0 ef a3 ac b2  bb c8 c3 d2 b2 ca c7 c8  |................|
000002f0  cb bc d2 b5 c4 d7 d4 d3  c9 a3 ac d3 da ca c7 b8  |................|
00000300  f7 c2 b7 cf eb d2 aa c5  c4 c9 e3 b5 c4 d3 ce bf  |................|
00000310  cd d6 bb c4 dc d3 f4 c3  c6 b5 c4 bb bb b8 f6 bd  |................|
00000320  c7 b6 c8 c8 a5 c5 c4 c9  e3 c5 ae c9 f1 a3 ac c3  |................|
00000330  e2 b5 c3 b0 d1 c4 b3 b8  f6 b2 bb c3 f7 c9 fa ce  |................|
00000340  ef c5 c4 bd f8 c8 a5 a3  ac c8 bb ba f3 c8 a5 c7  |................|
00000350  f3 70 73 b4 f3 c9 f1 b0  ef d7 c5 b0 d1 d5 e2 b8  |.ps.............|
00000360  f6 b2 bb c3 f7 c9 fa ce  ef 70 b5 f4 b5 c4 ca b1  |.........p......|
00000370  ba f2 a3 ac bb e1 b3 f6  cf d6 b8 f7 d6 d6 b8 e3  |................|
00000380  d0 a6 be b5 cd b7 a1 ad  a1 ad 0a 20 20 20 20 b8  |...........    .|
00000390  ba d4 f0 ce ac bb a4 b9  e3 b3 a1 b4 ce d0 f2 b5  |................|
000003a0  c4 be af b2 ec c3 c7 d2  b2 d7 a2 d2 e2 b5 bd c1  |................|
000003b0  cb d5 e2 b8 f6 c3 f7 cf  d4 cf f3 ca c7 b6 ab b7  |................|
000003c0  bd c8 cb b5 c4 bc d2 bb  ef a3 ac c8 bb b6 f8 b6  |................|
000003d0  d4 b7 bd d6 bb ca c7 b4  f4 b4 f4 b5 c4 d5 be d4  |................|
000003e0  da c4 c7 c0 ef a3 ac d3  d6 c3 bb d3 d0 d7 f6 ca  |................|
000003f0  b2 c3 b4 c6 e4 cb fb b5  c4 ca c2 c7 e9 a3 ac c5  |................|
00000400  e7 c8 aa b5 c4 cb ae b3  d8 c4 c7 c3 b4 c7 b3 d2  |................|
00000410  b2 b2 bb cc ab c8 dd d2  d7 d1 cd cb c0 c8 cb a3  |................|
00000420  ac c4 c7 d2 b2 d6 bb c4  dc cb e6 cb fb c8 a5 c1  |................|
00000430  cb a1 a3 0a 0a 20 20 20  20 0a 20 20 20 20 63 61  |.....    .    ca|
00000440  6c 6c 5f 6e 70 63 5f 63  68 61 6e 67 65 5f 6d 61  |ll_npc_change_ma|
00000450  70 0a 20 20 20 20                                 |p.    |
00000456

output

DEBUG: Found user's home dir: /Users/jschpp
DEBUG: Skipping ignore file /Users/jschpp/.agignore: not readable
DEBUG: Found user's global Git excludesfile: /Users/jschpp/.gitignore_global
DEBUG: Loading ignore file /Users/jschpp/.gitignore_global.
DEBUG: added ignore pattern *~ to root ignores
DEBUG: added ignore pattern .DS_Store to root ignores
DEBUG: added ignore pattern .classpath to root ignores
DEBUG: added ignore pattern .gitignore to root ignores
DEBUG: added ignore pattern bin/org to root ignores
DEBUG: added ignore pattern .texlipse to root ignores
DEBUG: added ignore pattern .project to root ignores
DEBUG: added ignore pattern .settings to root ignores
DEBUG: added ignore pattern org.eclipse.jdt.ui.prefs to root ignores
DEBUG: added ignore pattern .tags* to root ignores
DEBUG: added ignore pattern .tags1 to root ignores
DEBUG: Query is call_npc_change_map
DEBUG: PCRE Version: 8.38 2015-11-23
DEBUG: Using 7 workers
DEBUG: No CPU affinity support.
DEBUG: No CPU affinity support.
DEBUG: No CPU affinity support.
DEBUG: DEBUG: DEBUG: DEBUG: DEBUG: Worker 0 startedWorker 1 startedWorker 2 startedNo CPU affinity support.Worker 3 started




DEBUG: No CPU affinity support.
DEBUG: Worker 4 started
DEBUG: No CPU affinity support.
DEBUG: Worker 5 started
DEBUG: No CPU affinity support.
DEBUG: searching path . for call_npc_change_map
DEBUG: Worker 6 started
DEBUG: Skipping ignore file ./.agignore: not readable
DEBUG: Skipping ignore file ./.gitignore: not readable
DEBUG: Skipping ignore file ./.git/info/exclude: not readable
DEBUG: Skipping ignore file ./.hgignore: not readable
DEBUG: Skipping svn ignore file ./.svn/dir-prop-base
DEBUG: path_start . filename test.lua
DEBUG: pattern .DS_Store doesn't match path test.lua
DEBUG: pattern .classpath doesn't match path test.lua
DEBUG: pattern .gitignore doesn't match path test.lua
DEBUG: pattern .project doesn't match path test.lua
DEBUG: pattern .settings doesn't match path test.lua
DEBUG: pattern .tags1 doesn't match path test.lua
DEBUG: pattern .texlipse doesn't match path test.lua
DEBUG: pattern bin/org doesn't match path test.lua
DEBUG: pattern org.eclipse.jdt.ui.prefs doesn't match path test.lua
DEBUG: pattern *~ doesn't match file test.lua
DEBUG: pattern .tags* doesn't match file test.lua
DEBUG: test.lua not ignored
DEBUG: ./test.lua added to work queue
DEBUG: DEBUG: Worker 3 finished.Worker 2 finished.
DEBUG: 
DEBUG: DEBUG: DEBUG: Worker 1 finished.Worker 5 finished.DEBUG: Worker 6 finished.Worker 4 finished.

found suspicious byte 0xed at pos 6


DEBUG: found suspicious byte 0xb5 at pos 7
DEBUG: found suspicious byte 0xc0 at pos 9
DEBUG: found suspicious byte 0xce at pos 11
DEBUG: found suspicious byte 0xf7 at pos 12
DEBUG: found suspicious byte 0xb1 at pos 13
DEBUG: found suspicious byte 0xb4 at pos 14
DEBUG: found suspicious byte 0xc0 at pos 15
DEBUG: found suspicious byte 0xb3 at pos 16
DEBUG: found suspicious byte 0xb9 at pos 19
DEBUG: found suspicious byte 0xa3 at pos 23
DEBUG: found suspicious byte 0xac at pos 24
DEBUG: found suspicious byte 0xce at pos 26
DEBUG: found suspicious byte 0xcb at pos 28
DEBUG: found suspicious byte 0xe7 at pos 30
DEBUG: found suspicious byte 0xa3 at pos 33
DEBUG: found suspicious byte 0xac at pos 34
DEBUG: found suspicious byte 0xda at pos 38
DEBUG: found suspicious byte 0xc0 at pos 39
DEBUG: found suspicious byte 0xc0 at pos 43
DEBUG: found suspicious byte 0xb3 at pos 44
DEBUG: found suspicious byte 0xd0 at pos 46
DEBUG: found suspicious byte 0xb4 at pos 49
DEBUG: found suspicious byte 0xf3 at pos 50
DEBUG: found suspicious byte 0xb5 at pos 51
DEBUG: found suspicious byte 0xc0 at pos 52
DEBUG: found suspicious byte 0xba at pos 53
DEBUG: found suspicious byte 0xc6 at pos 55
DEBUG: found suspicious byte 0xc0 at pos 57
DEBUG: found suspicious byte 0xad at pos 58
DEBUG: found suspicious byte 0xb6 at pos 59
DEBUG: found suspicious byte 0xb4 at pos 61
DEBUG: found suspicious byte 0xf3 at pos 62
DEBUG: found suspicious byte 0xb5 at pos 63
DEBUG: found suspicious byte 0xc0 at pos 64
DEBUG: found suspicious byte 0xbd at pos 65
DEBUG: found suspicious byte 0xbb at pos 66
DEBUG: found suspicious byte 0xbd at pos 67
DEBUG: found suspicious byte 0xb5 at pos 71
DEBUG: found suspicious byte 0xce at pos 73
DEBUG: found suspicious byte 0xf7 at pos 74
DEBUG: found suspicious byte 0xb1 at pos 75
DEBUG: found suspicious byte 0xb4 at pos 76
DEBUG: found suspicious byte 0xc0 at pos 77
DEBUG: found suspicious byte 0xb3 at pos 78
DEBUG: found suspicious byte 0xb9 at pos 81
DEBUG: found suspicious byte 0xc7 at pos 86
DEBUG: found suspicious byte 0xed at pos 88
DEBUG: found suspicious byte 0xb5 at pos 89
DEBUG: found suspicious byte 0xc0 at pos 91
DEBUG: found suspicious byte 0xca at pos 93
DEBUG: found suspicious byte 0xd6 at pos 99
DEBUG: found suspicious byte 0xbe at pos 100
DEBUG: found suspicious byte 0xa3 at pos 101
DEBUG: found suspicious byte 0xac at pos 102
DEBUG: found suspicious byte 0xb8 at pos 103
DEBUG: found suspicious byte 0xa5 at pos 104
DEBUG: File ./test.lua is binary. Skipping...
DEBUG: Worker 0 finished.