Skip to content

Instantly share code, notes, and snippets.

@Transfusion

Transfusion/gist:b2075b7a08863c3e5b5afc96b119c29d

Created October 26, 2019 10:39
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save Transfusion/b2075b7a08863c3e5b5afc96b119c29d to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save Transfusion/b2075b7a08863c3e5b5afc96b119c29d to your computer and use it in GitHub Desktop.
Download ZIP
cchardet incremental detection on websites https://github.com/aio-libs/aiohttp/issues/4112
Raw
gistfile1.txt
chunk 5 reached
{'encoding': 'SHIFT_JIS', 'confidence': 0.9900000095367432}
Filename: memory_profiling_4112.py
Line # Mem usage Increment Line Contents
================================================
13 21.5 MiB 21.5 MiB @profile
14 def chunked_with_memoryview(chunk_size: int = 2 * 1024):
15 21.5 MiB 0.0 MiB detector = chardet.UniversalDetector()
16 21.5 MiB 0.0 MiB _body_memoryview = memoryview(_bytes)
17 # print(len(_bytes) == len(_body_memoryview))
18 21.5 MiB 0.0 MiB for i in range((len(_body_memoryview) // chunk_size) + 1):
19 21.5 MiB 0.0 MiB _chunk = _body_memoryview[i * chunk_size: (i + 1) * chunk_size]
20 21.5 MiB 0.0 MiB _chunk_bytes = _chunk.tobytes()
21 21.5 MiB 0.0 MiB detector.feed(_chunk_bytes)
22 21.5 MiB 0.0 MiB del _chunk_bytes
23 21.5 MiB 0.0 MiB if detector.done:
24 21.5 MiB 0.0 MiB print("chunk " + str(i) + " reached")
25 21.5 MiB 0.0 MiB break
26
27 21.5 MiB 0.0 MiB detector.close()
28 21.5 MiB 0.0 MiB print(detector.result)
SHIFT_JIS
Filename: memory_profiling_4112.py
Line # Mem usage Increment Line Contents
================================================
31 21.5 MiB 21.5 MiB @profile
32 def without_chunking():
33 21.5 MiB 0.0 MiB print(chardet.detect(_bytes)['encoding'])
chunk 20 reached
{'encoding': 'GB18030', 'confidence': 0.9900000095367432}
Filename: memory_profiling_4112.py
Line # Mem usage Increment Line Contents
================================================
13 21.9 MiB 21.9 MiB @profile
14 def chunked_with_memoryview(chunk_size: int = 2 * 1024):
15 21.9 MiB 0.0 MiB detector = chardet.UniversalDetector()
16 21.9 MiB 0.0 MiB _body_memoryview = memoryview(_bytes)
17 # print(len(_bytes) == len(_body_memoryview))
18 21.9 MiB 0.0 MiB for i in range((len(_body_memoryview) // chunk_size) + 1):
19 21.9 MiB 0.0 MiB _chunk = _body_memoryview[i * chunk_size: (i + 1) * chunk_size]
20 21.9 MiB 0.0 MiB _chunk_bytes = _chunk.tobytes()
21 21.9 MiB 0.0 MiB detector.feed(_chunk_bytes)
22 21.9 MiB 0.0 MiB del _chunk_bytes
23 21.9 MiB 0.0 MiB if detector.done:
24 21.9 MiB 0.0 MiB print("chunk " + str(i) + " reached")
25 21.9 MiB 0.0 MiB break
26
27 21.9 MiB 0.0 MiB detector.close()
28 21.9 MiB 0.0 MiB print(detector.result)
GB18030
Filename: memory_profiling_4112.py
Line # Mem usage Increment Line Contents
================================================
31 21.9 MiB 21.9 MiB @profile
32 def without_chunking():
33 21.9 MiB 0.0 MiB print(chardet.detect(_bytes)['encoding'])
{'encoding': 'KOI8-R', 'confidence': 0.7338306903839111}
Filename: memory_profiling_4112.py
Line # Mem usage Increment Line Contents
================================================
13 21.8 MiB 21.8 MiB @profile
14 def chunked_with_memoryview(chunk_size: int = 2 * 1024):
15 21.8 MiB 0.0 MiB detector = chardet.UniversalDetector()
16 21.8 MiB 0.0 MiB _body_memoryview = memoryview(_bytes)
17 # print(len(_bytes) == len(_body_memoryview))
18 21.8 MiB 0.0 MiB for i in range((len(_body_memoryview) // chunk_size) + 1):
19 21.8 MiB 0.0 MiB _chunk = _body_memoryview[i * chunk_size: (i + 1) * chunk_size]
20 21.8 MiB 0.0 MiB _chunk_bytes = _chunk.tobytes()
21 21.8 MiB 0.0 MiB detector.feed(_chunk_bytes)
22 21.8 MiB 0.0 MiB del _chunk_bytes
23 21.8 MiB 0.0 MiB if detector.done:
24 print("chunk " + str(i) + " reached")
25 break
26
27 21.8 MiB 0.0 MiB detector.close()
28 21.8 MiB 0.0 MiB print(detector.result)
KOI8-R
Filename: memory_profiling_4112.py
Line # Mem usage Increment Line Contents
================================================
31 21.8 MiB 21.8 MiB @profile
32 def without_chunking():
33 21.8 MiB 0.0 MiB print(chardet.detect(_bytes)['encoding'])
chunk 13 reached
{'encoding': 'UHC', 'confidence': 0.9900000095367432}
Filename: memory_profiling_4112.py
Line # Mem usage Increment Line Contents
================================================
13 22.6 MiB 22.6 MiB @profile
14 def chunked_with_memoryview(chunk_size: int = 2 * 1024):
15 22.6 MiB 0.0 MiB detector = chardet.UniversalDetector()
16 22.6 MiB 0.0 MiB _body_memoryview = memoryview(_bytes)
17 # print(len(_bytes) == len(_body_memoryview))
18 22.6 MiB 0.0 MiB for i in range((len(_body_memoryview) // chunk_size) + 1):
19 22.6 MiB 0.0 MiB _chunk = _body_memoryview[i * chunk_size: (i + 1) * chunk_size]
20 22.6 MiB 0.0 MiB _chunk_bytes = _chunk.tobytes()
21 22.6 MiB 0.0 MiB detector.feed(_chunk_bytes)
22 22.6 MiB 0.0 MiB del _chunk_bytes
23 22.6 MiB 0.0 MiB if detector.done:
24 22.6 MiB 0.0 MiB print("chunk " + str(i) + " reached")
25 22.6 MiB 0.0 MiB break
26
27 22.6 MiB 0.0 MiB detector.close()
28 22.6 MiB 0.0 MiB print(detector.result)
UHC
Filename: memory_profiling_4112.py
Line # Mem usage Increment Line Contents
================================================
31 22.6 MiB 22.6 MiB @profile
32 def without_chunking():
33 22.6 MiB 0.0 MiB print(chardet.detect(_bytes)['encoding'])
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment