3
Я написал небольшой тест, где я сравниваю различные методы конкатенации строк для ZOCache.tempfile.TemporaryFile vs. StringIO
Так выглядит здесь как tempfile.TemporaryFile быстрее, чем все остальное:
$ python src/ZOCache/tmp_benchmark.py
3.00407409668e-05 TemporaryFile
0.385630846024 SpooledTemporaryFile
0.299962997437 BufferedRandom
0.0849719047546 io.StringIO
0.113346099854 concat
Эталон код, который я использую:
#!/usr/bin/python
from __future__ import print_function
import io
import timeit
import tempfile
class Error(Exception):
pass
def bench_temporaryfile():
with tempfile.TemporaryFile(bufsize=10*1024*1024) as out:
for i in range(0, 100):
out.write(b"Value = ")
out.write(bytes(i))
out.write(b" ")
# Get string.
out.seek(0)
contents = out.read()
out.close()
# Test first letter.
if contents[0:5] != b"Value":
raise Error
def bench_spooledtemporaryfile():
with tempfile.SpooledTemporaryFile(max_size=10*1024*1024) as out:
for i in range(0, 100):
out.write(b"Value = ")
out.write(bytes(i))
out.write(b" ")
# Get string.
out.seek(0)
contents = out.read()
out.close()
# Test first letter.
if contents[0:5] != b"Value":
raise Error
def bench_BufferedRandom():
# 1. BufferedRandom
with io.open('out.bin', mode='w+b') as fp:
with io.BufferedRandom(fp, buffer_size=10*1024*1024) as out:
for i in range(0, 100):
out.write(b"Value = ")
out.write(bytes(i))
out.write(b" ")
# Get string.
out.seek(0)
contents = out.read()
# Test first letter.
if contents[0:5] != b'Value':
raise Error
def bench_stringIO():
# 1. Use StringIO.
out = io.StringIO()
for i in range(0, 100):
out.write(u"Value = ")
out.write(unicode(i))
out.write(u" ")
# Get string.
contents = out.getvalue()
out.close()
# Test first letter.
if contents[0] != 'V':
raise Error
def bench_concat():
# 2. Use string appends.
data = ""
for i in range(0, 100):
data += u"Value = "
data += unicode(i)
data += u" "
# Test first letter.
if data[0] != u'V':
raise Error
if __name__ == '__main__':
print(str(timeit.timeit('bench_temporaryfile()', setup="from __main__ import bench_temporaryfile", number=1000)) + " TemporaryFile")
print(str(timeit.timeit('bench_spooledtemporaryfile()', setup="from __main__ import bench_spooledtemporaryfile", number=1000)) + " SpooledTemporaryFile")
print(str(timeit.timeit('bench_BufferedRandom()', setup="from __main__ import bench_BufferedRandom", number=1000)) + " BufferedRandom")
print(str(timeit.timeit("bench_stringIO()", setup="from __main__ import bench_stringIO", number=1000)) + " io.StringIO")
print(str(timeit.timeit("bench_concat()", setup="from __main__ import bench_concat", number=1000)) + " concat")
EDIT Python3.4.3 + io.BytesIO
python3 ./src/ZOCache/tmp_benchmark.py
2.689500024644076e-05 TemporaryFile
0.30429405899985795 SpooledTemporaryFile
0.348170792000019 BufferedRandom
0.0764778530001422 io.BytesIO
0.05162201000030109 concat
Новый источник с io.BytesIO:
#!/usr/bin/python3
from __future__ import print_function
import io
import timeit
import tempfile
class Error(Exception):
pass
def bench_temporaryfile():
with tempfile.TemporaryFile() as out:
for i in range(0, 100):
out.write(b"Value = ")
out.write(bytes(str(i), 'utf-8'))
out.write(b" ")
# Get string.
out.seek(0)
contents = out.read()
out.close()
# Test first letter.
if contents[0:5] != b"Value":
raise Error
def bench_spooledtemporaryfile():
with tempfile.SpooledTemporaryFile(max_size=10*1024*1024) as out:
for i in range(0, 100):
out.write(b"Value = ")
out.write(bytes(str(i), 'utf-8'))
out.write(b" ")
# Get string.
out.seek(0)
contents = out.read()
out.close()
# Test first letter.
if contents[0:5] != b"Value":
raise Error
def bench_BufferedRandom():
# 1. BufferedRandom
with io.open('out.bin', mode='w+b') as fp:
with io.BufferedRandom(fp, buffer_size=10*1024*1024) as out:
for i in range(0, 100):
out.write(b"Value = ")
out.write(bytes(i))
out.write(b" ")
# Get string.
out.seek(0)
contents = out.read()
# Test first letter.
if contents[0:5] != b'Value':
raise Error
def bench_BytesIO():
# 1. Use StringIO.
out = io.BytesIO()
for i in range(0, 100):
out.write(b"Value = ")
out.write(bytes(str(i), 'utf-8'))
out.write(b" ")
# Get string.
contents = out.getvalue()
out.close()
# Test first letter.
if contents[0:5] != b'Value':
raise Error
def bench_concat():
# 2. Use string appends.
data = ""
for i in range(0, 100):
data += "Value = "
data += str(i)
data += " "
# Test first letter.
if data[0] != 'V':
raise Error
if __name__ == '__main__':
print(str(timeit.timeit('bench_temporaryfile()', setup="from __main__ import bench_temporaryfile", number=1000)) + " TemporaryFile")
print(str(timeit.timeit('bench_spooledtemporaryfile()', setup="from __main__ import bench_spooledtemporaryfile", number=1000)) + " SpooledTemporaryFile")
print(str(timeit.timeit('bench_BufferedRandom()', setup="from __main__ import bench_BufferedRandom", number=1000)) + " BufferedRandom")
print(str(timeit.timeit("bench_BytesIO()", setup="from __main__ import bench_BytesIO", number=1000)) + " io.BytesIO")
print(str(timeit.timeit("bench_concat()", setup="from __main__ import bench_concat", number=1000)) + " concat")
Это правда для каждой платформы? И если да, то почему?
EDIT: Результаты с фиксированным эталоном (и фиксированным кодом):
0.2675984420002351 TemporaryFile
0.28104681999866443 SpooledTemporaryFile
0.3555715570000757 BufferedRandom
0.10379689100045653 io.BytesIO
0.05650951399911719 concat
'GROANMODE = 1' Вы фактически не запускали временный пробный файл, это должно быть' timeit ('bench_temporaryfile()' '(с помощью функции parens для вызова функции). – tdelaney
@tdelaney: Nice catch. заметил, как подозрительно было время, но я просто предположил, что моя архаичная система не делает временные файлы эффективно. :-) Я включил это в свой ответ (и сделал его основной проблемой, потому что это так). – ShadowRanger
Спасибо @tdelaney и ShadowRanger. – pcdummy