问题:如何以相反的顺序读取文件?
如何使用python以相反的顺序读取文件?我想从最后一行读取文件。
How to read a file in reverse order using python? I want to read a file from last line to first line.
回答 0
for line in reversed(open("filename").readlines()):
print line.rstrip()
在Python 3中:
for line in reversed(list(open("filename"))):
print(line.rstrip())
for line in reversed(open("filename").readlines()):
print line.rstrip()
And in Python 3:
for line in reversed(list(open("filename"))):
print(line.rstrip())
回答 1
作为生成器编写的正确,有效的答案。
import os
def reverse_readline(filename, buf_size=8192):
"""A generator that returns the lines of a file in reverse order"""
with open(filename) as fh:
segment = None
offset = 0
fh.seek(0, os.SEEK_END)
file_size = remaining_size = fh.tell()
while remaining_size > 0:
offset = min(file_size, offset + buf_size)
fh.seek(file_size - offset)
buffer = fh.read(min(remaining_size, buf_size))
remaining_size -= buf_size
lines = buffer.split('\n')
# The first line of the buffer is probably not a complete line so
# we'll save it and append it to the last line of the next buffer
# we read
if segment is not None:
# If the previous chunk starts right from the beginning of line
# do not concat the segment to the last line of new chunk.
# Instead, yield the segment first
if buffer[-1] != '\n':
lines[-1] += segment
else:
yield segment
segment = lines[0]
for index in range(len(lines) - 1, 0, -1):
if lines[index]:
yield lines[index]
# Don't yield None if the file was empty
if segment is not None:
yield segment
A correct, efficient answer written as a generator.
import os
def reverse_readline(filename, buf_size=8192):
"""A generator that returns the lines of a file in reverse order"""
with open(filename) as fh:
segment = None
offset = 0
fh.seek(0, os.SEEK_END)
file_size = remaining_size = fh.tell()
while remaining_size > 0:
offset = min(file_size, offset + buf_size)
fh.seek(file_size - offset)
buffer = fh.read(min(remaining_size, buf_size))
remaining_size -= buf_size
lines = buffer.split('\n')
# The first line of the buffer is probably not a complete line so
# we'll save it and append it to the last line of the next buffer
# we read
if segment is not None:
# If the previous chunk starts right from the beginning of line
# do not concat the segment to the last line of new chunk.
# Instead, yield the segment first
if buffer[-1] != '\n':
lines[-1] += segment
else:
yield segment
segment = lines[0]
for index in range(len(lines) - 1, 0, -1):
if lines[index]:
yield lines[index]
# Don't yield None if the file was empty
if segment is not None:
yield segment
回答 2
这样的事情怎么样:
import os
def readlines_reverse(filename):
with open(filename) as qfile:
qfile.seek(0, os.SEEK_END)
position = qfile.tell()
line = ''
while position >= 0:
qfile.seek(position)
next_char = qfile.read(1)
if next_char == "\n":
yield line[::-1]
line = ''
else:
line += next_char
position -= 1
yield line[::-1]
if __name__ == '__main__':
for qline in readlines_reverse(raw_input()):
print qline
由于文件是按相反的顺序逐个字符读取的,因此即使在非常大的文件中也可以使用,只要将单独的行放入内存中即可。
How about something like this:
import os
def readlines_reverse(filename):
with open(filename) as qfile:
qfile.seek(0, os.SEEK_END)
position = qfile.tell()
line = ''
while position >= 0:
qfile.seek(position)
next_char = qfile.read(1)
if next_char == "\n":
yield line[::-1]
line = ''
else:
line += next_char
position -= 1
yield line[::-1]
if __name__ == '__main__':
for qline in readlines_reverse(raw_input()):
print qline
Since the file is read character by character in reverse order, it will work even on very large files, as long as individual lines fit into memory.
回答 3
您也可以使用python模块 file_read_backwards
。
通过pip install file_read_backwards
(v1.2.1)安装后,您可以通过以下方式以内存高效的方式向后(逐行)读取整个文件:
#!/usr/bin/env python2.7
from file_read_backwards import FileReadBackwards
with FileReadBackwards("/path/to/file", encoding="utf-8") as frb:
for l in frb:
print l
它支持“ utf-8”,“ latin-1”和“ ascii”编码。
也支持python3。可以在http://file-read-backwards.readthedocs.io/en/latest/readme.html上找到更多文档。
You can also use python module file_read_backwards
.
After installing it, via pip install file_read_backwards
(v1.2.1), you can read the entire file backwards (line-wise) in a memory efficient manner via:
#!/usr/bin/env python2.7
from file_read_backwards import FileReadBackwards
with FileReadBackwards("/path/to/file", encoding="utf-8") as frb:
for l in frb:
print l
It supports “utf-8″,”latin-1”, and “ascii” encodings.
Support is also available for python3. Further documentation can be found at http://file-read-backwards.readthedocs.io/en/latest/readme.html
回答 4
for line in reversed(open("file").readlines()):
print line.rstrip()
如果您使用的是Linux,则可以使用tac
command。
$ tac file
您可以在ActiveState的此处和此处找到2个食谱
for line in reversed(open("file").readlines()):
print line.rstrip()
If you are on linux, you can use tac
command.
$ tac file
2 recipes you can find in ActiveState here and here
回答 5
import re
def filerev(somefile, buffer=0x20000):
somefile.seek(0, os.SEEK_END)
size = somefile.tell()
lines = ['']
rem = size % buffer
pos = max(0, (size // buffer - 1) * buffer)
while pos >= 0:
somefile.seek(pos, os.SEEK_SET)
data = somefile.read(rem + buffer) + lines[0]
rem = 0
lines = re.findall('[^\n]*\n?', data)
ix = len(lines) - 2
while ix > 0:
yield lines[ix]
ix -= 1
pos -= buffer
else:
yield lines[0]
with open(sys.argv[1], 'r') as f:
for line in filerev(f):
sys.stdout.write(line)
import re
def filerev(somefile, buffer=0x20000):
somefile.seek(0, os.SEEK_END)
size = somefile.tell()
lines = ['']
rem = size % buffer
pos = max(0, (size // buffer - 1) * buffer)
while pos >= 0:
somefile.seek(pos, os.SEEK_SET)
data = somefile.read(rem + buffer) + lines[0]
rem = 0
lines = re.findall('[^\n]*\n?', data)
ix = len(lines) - 2
while ix > 0:
yield lines[ix]
ix -= 1
pos -= buffer
else:
yield lines[0]
with open(sys.argv[1], 'r') as f:
for line in filerev(f):
sys.stdout.write(line)
回答 6
接受的答案不适用于文件大而内存不足的情况(这种情况很少见)。
正如其他人所指出的那样,@ srohde的答案看起来不错,但存在下一个问题:
- 当我们可以传递文件对象并将其留给用户来决定应以哪种编码方式读取文件时,打开文件看起来是多余的,
即使我们重构为接受文件对象,它也不适用于所有编码:我们可以选择具有utf-8
编码和非ascii内容的文件,例如
й
通过buf_size
等于1
并将具有
UnicodeDecodeError: 'utf8' codec can't decode byte 0xb9 in position 0: invalid start byte
当然文字可能更大,但 buf_size
可能会被拾取,从而导致上述混淆错误,
- 我们无法指定自定义行分隔符,
- 我们不能选择保留行分隔符。
因此,考虑到所有这些问题,我编写了单独的函数:
- 一种适用于字节流的
- 第二个用于文本流,并将其基础字节流委托给第一个,并解码结果行。
首先,让我们定义下一个实用程序函数:
ceil_division
用于天花板分隔(与标准//
地板分隔相反,更多信息可以在此线程中找到)
def ceil_division(left_number, right_number):
"""
Divides given numbers with ceiling.
"""
return -(-left_number // right_number)
split
通过从右端给定的分隔符分割字符串并保持其能力:
def split(string, separator, keep_separator):
"""
Splits given string by given separator.
"""
parts = string.split(separator)
if keep_separator:
*parts, last_part = parts
parts = [part + separator for part in parts]
if last_part:
return parts + [last_part]
return parts
read_batch_from_end
从二进制流的右端读取批处理
def read_batch_from_end(byte_stream, size, end_position):
"""
Reads batch from the end of given byte stream.
"""
if end_position > size:
offset = end_position - size
else:
offset = 0
size = end_position
byte_stream.seek(offset)
return byte_stream.read(size)
之后,我们可以定义函数以相反的顺序读取字节流,例如
import functools
import itertools
import os
from operator import methodcaller, sub
def reverse_binary_stream(byte_stream, batch_size=None,
lines_separator=None,
keep_lines_separator=True):
if lines_separator is None:
lines_separator = (b'\r', b'\n', b'\r\n')
lines_splitter = methodcaller(str.splitlines.__name__,
keep_lines_separator)
else:
lines_splitter = functools.partial(split,
separator=lines_separator,
keep_separator=keep_lines_separator)
stream_size = byte_stream.seek(0, os.SEEK_END)
if batch_size is None:
batch_size = stream_size or 1
batches_count = ceil_division(stream_size, batch_size)
remaining_bytes_indicator = itertools.islice(
itertools.accumulate(itertools.chain([stream_size],
itertools.repeat(batch_size)),
sub),
batches_count)
try:
remaining_bytes_count = next(remaining_bytes_indicator)
except StopIteration:
return
def read_batch(position):
result = read_batch_from_end(byte_stream,
size=batch_size,
end_position=position)
while result.startswith(lines_separator):
try:
position = next(remaining_bytes_indicator)
except StopIteration:
break
result = (read_batch_from_end(byte_stream,
size=batch_size,
end_position=position)
+ result)
return result
batch = read_batch(remaining_bytes_count)
segment, *lines = lines_splitter(batch)
yield from reverse(lines)
for remaining_bytes_count in remaining_bytes_indicator:
batch = read_batch(remaining_bytes_count)
lines = lines_splitter(batch)
if batch.endswith(lines_separator):
yield segment
else:
lines[-1] += segment
segment, *lines = lines
yield from reverse(lines)
yield segment
最后,可以将文本文件反转功能定义如下:
import codecs
def reverse_file(file, batch_size=None,
lines_separator=None,
keep_lines_separator=True):
encoding = file.encoding
if lines_separator is not None:
lines_separator = lines_separator.encode(encoding)
yield from map(functools.partial(codecs.decode,
encoding=encoding),
reverse_binary_stream(
file.buffer,
batch_size=batch_size,
lines_separator=lines_separator,
keep_lines_separator=keep_lines_separator))
测验
准备工作
我已经使用fsutil
command生成了4个文件:
- 没有内容的empty.txt,大小为0MB
- tiny.txt,大小为1MB
- small.txt,大小为10MB
- large.txt,大小为50MB
我也重构了@srohde解决方案以使用文件对象而不是文件路径。
测试脚本
from timeit import Timer
repeats_count = 7
number = 1
create_setup = ('from collections import deque\n'
'from __main__ import reverse_file, reverse_readline\n'
'file = open("{}")').format
srohde_solution = ('with file:\n'
' deque(reverse_readline(file,\n'
' buf_size=8192),'
' maxlen=0)')
azat_ibrakov_solution = ('with file:\n'
' deque(reverse_file(file,\n'
' lines_separator="\\n",\n'
' keep_lines_separator=False,\n'
' batch_size=8192), maxlen=0)')
print('reversing empty file by "srohde"',
min(Timer(srohde_solution,
create_setup('empty.txt')).repeat(repeats_count, number)))
print('reversing empty file by "Azat Ibrakov"',
min(Timer(azat_ibrakov_solution,
create_setup('empty.txt')).repeat(repeats_count, number)))
print('reversing tiny file (1MB) by "srohde"',
min(Timer(srohde_solution,
create_setup('tiny.txt')).repeat(repeats_count, number)))
print('reversing tiny file (1MB) by "Azat Ibrakov"',
min(Timer(azat_ibrakov_solution,
create_setup('tiny.txt')).repeat(repeats_count, number)))
print('reversing small file (10MB) by "srohde"',
min(Timer(srohde_solution,
create_setup('small.txt')).repeat(repeats_count, number)))
print('reversing small file (10MB) by "Azat Ibrakov"',
min(Timer(azat_ibrakov_solution,
create_setup('small.txt')).repeat(repeats_count, number)))
print('reversing large file (50MB) by "srohde"',
min(Timer(srohde_solution,
create_setup('large.txt')).repeat(repeats_count, number)))
print('reversing large file (50MB) by "Azat Ibrakov"',
min(Timer(azat_ibrakov_solution,
create_setup('large.txt')).repeat(repeats_count, number)))
注意:我已经使用了collections.deque
类来耗尽生成器。
产出
对于Windows 10上的PyPy 3.5:
reversing empty file by "srohde" 8.31e-05
reversing empty file by "Azat Ibrakov" 0.00016090000000000028
reversing tiny file (1MB) by "srohde" 0.160081
reversing tiny file (1MB) by "Azat Ibrakov" 0.09594989999999998
reversing small file (10MB) by "srohde" 8.8891863
reversing small file (10MB) by "Azat Ibrakov" 5.323388100000001
reversing large file (50MB) by "srohde" 186.5338368
reversing large file (50MB) by "Azat Ibrakov" 99.07450229999998
对于Windows 10上的CPython 3.5:
reversing empty file by "srohde" 3.600000000000001e-05
reversing empty file by "Azat Ibrakov" 4.519999999999958e-05
reversing tiny file (1MB) by "srohde" 0.01965560000000001
reversing tiny file (1MB) by "Azat Ibrakov" 0.019207699999999994
reversing small file (10MB) by "srohde" 3.1341862999999996
reversing small file (10MB) by "Azat Ibrakov" 3.0872588000000007
reversing large file (50MB) by "srohde" 82.01206720000002
reversing large file (50MB) by "Azat Ibrakov" 82.16775059999998
因此,我们可以看到它的性能像原始解决方案一样,但它更具通用性,并且没有上面列出的缺点。
广告
我已经将此添加到具有许多经过良好测试的功能/迭代实用程序0.3.0
的lz
软件包版本(需要Python 3.5 +)中。
可以像
import io
from lz.iterating import reverse
...
with open('path/to/file') as file:
for line in reverse(file, batch_size=io.DEFAULT_BUFFER_SIZE):
print(line)
它支持所有标准编码(可能utf-7
是因为我很难定义一种策略来生成可编码的字符串)。
Accepted answer won’t work for cases with large files that won’t fit in memory (which is not a rare case).
As it was noted by others, @srohde answer looks good, but it has next issues:
- openning file looks redundant, when we can pass file object & leave it to user to decide in which encoding it should be read,
even if we refactor to accept file object, it won’t work for all encodings: we can choose file with utf-8
encoding and non-ascii contents like
й
pass buf_size
equal to 1
and will have
UnicodeDecodeError: 'utf8' codec can't decode byte 0xb9 in position 0: invalid start byte
of course text may be larger but buf_size
may be picked up so it’ll lead to obfuscated error like above,
- we can’t specify custom line separator,
- we can’t choose to keep line separator.
So considering all these concerns I’ve written separate functions:
- one which works with byte streams,
- second one which works with text streams and delegates its underlying byte stream to the first one and decodes resulting lines.
First of all let’s define next utility functions:
ceil_division
for making division with ceiling (in contrast with standard //
division with floor, more info can be found in this thread)
def ceil_division(left_number, right_number):
"""
Divides given numbers with ceiling.
"""
return -(-left_number // right_number)
split
for splitting string by given separator from right end with ability to keep it:
def split(string, separator, keep_separator):
"""
Splits given string by given separator.
"""
parts = string.split(separator)
if keep_separator:
*parts, last_part = parts
parts = [part + separator for part in parts]
if last_part:
return parts + [last_part]
return parts
read_batch_from_end
to read batch from the right end of binary stream
def read_batch_from_end(byte_stream, size, end_position):
"""
Reads batch from the end of given byte stream.
"""
if end_position > size:
offset = end_position - size
else:
offset = 0
size = end_position
byte_stream.seek(offset)
return byte_stream.read(size)
After that we can define function for reading byte stream in reverse order like
import functools
import itertools
import os
from operator import methodcaller, sub
def reverse_binary_stream(byte_stream, batch_size=None,
lines_separator=None,
keep_lines_separator=True):
if lines_separator is None:
lines_separator = (b'\r', b'\n', b'\r\n')
lines_splitter = methodcaller(str.splitlines.__name__,
keep_lines_separator)
else:
lines_splitter = functools.partial(split,
separator=lines_separator,
keep_separator=keep_lines_separator)
stream_size = byte_stream.seek(0, os.SEEK_END)
if batch_size is None:
batch_size = stream_size or 1
batches_count = ceil_division(stream_size, batch_size)
remaining_bytes_indicator = itertools.islice(
itertools.accumulate(itertools.chain([stream_size],
itertools.repeat(batch_size)),
sub),
batches_count)
try:
remaining_bytes_count = next(remaining_bytes_indicator)
except StopIteration:
return
def read_batch(position):
result = read_batch_from_end(byte_stream,
size=batch_size,
end_position=position)
while result.startswith(lines_separator):
try:
position = next(remaining_bytes_indicator)
except StopIteration:
break
result = (read_batch_from_end(byte_stream,
size=batch_size,
end_position=position)
+ result)
return result
batch = read_batch(remaining_bytes_count)
segment, *lines = lines_splitter(batch)
yield from reverse(lines)
for remaining_bytes_count in remaining_bytes_indicator:
batch = read_batch(remaining_bytes_count)
lines = lines_splitter(batch)
if batch.endswith(lines_separator):
yield segment
else:
lines[-1] += segment
segment, *lines = lines
yield from reverse(lines)
yield segment
and finally a function for reversing text file can be defined like:
import codecs
def reverse_file(file, batch_size=None,
lines_separator=None,
keep_lines_separator=True):
encoding = file.encoding
if lines_separator is not None:
lines_separator = lines_separator.encode(encoding)
yield from map(functools.partial(codecs.decode,
encoding=encoding),
reverse_binary_stream(
file.buffer,
batch_size=batch_size,
lines_separator=lines_separator,
keep_lines_separator=keep_lines_separator))
Tests
Preparations
I’ve generated 4 files using fsutil
command:
- empty.txt with no contents, size 0MB
- tiny.txt with size of 1MB
- small.txt with size of 10MB
- large.txt with size of 50MB
also I’ve refactored @srohde solution to work with file object instead of file path.
Test script
from timeit import Timer
repeats_count = 7
number = 1
create_setup = ('from collections import deque\n'
'from __main__ import reverse_file, reverse_readline\n'
'file = open("{}")').format
srohde_solution = ('with file:\n'
' deque(reverse_readline(file,\n'
' buf_size=8192),'
' maxlen=0)')
azat_ibrakov_solution = ('with file:\n'
' deque(reverse_file(file,\n'
' lines_separator="\\n",\n'
' keep_lines_separator=False,\n'
' batch_size=8192), maxlen=0)')
print('reversing empty file by "srohde"',
min(Timer(srohde_solution,
create_setup('empty.txt')).repeat(repeats_count, number)))
print('reversing empty file by "Azat Ibrakov"',
min(Timer(azat_ibrakov_solution,
create_setup('empty.txt')).repeat(repeats_count, number)))
print('reversing tiny file (1MB) by "srohde"',
min(Timer(srohde_solution,
create_setup('tiny.txt')).repeat(repeats_count, number)))
print('reversing tiny file (1MB) by "Azat Ibrakov"',
min(Timer(azat_ibrakov_solution,
create_setup('tiny.txt')).repeat(repeats_count, number)))
print('reversing small file (10MB) by "srohde"',
min(Timer(srohde_solution,
create_setup('small.txt')).repeat(repeats_count, number)))
print('reversing small file (10MB) by "Azat Ibrakov"',
min(Timer(azat_ibrakov_solution,
create_setup('small.txt')).repeat(repeats_count, number)))
print('reversing large file (50MB) by "srohde"',
min(Timer(srohde_solution,
create_setup('large.txt')).repeat(repeats_count, number)))
print('reversing large file (50MB) by "Azat Ibrakov"',
min(Timer(azat_ibrakov_solution,
create_setup('large.txt')).repeat(repeats_count, number)))
Note: I’ve used collections.deque
class to exhaust generator.
Outputs
For PyPy 3.5 on Windows 10:
reversing empty file by "srohde" 8.31e-05
reversing empty file by "Azat Ibrakov" 0.00016090000000000028
reversing tiny file (1MB) by "srohde" 0.160081
reversing tiny file (1MB) by "Azat Ibrakov" 0.09594989999999998
reversing small file (10MB) by "srohde" 8.8891863
reversing small file (10MB) by "Azat Ibrakov" 5.323388100000001
reversing large file (50MB) by "srohde" 186.5338368
reversing large file (50MB) by "Azat Ibrakov" 99.07450229999998
For CPython 3.5 on Windows 10:
reversing empty file by "srohde" 3.600000000000001e-05
reversing empty file by "Azat Ibrakov" 4.519999999999958e-05
reversing tiny file (1MB) by "srohde" 0.01965560000000001
reversing tiny file (1MB) by "Azat Ibrakov" 0.019207699999999994
reversing small file (10MB) by "srohde" 3.1341862999999996
reversing small file (10MB) by "Azat Ibrakov" 3.0872588000000007
reversing large file (50MB) by "srohde" 82.01206720000002
reversing large file (50MB) by "Azat Ibrakov" 82.16775059999998
So as we can see it performs like original solution, but is more general and free of its disadvantages listed above.
Advertisement
I’ve added this to 0.3.0
version of lz
package (requires Python 3.5+) that have many well-tested functional/iterating utilities.
Can be used like
import io
from lz.iterating import reverse
...
with open('path/to/file') as file:
for line in reverse(file, batch_size=io.DEFAULT_BUFFER_SIZE):
print(line)
It supports all standard encodings (maybe except utf-7
since it is hard for me to define a strategy for generating strings encodable with it).
回答 7
在这里,您可以找到我的实现,可以通过更改“ buffer”变量来限制ram的使用,这是一个程序在开始时打印空行的错误。
如果没有新行超过缓冲区字节,内存使用也会增加,“ leak”变量将一直增加,直到看到新行(“ \ n”)为止。
这也适用于大于我的总内存的16 GB文件。
import os,sys
buffer = 1024*1024 # 1MB
f = open(sys.argv[1])
f.seek(0, os.SEEK_END)
filesize = f.tell()
division, remainder = divmod(filesize, buffer)
line_leak=''
for chunk_counter in range(1,division + 2):
if division - chunk_counter < 0:
f.seek(0, os.SEEK_SET)
chunk = f.read(remainder)
elif division - chunk_counter >= 0:
f.seek(-(buffer*chunk_counter), os.SEEK_END)
chunk = f.read(buffer)
chunk_lines_reversed = list(reversed(chunk.split('\n')))
if line_leak: # add line_leak from previous chunk to beginning
chunk_lines_reversed[0] += line_leak
# after reversed, save the leakedline for next chunk iteration
line_leak = chunk_lines_reversed.pop()
if chunk_lines_reversed:
print "\n".join(chunk_lines_reversed)
# print the last leaked line
if division - chunk_counter < 0:
print line_leak
Here you can find my my implementation, you can limit the ram usage by changing the “buffer” variable, there is a bug that the program prints an empty line in the beginning.
And also ram usage may be increase if there is no new lines for more than buffer bytes, “leak” variable will increase until seeing a new line (“\n”).
This is also working for 16 GB files which is bigger then my total memory.
import os,sys
buffer = 1024*1024 # 1MB
f = open(sys.argv[1])
f.seek(0, os.SEEK_END)
filesize = f.tell()
division, remainder = divmod(filesize, buffer)
line_leak=''
for chunk_counter in range(1,division + 2):
if division - chunk_counter < 0:
f.seek(0, os.SEEK_SET)
chunk = f.read(remainder)
elif division - chunk_counter >= 0:
f.seek(-(buffer*chunk_counter), os.SEEK_END)
chunk = f.read(buffer)
chunk_lines_reversed = list(reversed(chunk.split('\n')))
if line_leak: # add line_leak from previous chunk to beginning
chunk_lines_reversed[0] += line_leak
# after reversed, save the leakedline for next chunk iteration
line_leak = chunk_lines_reversed.pop()
if chunk_lines_reversed:
print "\n".join(chunk_lines_reversed)
# print the last leaked line
if division - chunk_counter < 0:
print line_leak
回答 8
感谢您的回答@srohde。它使用“ is”运算符检查换行符时有一个小错误,并且我无法对信誉为1的答案进行评论。我也想管理外部文件的打开,因为这使我可以将杂物嵌入luigi任务。
我需要更改的形式如下:
with open(filename) as fp:
for line in fp:
#print line, # contains new line
print '>{}<'.format(line)
我想更改为:
with open(filename) as fp:
for line in reversed_fp_iter(fp, 4):
#print line, # contains new line
print '>{}<'.format(line)
这是修改后的答案,需要文件句柄并保留换行符:
def reversed_fp_iter(fp, buf_size=8192):
"""a generator that returns the lines of a file in reverse order
ref: https://stackoverflow.com/a/23646049/8776239
"""
segment = None # holds possible incomplete segment at the beginning of the buffer
offset = 0
fp.seek(0, os.SEEK_END)
file_size = remaining_size = fp.tell()
while remaining_size > 0:
offset = min(file_size, offset + buf_size)
fp.seek(file_size - offset)
buffer = fp.read(min(remaining_size, buf_size))
remaining_size -= buf_size
lines = buffer.splitlines(True)
# the first line of the buffer is probably not a complete line so
# we'll save it and append it to the last line of the next buffer
# we read
if segment is not None:
# if the previous chunk starts right from the beginning of line
# do not concat the segment to the last line of new chunk
# instead, yield the segment first
if buffer[-1] == '\n':
#print 'buffer ends with newline'
yield segment
else:
lines[-1] += segment
#print 'enlarged last line to >{}<, len {}'.format(lines[-1], len(lines))
segment = lines[0]
for index in range(len(lines) - 1, 0, -1):
if len(lines[index]):
yield lines[index]
# Don't yield None if the file was empty
if segment is not None:
yield segment
Thanks for the answer @srohde. It has a small bug checking for newline character with ‘is’ operator, and I could not comment on the answer with 1 reputation. Also I’d like to manage file open outside because that enables me to embed my ramblings for luigi tasks.
What I needed to change has the form:
with open(filename) as fp:
for line in fp:
#print line, # contains new line
print '>{}<'.format(line)
I’d love to change to:
with open(filename) as fp:
for line in reversed_fp_iter(fp, 4):
#print line, # contains new line
print '>{}<'.format(line)
Here is a modified answer that wants a file handle and keeps newlines:
def reversed_fp_iter(fp, buf_size=8192):
"""a generator that returns the lines of a file in reverse order
ref: https://stackoverflow.com/a/23646049/8776239
"""
segment = None # holds possible incomplete segment at the beginning of the buffer
offset = 0
fp.seek(0, os.SEEK_END)
file_size = remaining_size = fp.tell()
while remaining_size > 0:
offset = min(file_size, offset + buf_size)
fp.seek(file_size - offset)
buffer = fp.read(min(remaining_size, buf_size))
remaining_size -= buf_size
lines = buffer.splitlines(True)
# the first line of the buffer is probably not a complete line so
# we'll save it and append it to the last line of the next buffer
# we read
if segment is not None:
# if the previous chunk starts right from the beginning of line
# do not concat the segment to the last line of new chunk
# instead, yield the segment first
if buffer[-1] == '\n':
#print 'buffer ends with newline'
yield segment
else:
lines[-1] += segment
#print 'enlarged last line to >{}<, len {}'.format(lines[-1], len(lines))
segment = lines[0]
for index in range(len(lines) - 1, 0, -1):
if len(lines[index]):
yield lines[index]
# Don't yield None if the file was empty
if segment is not None:
yield segment
回答 9
一个简单的函数来创建另一个反转的文件(仅适用于Linux):
import os
def tac(file1, file2):
print(os.system('tac %s > %s' % (file1,file2)))
如何使用
tac('ordered.csv', 'reversed.csv')
f = open('reversed.csv')
a simple function to create a second file reversed (linux only):
import os
def tac(file1, file2):
print(os.system('tac %s > %s' % (file1,file2)))
how to use
tac('ordered.csv', 'reversed.csv')
f = open('reversed.csv')
回答 10
回答 11
使用open(“ filename”)as f:
print(f.read()[::-1])
with open(“filename”) as f:
print(f.read()[::-1])
回答 12
def reverse_lines(filename):
y=open(filename).readlines()
return y[::-1]
def reverse_lines(filename):
y=open(filename).readlines()
return y[::-1]
回答 13
with
处理文件时请务必使用,因为它可以为您处理所有事情:
with open('filename', 'r') as f:
for line in reversed(f.readlines()):
print line
或在Python 3中:
with open('filename', 'r') as f:
for line in reversed(list(f.readlines())):
print(line)
Always use with
when working with files as it handles everything for you:
with open('filename', 'r') as f:
for line in reversed(f.readlines()):
print line
Or in Python 3:
with open('filename', 'r') as f:
for line in reversed(list(f.readlines())):
print(line)
回答 14
您需要首先以读取格式打开文件,将其保存到变量,然后以写入格式打开第二个文件,在该文件中您将使用[::-1]切片来写入或附加变量,从而完全反转文件。您还可以使用readlines()使其成为一行列表,您可以对其进行操作
def copy_and_reverse(filename, newfile):
with open(filename) as file:
text = file.read()
with open(newfile, "w") as file2:
file2.write(text[::-1])
you would need to first open your file in read format, save it to a variable, then open the second file in write format where you would write or append the variable using a the [::-1] slice, completely reversing the file. You can also use readlines() to make it into a list of lines, which you can manipulate
def copy_and_reverse(filename, newfile):
with open(filename) as file:
text = file.read()
with open(newfile, "w") as file2:
file2.write(text[::-1])
回答 15
大多数答案都需要在执行任何操作之前先读取整个文件。此样本从头开始读取越来越大的样本。
在编写此答案时,我仅看到MuratYükselen的答案。几乎一样,我认为这是一件好事。下面的示例还处理\ r,并在每个步骤中增加其缓冲区大小。我也有一些单元测试来备份此代码。
def readlines_reversed(f):
""" Iterate over the lines in a file in reverse. The file must be
open in 'rb' mode. Yields the lines unencoded (as bytes), including the
newline character. Produces the same result as readlines, but reversed.
If this is used to reverse the line in a file twice, the result is
exactly the same.
"""
head = b""
f.seek(0, 2)
t = f.tell()
buffersize, maxbuffersize = 64, 4096
while True:
if t <= 0:
break
# Read next block
buffersize = min(buffersize * 2, maxbuffersize)
tprev = t
t = max(0, t - buffersize)
f.seek(t)
lines = f.read(tprev - t).splitlines(True)
# Align to line breaks
if not lines[-1].endswith((b"\n", b"\r")):
lines[-1] += head # current tail is previous head
elif head == b"\n" and lines[-1].endswith(b"\r"):
lines[-1] += head # Keep \r\n together
elif head:
lines.append(head)
head = lines.pop(0) # can be '\n' (ok)
# Iterate over current block in reverse
for line in reversed(lines):
yield line
if head:
yield head
Most of the answers need to read the whole file before doing anything. This sample reads increasingly large samples from the end.
I only saw Murat Yükselen’s answer while writing this answer. It’s nearly the same, which I suppose is a good thing. The sample below also deals with \r and increases its buffersize at each step. I also have some unit tests to back this code up.
def readlines_reversed(f):
""" Iterate over the lines in a file in reverse. The file must be
open in 'rb' mode. Yields the lines unencoded (as bytes), including the
newline character. Produces the same result as readlines, but reversed.
If this is used to reverse the line in a file twice, the result is
exactly the same.
"""
head = b""
f.seek(0, 2)
t = f.tell()
buffersize, maxbuffersize = 64, 4096
while True:
if t <= 0:
break
# Read next block
buffersize = min(buffersize * 2, maxbuffersize)
tprev = t
t = max(0, t - buffersize)
f.seek(t)
lines = f.read(tprev - t).splitlines(True)
# Align to line breaks
if not lines[-1].endswith((b"\n", b"\r")):
lines[-1] += head # current tail is previous head
elif head == b"\n" and lines[-1].endswith(b"\r"):
lines[-1] += head # Keep \r\n together
elif head:
lines.append(head)
head = lines.pop(0) # can be '\n' (ok)
# Iterate over current block in reverse
for line in reversed(lines):
yield line
if head:
yield head
回答 16
逐行读取文件,然后以相反顺序将其添加到列表中。
这是代码示例:
reverse = []
with open("file.txt", "r") as file:
for line in file:
line = line.strip()
reverse[0:0] = line
Read the file line by line and then add it on a list in reverse order.
Here is an example of code :
reverse = []
with open("file.txt", "r") as file:
for line in file:
line = line.strip()
reverse[0:0] = line
回答 17
import sys
f = open(sys.argv[1] , 'r')
for line in f.readlines()[::-1]:
print line
import sys
f = open(sys.argv[1] , 'r')
for line in f.readlines()[::-1]:
print line
回答 18
def previous_line(self, opened_file):
opened_file.seek(0, os.SEEK_END)
position = opened_file.tell()
buffer = bytearray()
while position >= 0:
opened_file.seek(position)
position -= 1
new_byte = opened_file.read(1)
if new_byte == self.NEW_LINE:
parsed_string = buffer.decode()
yield parsed_string
buffer = bytearray()
elif new_byte == self.EMPTY_BYTE:
continue
else:
new_byte_array = bytearray(new_byte)
new_byte_array.extend(buffer)
buffer = new_byte_array
yield None
使用:
opened_file = open(filepath, "rb")
iterator = self.previous_line(opened_file)
line = next(iterator) #one step
close(opened_file)
def previous_line(self, opened_file):
opened_file.seek(0, os.SEEK_END)
position = opened_file.tell()
buffer = bytearray()
while position >= 0:
opened_file.seek(position)
position -= 1
new_byte = opened_file.read(1)
if new_byte == self.NEW_LINE:
parsed_string = buffer.decode()
yield parsed_string
buffer = bytearray()
elif new_byte == self.EMPTY_BYTE:
continue
else:
new_byte_array = bytearray(new_byte)
new_byte_array.extend(buffer)
buffer = new_byte_array
yield None
to use:
opened_file = open(filepath, "rb")
iterator = self.previous_line(opened_file)
line = next(iterator) #one step
close(opened_file)
回答 19
我不得不在一段时间前使用以下代码。它通过管道传递给外壳。恐怕我没有完整的脚本了。如果您使用的是unixish操作系统,则可以使用“ tac”,但是,例如在Mac OSX上,tac命令不起作用,请使用tail -r。以下代码段测试了您所使用的平台,并相应地调整了命令
# We need a command to reverse the line order of the file. On Linux this
# is 'tac', on OSX it is 'tail -r'
# 'tac' is not supported on osx, 'tail -r' is not supported on linux.
if sys.platform == "darwin":
command += "|tail -r"
elif sys.platform == "linux2":
command += "|tac"
else:
raise EnvironmentError('Platform %s not supported' % sys.platform)
I had to do this some time ago and used the below code. It pipes to the shell. I am afraid i do not have the complete script anymore. If you are on a unixish operating system, you can use “tac”, however on e.g. Mac OSX tac command does not work, use tail -r. The below code snippet tests for which platform you’re on, and adjusts the command accordingly
# We need a command to reverse the line order of the file. On Linux this
# is 'tac', on OSX it is 'tail -r'
# 'tac' is not supported on osx, 'tail -r' is not supported on linux.
if sys.platform == "darwin":
command += "|tail -r"
elif sys.platform == "linux2":
command += "|tac"
else:
raise EnvironmentError('Platform %s not supported' % sys.platform)