-
-
Save nicwolff/b4da6ec84ba9c23c8e59 to your computer and use it in GitHub Desktop.
| import os | |
| import sys | |
| from xml.sax import parse | |
| from xml.sax.saxutils import XMLGenerator | |
| class CycleFile(object): | |
| def __init__(self, filename): | |
| self.basename, self.ext = os.path.splitext(filename) | |
| self.index = 0 | |
| self.open_next_file() | |
| def open_next_file(self): | |
| self.index += 1 | |
| self.file = open(self.name(), 'w') | |
| def name(self): | |
| return '%s%s%s' % (self.basename, self.index, self.ext) | |
| def cycle(self): | |
| self.file.close() | |
| self.open_next_file() | |
| def write(self, str): | |
| self.file.write(str) | |
| def close(self): | |
| self.file.close() | |
| class XMLBreaker(XMLGenerator): | |
| def __init__(self, break_into=None, break_after=1000, out=None, *args, **kwargs): | |
| XMLGenerator.__init__(self, out, encoding='utf-8', *args, **kwargs) | |
| self.out_file = out | |
| self.break_into = break_into | |
| self.break_after = break_after | |
| self.context = [] | |
| self.count = 0 | |
| def startElement(self, name, attrs): | |
| XMLGenerator.startElement(self, name, attrs) | |
| self.context.append((name, attrs)) | |
| def endElement(self, name): | |
| XMLGenerator.endElement(self, name) | |
| self.context.pop() | |
| if name == self.break_into: | |
| self.count += 1 | |
| if self.count == self.break_after: | |
| self.count = 0 | |
| for element in reversed(self.context): | |
| self.out_file.write("\n") | |
| XMLGenerator.endElement(self, element[0]) | |
| self.out_file.cycle() | |
| XMLGenerator.startDocument(self) | |
| for element in self.context: | |
| XMLGenerator.startElement(self, *element) | |
| filename, break_into, break_after = sys.argv[1:] | |
| parse(filename, XMLBreaker(break_into, int(break_after), out=CycleFile(filename))) |
Your script has been very useful once I figured out a little more python. One more question. If it comes across an XML that is missing a tag, it will crash. Fair enough. Where can I add try / except to explicitly ask this script to release the file that was being written when it came across a bad xml file with missing tag?
I'm trying to automate splitting. In case splitting fails, I want to automatically delete the splits that were created. Currently, I'm able to delete all splits except for the last one where the script crashed, at least until the python script is done running.
Can you please let me know where you are giving you the input file name, the element to split on etc..,
Great job.
Thank you very much!
I think there is a bug in this script, in that it is not able to distinguish between elements named the same, that occur at different levels. If you feed it:
python38 my.xml name 25000
Where my.xml look like:
<root>
<name>
<cd>
<name>
</name>
</cd>
<cassette>
<name>
</name>
</cassette>
</name>
</root>
The script believes that there are three name elements, where you probably only want it to believe there is one. So it consequently splits the files in strange positions, and you'll probably end up with more records than you thought you had.
@positivity13
for those who are getting the following error ,
self.file.write(str)
TypeError: write() argument must be str, not bytes
change the write function into
def write(self, str_text):
self.file.write(str_text.decode("utf-8") if isinstance(str_text,bytes) else str_text)
Thank you so much. It works very well.
@positivity13 for those who are getting the following error ,
self.file.write(str) TypeError: write() argument must be str, not byteschange the write function into
def write(self, str_text): self.file.write(str_text.decode("utf-8") if isinstance(str_text,bytes) else str_text)
Thanks!
Hello Nic,
A licence would be great. I would like to know whether or not I can reuse your code and what are the terms.
Cheers!
R
Thanks for sharing the script! One issue I got is that the last splitted file always failed to open, the error looked something like 'error on line {N} at column {C}: Premature end of data in tag debt line {N}'. Is there anywhere in the script that can be modified to resolve this issue?
Thanks for sharing this script! It works so well!
I am in the same boat, can you please let me know if you have got any solution.