Created
March 2, 2020 01:58
-
-
Save MrJeremyHobbs/012c5a8182ff054d35d0187479ea4410 to your computer and use it in GitHub Desktop.
PyMarc example with permission reading mode
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/python3 | |
import os | |
import logging | |
from pymarc import MARCReader | |
from tqdm import tqdm | |
def main(): | |
# clean-up | |
try: | |
os.system('cls') | |
os.system('del testing.log') | |
except: | |
pass | |
# logging | |
logging.basicConfig(filename='testing.log',level=logging.DEBUG) | |
# progressbar | |
total_records = 3715163 | |
pbar = tqdm(total=total_records) | |
# open marc file and read contents | |
# change the slice to limit files | |
for filename in os.listdir('.//data'): # <------- check this limiter | |
counter = 0 | |
with open(f'data/{filename}', 'rb') as fh: | |
reader = MARCReader(fh, to_unicode=False, utf8_handling='ignore', | |
permissive=True) | |
# progressbar | |
pbar.set_description(f"Reading {filename}") | |
# loop through records | |
for record in reader: | |
# check for errors | |
if record is None: | |
logging.info('BAD RECORD FOUND') | |
# read through records | |
counter += 15 | |
pbar.update(1) | |
# log totals | |
logging.info(f'{filename} total: {counter}') | |
# finish | |
pbar.set_description(f"Finished") | |
# top-level | |
if __name__ == "__main__": | |
main() |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
When I try your python script, I get the following error:
[root@libvm15 publishToOCLC]# python pymarc_permissive_reader.py
File "pymarc_permissive_reader.py", line 26
with open(f'data/{filename}', 'rb') as fh:
^
SyntaxError: invalid syntax