crawlingResultToCSV.py

import os
import csv

searchKeywords = ["P2P","금융"]

def main():
    for dirname, dirnames, filenames in os.walk('.'):
        for subdirname in dirnames:
            if subdirname == "news":
                print("in " + os.path.join(dirname, subdirname))
                filter(os.path.join(dirname, subdirname))

def filter(newsDirname):
    for dirname, dirnames, filenames in os.walk(newsDirname):
        csvfilename = dirname.split('/')[-1]
        if os.path.exists(csvfilename+'.csv'):
            print(csvfilename+'.csv already exist. overiding file...')
        else:
            print(csvfilename+'.csv have just been created.')
        csvFile = open(csvfilename+'.csv', 'w', encoding='utf-8', newline='')
        wr = csv.writer(csvFile)

        for filename in filenames:
            if filename.endswith("index"):
                print(os.path.join(dirname, filename))
                f = open(os.path.join(dirname, filename), 'r', encoding='UTF8')
                lines = f.readlines()
                numOfSearched = 0
                for idx,line in enumerate(lines):
                    if len(line) < 30:
                        continue
                    (header, title, content) = line.split('|||||||')
                    if all([elem in content for elem in searchKeywords]):
                        numOfSearched += 1
                        wr.writerow(parseMetadata(header,title,content))
                f.close()
                print(numOfSearched,"articles searched")
        csvFile.close()

def parseMetadata(metadata,title,content):
    metadataList = metadata.split()[0].split('/')
    # return [metadataList[1]+"-"+metadataList[2]+"-"+metadataList[3],metadataList[0],content.strip()]
    return [metadataList[1]+"-"+metadataList[2]+"-"+metadataList[3],metadataList[0],metadataList[4],title.strip(),content.strip()]

if __name__ == "__main__":
    main()
    print("Done!")