Function to add program and baseurl fields to raw httpx json output

brevity-process-httpx.py

def processHttpx(programName, refinedBucketPath, inputBucketPath, presentationBucketPath, operationName, programInputBucketPath):
    fileName = programName + '-httpx-' + operationName + '.json'
    presentationFilePath = presentationBucketPath + 'httpx-json/' + fileName
    
    df = pd.read_json(presentationFilePath, lines=True)
    df['program'] = programName
    
    if (operationName == 'initial'):
        storePathUrl = programInputBucketPath + programName + '/' + programName + '-httpx.csv'
        df.to_csv(storePathUrl, header=False, index=False, sep='\n')
    
        storePathUrl = programInputBucketPath + programName + '/' + programName + '-urls-base.csv'
        dfUrls = df.drop_duplicates(subset=['url'])
        dfUrls['url'].to_csv(storePathUrl, header=False, index=False, sep='\n')
    
    from urllib.parse import urlparse

    def _parseUrlRoot(urlvalue):
        cleanurl = urlparse(urlvalue).netloc
        return cleanurl

    def _parseUrlBase(urlvalue):
        baseurl = urlparse(urlvalue)#.netloc
        baseurl = baseurl.scheme + '://' + baseurl.netloc + baseurl.path
        return baseurl

    df['domain'] = df['url'].apply(_parseUrlRoot)
    df['baseurl'] = df['url'].apply(_parseUrlBase)
    fileOutputName = programName + '-httpx.json'
    outputPath = presentationBucketPath + 'httpx/' + fileOutputName
    
    # Check if there is already output so that it is not overwritten
    try:
        dfInitialHttpx = pd.read_json(outputPath, lines=True)
        df = dfInitialHttpx.append(df)
        df = df.drop_duplicates(subset=['url'], keep='last')
    except:
        print('No initial httpx output')
      
    df.to_json(outputPath, orient='records', lines=True)

    if (operationName == 'initial'):
        fileOutputCrawl = programName + '-httpx-crawl.csv'
        storePathUrl = inputBucketPath + programName + '/' + fileOutputName
        df['url'].to_csv(storePathUrl, header=False, index=False, sep='\n')
    return 'Success'