conversation table json dumps of 100 records in each json collection

conversation_json_Dumps.py

# invoice info  -    19,424
print('iterate with index - steps of 100')

import mysql.connector
from mysql.connector import Error
import logging
import re
import time
from datetime import datetime , timedelta
import json

now = datetime.now()

fileName = now.strftime('%Y_%m_%d_%H_%M_%S')
logging.basicConfig(filename="logs\\shardingJSONS_conversation_1_"+fileName+"_LOGS.log", level=logging.INFO, datefmt='%Y-%m-%d %H:%M:%S')

startTime = now
print('script started at : {}'.format(startTime.strftime('%Y-%m-%d %H:%M:%S')))
logging.info('script started at : {}\n'.format(startTime.strftime('%Y-%m-%d %H:%M:%S')))

print('creating shards of huge amount of data\n')

sizeOfShard = 100
index = 0
counter = 0
#lastIndex = 479,300 - conversation
lastIndex =   479300 

alpha = re.compile(r"[a-z|A-Z]|[!@#$%^&*/]")


queryConversation = "select s_n,with_email,user_id,added_on,msg,followup_on FROM conversation"

#
# if query gives null response add condition 
#
#       try an outrage query
#   build condtion logic for that response
#


for i in range(index , lastIndex ,sizeOfShard):
    query = ''
    
    print(' ------------- {} -------------- '.format(counter))
    logging.info(' ------------- {} -------------- '.format(counter))

    # print('{} -> running for limit {},{}'.format(counter,i,sizeOfShard))
    print('{} ->  limit {} OFFSET {}'.format(counter,sizeOfShard , i))
    offSet = ' LIMIT {} OFFSET {} '.format(sizeOfShard,i)
    
    with mysql.connector.connect() as connectPointer:
        with connectPointer.cursor() as aCursor:
            query = queryConversation+offSet
            print('running - ',query)
            print(' . .  fetching conversation . . ')
            logging.info('\t. fetching conversation: {}'.format(query))
            aCursor.execute(query)
            result = aCursor.fetchall()
            if not result:
                print('--> no  conversation left in table <--')
                logging.info('__________  no conversation left to write  __________')
            else:
                # payloads for invoice info ---------------------------------
                # Id,invoice_no,invoice_of,user_id,added_on,Email,payment_in,sale_rule,sale_amount 
                conv_Collection_List = []
                for rows in result:
                    convCollection = {}
                    name = str(rows[0])
                    s_n = rows[0]
                    with_email = rows[1]
                    user_id = rows[2]
                    msg = rows[4]
                    followup_on = str(rows[-1])
                    
                    convCollection['Name'] = name
                    convCollection['s_n'] = s_n
                    convCollection['msg'] = msg
                    convCollection['user_id'] = user_id
                    convCollection['with_email'] = with_email
                    
                    del name
                    del s_n
                    del with_email
                    del user_id
                    del msg
                    
                    invInfoColl = {}
                    invInfoColl['s_n'] = rows[0]
                    invInfoColl['Name'] = str(rows[0])
                    added_on = rows[3]
                    if added_on == None:
                        convCollection['added_on'] = None                       
                    else:
                        convCollection['added_on'] = str(added_on).replace(' ','T') + '+05:30'
                    if len (followup_on) < 5:
                        convCollection['followup_on'] = None
                    else:
                        convCollection['followup_on'] = followup_on.replace(' ','T') + '+05:30'
                    
                    del added_on
                    del followup_on
                    
                    conv_Collection_List.append(convCollection)

                conversationFile = 'conversations\\conversation_index_{}.json'.format(i)
                print(f' . .. writing to {conversationFile}')
    
                with open(conversationFile,'w',encoding='utf-8') as clientJson:
                        clientJson.write(json.dumps(
                            {"data":conv_Collection_List},indent=4,default=str,sort_keys=True
                            ))
                
                print(f' . .. check {conversationFile}')
                logging.info(f' . .. check {conversationFile}')
    time.sleep(2)
    counter+=1
    
logging.info(' {0} {0} '.format(' - - - - - - - - - - - - - - - - - - - - - - - - - - - '))

now =  datetime.now()
endTime = now
print('script ended at : {}'.format(endTime.strftime('%Y-%m-%d %H:%M:%S')))
logging.info('script ended at : {}'.format(endTime.strftime('%Y-%m-%d %H:%M:%S')))