invoice info json dumps records of 100

invoiceInfo_json_Dumps.py

# invoice info  -    19,424
print('iterate with index - steps of 100')

import mysql.connector
from mysql.connector import Error
import logging
import re
import time
from datetime import datetime , timedelta
import json

now = datetime.now()

fileName = now.strftime('%Y_%m_%d_%H_%M_%S')
logging.basicConfig(filename="logs\\shardingJSONS_invoiceInfo_2_"+fileName+"_LOGS.log", level=logging.INFO, datefmt='%Y-%m-%d %H:%M:%S')

startTime = now
print('script started at : {}'.format(startTime.strftime('%Y-%m-%d %H:%M:%S')))
logging.info('script started at : {}\n'.format(startTime.strftime('%Y-%m-%d %H:%M:%S')))

print('creating shards of huge amount of data\n')

sizeOfShard = 100
index = 0
counter = 0
#lastIndex = 479,300
# invoice items /* 28,577 */
#  19,424 - invoice info
lastIndex =   19500 # test with leads table ~ around 96.5K records

alpha = re.compile(r"[a-z|A-Z]|[!@#$%^&*/]")


invoiceInfo = 'select * from invoice_info'

#
# if query gives null response add condition 
#
#       try an outrage query
#   build condtion logic for that response
#


for i in range(index , lastIndex ,sizeOfShard):
    query = ''
    
    print(' ------------- {} -------------- '.format(counter))
    logging.info(' ------------- {} -------------- '.format(counter))

    # print('{} -> running for limit {},{}'.format(counter,i,sizeOfShard))
    print('{} ->  limit {} OFFSET {}'.format(counter,sizeOfShard , i))
    offSet = ' LIMIT {} OFFSET {} '.format(sizeOfShard,i)
    
    with mysql.connector.connect() as connectPointer:
        with connectPointer.cursor() as aCursor:
            query = invoiceInfo+offSet
            print('running - ',query)
            print(' . .  fetching invoice info . . ')
            logging.info('\t. fetching invoice info : {}'.format(query))
            aCursor.execute(query)
            result = aCursor.fetchall()
            if not result:
                print('--> no invoice info left in table <--')
                logging.info('__________  no invoice info left to write  __________')
            else:
                # payloads for invoice info ---------------------------------
                # Id,invoice_no,invoice_of,user_id,added_on,Email,payment_in,sale_rule,sale_amount 
                invoiceInfoCollList = []
                for rows in result:
                    invInfoColl = {}
                    invInfoColl['local_id'] = rows[0]
                    invInfoColl['Name'] = str(rows[0])
                    invInfoColl['invoice_no'] = rows[1]
                    invInfoColl['invoice_of'] = rows[2]
                    invInfoColl['Customer_Name'] = rows[3]
                    invInfoColl['Company_Name'] = rows[4]
                    invInfoColl['Customer_Designation'] = rows[5]
                    invInfoColl['Address'] = rows[6]
                    invInfoColl['City'] = rows[7]
                    invInfoColl['State'] = rows[8]
                    invInfoColl['Tel_No'] = rows[9]
                    added_on = rows[14]
                    if added_on == None:
                        invInfoColl['added_on'] = None                       
                    else:
                        invInfoColl['added_on'] = str(added_on).replace(' ','T') + '+05:30'
                    invInfoColl['Email'] = rows[12]
                    invInfoColl['payment_in'] = rows[17]
                    invInfoColl['sale_rule'] = rows[23]
                    invInfoColl['sale_amount'] = rows[24]
                    invInfoColl['user_id'] = rows[13]
                    payment_on = rows[16]
                    if payment_on == None:
                        invInfoColl['payment_on'] = None
                    else:
                        invInfoColl['payment_on'] = str(payment_on)
                    invInfoColl['payment_detail'] = rows[15]
                    invInfoColl['Address'] = rows[6]
                    invInfoColl['payment_in'] = rows[17]
                    invInfoColl['data_detail'] = rows[18]
                    invInfoColl['tds_percentage'] = rows[22]
                    invInfoColl['sale_rule'] = rows[23]
                    invInfoColl['sale_amount'] = rows[24]
                    invInfoColl['amount_recieved'] = rows[19]
                    data_sent_on = str(rows[20])
                    if len (data_sent_on) < 5:
                        invInfoColl['data_sent_on'] = None
                    else:
                        invInfoColl['data_sent_on'] = data_sent_on.replace(' ','T') + '+05:30'
                    invoiceInfoCollList.append(invInfoColl)
                    
                    del added_on
                    del payment_on

                invoiceInfoFile = 'invoiceInfo\\invoiceInfo_index_{}.json'.format(i)
                with open(invoiceInfoFile,'w',encoding='utf-8') as clientJson:
                        clientJson.write(json.dumps(
                            {"data":invoiceInfoCollList},indent=4,default=str,sort_keys=True
                            ))
                
                print(f' . .. check {invoiceInfoFile}')
                logging.info(f' . .. check {invoiceInfoFile}')
    time.sleep(2)
    counter+=1
logging.info(' {0} {0} '.format(' - - - - - - - - - - - - - - - - - - - - - - - - - - - '))

now =  datetime.now()
endTime = now
print('script ended at : {}'.format(endTime.strftime('%Y-%m-%d %H:%M:%S')))
logging.info('script ended at : {}'.format(endTime.strftime('%Y-%m-%d %H:%M:%S')))