Data Ingestion #python #DataScience

Data_Ingestion.py

def get_data_lists(selected_col=[], indvGroupOnly=False):

  	# Data input
    xls = pd.ExcelFile("final_data_NMR_DILC_comb.xlsx")
    day1_org = pd.read_excel(xls,"Day1")
    day3_org = pd.read_excel(xls,"Day3")
    day10_org = pd.read_excel(xls,"Day10-11")

#     print(selected_col)
    if len(selected_col)>0:
        day1 = day1_org[selected_col]
        day3 = day3_org[selected_col]
        day10 = day10_org[selected_col]
    else:
        day1 = deepcopy(day1_org).drop(['Sample Name'],axis=1)
        day3 = deepcopy(day3_org).drop(['Sample Name'],axis=1)
        day10 = deepcopy(day10_org).drop(['Sample Name'],axis=1)

	#features
    l_features_org = list(day1.columns)

   	#Get the values for each specific group
    D1H=day1[1:11].values[:,:].astype(float)
    D1P=day1[13:23].values[:,:].astype(float)
    D1N=day1[25:35].values[:,:].astype(float)
    
    D3P=day3[1:11].values[:,:].astype(float)
    D3N=day3[13:23].values[:,:].astype(float)

    D10P=day1[1:10].values[:,:].astype(float)

    
    # Make groups sets and comparisions
    D1HvP = np.vstack([D1H,D1P])
    l1HvP = [0]*10 + [1]*10

    D1HNvP = np.vstack([D1H,D1N,D1P])
    l1HNvP = [0]*20 + [1]*10

    D1NvP = np.vstack([D1N,D1P])
    l1NvP = [0]*10 + [1]*10

    D3NvP = np.vstack([D3N,D3P])
    l3NvP = [0]*10 + [1]*10

    D1v3P = np.vstack([D1P,D3P])
    l1v3P = [0]*10 + [1]*10

    D1v3N = np.vstack([D1N,D3N])
    l1v3N = [0]*10 + [1]*10

    D13Pv13N = np.vstack([D1P,D3P,D1N,D3N])
    l13Pv13N = [0]*10 + [0]*10 + [1]*10 + [1]*10

    D1v3v10P = np.vstack([D1P[:-1],D3P[:-1],D10P])
    l1v3v10P = [0]*9 + [1]*9 + [2]*9
    
    if indvGroupOnly:
      l_mat_raw = [D1H,D1P,D1N,D3P,D3N,D10P]
      l_title_raw = ["D1H","D1P","D1N","D3P","D3N","D10P"]
      return l_mat_raw
   	else:
      l_mat = [D1HvP,D1NvP,D1HNvP,D3NvP,D1v3P,D1v3N,D13Pv13N,D1v3v10P]
      l_label = [l1HvP,l1NvP,l1HNvP,l3NvP,l1NvP,l1v3N,l13Pv13N,l1v3v10P]
      l_title = ["D1HvP","D1NvP","D1HNvP","D3NvP","D1v3P","D1v3N","D13Pv13N","D1v3v10P"]
      return l_mat,l_label,l_title,l_features_org
    
l_mat,l_label,l_title,l_features_org = get_data_lists()