Conduit

script.hs

#!/usr/bin/env stack
-- stack runghc --resolver lts-12.5 --stack-yaml ../stack.yaml 

{-# LANGUAGE OverloadedStrings #-}
import qualified Data.ByteString.Char8     as S8
import Conduit
import Data.CSV.Conduit
import qualified Data.Conduit.Combinators  as CC
import qualified Data.Conduit.Binary       as CB
import qualified Data.Conduit.List         as CL
import qualified Data.Map                  as Map
import Data.Map (Map)
import Data.Text (Text)
import Data.Maybe (maybe)
import Data.Foldable (traverse_)
import Control.Monad (join)
import Control.Monad.IO.Class
import Control.Monad.Trans.Resource (MonadResource, MonadThrow)
import Text.Read (readMaybe)
import SyncConfig 

type SFDCField = String 
type SAPField = String 
type SAPValue = String 
type Index = String
type TableIndex = (SyncTableConfig, Index)
-- Map Index (Map SFDCField SAPValue)

syncConfig = SyncConfig sfConfig [maraTable]

sfConfig = SfdcConfig "SARS_Material_Master__c" "sfdc.csv"

maraTable  = SyncTableConfig "MARA" "MOCK_DATA.csv" ["Plant", "Material"] ["Plant__c", "Material__c"] ["WERKS"] ["WERKS__c"]
maraTable2 = SyncTableConfig "MARA" "mara2.csv" ["Plant", "Material"] ["Plant__c", "Material__c"] ["WERKS"] ["WERKS__c"]

-- (Map TableIndex (Map SFDCField SAPValue))

getDataSync :: [SyncTableConfig] -> IO [(Map Text (Map Text Text))]
getDataSync ls = runConduitRes $ execSourceTables ls 

execSourceTables :: (Monad m, MonadResource m, MonadThrow m ) => [SyncTableConfig] -> ConduitT i o m [(Map Text (Map Text Text))]
execSourceTables ls = sequenceConduits $ sourceTables ls
    
sourceTables :: (Monad m, MonadResource m, MonadThrow m ) => [SyncTableConfig] -> [ConduitT i o m (Map Text (Map Text Text))]    
sourceTables ls = fmap readTable ls

readTable :: (MonadThrow m, Monad m, MonadResource m) => SyncTableConfig -> ConduitT i o m (Map Text (Map Text Text))
readTable sfconf = sapSource fieldsMap sfconf .| CC.fold   -- I believe this part of the code is messing up everything.
    where fieldsMap = Map.fromList $ zip (sfdcObjectFields sfconf) (sapTableFields sfconf)

sapSource :: (Monad m, MonadResource m, MonadThrow m ) => Map Text Text -> SyncTableConfig -> ConduitT i (Map Text (Map Text Text)) m ()
sapSource fldsMap sfconf = sourceFile (sapTableFile sfconf) 
                         .| intoCSV (defCSVSettings {csvSep = ',', csvQuoteChar = Nothing})
                         .| buildSapIndexC (sapTableIndex sfconf)
                         .| buildSapTableC fldsMap 

buildSapIndexC :: (Monad m, MonadResource m, MonadThrow m ) => [Text] -> ConduitT (MapRow Text) (MapRow Text, Text) m ()
buildSapIndexC lindex = do
    maybeMaprow <- await
    let maybeResult = maybeMaprow >>= \maprow -> do 
            index <- foldl (\b a -> Map.lookup a maprow >>= (flip fmap b) . mappend) (Just mempty) lindex
            pure $ (maprow, index) 
    maybe (return ()) yield maybeResult
    
buildSapTableC :: (Monad m, MonadResource m, MonadThrow m ) => Map Text Text -> ConduitT (MapRow Text, Text) (Map Text (Map Text Text)) m ()
buildSapTableC fldsMap  = do 
    maybeTuple <- await
    let maybeResult = maybeTuple >>= \(maprow, index) -> do
        Map.singleton index <$> traverse (flip Map.lookup maprow) fldsMap
    maybe (return ()) yield maybeResult 

reverseMap :: Monad m => ConduitT (MapRow Text) (MapRow Text) m ()
reverseMap = CL.map $ Map.fromList . reverse . Map.toList 

main :: IO ()
main = do
    l <- getDataSync [maraTable, maraTable2]
    traverse_ (putStrLn . show) l
    -- runConduitRes $
    --     sfdcSource (sfdcConfig syncConfig) .| mapM_C (liftIO . print) 

SyncConfig.hs

{-# LANGUAGE DeriveGeneric #-}
{-# LANGUAGE LambdaCase #-}

module SyncConfig where 

import GHC.Generics (Generic)
import Data.Aeson (FromJSON(..), genericParseJSON, defaultOptions, fieldLabelModifier)
import Data.Yaml (ParseException, decodeFileEither)
import Data.Text (Text)

data SyncConfig = SyncConfig { 
      sfdcConfig      :: SfdcConfig
    , syncTableConfig :: [SyncTableConfig]
    } deriving (Generic, Show)

data SfdcConfig = SfdcConfig { 
        sfdcObject       :: Text
      , sfdcFile         :: FilePath 
      } deriving (Generic, Show)

data SyncTableConfig = SyncTableConfig { 
        sapTable         :: Text 
      , sapTableFile     :: FilePath
      , sapTableIndex    :: [Text]
      , sfdcObjectIndex  :: [Text]
      , sapTableFields   :: [Text]
      , sfdcObjectFields :: [Text]
      } deriving (Generic, Show)

instance FromJSON SyncConfig where     
    parseJSON = genericParseJSON defaultOptions { fieldLabelModifier = syncConfigFieldRename }

instance FromJSON SfdcConfig where 
    parseJSON = genericParseJSON defaultOptions { fieldLabelModifier = sfdcConfigFieldRename }

instance FromJSON SyncTableConfig where 
    parseJSON = genericParseJSON defaultOptions { fieldLabelModifier = syncTableConfigFieldRename }


readSyncConfig :: FilePath -> IO (Either ParseException SyncConfig)
readSyncConfig = decodeFileEither

syncConfigFieldRename :: String -> String 
syncConfigFieldRename = \case 
    "sfdcConfig"      -> "sfdc-config"
    "syncTableConfig" -> "sync-table-config"

sfdcConfigFieldRename :: String -> String 
sfdcConfigFieldRename = \case 
    "sfdcObject" -> "sfd  c-object"
    "sfdcFile"   -> "sfdc-file"

syncTableConfigFieldRename :: String -> String 
syncTableConfigFieldRename = \case 
    "sapTable"         -> "sap-table"
    "sapTableFile"     -> "sap-table-file"
    "sapTableIndex"    -> "sap-table-index"
    "sfdcObjectIndex"  -> "sfdc-object-index"
    "sapTableFields"   -> "sap-table-fields"
    "sfdcObjectFields" -> "sfdc-object-fields"