Skip to content

Instantly share code, notes, and snippets.

@dmateusp
Last active May 19, 2019 18:58
Show Gist options
  • Save dmateusp/aa2ebcbb973e307f77d665739e589809 to your computer and use it in GitHub Desktop.
Save dmateusp/aa2ebcbb973e307f77d665739e589809 to your computer and use it in GitHub Desktop.
Expanding on examples from https://github.com/haskell-works/avro, using Avro with Haskell
-- run: stack ghci --package avro --package text --package bytestring --ghc-options -XOverloadedStrings
-- :load AvroLearn.hs
{-# LANGUAGE OverloadedStrings #-}
import Control.Monad ((<=<))
import Data.Avro (FromAvro (..), HasAvroSchema (..),
Schema, ToAvro (..), badValue,
decodeContainer, encodeContainer, record,
(.:), (.=))
import qualified Data.Avro.Schema as S (Field (..), Order (..), Result (..),
Type (..), TypeName (..), mkEnum)
import Data.Avro.Types as T (Value (..))
import qualified Data.ByteString.Lazy as B (readFile, writeFile)
import Data.Text (Text, pack)
-- helper func
toEnumResult :: (Enum a, Bounded a) => Int -> S.Result a
toEnumResult i =
let r = toEnum i
maxEnum = maxBound `asTypeOf` r
minEnum = minBound `asTypeOf` r
in if i >= fromEnum minEnum && i <= fromEnum maxEnum
then S.Success r
else S.Error $ (show i) <> "is outside Enum range"
-- Avro namespace
wowAnalyticsAvroNamespace :: [Text]
wowAnalyticsAvroNamespace = ["io", "github", "dmateusp", "wow_analytics"]
-- Gender
data Gender
= Male
| Female
deriving (Show, Enum, Bounded)
possibleGenders :: [Gender]
possibleGenders = [Male, Female]
instance HasAvroSchema Gender where
schema = return genderSchema
instance ToAvro Gender where
toAvro = flip (T.Enum genderSchema) "Gender" . fromEnum
instance FromAvro Gender where
fromAvro (Enum _ i _) = toEnumResult i
fromAvro notEnum = badValue notEnum "Gender"
genderSchema :: Schema
genderSchema = S.mkEnum typeName aliases documentation symbols
where
typeName = S.TN "wow_gender" wowAnalyticsAvroNamespace
aliases = [S.TN "gender" wowAnalyticsAvroNamespace]
documentation = Just "Genders that a World of Warcraft character can have"
symbols = pack . show <$> possibleGenders
-- WowCharacter
data WowCharacter = WowCharacter
{ name :: Text
, level :: Int
, gender :: Gender
} deriving (Show)
instance HasAvroSchema WowCharacter where
schema = return wowCharacterSchema
instance ToAvro WowCharacter where
toAvro w = record wowCharacterSchema ["name" .= name w, "level" .= level w, "gender" .= gender w]
instance FromAvro WowCharacter
-- Record has a Schema (that we ignore here) and a Hashmap of (field .: values) (r)
where
fromAvro (Record _ r) = WowCharacter <$> r .: "name" <*> r .: "level" <*> r .: "gender"
fromAvro r' = badValue r' "WowCharacter"
wowCharacterSchema :: Schema
wowCharacterSchema =
S.Record
{ S.name = S.TN "wow_character" wowAnalyticsAvroNamespace
, S.aliases = [S.TN "character" wowAnalyticsAvroNamespace]
, S.doc = Just "A Word of Warcraft character"
, S.order = Just S.Ignore
, S.fields =
[ S.Field
{ S.fldName = "name"
, S.fldAliases = []
, S.fldDoc = Just "Name of the character"
, S.fldOrder = Nothing
, S.fldType = S.String
, S.fldDefault = Nothing
}
, S.Field
{ S.fldName = "level"
, S.fldAliases = ["lvl"]
, S.fldDoc = Just "Level of the character"
, S.fldOrder = Nothing
, S.fldType = S.Int
, S.fldDefault = Just (T.Int 1)
}
, S.Field
{ S.fldName = "gender"
, S.fldAliases = []
, S.fldDoc = Just "Gender of the character"
, S.fldOrder = Nothing
, S.fldType = genderSchema
, S.fldDefault = Nothing
}
]
}
-- note that we use encodeContainer and not encode
-- encodeContainer is for files (ContainerFiles in Avro), and
-- encode is for messages, see https://github.com/haskell-works/avro/issues/106
writeExamples :: IO ()
writeExamples =
B.writeFile "wow_characters.avro" <=< encodeContainer $
[[WowCharacter "Ironsword" 1 Male, WowCharacter "Uncons" 70 Female]]
-- in ghci: writeExamples
-- file can be read with: java -jar /path/to/avro-tools-1.8.2.jar tojson --pretty /path/to/wow_characters.avro
{-{
"name" : "Ironsword",
"level" : 1,
"gender" : "Male"
}
{
"name" : "Uncons",
"level" : 70,
"gender" : "Female"
}-}
roundTrip :: IO ([[WowCharacter]])
roundTrip = writeExamples >> decodeContainer <$> (B.readFile "wow_characters.avro")
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment