Last active
May 19, 2019 18:58
-
-
Save dmateusp/aa2ebcbb973e307f77d665739e589809 to your computer and use it in GitHub Desktop.
Expanding on examples from https://github.com/haskell-works/avro, using Avro with Haskell
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
-- run: stack ghci --package avro --package text --package bytestring --ghc-options -XOverloadedStrings | |
-- :load AvroLearn.hs | |
{-# LANGUAGE OverloadedStrings #-} | |
import Control.Monad ((<=<)) | |
import Data.Avro (FromAvro (..), HasAvroSchema (..), | |
Schema, ToAvro (..), badValue, | |
decodeContainer, encodeContainer, record, | |
(.:), (.=)) | |
import qualified Data.Avro.Schema as S (Field (..), Order (..), Result (..), | |
Type (..), TypeName (..), mkEnum) | |
import Data.Avro.Types as T (Value (..)) | |
import qualified Data.ByteString.Lazy as B (readFile, writeFile) | |
import Data.Text (Text, pack) | |
-- helper func | |
toEnumResult :: (Enum a, Bounded a) => Int -> S.Result a | |
toEnumResult i = | |
let r = toEnum i | |
maxEnum = maxBound `asTypeOf` r | |
minEnum = minBound `asTypeOf` r | |
in if i >= fromEnum minEnum && i <= fromEnum maxEnum | |
then S.Success r | |
else S.Error $ (show i) <> "is outside Enum range" | |
-- Avro namespace | |
wowAnalyticsAvroNamespace :: [Text] | |
wowAnalyticsAvroNamespace = ["io", "github", "dmateusp", "wow_analytics"] | |
-- Gender | |
data Gender | |
= Male | |
| Female | |
deriving (Show, Enum, Bounded) | |
possibleGenders :: [Gender] | |
possibleGenders = [Male, Female] | |
instance HasAvroSchema Gender where | |
schema = return genderSchema | |
instance ToAvro Gender where | |
toAvro = flip (T.Enum genderSchema) "Gender" . fromEnum | |
instance FromAvro Gender where | |
fromAvro (Enum _ i _) = toEnumResult i | |
fromAvro notEnum = badValue notEnum "Gender" | |
genderSchema :: Schema | |
genderSchema = S.mkEnum typeName aliases documentation symbols | |
where | |
typeName = S.TN "wow_gender" wowAnalyticsAvroNamespace | |
aliases = [S.TN "gender" wowAnalyticsAvroNamespace] | |
documentation = Just "Genders that a World of Warcraft character can have" | |
symbols = pack . show <$> possibleGenders | |
-- WowCharacter | |
data WowCharacter = WowCharacter | |
{ name :: Text | |
, level :: Int | |
, gender :: Gender | |
} deriving (Show) | |
instance HasAvroSchema WowCharacter where | |
schema = return wowCharacterSchema | |
instance ToAvro WowCharacter where | |
toAvro w = record wowCharacterSchema ["name" .= name w, "level" .= level w, "gender" .= gender w] | |
instance FromAvro WowCharacter | |
-- Record has a Schema (that we ignore here) and a Hashmap of (field .: values) (r) | |
where | |
fromAvro (Record _ r) = WowCharacter <$> r .: "name" <*> r .: "level" <*> r .: "gender" | |
fromAvro r' = badValue r' "WowCharacter" | |
wowCharacterSchema :: Schema | |
wowCharacterSchema = | |
S.Record | |
{ S.name = S.TN "wow_character" wowAnalyticsAvroNamespace | |
, S.aliases = [S.TN "character" wowAnalyticsAvroNamespace] | |
, S.doc = Just "A Word of Warcraft character" | |
, S.order = Just S.Ignore | |
, S.fields = | |
[ S.Field | |
{ S.fldName = "name" | |
, S.fldAliases = [] | |
, S.fldDoc = Just "Name of the character" | |
, S.fldOrder = Nothing | |
, S.fldType = S.String | |
, S.fldDefault = Nothing | |
} | |
, S.Field | |
{ S.fldName = "level" | |
, S.fldAliases = ["lvl"] | |
, S.fldDoc = Just "Level of the character" | |
, S.fldOrder = Nothing | |
, S.fldType = S.Int | |
, S.fldDefault = Just (T.Int 1) | |
} | |
, S.Field | |
{ S.fldName = "gender" | |
, S.fldAliases = [] | |
, S.fldDoc = Just "Gender of the character" | |
, S.fldOrder = Nothing | |
, S.fldType = genderSchema | |
, S.fldDefault = Nothing | |
} | |
] | |
} | |
-- note that we use encodeContainer and not encode | |
-- encodeContainer is for files (ContainerFiles in Avro), and | |
-- encode is for messages, see https://github.com/haskell-works/avro/issues/106 | |
writeExamples :: IO () | |
writeExamples = | |
B.writeFile "wow_characters.avro" <=< encodeContainer $ | |
[[WowCharacter "Ironsword" 1 Male, WowCharacter "Uncons" 70 Female]] | |
-- in ghci: writeExamples | |
-- file can be read with: java -jar /path/to/avro-tools-1.8.2.jar tojson --pretty /path/to/wow_characters.avro | |
{-{ | |
"name" : "Ironsword", | |
"level" : 1, | |
"gender" : "Male" | |
} | |
{ | |
"name" : "Uncons", | |
"level" : 70, | |
"gender" : "Female" | |
}-} | |
roundTrip :: IO ([[WowCharacter]]) | |
roundTrip = writeExamples >> decodeContainer <$> (B.readFile "wow_characters.avro") |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment