xentropy.hs

#!/usr/bin/env stack
-- stack --install-ghc runghc
import qualified Data.Map as M

type Distribution = M.Map String Float

-- letter frequencies from https://en.wikipedia.org/wiki/Letter_frequency
dist :: Distribution
dist = M.fromList [("a", 0.08167), ("b", 0.01492), ("c", 0.02782), ("d", 0.04253), ("e", 0.12702), ("f",0.02228),
                   ("g", 0.02015), ("h",0.06094), ("i",0.06966), ("j",0.00153), ("k",0.00772), ("l",0.04025),
                   ("m",0.02406), ("n",0.06749), ("o",0.07507), ("p",0.01929), ("q",0.00095), ("r",0.05987),
                   ("s",0.06327), ("t",0.09056), ("u",0.02758), ("v",0.00978), ("w",0.02360), ("x",0.00150), ("y",0.01974), ("z",0.00074) ]

logmo :: (Float,Float) -> Float
logmo (m,o) = -1 * m * log o

-- given two probabilty distributions, calculate the cross entropy.
xentropy :: Distribution -> Distribution -> Float
xentropy model observed = let m_prob = map snd (M.toList model)
                              o_prob = map snd (M.toList observed)
                              mo = zip m_prob o_prob
                              logloss = sum $ map logmo mo
                           in logloss

english_xentropt = xentropy dist dist
main :: IO ()
main = print english_xentropt

--Claude Shannon calculated 2.62 bits / letter http://people.seas.harvard.edu/~jones/cscie129/papers/stanford_info_paper/entropy_of_english_9.htm
--we get 2.89