functor1.fs

// Functor instances. Define Functor as a single case discriminated union
// to help with overload resolution.
type Functor = Functor with
    static member fmap(Functor, mapping, option) = Option.map mapping option
    static member fmap(Functor, mapping, list) = List.map mapping list

// Helper function to resolve Functor overload. Needed because we can't specify a concrete type
// such as Functor in a statically-resolved type parameter constraint, so we instead pass in an
// instance of Functor to resolve the ^o parameter. Also notice we have ^c and ^d instead of
// ^f< ^a > and ^f < ^b > because F# doesn't allow this.
let inline fmapHelper overloads mapping instance = 
    ((^o or ^c): (static member fmap : ^o * (^a -> ^b) * ^c -> ^d) (overloads, mapping, instance))
 
// Finally, define fmap
let inline fmap mapping instance = fmapHelper Functor mapping instance

// It works!
fmap ((*)2) [1;2;3]
fmap ((*)2) (Some 3)

// FsControl library: https://github.com/gmpl/FsControl

// In fairness, it's far easier to just define an interface if you control the type.
// The advantage of this is that you can add "instances" to types after the fact.

functor2.fs

// Here's another version that uses an operator overloading trick to avoid the 
// helper and specifying the constraints explicitly.
type Functor = Functor with
    static member ($) (Functor, instance) = fun mapping -> Option.map mapping instance
    static member ($) (Functor, instance) = fun mapping -> List.map mapping instance
    
let inline fmap mapping instance  = (Functor $ instance) mapping

fmap ((*)2) [1;2;3]
fmap ((*)2) (Some 3)