abc.py

# Works

class Foo:
  def __init__(self, sc, df):
    self.sc = sc # a spark context
    self.df = df # a spark dataframe
    
    f = udf(lambda x: Foo.do_something(x), ArrayType(StringType()))
    
    self.new_df = self.df.withColum('bar', f(self.df.foo))
    
  @staticmethod
  def do_something(data):
    return data.split('.')
  
  
  # Doesn't work
  
  from abc import ABCMeta, abstractmethod
  
  class Abstract:
    __metaclass__ = ABCMeta
    def __init__(self, sc, df):
      self.sc = sc
      self.df = df
    
    @abstractmethod
    def process(self, data):
      pass
    
    
  class Foo(Abstract):
    def __init__(self, sc, df):
      Abstract.__init__(self, sc, df)
      
      self.new_df = self.process(self.df)
      
    def process(self, data):
      f = udf(lambda x: Foo.do_something(x), ArrayType(StringType()))
    
      return data.withColum('bar', f(data.foo))
      
      
    @staticmethod
    def do_something(data):
      return data.split('.')