Sparkimalz

sparkimalz.py

from pyspark.sql import Row

spark.conf.set('spark.sql.parquet.writeLegacyFormat', 'false')
spark.conf.set('spark.sql.parquet.compression.codec', 'uncompressed')

sc = spark.sparkContext
df = spark.createDataFrame(
    sc.parallelize(range(1, 100)
).map(lambda i: Row(value=i)))

df.select(df.value.cast('decimal(4, 2)')).write.parquet('/mounted/int32_decimal.parquet')
df.select(df.value.cast('decimal(10, 2)')).write.parquet('/mounted/int64_decimal.parquet')
df.select(df.value.cast('decimal(25, 2)')).write.parquet('/mounted/fixed_length_decimal.parquet')

spark.conf.set('spark.sql.parquet.writeLegacyFormat', 'true')
df.select(df.value.cast('decimal(13, 2)')).write.parquet('/mounted/fixed_length_decimal_legacy.parquet')