This is a collection of the things I believe about software development. I have worked for years building backend and data processing systems, so read the below within that context.
Agree? Disagree? Feel free to let me know at @JanStette. See also my blog at www.janvsmachine.net.
Keep it simple, stupid. You ain't gonna need it.
| package backfiller | |
| import aws.AWS | |
| import cats.effect.{ExitCode, IO, IOApp} | |
| import com.amazonaws.AmazonServiceException | |
| import com.amazonaws.retry.RetryUtils | |
| import com.gu.emr.ClusterManager.ClusterID | |
| import com.gu.emr.EmrClusterManager | |
| import com.gu.emr.model.{ClusterDefinition, EmrStep, RunConfiguration} | |
| import fs2.{Chunk, Pure, Stream} |
| import spark.implicits._ | |
| // Input data | |
| val identity = | |
| spark | |
| .table("clean.identity") | |
| .select("id", "account_created", "last_active_date", "primary_email_address") | |
| .cache() |
| def eval(expr: Expr, env: Env): Expr = expr match { | |
| // ... | |
| case "cons" :: arg1 :: arg2 :: Nil => eval(arg2, env) match { | |
| case l: List[Expr] => eval(arg1, env) :: l | |
| case arg => throw new IllegalArgumentException(s"Second argument to 'cons' should be list, was: '$arg'") | |
| } | |
| case "cons" :: args => throw new IllegalArgumentException(s"Invalid arguments to 'cons': $args") | |
| case "null?" :: head :: Nil => eval(head, env) == List() | |
| case "null?" :: _ => throw new IllegalArgumentException("Expected exactly one argument for 'null?'") | |
| // ... |