Arturo Gonzalez al102964

Data scientist, looking forward to make the world a better place through data.

al102964 / main.yml

Created May 9, 2021 23:01

	name: CI

	# Controls when the action will run.
	on:
	# Triggers the workflow on push or pull request events but only for the master branch
	push:
	branches: [ master ]
	pull_request:
	branches: [ master ]

al102964 / main.yml

Created May 9, 2021 23:01

	name: CI

	# Controls when the action will run.
	on:
	# Triggers the workflow on push or pull request events but only for the master branch
	push:
	branches: [ master ]
	pull_request:
	branches: [ master ]

al102964 / main.yml

Created May 9, 2021 23:00

al102964 / SparkCopyPostgres.scala

Created April 6, 2021 04:58 — forked from longcao/SparkCopyPostgres.scala

COPY Spark DataFrame rows to PostgreSQL (via JDBC)

	import java.io.InputStream

	import org.apache.spark.sql.execution.datasources.jdbc.JdbcUtils
	import org.apache.spark.sql.{ DataFrame, Row }

	import org.postgresql.copy.CopyManager
	import org.postgresql.core.BaseConnection

	val jdbcUrl = s"jdbc:postgresql://..." // db credentials elided
	val connectionProperties = {

al102964 / riiaa-manifest.json

Created August 25, 2020 22:52

	{
	"fileLocations":[
	{"URIPrefixes":["s3://bucket-name/archivo.csv"]}
	],
	"globalUploadSettings":{
	"format": "CSV",
	"delimiter":",",
	"containsHeader":true
	}
	}

al102964 / riiaa-databricks-consumo.py

Last active August 25, 2020 22:48

	from pyspark.sql import Window
	import pyspark.sql.functions as f

	w = Window.orderBy("start")

	consumo = df.select("central","id_dispositivo","valor","timestamp","tarifa")\
	.filter("id_dispositivo = 'eta'")\
	.groupBy(f.window("timestamp", "60 minutes").alias("intervalo"),"tarifa")\
	.agg(f.max("valor").alias("consumo_max_hora"))\
	.select("intervalo.start","tarifa","consumo_max_hora")\

al102964 / riiaa-databricks-show-data.py

Created August 25, 2020 19:21

	from pyspark.sql.types import *

	schema = StructType([
	StructField("central", StringType()),
	StructField("gateway", StringType()),
	StructField("board", StringType()),
	StructField("id_dispositivo", StringType()),
	StructField("valor", FloatType()),
	StructField("timestamp", TimestampType()),
	StructField("tarifa", StringType()),

al102964 / riiaa-databricks-show-s3-data.py

Created August 25, 2020 19:08

display(dbutils.fs.ls('/mnt/s3data/data/'))

al102964 / riiaa-databricks-mount-s3.py

Last active August 25, 2020 19:07

	import urllib

	#Llaves de acceso a bucket de trabajo
	ACCESS_KEY = "LLAVE"
	SECRET_KEY = "SECRET"
	AWS_BUCKET_NAME = "al102964-iot-data"

	ENCODED_SECRET_KEY = urllib.parse.quote(SECRET_KEY,"")

	MOUNT_NAME = "s3data"

al102964 / pps_mean.py

Created June 19, 2020 09:35

	print("Mean predictive power score value: {}".format(pps.matrix(df).abs().unstack().sort_values(ascending=False).drop_duplicates().mean()))
	print("Mean correlation value: {}".format(df.corr().abs().unstack().sort_values(ascending=False).drop_duplicates().mean()))