Dipanjan (DJ) Sarkar dipanjanS

Data Science Lead, Google Dev Expert - ML, Author

dipanjanS / spark_log_analytics_44.py

Created April 10, 2019 15:31

	c = sns.catplot(x='day', y='count',
	data=errors_by_date_sorted_pd_df,
	kind='point', height=5, aspect=1.5)

dipanjanS / spark_log_analytics_43.py

Created April 10, 2019 15:30

	errors_by_date_sorted_df = (not_found_df
	.groupBy(F.dayofmonth('time').alias('day'))
	.count()
	.sort("day"))

	errors_by_date_sorted_pd_df = errors_by_date_sorted_df.toPandas()
	errors_by_date_sorted_pd_df

dipanjanS / spark_log_analytics_42.py

Created April 10, 2019 15:27

	hosts_404_count_df = (not_found_df
	.groupBy("host")
	.count()
	.sort("count", ascending=False)
	.limit(20))

	hosts_404_count_df.show(truncate=False)

dipanjanS / spark_log_analytics_41.py

Last active April 10, 2019 15:26

	endpoints_404_count_df = (not_found_df
	.groupBy("endpoint")
	.count()
	.sort("count", ascending=False)
	.limit(20))

	endpoints_404_count_df.show(truncate=False)

dipanjanS / spark_log_analytics_40.py

Created April 9, 2019 18:24

	not_found_df = logs_df.filter(logs_df["status"] == 404).cache()
	print(('Total 404 responses: {}').format(not_found_df.count()))

dipanjanS / spark_log_analytics_39.py

Created April 9, 2019 18:22

	c = sns.catplot(x='day', y='avg_reqs',
	data=avg_daily_reqests_per_host_df,
	kind='point', height=5, aspect=1.5)

dipanjanS / spark_log_analytics_38.py

Created April 9, 2019 18:20

	daily_hosts_df = (host_day_distinct_df
	.groupBy('day')
	.count()
	.select(col("day"),
	col("count").alias("total_hosts")))

	total_daily_reqests_df = (logs_df
	.select(F.dayofmonth("time")
	.alias("day"))
	.groupBy("day")

dipanjanS / spark_log_analytics_37.py

Created April 9, 2019 18:08

dipanjanS / spark_log_analytics_36.py

Created April 9, 2019 18:06

	def_mr = pd.get_option('max_rows')
	pd.set_option('max_rows', 10)

	daily_hosts_df = (host_day_distinct_df
	.groupBy('day')
	.count()
	.sort("day"))

	daily_hosts_df = daily_hosts_df.toPandas()
	daily_hosts_df

dipanjanS / spark_log_analytics_35.py

Created April 9, 2019 18:03

	host_day_distinct_df = (host_day_df
	.dropDuplicates())
	host_day_distinct_df.show(5, truncate=False)