Skip to content

Instantly share code, notes, and snippets.

@julian-west
julian-west / great_expectations.yml
Created December 3, 2021 21:13
Great Expectations yaml configuration file for remote stores on Google Cloud Storage
config_version: 3
datasources:
my_gcs_datasource:
class_name: Datasource
module_name: "great_expectations.datasource"
execution_engine:
class_name: PandasExecutionEngine
data_connectors:
default_runtime_data_connector_name:
@julian-west
julian-west / main.py
Last active January 3, 2024 19:44
Cloud Function using Great Expectations for data validation
"""Great Expectations Checkpoint"""
import logging
import os
from typing import Any, Dict
from great_expectations.checkpoint import SimpleCheckpoint
from great_expectations.checkpoint.types.checkpoint_result import CheckpointResult
from great_expectations.core.batch import RuntimeBatchRequest
from great_expectations.data_context import BaseDataContext
from great_expectations.data_context.types.base import DataContextConfig
@julian-west
julian-west / .pre-commit-config.yaml
Last active November 30, 2023 04:43
Example .pre-commit-config.yaml file
repos:
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v3.2.0
hooks:
- id: trailing-whitespace
- id: mixed-line-ending
- id: check-added-large-files
args: ['--maxkb=1000']
- id: end-of-file-fixer
- id: requirements-txt-fixer
# ---------------------------------------
# Plot graph
# ---------------------------------------
# edges
tracer = go.Scatter(x=Xedges, y=Yedges,
mode='lines',
line= dict(color='#DCDCDC', width=1),
hoverinfo='none',
showlegend=False)
# ---------------------------------------
# Get statistics for tooltip
# ---------------------------------------
# make list of node labels.
node_label = list(mst.nodes())
# calculate annualised returns, annualised volatility and round to 2dp
annual_vol, annual_ret, annual_vol_2dp, annual_ret_2dp = calculate_stats()
# get top and bottom 3 correlations for each node
top_3_corrs, bottom_3_corrs = get_top_and_bottom_three()
def get_coordinates(G=mst):
"""Returns the positions of nodes and edges in a format for Plotly to draw the network"""
# get list of node positions
pos = nx.fruchterman_reingold_layout(mst)
Xnodes = [pos[n][0] for n in mst.nodes()]
Ynodes = [pos[n][1] for n in mst.nodes()]
Xedges = []
Yedges = []
def convert_rankings_to_string(ranking):
"""
Concatenates list of node and correlation into a single string which is the
preferred format for the plotly tooltip.
Inserts html "<br>" inbetween each item in order to add a new line in the tooltip
"""
s = ''
for r in ranking:
s += r + "<br>"
#create minimum spanning tree layout from Gx (after small correlations have been removed)
mst = nx.minimum_spanning_tree(Gx)
edge_colours = []
#assign edge colours
for key, value in nx.get_edge_attributes(mst, 'correlation').items():
edge_colours.append(assign_colour(value))
# draw improved graph
nx.draw(Gx, pos=nx.fruchterman_reingold_layout(Gx), with_labels=True,
node_size=node_size, node_color="#e1575c", edge_color=edge_colours,
width = edge_width)
plt.title("Asset price correlations - Fruchterman-Reingold layout",fontdict=font_dict)
plt.show()
# draw improved graph
sns.set(rc={'figure.figsize': (9, 9)})
font_dict = {'fontsize': 18}
nx.draw(Gx, pos=nx.circular_layout(Gx), with_labels=True,
node_size=node_size, node_color="#e1575c", edge_color=edge_colours,
width=edge_width)
plt.title("Asset price correlations", fontdict=font_dict)
plt.show()