DANGerous tommydangerous

🧙

Fire!

Making data more accessible.

tommydangerous / mage_delta-lake_read_battle-history_partitioned.py

Last active May 11, 2023 18:34

	from deltalake import DeltaTable


	dt = DeltaTable(
	# Change this to your unique URI from a previous step
	# if you’re using your own AWS credentials.
	's3://mage-demo-public/magic-energy-and-battle-history/1337',
	storage_options={
	'AWS_ACCESS_KEY_ID': '...',
	'AWS_SECRET_ACCESS_KEY': '...',

tommydangerous / mage_delta-lake_read_battle-history_versioned.py

Created May 11, 2023 18:11

	from deltalake import DeltaTable


	dt = DeltaTable(
	# Change this to your unique URI from a previous step
	# if you’re using your own AWS credentials.
	's3://mage-demo-public/battle-history-versioned/1337',
	storage_options={
	'AWS_ACCESS_KEY_ID': '...',
	'AWS_SECRET_ACCESS_KEY': '...',

tommydangerous / mage_delta-lake_upload_multiple_battle-history.py

Last active May 11, 2023 18:01

	from deltalake.writer import write_deltalake


	# ['Aiur', 'Eos', 'Gaia', 'Kamigawa', 'Korhal', 'Ravnica']
	planets = list(sorted(set(df['planet'].values)))

	# Loop through each planet
	for planet in planets:
	# Select a subset of the battle history data for a single planet
	planet_df = df.query(f"`planet` == '{planet}'")

tommydangerous / mage_delta-lake_export_data_into_avatar.py

Last active May 11, 2023 18:18

	from deltalake.writer import write_deltalake


	@data_exporter
	def export_data(combined_data, args, *kwargs):
	write_deltalake(
	# Change this URI to your own unique URI
	's3://mage-demo-public/magic-energy-and-battle-history/1337',
	data=combined_data,
	mode='overwrite',

tommydangerous / mage_delta-lake_combine_magic_energy_and_battle_history.py

Last active May 9, 2023 23:46

	from deltalake import DeltaTable
	import pandas as pd


	@transformer
	def transform(magic_energy, args, *kwargs):
	dt = DeltaTable(
	# Change this to your unique URI from a previous step
	# if you’re using your own AWS credentials.
	's3://mage-demo-public/battle-history/1337',

tommydangerous / mage_delta-lake_load_magic_energy.py

Last active May 9, 2023 23:30

	from deltalake import DeltaTable


	@data_loader
	def load_data(args, *kwargs):
	dt = DeltaTable(
	's3://mage-demo-public/magic-energy/1337',
	storage_options={
	'AWS_ACCESS_KEY_ID': '...',
	'AWS_SECRET_ACCESS_KEY': '...',

tommydangerous / mage_delta-lake_read_battle-history.py

Last active May 9, 2023 23:42

	from deltalake import DeltaTable


	dt = DeltaTable(
	# Change this to your unique URI from a previous step
	# if you’re using your own AWS credentials.
	's3://mage-demo-public/battle-history/1337',
	storage_options={
	'AWS_ACCESS_KEY_ID': '...',
	'AWS_SECRET_ACCESS_KEY': '...',

tommydangerous / mage_delta-lake_upload_battle-history.py

Last active May 11, 2023 17:49

	from deltalake.writer import write_deltalake


	write_deltalake(
	# Change this URI to your own unique URI
	's3://mage-demo-public/battle-history/1337',
	data=df,
	mode='overwrite',
	overwrite_schema=True,
	storage_options={

tommydangerous / mage_delta-lake_download_battle-history.py

Last active May 9, 2023 22:52

	import pandas as pd


	df = pd.read_csv(
	'https://raw.githubusercontent.com/mage-ai/datasets/master/battle_history.csv',
	)

tommydangerous / create_matrix.py

Created June 17, 2021 05:36

create_matrix.py