yassineAlouini

import pandas as pd

def compare_two_dfs(input_df_1, input_df_2):
    df_1, df_2 = input_df_1.copy(), input_df_2.copy()
    ne_stacked = (df_1 != df_2).stack()
    changed = ne_stacked[ne_stacked]
    changed.index.names = ['id', 'col']
    difference_locations = np.where(df_1 != df_2)

    changed_from = df_1.values[difference_locations]

yassineAlouini

import s3fs
import pandas as pd

def get_csv_from_s3(folder_path):
    dfs = []
    s3 = s3fs.S3FileSystem()
    for fp in s3.ls(folder_path):
        if '.csv' in fp:
            with s3.open(fp) as s3f:
                with bz2file.open(s3f) as f:

yassineAlouini

import geopandas as gpd
import pandas as pd

def multipolygon_to_polygon(input_gdf):
    """Break each MultiPolygon to its constituting Polygons, create a new polygons DataFrame and then
       append it to the initial GeoDataFrame
    """
    gdf = input_gdf.copy()
    multipolygon_gdf = gdf.loc[lambda df: df.geometry.geom_type == 'MultiPolygon', :]
    for row_index, row in multipolygon_gdf.iterrows():

yassineAlouini

import bz2
import glob


for fp in glob.iglob('path/to_files'):
    with open(fp, 'rb') as f_input:
        with bz2.BZ2File(fp + '.bz2', 'wb', compresslevel=9) as f_output:
            f_output.write(f_input.read())

yassineAlouini

import s3fs

s3 = s3fs.S3FileSystem()
S3_BUCKET = ''
BASE_KEY = ''

def get_data_from_s3(file_name):
    key = BASE_KEY.format('data', file_name)
    s3.get('s3://{}/{}'.format(S3_BUCKET, key), '/tmp/{}'.format(file_name))
    dfs = []

yassineAlouini

# You need to install xlsxwriter: pip install xlsxwriter
import pandas as pd
import StringIO


def save_df_to_xls(input_df, fp, sheet_name='report'):
    df = input_df.copy()
    writer = pd.ExcelWriter(fp, engine='xlsxwriter', 
                            datetime_format='d/mm/yyyy') # Datatime format for french users
    df.to_excel(writer, index=False, sheet_name=sheet_name, encoding='utf-8) # Encoding for french users

yassineAlouini

import pandas as pd
import geopandas as gpd
from shapely.geometry import Point
import json


def df_to_gdf(input_df):
    """
    Convert a DataFrame with longitude and latitude columns
    to a GeoDataFrame.

yassineAlouini

Keybase proof

I hereby claim:

• I am yassineAlouini on github.
• I am yassinealouini (https://keybase.io/yassinealouini) on keybase.
• I have a public key whose fingerprint is 638C BEFB DFA5 9085 67F1 7C16 BCA5 905F 4A55 3726

To claim this, I am signing this object:

yassineAlouini

license: mit

yassineAlouini

Built with blockbuilder.org