Last active
December 15, 2021 18:51
-
-
Save rg3915/5fb3a2e7338115bc92e82b7a9a2b372b to your computer and use it in GitHub Desktop.
Annotations of Pandas DataFrame
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| { | |
| "cells": [ | |
| { | |
| "cell_type": "code", | |
| "execution_count": 1, | |
| "metadata": {}, | |
| "outputs": [], | |
| "source": [ | |
| "import pandas as pd\n", | |
| "filename = '/home/dev/fix/348670_ion_annotations.tsv'" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 2, | |
| "metadata": {}, | |
| "outputs": [], | |
| "source": [ | |
| "df = pd.read_csv(filename, sep='\\t')" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 3, | |
| "metadata": {}, | |
| "outputs": [ | |
| { | |
| "data": { | |
| "text/html": [ | |
| "<div>\n", | |
| "<style>\n", | |
| " .dataframe thead tr:only-child th {\n", | |
| " text-align: right;\n", | |
| " }\n", | |
| "\n", | |
| " .dataframe thead th {\n", | |
| " text-align: left;\n", | |
| " }\n", | |
| "\n", | |
| " .dataframe tbody tr th {\n", | |
| " vertical-align: top;\n", | |
| " }\n", | |
| "</style>\n", | |
| "<table border=\"1\" class=\"dataframe\">\n", | |
| " <thead>\n", | |
| " <tr style=\"text-align: right;\">\n", | |
| " <th></th>\n", | |
| " <th>chrom</th>\n", | |
| " <th>pos</th>\n", | |
| " <th>_id</th>\n", | |
| " <th>ref</th>\n", | |
| " <th>alt</th>\n", | |
| " <th>qual</th>\n", | |
| " <th>filter</th>\n", | |
| " <th>alt_pos_strand</th>\n", | |
| " <th>dbsnp_bin</th>\n", | |
| " <th>dbsnp_chrom</th>\n", | |
| " <th>...</th>\n", | |
| " <th>gnomad_an_popmax</th>\n", | |
| " <th>gnomad_af_popmax</th>\n", | |
| " <th>gnomad_dp_median</th>\n", | |
| " <th>gnomad_dref_median</th>\n", | |
| " <th>gnomad_gq_median</th>\n", | |
| " <th>gnomad_ab_median</th>\n", | |
| " <th>gnomad_as_rf</th>\n", | |
| " <th>gnomad_as_filterstatus</th>\n", | |
| " <th>fdvr</th>\n", | |
| " <th>hs_only</th>\n", | |
| " </tr>\n", | |
| " </thead>\n", | |
| " <tbody>\n", | |
| " <tr>\n", | |
| " <th>0</th>\n", | |
| " <td>4</td>\n", | |
| " <td>0</td>\n", | |
| " <td>19</td>\n", | |
| " <td>1</td>\n", | |
| " <td>1</td>\n", | |
| " <td>0</td>\n", | |
| " <td>4</td>\n", | |
| " <td>1</td>\n", | |
| " <td>0</td>\n", | |
| " <td>4</td>\n", | |
| " <td>...</td>\n", | |
| " <td>0</td>\n", | |
| " <td>0</td>\n", | |
| " <td>0</td>\n", | |
| " <td>0</td>\n", | |
| " <td>0</td>\n", | |
| " <td>0</td>\n", | |
| " <td>0</td>\n", | |
| " <td>0</td>\n", | |
| " <td>0</td>\n", | |
| " <td>0</td>\n", | |
| " </tr>\n", | |
| " <tr>\n", | |
| " <th>1</th>\n", | |
| " <td>5</td>\n", | |
| " <td>0</td>\n", | |
| " <td>19</td>\n", | |
| " <td>1</td>\n", | |
| " <td>1</td>\n", | |
| " <td>0</td>\n", | |
| " <td>4</td>\n", | |
| " <td>1</td>\n", | |
| " <td>0</td>\n", | |
| " <td>5</td>\n", | |
| " <td>...</td>\n", | |
| " <td>0</td>\n", | |
| " <td>0</td>\n", | |
| " <td>0</td>\n", | |
| " <td>0</td>\n", | |
| " <td>0</td>\n", | |
| " <td>0</td>\n", | |
| " <td>0</td>\n", | |
| " <td>0</td>\n", | |
| " <td>0</td>\n", | |
| " <td>0</td>\n", | |
| " </tr>\n", | |
| " <tr>\n", | |
| " <th>2</th>\n", | |
| " <td>5</td>\n", | |
| " <td>0</td>\n", | |
| " <td>18</td>\n", | |
| " <td>1</td>\n", | |
| " <td>1</td>\n", | |
| " <td>0</td>\n", | |
| " <td>4</td>\n", | |
| " <td>1</td>\n", | |
| " <td>0</td>\n", | |
| " <td>5</td>\n", | |
| " <td>...</td>\n", | |
| " <td>0</td>\n", | |
| " <td>0</td>\n", | |
| " <td>0</td>\n", | |
| " <td>0</td>\n", | |
| " <td>0</td>\n", | |
| " <td>0</td>\n", | |
| " <td>0</td>\n", | |
| " <td>0</td>\n", | |
| " <td>0</td>\n", | |
| " <td>0</td>\n", | |
| " </tr>\n", | |
| " <tr>\n", | |
| " <th>3</th>\n", | |
| " <td>5</td>\n", | |
| " <td>0</td>\n", | |
| " <td>19</td>\n", | |
| " <td>1</td>\n", | |
| " <td>1</td>\n", | |
| " <td>0</td>\n", | |
| " <td>4</td>\n", | |
| " <td>1</td>\n", | |
| " <td>0</td>\n", | |
| " <td>5</td>\n", | |
| " <td>...</td>\n", | |
| " <td>0</td>\n", | |
| " <td>0</td>\n", | |
| " <td>0</td>\n", | |
| " <td>0</td>\n", | |
| " <td>0</td>\n", | |
| " <td>0</td>\n", | |
| " <td>0</td>\n", | |
| " <td>0</td>\n", | |
| " <td>0</td>\n", | |
| " <td>0</td>\n", | |
| " </tr>\n", | |
| " <tr>\n", | |
| " <th>4</th>\n", | |
| " <td>5</td>\n", | |
| " <td>0</td>\n", | |
| " <td>19</td>\n", | |
| " <td>1</td>\n", | |
| " <td>1</td>\n", | |
| " <td>0</td>\n", | |
| " <td>4</td>\n", | |
| " <td>1</td>\n", | |
| " <td>0</td>\n", | |
| " <td>5</td>\n", | |
| " <td>...</td>\n", | |
| " <td>5</td>\n", | |
| " <td>11</td>\n", | |
| " <td>2</td>\n", | |
| " <td>11</td>\n", | |
| " <td>2</td>\n", | |
| " <td>11</td>\n", | |
| " <td>11</td>\n", | |
| " <td>4</td>\n", | |
| " <td>0</td>\n", | |
| " <td>0</td>\n", | |
| " </tr>\n", | |
| " <tr>\n", | |
| " <th>5</th>\n", | |
| " <td>5</td>\n", | |
| " <td>0</td>\n", | |
| " <td>19</td>\n", | |
| " <td>1</td>\n", | |
| " <td>1</td>\n", | |
| " <td>0</td>\n", | |
| " <td>4</td>\n", | |
| " <td>1</td>\n", | |
| " <td>0</td>\n", | |
| " <td>5</td>\n", | |
| " <td>...</td>\n", | |
| " <td>11</td>\n", | |
| " <td>23</td>\n", | |
| " <td>6</td>\n", | |
| " <td>25</td>\n", | |
| " <td>5</td>\n", | |
| " <td>23</td>\n", | |
| " <td>23</td>\n", | |
| " <td>9</td>\n", | |
| " <td>0</td>\n", | |
| " <td>0</td>\n", | |
| " </tr>\n", | |
| " <tr>\n", | |
| " <th>6</th>\n", | |
| " <td>5</td>\n", | |
| " <td>0</td>\n", | |
| " <td>19</td>\n", | |
| " <td>1</td>\n", | |
| " <td>1</td>\n", | |
| " <td>0</td>\n", | |
| " <td>4</td>\n", | |
| " <td>1</td>\n", | |
| " <td>0</td>\n", | |
| " <td>5</td>\n", | |
| " <td>...</td>\n", | |
| " <td>10</td>\n", | |
| " <td>23</td>\n", | |
| " <td>5</td>\n", | |
| " <td>24</td>\n", | |
| " <td>5</td>\n", | |
| " <td>23</td>\n", | |
| " <td>23</td>\n", | |
| " <td>9</td>\n", | |
| " <td>0</td>\n", | |
| " <td>0</td>\n", | |
| " </tr>\n", | |
| " <tr>\n", | |
| " <th>7</th>\n", | |
| " <td>5</td>\n", | |
| " <td>0</td>\n", | |
| " <td>19</td>\n", | |
| " <td>1</td>\n", | |
| " <td>1</td>\n", | |
| " <td>0</td>\n", | |
| " <td>4</td>\n", | |
| " <td>1</td>\n", | |
| " <td>0</td>\n", | |
| " <td>5</td>\n", | |
| " <td>...</td>\n", | |
| " <td>10</td>\n", | |
| " <td>23</td>\n", | |
| " <td>5</td>\n", | |
| " <td>24</td>\n", | |
| " <td>5</td>\n", | |
| " <td>23</td>\n", | |
| " <td>23</td>\n", | |
| " <td>9</td>\n", | |
| " <td>0</td>\n", | |
| " <td>0</td>\n", | |
| " </tr>\n", | |
| " <tr>\n", | |
| " <th>8</th>\n", | |
| " <td>5</td>\n", | |
| " <td>0</td>\n", | |
| " <td>19</td>\n", | |
| " <td>1</td>\n", | |
| " <td>1</td>\n", | |
| " <td>0</td>\n", | |
| " <td>4</td>\n", | |
| " <td>1</td>\n", | |
| " <td>0</td>\n", | |
| " <td>5</td>\n", | |
| " <td>...</td>\n", | |
| " <td>4</td>\n", | |
| " <td>11</td>\n", | |
| " <td>2</td>\n", | |
| " <td>12</td>\n", | |
| " <td>2</td>\n", | |
| " <td>11</td>\n", | |
| " <td>11</td>\n", | |
| " <td>4</td>\n", | |
| " <td>0</td>\n", | |
| " <td>0</td>\n", | |
| " </tr>\n", | |
| " <tr>\n", | |
| " <th>9</th>\n", | |
| " <td>5</td>\n", | |
| " <td>0</td>\n", | |
| " <td>19</td>\n", | |
| " <td>1</td>\n", | |
| " <td>1</td>\n", | |
| " <td>0</td>\n", | |
| " <td>4</td>\n", | |
| " <td>1</td>\n", | |
| " <td>0</td>\n", | |
| " <td>5</td>\n", | |
| " <td>...</td>\n", | |
| " <td>5</td>\n", | |
| " <td>11</td>\n", | |
| " <td>2</td>\n", | |
| " <td>11</td>\n", | |
| " <td>2</td>\n", | |
| " <td>11</td>\n", | |
| " <td>11</td>\n", | |
| " <td>4</td>\n", | |
| " <td>0</td>\n", | |
| " <td>0</td>\n", | |
| " </tr>\n", | |
| " <tr>\n", | |
| " <th>10</th>\n", | |
| " <td>5</td>\n", | |
| " <td>0</td>\n", | |
| " <td>19</td>\n", | |
| " <td>1</td>\n", | |
| " <td>1</td>\n", | |
| " <td>0</td>\n", | |
| " <td>4</td>\n", | |
| " <td>1</td>\n", | |
| " <td>0</td>\n", | |
| " <td>5</td>\n", | |
| " <td>...</td>\n", | |
| " <td>0</td>\n", | |
| " <td>0</td>\n", | |
| " <td>0</td>\n", | |
| " <td>0</td>\n", | |
| " <td>0</td>\n", | |
| " <td>0</td>\n", | |
| " <td>0</td>\n", | |
| " <td>0</td>\n", | |
| " <td>0</td>\n", | |
| " <td>0</td>\n", | |
| " </tr>\n", | |
| " <tr>\n", | |
| " <th>11</th>\n", | |
| " <td>5</td>\n", | |
| " <td>0</td>\n", | |
| " <td>19</td>\n", | |
| " <td>1</td>\n", | |
| " <td>1</td>\n", | |
| " <td>0</td>\n", | |
| " <td>4</td>\n", | |
| " <td>1</td>\n", | |
| " <td>0</td>\n", | |
| " <td>5</td>\n", | |
| " <td>...</td>\n", | |
| " <td>6</td>\n", | |
| " <td>11</td>\n", | |
| " <td>3</td>\n", | |
| " <td>12</td>\n", | |
| " <td>2</td>\n", | |
| " <td>11</td>\n", | |
| " <td>11</td>\n", | |
| " <td>4</td>\n", | |
| " <td>0</td>\n", | |
| " <td>0</td>\n", | |
| " </tr>\n", | |
| " <tr>\n", | |
| " <th>12</th>\n", | |
| " <td>5</td>\n", | |
| " <td>0</td>\n", | |
| " <td>19</td>\n", | |
| " <td>1</td>\n", | |
| " <td>1</td>\n", | |
| " <td>0</td>\n", | |
| " <td>4</td>\n", | |
| " <td>1</td>\n", | |
| " <td>0</td>\n", | |
| " <td>5</td>\n", | |
| " <td>...</td>\n", | |
| " <td>4</td>\n", | |
| " <td>11</td>\n", | |
| " <td>2</td>\n", | |
| " <td>12</td>\n", | |
| " <td>2</td>\n", | |
| " <td>11</td>\n", | |
| " <td>11</td>\n", | |
| " <td>4</td>\n", | |
| " <td>0</td>\n", | |
| " <td>0</td>\n", | |
| " </tr>\n", | |
| " </tbody>\n", | |
| "</table>\n", | |
| "<p>13 rows × 265 columns</p>\n", | |
| "</div>" | |
| ], | |
| "text/plain": [ | |
| " chrom pos _id ref alt qual filter alt_pos_strand dbsnp_bin \\\n", | |
| "0 4 0 19 1 1 0 4 1 0 \n", | |
| "1 5 0 19 1 1 0 4 1 0 \n", | |
| "2 5 0 18 1 1 0 4 1 0 \n", | |
| "3 5 0 19 1 1 0 4 1 0 \n", | |
| "4 5 0 19 1 1 0 4 1 0 \n", | |
| "5 5 0 19 1 1 0 4 1 0 \n", | |
| "6 5 0 19 1 1 0 4 1 0 \n", | |
| "7 5 0 19 1 1 0 4 1 0 \n", | |
| "8 5 0 19 1 1 0 4 1 0 \n", | |
| "9 5 0 19 1 1 0 4 1 0 \n", | |
| "10 5 0 19 1 1 0 4 1 0 \n", | |
| "11 5 0 19 1 1 0 4 1 0 \n", | |
| "12 5 0 19 1 1 0 4 1 0 \n", | |
| "\n", | |
| " dbsnp_chrom ... gnomad_an_popmax gnomad_af_popmax \\\n", | |
| "0 4 ... 0 0 \n", | |
| "1 5 ... 0 0 \n", | |
| "2 5 ... 0 0 \n", | |
| "3 5 ... 0 0 \n", | |
| "4 5 ... 5 11 \n", | |
| "5 5 ... 11 23 \n", | |
| "6 5 ... 10 23 \n", | |
| "7 5 ... 10 23 \n", | |
| "8 5 ... 4 11 \n", | |
| "9 5 ... 5 11 \n", | |
| "10 5 ... 0 0 \n", | |
| "11 5 ... 6 11 \n", | |
| "12 5 ... 4 11 \n", | |
| "\n", | |
| " gnomad_dp_median gnomad_dref_median gnomad_gq_median gnomad_ab_median \\\n", | |
| "0 0 0 0 0 \n", | |
| "1 0 0 0 0 \n", | |
| "2 0 0 0 0 \n", | |
| "3 0 0 0 0 \n", | |
| "4 2 11 2 11 \n", | |
| "5 6 25 5 23 \n", | |
| "6 5 24 5 23 \n", | |
| "7 5 24 5 23 \n", | |
| "8 2 12 2 11 \n", | |
| "9 2 11 2 11 \n", | |
| "10 0 0 0 0 \n", | |
| "11 3 12 2 11 \n", | |
| "12 2 12 2 11 \n", | |
| "\n", | |
| " gnomad_as_rf gnomad_as_filterstatus fdvr hs_only \n", | |
| "0 0 0 0 0 \n", | |
| "1 0 0 0 0 \n", | |
| "2 0 0 0 0 \n", | |
| "3 0 0 0 0 \n", | |
| "4 11 4 0 0 \n", | |
| "5 23 9 0 0 \n", | |
| "6 23 9 0 0 \n", | |
| "7 23 9 0 0 \n", | |
| "8 11 4 0 0 \n", | |
| "9 11 4 0 0 \n", | |
| "10 0 0 0 0 \n", | |
| "11 11 4 0 0 \n", | |
| "12 11 4 0 0 \n", | |
| "\n", | |
| "[13 rows x 265 columns]" | |
| ] | |
| }, | |
| "execution_count": 3, | |
| "metadata": {}, | |
| "output_type": "execute_result" | |
| } | |
| ], | |
| "source": [ | |
| "# Como ver o tamanho de todos os campos\n", | |
| "df.applymap(lambda el: len(el) if isinstance(el, str) else 0)" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "metadata": {}, | |
| "outputs": [], | |
| "source": [] | |
| } | |
| ], | |
| "metadata": { | |
| "kernelspec": { | |
| "display_name": "Django Shell-Plus", | |
| "language": "python", | |
| "name": "django_extensions" | |
| }, | |
| "language_info": { | |
| "codemirror_mode": { | |
| "name": "ipython", | |
| "version": 3 | |
| }, | |
| "file_extension": ".py", | |
| "mimetype": "text/x-python", | |
| "name": "python", | |
| "nbconvert_exporter": "python", | |
| "pygments_lexer": "ipython3", | |
| "version": "3.5.2" | |
| } | |
| }, | |
| "nbformat": 4, | |
| "nbformat_minor": 2 | |
| } |
Para transformar Query do Django em DataFrame:
df = pd.DataFrame(list(User.objects.all().values()))
Pra mostrar somente algumas colunas:
df[['id', 'email']]
Transformando todos os campos do DataFrame num jSON
df.T.apply(dict).tolist()
Renomeando colunas
df.rename(columns={'old_name': 'new_name'})
Selecionando as colunas de 10 em 10
df.iloc[:, :10]
com transposição
df.T.iloc[:10]
Novo df com algumas colunas.
new = old.filter(['A','B','D'], axis=1)
Como não inserir chaves com valor nulo no dicionário:
df.T.apply(lambda x: dict(x.dropna())).tolist()
Retornando os valores cujo tamanho da string seja maior que...
df[df['foo'].str.len() > 50]['foo']
Mostrando o tamanho de cada célula
df[df['foo'].str.len() > 50]['foo'].str.len()
Mostrando o tamanho de cada célula da coluna
df['foo_len'] = df['foo'].apply(len)
df[['foo_len', 'foo']]
Ou
df['foo'].str.len()
Suponha que você tenha Cliente e Obra.
Pegando o ID do Cliente que está em Obra e trocando pelo nome do Cliente que está no outro DataFrame.
# JSON do Cliente com IDCliente e Cliente
dict_cliente = df_cliente[['IDCliente', 'Cliente']].T.apply(dict).tolist()
[
{'IDCliente': 288, 'Cliente': 'Cliente Um'},
{'IDCliente': 1, 'Cliente': 'Cliente Dois'},
{'IDCliente': 959, 'Cliente': 'Cliente Três'},
]
Montando o dicionário que será usado como busca de cada Cliente a partir do seu ID.
_dict_cliente = {}
for item in dict_cliente:
_dict_cliente[item['IDCliente']] = item['Cliente']
_dict_cliente
{
288: 'Cliente Um',
1: 'Cliente Dois',
959: 'Cliente Três',
}
A partir desse dicionário fazemos a busca no outro DataFrame.
for row in df.itertuples():
nome_cliente = _dict_cliente.get(row.IDCliente)
print(row.IDCliente, nome_cliente)
288 Cliente Um
1 Cliente Dois
959 Cliente Três
Retornando o valor máximo agrupado por ano.
df.groupby(['Ano'])['NumeroOrcamento'].max()
df.reset_index()
Verificando data vazia:
for row in df.itertuples():
if row.DataOrcamento is pd.NaT:
print('Vazio')
else:
print(row.DataOrcamento)
Definindo vários fillna diferentes por coluna:
values = {'last_name': '', 'occupation': '', 'age': 0}
df = df.fillna(value=values)
df.head()
Se tiver problema com liblzma, faça um downgrade do Pandas para pandas==0.24.2.
https://stackoverflow.com/a/57115325
Retorna o tamanho do maior objeto de cada coluna.
dict_sizes = {}
for col in df.columns:
try:
print(f'{col} max length: {df[col].map(len).max()}\n')
dict_sizes[col] = df[col].map(len).max()
except Exception as e:
raise e
dict_sizes
dtype example
df['estoque'] =df['estoque'].fillna(0).astype(int)
Pandas Dataframe df to Django
https://www.laurivan.com/save-pandas-dataframe-as-django-model/
Produto.objects.bulk_create(
Produto(**item) for item in df.to_dict('records')
)
Definindo os tipos das colunas com dtype
dict_types_annot = {
'produto': str,
'ncm': str,
'preco': float,
'estoque': 'Int64',
}
# Define os tipos das colunas
dff = df.astype(dict_types_annot, errors='ignore')
# Troca 'nan' por None e float por None.
dff = dff.replace({'nan': None, float('nan'): None})
dff.to_dict('records')
Produto.objects.bulk_create(
Produto(**item) for item in dff.to_dict('records')
)
Intersecção de dataframes
import pandas as pd
import numpy as np
import datetime
from random import randint
df1 = pd.DataFrame({
'letters': ['A', 'B', 'C', 'D', 'E', 'J', 'K', 'M'],
'B': np.random.randint(0, 10, 8),
})
df1
df2 = pd.DataFrame({
'letters': ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'U', 'Z'],
'B': np.random.randint(0, 10, 26),
})
df2
# Retorna o que tem de comum nos dois dataframes.
pd.merge(df1, df2, how='inner', on='fruits')
# Retorna o que tem de comum, considerando o df1.
pd.merge(df1, df2, how='left', on='fruits')
# Retorna o que tem de comum, considerando o df2.
pd.merge(df1, df2, how='right', on='fruits')
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Código que substitui da célula [6] em diante do
separe_email.ipynb:Um detalhe é a falta de um índice único dos registros. Então, supondo o índice automático decorrente da importação do arquivo Excel como ID válido, esse ID é o usado no "join" (
merge).