Skip to content

Instantly share code, notes, and snippets.

View rdemorais's full-sized avatar
🏠
Working from home

Rafael de Morais rdemorais

🏠
Working from home
View GitHub Profile
@rdemorais
rdemorais / remove_names.py
Created November 15, 2023 15:52
Remover nomes do texto
data = [
{
"text": "paciente diagnosticado com dm , nega has . paciente acompanhado da mãe , dona Maria Fagundes",
"clinical_entities": [
{
"entity": "dm",
"entity_tokens": ["dm"],
"label": "DISEASE",
"start": 27,
"end": 29
@rdemorais
rdemorais / gist:bba4c28048098c9a5bcaab525fa9e3c9
Created October 4, 2023 16:50
Finalize stuck namespace - K8s
NAMESPACE=ingress-nginx; kubectl get namespace $NAMESPACE -o json | jq 'del(.spec.finalizers[0])' | kubectl replace --raw "/api/v1/namespaces/$NAMESPACE/finalize" -f -
# console:
# export AWS_ACCESS_KEY_ID=AWS_KEY
# export AWS_SECRET_ACCESS_KEY=AWS_SECRET
from tqdm import tqdm
import boto3
import os
s3 = boto3.client('s3')
@rdemorais
rdemorais / blogset_br.py
Created February 21, 2023 17:19
Código para criação do dataset Blogset
import gzip
import shutil
from unicodedata import normalize
from bs4 import BeautifulSoup
import ftfy
import re
from tqdm import tqdm
import csv
import pgp from "pg-promise";
const connections = [];
export default class Redshift {
static async getConnection() {
const dbName = "myDb";
if (!connections[dbName]) {
const dbUser = "dbUser";
@rdemorais
rdemorais / progress.py
Created July 23, 2022 12:43
Get download progress with boto3
import sys
import threading
class ProgressPercentage(object):
''' Progress Class
Class for calculating and displaying download progress
'''
def __init__(self, client, bucket, filename):
''' Initialize
@rdemorais
rdemorais / transformer.py
Created July 23, 2022 12:31
Import transformer model into spacy v3
from thinc.api import Config
import spacy
DEFAULT_CONFIG_STR = """
[transformer]
max_batch_items = 4096
[transformer.set_extra_annotations]
@annotation_setters = "spacy-transformers.null_annotation_setter.v1"
@rdemorais
rdemorais / time_dim.sql
Created February 20, 2022 14:35
Create time dimension
INSERT INTO dm_time (co_dm_time, epoch, day_suffix, day_name, day_of_week, day_of_month, day_of_quarter, day_of_year, week_of_month, week_of_year, week_of_year_iso, month_actual, month_name, month_name_abbreviated, quarter_actual, quarter_name, year_actual, first_day_of_week, last_day_of_week, first_day_of_month, last_day_of_month, first_day_of_quarter, last_day_of_quarter, first_day_of_year, last_day_of_year, mmyyyy, mmddyyyy, weekend_indr)
SELECT datum AS date_actual,
EXTRACT(EPOCH FROM datum) AS epoch,
TO_CHAR(datum, 'fmDDth') AS day_suffix,
TO_CHAR(datum, 'TMDay') AS day_name,
EXTRACT(ISODOW FROM datum) AS day_of_week,
EXTRACT(DAY FROM datum) AS day_of_month,
datum - DATE_TRUNC('quarter', datum)::DATE + 1 AS day_of_quarter,
EXTRACT(DOY FROM datum) AS day_of_year,
TO_CHAR(datum, 'W')::INT AS week_of_month,
@rdemorais
rdemorais / copy_text_to_pod.sh
Created November 1, 2021 16:03
kubectl copy without cp
#!/usr/bin/env bash
function copy_text_to_pod() {
namespace=$1
pod_name=$2
src_filename=$3
dest_filename=$4
base64_text=`cat $src_filename | base64`
kubectl --kubeconfig=conasems-config.yml exec -n $namespace $pod_name -- bash -c "echo \"$base64_text\" | base64 -d > $dest_filename"
@rdemorais
rdemorais / git-remove-branches.txt
Created August 8, 2020 16:06
Remover local branches
git branch -D `git branch --merged | grep -v \* | xargs`