Python

aws_rek.md

aws setup

• install aws cli from https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html

• aws configure

  • AWS -> Security credentials -> Access Keys -> Create
  • Key ID
  • Access Key
  • Def region name: eu-west-1

• aws rekognition create-collection --collection-id COLLECTION_NAME --region eu-west-1

  • create a collection in aws rekognition

• aws dynamodb create-table --table-name COLLECTION_NAME --attribute-definitions AttributeName=RekognitionId,AttributeType=S --key-schema AttributeName=RekognitionId,KeyType=HASH --provisioned-throughput ReadCapacityUnits=1,WriteCapacityUnits=1 --region eu-west-1

  • create a table in dynamoDb to store faceprint-id and key

• aws s3 mb s3://UNIQUE_BUCKET_NAME

  • bucket to store images

• IAM Role for the lambda function

  • IAM -> Roles ->
  • trusted entity type: AWS Service
  • service: Lambda
  • next -> name role -> create
  • edit policy: roles list find role -> click on it... -> create inline policy - json tab

{
  "Version": "2012-10-17",
  "Statement": [
      {
          "Effect": "Allow",
          "Action": [
              "logs:CreateLogGroup",
              "logs:CreateLogStream",
              "logs:PutLogEvents"
          ],
          "Resource": "arn:aws:logs:*:*:*"
      },
      {
          "Effect": "Allow",
          "Action": [
              "s3:GetObject"
          ],
          "Resource": [
              "arn:aws:s3:::BUCKET_NAME/*"
          ]
      },
      {
          "Effect": "Allow",
          "Action": [
              "dynamodb:PutItem"
          ],
          "Resource": [
              "[DYNAMOdb_ARN_FROM_CONSOLE_NOT_LOG]"
          ]
      },
      {
          "Effect": "Allow",
          "Action": [
              "rekognition:IndexFaces"
          ],
          "Resource": "*"
      }
  ]
}

• Write Lambda func
  • search and create lambda func
  • choose appropriate role
  • in triggers -> select S3 bucket -> select on-create

from __future__ import print_function

import boto3
from decimal import Decimal
import json
import urllib

print('Loading function')

dynamodb = boto3.client('dynamodb')
s3 = boto3.client('s3')
rekognition = boto3.client('rekognition')

# --------------- Helper Functions ------------------

def index_faces(bucket, key):
  response = rekognition.index_faces(
      Image={"S3Object":
          {"Bucket": bucket,
          "Name": key}},
          CollectionId="COLLECTION_NAME")
  return response
    
def update_index(tableName,faceId, fullName):
  response = dynamodb.put_item(
    TableName=tableName,
    Item={
      'RekognitionId': {'S': faceId},
      'FullName': {'S': fullName}
      }
    ) 
    
# --------------- Main handler ------------------

def lambda_handler(event, context):
  # Get the object from the event
  bucket = event['Records'][0]['s3']['bucket']['name']
  print("Records: ",event['Records'])
  key = event['Records'][0]['s3']['object']['key']
  print("Key: ",key)
  # key = key.encode()
  # key = urllib.parse.unquote_plus(key)

    try:
      # Calls Amazon Rekognition IndexFaces API to detect faces in S3 object 
      # to index faces into specified collection
        
      response = index_faces(bucket, key)
        
      # Commit faceId and full name object metadata to DynamoDB
      if response['ResponseMetadata']['HTTPStatusCode'] == 200:
        faceId = response['FaceRecords'][0]['Face']['FaceId']

        ret = s3.head_object(Bucket=bucket,Key=key)
        personFullName = ret['Metadata']['fullname']

        update_index('COLLECTION_NAME',faceId,personFullName)

      # Print response to console
      print(response)

      return response
    except Exception as e:
      print(e)
      print("Error processing object {} from bucket {}. ".format(key, bucket))
      raise e

python_A_batchInsert_GMCA.md

import json
from google.oauth2 import service_account
from googleapiclient.errors import HttpError
from googleapiclient.discovery import build

# api KEy

# Set the API client credentials
creds = service_account.Credentials.from_service_account_file(
    'content-api-key.json',
    scopes=['https://www.googleapis.com/auth/content'])

# Set the Merchant Center account ID and the target country
merchant_id = 'MERCHAND_ID'
target_country = 'COUNTRY_ID'

# Load the product data from a JSON file
with open('./products.json', 'r') as f:
    product_data = json.load(f)

# Build the batch request body
batch_request_body = {
    'entries': [
        {
            'batchId': str(i),
            'merchantId': merchant_id,
            'method': 'insert',
            'product': product
        } for i, product in enumerate(product_data["entries"])
    ]
}

# print(batch_request_body)

# Initialize the Content API client
content_api = build('content', 'v2.1', credentials=creds)

# Send the batch request
try:
    request = content_api.products().custombatch(body=batch_request_body)
    response = request.execute()
    print(response)
except HttpError as error:
    print(f'An HTTP error {error.resp.status} occurred:\n{error.content}')

python_csv_read_save_imgas.md

from csv import reader
import requests
from PIL import Image
import io

def getImage(sku, url):
  resp = requests.get(url)
  try:
    img_bytes = io.BytesIO(resp.content)
    img = Image.open(img_bytes)
    img.save("./img/" + sku + ".png")
  except:
    print(f"[FAILED] {sku}")

# open file in read mode
with open('test.csv', 'r', encoding="utf8") as read_obj:
    csv_reader = reader(read_obj)
    i = 0
    for row in csv_reader:
      print('on row: %s' % (i))
      if row[4] != "":
        getImage(row[0], row[4])
      i += 1

import csv, sqlite3

con = sqlite3.connect(":memory:") # change to 'sqlite:///your_filename.db'
cur = con.cursor()
cur.execute("CREATE TABLE t (col1, col2);") # use your column names here

with open('data.csv','r') as fin: # `with` statement available in 2.5+
    # csv.DictReader uses first line in file for column headings by default
    dr = csv.DictReader(fin) # comma is default delimiter
    to_db = [(i['col1'], i['col2']) for i in dr]

cur.executemany("INSERT INTO t (col1, col2) VALUES (?, ?);", to_db)
con.commit()
con.close()

python_meetup_13Jan21.md

Meetup

Learning from Text https://github.com/MichaelMDowling/TopicModel/

Dataset: proquest

Tweets are hard to work with for topic modeling, because they are too short.

Topic Modeling

Uncovering the hidden or latent topics across a set of documents.

python_sqlite3.md

import sqlite3
from employee import Employee

# conn = sqlite3.connect("employees.db")
# testing
conn = sqlite3.connect(":memory:")

c = conn.cursor()

def insert_emp(emp):
  with conn:
    c.execute("""
    INSERT INTO employees
      VALUES (:first, :last, :pay)
    """, {'first': emp.first, 'last': emp.last, 'pay': emp.pay}
    )

def get_all():
  c.execute("""
  SELECT * FROM employees
  """
  )
  return c.fetchall()

def get_emps_by_name(lastname):
  c.execute("""
  SELECT * FROM employees
    WHERE last=:last
  """, {'last': lastname}
  )
  return c.fetchall()

def update_pay(emp, pay):
  with conn:
    c.execute("""
    UPDATE employees
      SET pay=:pay
      WHERE first=:first
      AND last=:last
    """, {'first': emp.first, 'last': emp.last, 'pay': pay}
    )

def remove(emp):
  with conn:
    c.execute("""
    DELETE FROM employees
      WHERE first=:first
      AND last=:last
    """, {'first': emp.first, 'last': emp.last}
    )


CREATE_TBL = """
CREATE TABLE employees (
  first text,
  last text,
  pay integer
)
"""
ADD_EMPLOYEE = """
INSERT INTO employees
  VALUES ('Mary', 'Neil', 5000)
"""
SELECT_ALL = """
SELECT * FROM employees
"""

c.execute(CREATE_TBL)
conn.commit()

emp_1 = Employee('John', 'Doe', 80000)
emp_2 = Employee('Jane', 'Doe', 90000)

insert_emp(emp_1)
insert_emp(emp_2)

update_pay(emp_1, 55000)
print(get_emps_by_name('Doe'))

print(get_all())

remove(emp_2)

print(get_all())

# c.execute("""
# INSERT INTO employees
#   VALUES (?, ?, ?)
# """, (emp_1.first, emp_1.last, emp_1.pay)
# )

# c.execute("""
# INSERT INTO employees
#   VALUES (:first, :last, :pay)
# """, {'first': emp_2.first, 'last': emp_2.last, 'pay': emp_2.pay}
# )
# conn.commit()

# c.execute("SELECT * FROM employees WHERE last=:last", {'last': 'Doe'})
# c.fetchone()
# c.fetchmany(4)
# print(c.fetchall())

# conn.commit()



conn.close()

python_tkinter_basics.md

weather app

import tkinter as tk
from dotenv import dotenv_values
import requests

key = dotenv_values(".env")
# key['API_KEY']

HEIGHT = 500
WIDTH = 600

def format_weather(weather):
  try:
    name = weather['name']
    cntr = weather['sys']['country']
    desc = weather['weather'][0]['description']
    temp = weather['main']['temp']
    final_str = 'City: %s\nCountry: %s\nConditions: %s\nTemperature: %i' % (name, cntr, desc, temp)
  except:
    final_str = 'problem retrieving data'
  return final_str

def get_weather(city):
  print("get data for: ", city)
# IN PRODUCTION REPLACE THIS WITH ACTUAL KEY:
  api_key = key['API_KEY']
  # api_key = ""
  url = 'https://api.openweathermap.org/data/2.5/weather'
  params = {
    'APPID': api_key,
    'q': city,
    'units': 'metric'
  }
  response = requests.get(url, params=params)
  weather = response.json()
  label['text'] = format_weather(weather)


root = tk.Tk()

canvas = tk.Canvas(root, height=HEIGHT, width=WIDTH)
canvas.pack()

bgi = tk.PhotoImage(file="bg.png")
bgl = tk.Label(root, image=bgi)
bgl.place(
  relheight=1,
  relwidth=1
)

frame = tk.Frame(root, bg="#80c1ff", bd=5)
frame.place(
  relx=0.5,
  rely=0.1,
  relwidth=0.75,
  relheight=0.1,
  anchor="n"
  )

entry = tk.Entry(frame, font=("Arial", 15))
entry.place(
  relwidth=0.65,
  relheight=1
  )

button = tk.Button(
  frame,
  font=("Arial", 14),
  text="Get Data",
  command=lambda: get_weather(entry.get())
  )
button.place(
  relx=0.675,
  relwidth=0.325,
  relheight=1
  )

lower_frame = tk.Frame(root, bg="#80c1ff", bd=5)
lower_frame.place(
  relx=0.5,
  rely=.25,
  relwidth=.75,
  relheight=0.6,
  anchor="n"
)

label = tk.Label(
  lower_frame,
  text="Label",
  bg="white",
  bd=4,
  justify="left",
  font=("Arial", 18),
  anchor="nw"
  )
label.place(
  relwidth=1,
  relheight=1
  )

root.mainloop()

basic layouts

Tutorials Point

import tkinter as tk

HEIGHT = 500
WIDTH = 350

root = tk.Tk()

canvas = tk.Canvas(root, height=HEIGHT, width=WIDTH)
canvas.pack()

frame = tk.Frame(root, bg="#80c1ff")
frame.place(relx=0.05, rely=0.05, relwidth=0.9, relheight=0.9)

button = tk.Button(frame, text="Button", bg="#345222")
button.place(relx=0, rely=0, relwidth=0.25, relheight=0.25)

label = tk.Label(frame, text="Label", bg="yellow")
label.place(relx=0.3, rely=0, relwidth=0.45, relheight=0.25)

entry = tk.Entry(frame, bg="green")
entry.place(relx=0, rely= 0.3, relwidth=0.5, relheight=0.25)

root.mainloop()

python_tkinter_tabs.md

import tkinter as tk
from tkinter import ttk
import tksheet
import random

lowercase = 'abcdefghijklmnopqrstuvwxyz'
uppercase = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
digits = '0123456789'
punctuation = '!"#$%&()*+,-./:;<=>?@[]^_{|}~' # excluded \'
allowed = lowercase + uppercase + digits + punctuation

def pass_gen():
  choice = random.SystemRandom().choice
  password_chars = [
    choice(lowercase),
    choice(uppercase),
    choice(digits),
    choice(punctuation)
  ] + random.sample(allowed, 14)
  res = ''.join(password_chars)
  passwordEntryTabTwo.delete(0,"end")
  passwordEntryTabTwo.insert(0,res)
  print("[BTN] generate password: %s" % (res))

def item_select(event):
  pass


# ----- FORM -----

form = tk.Tk()
form.title("Py Password Vault")
form.geometry("500x400")

tab_parent = ttk.Notebook(form)

tab1 = ttk.Frame(tab_parent)
tab2 = ttk.Frame(tab_parent)

tab_parent.add(tab1, text="All")
tab_parent.add(tab2, text="New")

# ----- TAB 1 -----

sheet = tksheet.Sheet(tab1)
sheet.set_sheet_data([[f"{ri+cj}" for cj in range(3)] for ri in range(1)])
sheet.enable_bindings((
  "single_select",
  "copy"
  ))
sheet.place(
  relx=0.01,
  rely=0.01,
  relwidth=1,
  relheight=1,
  anchor="nw"
)

# ----- TAB 2 -----
appLabelTabTwo = tk.Label(tab2, text="App")
appLabelTabTwo.grid(row=0, column=0, padx=10, pady=10)

appEntryTabTwo = tk.Entry(tab2)
appEntryTabTwo.grid(row=0, column=1, padx=10, pady=10)

usernameLabelTabTwo = tk.Label(tab2, text="User Name")
usernameLabelTabTwo.grid(row=1, column=0, padx=10, pady=10)

usernameEntryTabTwo = tk.Entry(tab2)
usernameEntryTabTwo.grid(row=1, column=1, padx=10, pady=10)

passwordLabelTabTwo = tk.Label(tab2, text="Password")
passwordLabelTabTwo.grid(row=2, column=0, padx=10, pady=10)

passwordEntryTabTwo = tk.Entry(tab2)
passwordEntryTabTwo.grid(row=2, column=1, padx=10, pady=10)

generatePasswordButtonTabTwo = tk.Button(
  tab2,
  text="Generate",
  command=lambda: pass_gen()
  )
generatePasswordButtonTabTwo.grid(row=2, column=3, padx=10, pady=10)

tab_parent.pack(expand=1, fill="both")
form.mainloop()