khaerulumam42

uuid	collection_id	embedding	document	cmetadata	custom_id
3fe65304-9bb0-4c11-939b-351387326d47	1ba42d4a-2e21-4933-ba29-5c32355ad22d	[0.50184065,-0.17211534,-0.013504442, ... ]	united kingdom	{"category": "country"}	b2dabee7-8ab5-11ee-814b-155d814ce4ad
466ae9c5-d043-44cf-a56c-896678a85c41	1ba42d4a-2e21-4933-ba29-5c32355ad22d	[0.07344677,0.2832106,-0.041023944, ...]	cats	{"category": "animal"}	b2dabed8-8ab5-11ee-814b-155d814ce4ad
2574f0e3-9994-444f-9ed2-bb03cf04201c	1ba42d4a-2e21-4933-ba29-5c32355ad22d	[0.09758755,0.11665281,-0.30114746, ...]	germany	{"category": "country"}	b2dabee8-8ab5-11ee-814b

khaerulumam42

data_string = '''cats, animal
dogs, animal
lions, animal
elephants, animal
penguins, animal
dolphins, animal
pandas, animal
tigers, animal
horses, animal
koalas, animal

khaerulumam42

results = pgvector.similarity_search_with_relevance_scores('duck', k=5)

khaerulumam42

from langchain.vectorstores.pgvector import PGVector, DistanceStrategy
from langchain.embeddings import HuggingFaceEmbeddings

model = 'bert-base-uncased'
embedding_model = HuggingFaceEmbeddings(model_name=model)

COLLECTION_NAME = 'top_10_dataset'
CONNECTION_STRING = PGVector.connection_string_from_db_params(
    driver='psycopg2',
    host=os.environ.get('PGVECTOR_HOST'),

khaerulumam42

from datetime import datetime
import time
import signal
import logging

class GracefulKiller:
    def __init__(self):
        self.kill_now = False
        signal.signal(signal.SIGINT, self.exit_gracefully)
        signal.signal(signal.SIGTERM, self.exit_gracefully)

khaerulumam42

sudo docker ps
>>> CONTAINER ID        IMAGE                        COMMAND                  CREATED             STATUS              PORTS                                                NAMES
>>> 776c6e63c95c        producer_consumer_consumer   "/usr/bin/dumb-init …"   15 seconds ago      Up 14 seconds                                                            consumer
>>> 09c11d829ace        producer_consumer_producer   "/usr/bin/dumb-init …"   15 seconds ago      Up 13 seconds       0.0.0.0:8000->8000/tcp                               producer
>>> 79f87828de3f        wurstmeister/zookeeper       "/bin/sh -c '/usr/sb…"   16 minutes ago      Up 16 minutes       22/tcp, 2888/tcp, 3888/tcp, 0.0.0.0:2182->2181/tcp   zookeeper
>>> 2e3b815047ed        wurstmeister/kafka           "start-kafka.sh"         16 minutes ago      Up 16 minutes       0.0.0.0:9092->9092/tcp                               kafka

khaerulumam42

sudo docker ps
>>> CONTAINER ID        IMAGE                    COMMAND                  CREATED             STATUS              PORTS                                                NAMES
>>> 79f87828de3f        wurstmeister/zookeeper   "/bin/sh -c '/usr/sb…"   42 seconds ago      Up 38 seconds       22/tcp, 2888/tcp, 3888/tcp, 0.0.0.0:2182->2181/tcp   zookeeper
>>> 2e3b815047ed        wurstmeister/kafka       "start-kafka.sh"         42 seconds ago      Up 39 seconds       0.0.0.0:9092->9092/tcp                               kafka

khaerulumam42

sudo docker network create tutorial
>>> a9487a8f771f607b4fe59f9e57580f5771a25fe6c606345844dee72ed8b12975

sudo docker network ls
>>> NETWORK ID          NAME                   DRIVER              SCOPE
>>> a9487a8f771f        tutorial               bridge              local

sudo docker network inspect tutorial
>>> [
>>>     {

khaerulumam42

version: "3.6"

services:
  producer:
    build:
        context: .
        dockerfile: Dockerfile-producer
    networks:
      - outside
    container_name: producer

khaerulumam42

FROM python:3.9-slim-buster as builder

LABEL MAINTAINER "Khaerul Umam <[email protected]>"
LABEL DESCRIPTION "consumer"

RUN apt-get update && apt-get upgrade -y \
    && apt-get install -y python3 python3-pip \
    python3-setuptools gfortran liblapack-dev liblapack3 nano \
    default-libmysqlclient-dev\
    supervisor locales \