davidmezzetti

# Print index info before (info() is also new!)
embeddings.info()

# Reindex
embeddings.reindex({"path": "sentence-transformers/paraphrase-MiniLM-L3-v2"})

print("------")

# Print index info after
embeddings.info()

davidmezzetti

# Save index as tar.xz
embeddings.save("index.tar.xz")
#tar -tvJf index.tar.xz
#echo
#xz -l index.tar.xz
#echo

# Reload index
embeddings.load("index.tar.xz")

davidmezzetti

import numpy as np
import requests

def transform(inputs):
  response = requests.post("https://api-inference.huggingface.co/pipeline/feature-extraction/sentence-transformers/nli-mpnet-base-v2",
                           json={"inputs": inputs})

  return np.array(response.json(), dtype=np.float32)

# Index data using vectors from Inference API

davidmezzetti

# pip install spacy --upgrade
# python -m spacy download en_core_web_md

import spacy

# Load spacy
nlp = spacy.load("en_core_web_md")

def transform(inputs):
    return [result.vector for result in nlp.pipe(inputs)]

davidmezzetti

embeddings:
  path: sentence-transformers/nli-mpnet-base-v2
  content: true
tabular:
  idcolumn: url
  textcolumns:
  - title
workflow:
  index:
    tasks:

davidmezzetti

summary:
  path: sshleifer/distilbart-cnn-12-6
textractor:
  join: true
  lines: false
  minlength: 100
  paragraphs: true
  sentences: false
workflow:
  summary:

davidmezzetti

translation: {}
workflow:
  translate:
    tasks:
    - action: translation
      args:
      - fr

davidmezzetti

summary:
  path: sshleifer/distilbart-cnn-12-6
textractor:
  join: true
  lines: false
  minlength: 100
  paragraphs: true
  sentences: false
translation: {}
workflow:

davidmezzetti

embeddings:
  path: sentence-transformers/nli-mpnet-base-v2
  content: true
  functions:
    - {name: translation, argcount: 2, function: translation}
tabular:
  idcolumn: url
  textcolumns:
  - title
translation:

davidmezzetti

from txtai import Embeddings

data = [
  "US tops 5 million confirmed virus cases",
  "Canada's last fully intact ice shelf has suddenly collapsed, forming a Manhattan-sized iceberg",
  "Beijing mobilises invasion craft along coast as Taiwan tensions escalate",
  "The National Park Service warns against sacrificing slower friends in a bear attack",
  "Maine man wins $1M from $25 lottery ticket",
  "Make huge profits without work, earn up to $100,000 a day"
]