sergiospagnuolo

library(httr)
library(stringr)

# Agentes de AI, mapeados do nytimes.com/robots.txt
ai_keywords <- c(
  "GPTBot", "ChatGPT-User", "PerplexityBot", "Amazonbot", "ClaudeBot", 
  "Omgilibot", "FacebookBot", "Applebot", "Applebot-Extended", "anthropic-ai", "Bytespider", 
  "Claude-Web", "YouBot", "CCBot", "Google-Extended", "Quora-Bot", "Meta-ExternalAgent"
)

dannguyen

Extracting financial disclosure reports and police blotter narratives using OpenAI's Structured Output

tl;dr this demo shows how to call OpenAI's gpt-4o-mini model, provide it with URL of a screenshot of a document, and extract data that follows a schema you define. The results are pretty solid even with little effort in defining the data — and no effort doing data prep. OpenAI's API could be a cost-efficient tool for large scale data gathering projects involving public documents.

OpenAI announced Structured Outputs for its API, a feature that allows users to specify the fields and schema of extracted data, and guarantees that the JSON output will follow that specification.

For example, given a Congressional financial disclosure report, with assets defined in a table like this:

jsoma

function onOpen() {
  const ui = SpreadsheetApp.getUi();
  // Adds a custom menu to the Google Sheets UI
  ui.createMenu('Checking helper')
    .addItem('Create Sample', 'showStratificationPrompt')
    .addToUi();
}

function showStratificationPrompt() {
  const ui = SpreadsheetApp.getUi();

jkeefe

{
    "Version": "2012-10-17",
    "Statement": [
        {
            "Effect": "Allow",
            "Action": [
                "s3:ListAllMyBuckets"
            ],
            "Resource": "arn:aws:s3:::*"
        },

jkeefe

[
    {
        "AllowedHeaders": [
            "*"
        ],
        "AllowedMethods": [
            "GET",
            "HEAD"
        ],
        "AllowedOrigins": [

turicas

johnymontana

CREATE CONSTRAINT IF NOT EXISTS FOR (n:Parcel) REQUIRE n.neo4jImportId IS UNIQUE;
CREATE CONSTRAINT IF NOT EXISTS FOR (n:Subject) REQUIRE n.neo4jImportId IS UNIQUE;
CREATE CONSTRAINT IF NOT EXISTS FOR (n:Bill) REQUIRE n.neo4jImportId IS UNIQUE;
CREATE CONSTRAINT IF NOT EXISTS FOR (n:Committee) REQUIRE n.neo4jImportId IS UNIQUE;
CREATE CONSTRAINT IF NOT EXISTS FOR (n:Legislator) REQUIRE n.neo4jImportId IS UNIQUE;
CREATE CONSTRAINT IF NOT EXISTS FOR (n:Trip) REQUIRE n.neo4jImportId IS UNIQUE;
CREATE CONSTRAINT IF NOT EXISTS FOR (n:Organization) REQUIRE n.neo4jImportId IS UNIQUE;
CREATE CONSTRAINT IF NOT EXISTS FOR (n:Destination) REQUIRE n.neo4jImportId IS UNIQUE;
CALL apoc.import.json("https://cdn.neo4jlabs.com/data/landgraph/landgraph.json");

fernandobarbalho

library(readxl)
library(ckanr)
library(purrr)

ckanr::package_search()

package<- ckanr::package_show(id= "22d13d17-bf69-4a1a-add2-25cc1e25f2d7", 
                              url= "https://www.tesourotransparente.gov.br/ckan") #busca todos os dados do dataset que se refere aos dados de COFOG

svavassori

import sys
import csv
import json


# Converts the JSON output of a PowerBI query to a CSV file
def extract(input_file, output_file):
    input_json = read_json(input_file)
    data = input_json["results"][0]["result"]["data"]
    dm0 = data["dsr"]["DS"][0]["PH"][0]["DM0"]

morkapronczay

import geopandas as gpd
import folium
import matplotlib.pyplot as plt

# read date
url = 'https://geoservicos.pbh.gov.br/geoserver/wfs?service=WFS&version=1.0.0&request=GetFeature&typeName=ide_bhgeo:BAIRRO&srsName=EPSG:31983&outputFormat=application%2Fjson'
gdf = gpd.read_file(url)

# check if data is right
fig, ax = plt.subplots(figsize=(10,10))