Some building stuff

buildings.py

# LLM stuff
import outlines
import torch
from transformers import AutoProcessor
from pydantic import BaseModel, Field
from typing import Literal, Optional, List

# Image stuff
from PIL import Image
import requests

# Rich for pretty printing
from rich import print

# To use Qwen-2-VL:
# from transformers import Qwen2VLForConditionalGeneration
# model_name = "Qwen/Qwen2-VL-7B-Instruct"
# model_class = Qwen2VLForConditionalGeneration

from transformers import AutoModelForImageTextToText
model_name = "HuggingFaceTB/SmolVLM-Instruct"
model_class = AutoModelForImageTextToText

model = outlines.models.transformers_vision(
    model_name,
    model_class=model_class,
    device="cuda",
    # model_kwargs={
    #     "device_map": "auto",
    #     "torch_dtype": torch.bfloat16,
    # },
    # processor_kwargs={
    #     "device": "cuda", # set to "cpu" if you don't have a GPU
    # },
)
def load_and_resize_image(image_path, max_size=1024):
    """
    Load and resize an image while maintaining aspect ratio

    Args:
        image_path: Path to the image file
        max_size: Maximum dimension (width or height) of the output image

    Returns:
        PIL Image: Resized image
    """
    image = Image.open(image_path)

    # Get current dimensions
    width, height = image.size

    # Calculate scaling factor
    scale = min(max_size / width, max_size / height)

    # Only resize if image is larger than max_size
    if scale < 1:
        new_width = int(width * scale)
        new_height = int(height * scale)
        image = image.resize((new_width, new_height), Image.Resampling.LANCZOS)

    return image

image = load_and_resize_image("input_building_streetview.png")


class Building(BaseModel):
    image_description: str

    area_reasoning: str
    area: Literal["< 55m2", "55 - 124m2", "125 - 249m2", "250 - 629m2", "630 - 3100m2", "> 3100m2"]
    area_confidence: Literal["High", "Medium", "Low"]

    facade_reasoning: str
    facade: Literal["No exterior wall material, structural material showing", "Concrete", "Glass", "Earth", "Masonry", "Metal", "Vegetative", "Wood", "Stucco", "Plastic/vinyl, various", "Material of exterior wall, other"]
    facade_confidence: Literal["High", "Medium", "Low"]

    floors_reasoning: str
    floors: Literal["1 Floor", "2 Floors", "3-7 Floors", "> 8 Floors"]
    floors_confidence: Literal["High", "Medium", "Low"]

    foundation_reasoning: str
    foundation: Literal["Shallow foundation, with lateral capacity", "Shallow foundation, no lateral capacity", "Deep foundation, with lateral capacity", "Deep foundation, no lateral capacity", "Other"]
    foundation_confidence: Literal["High", "Medium", "Low"]

    lateral_load_resisting_system_reasoning: str
    lateral_load_resisting_system: Literal["No lateral load-resisting system", "Moment frame", "Infilled frame", "Braced", "Post and beam", "Wall", "Dual frame-wall system", "Flat slab/plate or waffle slab", "Infilled flat slab/plate or infilled waffle slab", "Hybrid lateral load-resisting system", "Other lateral load-resisting system"]
    lateral_load_resisting_system_confidence: Literal["High", "Medium", "Low"]

    material_of_lateral_load_resisting_system_reasoning: str
    material_of_lateral_load_resisting_system: Literal["Concrete, unreinforced", "Concrete, reinforced", "Concrete, composite with steel section", "Steel", "Metal (except steel)", "Masonry, unreinforced", "Masonry, confined", "Masonry, reinforced", "Earth, unreinforced", "Earth, reinforced", "Wood", "Other material"]
    material_of_lateral_load_resisting_system_confidence: Literal["High", "Medium", "Low"]

    purpose_reasoning: str
    purpose: Literal["Residential", "Commercial and public", "Mixed use", "Industrial", "Agriculture", "Assembly", "Government", "Education", "Other occupancy type"]
    purpose_confidence: Literal["High", "Medium", "Low"]

    roof_material_reasoning: str
    roof_material: Literal["Concrete roof without additional covering", "Clay or concrete tile", "Fibre cement or metal tile", "Membrane roofing", "Slate", "Stone slab", "Metal or asbestos sheets", "Wooden and asphalt shingles", "Vegetative", "Earthen", "Solar panelled roofs", "Roof covering, other"]
    roof_material_confidence: Literal["High", "Medium", "Low"]

    roof_shape_reasoning: str
    roof_shape: Literal["Flat", "Pitched with gable ends", "Pitched and hipped", "Pitched with dormers", "Monopitch", "Sawtooth", "Curved", "Complex regular", "Complex irregular", "Roof shape, other"]
    roof_shape_confidence: Literal["High", "Medium", "Low"]

    year_built_reasoning: str
    year_built: Literal["< 1880", "1880 - 1919", "1920 - 1944", "1945 - 1968", "1969 - 1988", "1990 - 2008", "> 2008"]
    year_built_confidence: Literal["High", "Medium", "Low"]

# Set up the content you want to send to the model
messages = [
    {
        "role": "user",
        "content": [
            {
                # The image is provided as a PIL Image object
                "type": "image",
                "image": image,
            },
            {
                "type": "text",
                "text": f"""
                You review images of buildings and extract information about the building.
                The information you extract is used to classify the building in the taxonomy of the OpenQuake project.
                The taxonomy can be found at https://taxonomy.openquake.org/

                You must provide reasoning for each of the fields you extract. 

                Available fields and their descriptions are:
                - area
                    - < 55m2
                    - 55 - 124m2
                    - 125 - 249m2
                    - 250 - 629m2
                    - 630 - 3100m2
                    - > 3100m2
                - facade
                    - No exterior wall material, structural material showing
                    - Concrete
                    - Glass
                    - Earth
                    - Masonry
                    - Metal
                    - Vegetative
                    - Wood
                    - Stucco
                    - Plastic/vinyl, various
                    - Material of exterior wall, other
                - floors
                    - 1 Floor
                    - 2 Floors
                    - 3-7 Floors
                    - > 8 Floors
                - foundation
                    - Shallow foundation, with lateral capacity
                    - Shallow foundation, no lateral capacity
                    - Deep foundation, with lateral capacity
                    - Deep foundation, no lateral capacity
                    - Other
                - lateral load resisting system
                    - No lateral load-resisting system
                    - Moment frame
                    - Infilled frame
                    - Braced
                    - Post and beam
                    - Wall
                    - Dual frame-wall system
                    - Flat slab/plate or waffle slab
                    - Infilled flat slab/plate or infilled waffle slab
                    - Hybrid lateral load-resisting system
                    - Other lateral load-resisting system
                - material of lateral load resisting system
                    - Concrete, unreinforced
                    - Concrete, reinforced
                    - Concrete, composite with steel section
                    - Steel
                    - Metal (except steel)
                    - Masonry, unreinforced
                    - Masonry, confined
                    - Masonry, reinforced
                    - Earth, unreinforced
                    - Earth, reinforced
                    - Wood
                    - Other material
                - purpose
                    - Residential
                    - Commercial and public
                    - Mixed use
                    - Industrial
                    - Agriculture
                    - Assembly
                    - Government
                    - Education
                    - Other occupancy type
                - roof material
                    - Concrete roof without additional covering
                    - Clay or concrete tile
                    - Fibre cement or metal tile
                    - Membrane roofing
                    - Slate
                    - Stone slab
                    - Metal or asbestos sheets
                    - Wooden and asphalt shingles
                    - Vegetative
                    - Earthen
                    - Solar panelled roofs
                    - Tensile membrane or fabric roof
                    - Roof covering, other
                - roof shape
                    - Flat
                    - Pitched with gable ends
                    - Pitched and hipped
                    - Pitched with dormers
                    - Monopitch
                    - Sawtooth
                    - Curved
                    - Complex regular
                    - Complex irregular
                    - Roof shape, other
                - year built
                    - < 1880
                    - 1880 - 1919
                    - 1920 - 1944
                    - 1945 - 1968
                    - 1969 - 1988
                    - 1990 - 2008
                    - > 2008

                You must provide reasoning for each of the fields you extract. 
                The reasoning should be a short explanation of why you chose the field you did.
                The reasoning should be no more than 100 words.

                Return the information in JSON format.
            """},
        ],
    }
]

# Convert the messages to the final prompt
processor = AutoProcessor.from_pretrained(model_name)
prompt = processor.apply_chat_template(
    messages, tokenize=False, add_generation_prompt=True
)

# Prepare a function to process receipts
building_generator = outlines.generate.json(
    model,
    Building,

    # Greedy sampling is a good idea for numeric
    # data extraction -- no randomness.
    sampler=outlines.samplers.greedy()
)

# Generate the building information
result = building_generator(prompt, [image])
print(result)

# Also print the JSON
result_json = result.model_dump_json()
print(result_json)

# category,code,name,link
# area,ar1,< 55m2,
# area,ar2,55 - 124m2,
# area,ar3,125 - 249m2,
# area,ar4,250 - 629m2,
# area,ar5,630 - 3100m2,
# area,ar6,> 3100m2,
# façade,ewno,"No exterior wall material, structural material showing",https://taxonomy.openquake.org/?q=ewno
# façade,ewc,Concrete,https://taxonomy.openquake.org/?q=ewc
# façade,ewg,Glass,https://taxonomy.openquake.org/?q=ewg
# façade,ewe,Earth,https://taxonomy.openquake.org/?q=ewe
# façade,ewma,Masonry,https://taxonomy.openquake.org/?q=ewma
# façade,ewme,Metal,https://taxonomy.openquake.org/?q=ewme
# façade,ewv,Vegetative,https://taxonomy.openquake.org/?q=ewv
# façade,eww,Wood,https://taxonomy.openquake.org/?q=eww
# façade,ewst,Stucco,https://taxonomy.openquake.org/?q=ewst
# façade,ewpl,"Plastic/vinyl, various",https://taxonomy.openquake.org/?q=ewpl
# façade,ewcb,Cement-based boards,https://taxonomy.openquake.org/?q=ewcb
# façade,ewo,"Material of exterior wall, other",https://taxonomy.openquake.org/?q=ewo
# floors,h1,1 Floor,
# floors,h2,2 Floors,
# floors,hbet3-7,3-7 Floors,
# floors,hbet8,> 8 Floors,
# foundation,fossl,"Shallow foundation, with lateral capacity",https://taxonomy.openquake.org/?q=fossl
# foundation,fosn,"Shallow foundation, no lateral capacity",https://taxonomy.openquake.org/?q=fosn
# foundation,fosdl,"Deep foundation, with lateral capacity",https://taxonomy.openquake.org/?q=fosdl
# foundation,fosdn,"Deep foundation, no lateral capacity",https://taxonomy.openquake.org/?q=fosdn
# foundation,foso,"Foundation, other",https://taxonomy.openquake.org/?q=foso
# lateral load resisting system,ln,No lateral load-resisting system,https://taxonomy.openquake.org/?q=ln
# lateral load resisting system,lfm,Moment frame,https://taxonomy.openquake.org/?q=lfm
# lateral load resisting system,lfinf,Infilled frame,https://taxonomy.openquake.org/?q=lfinf
# lateral load resisting system,lfbr,Braced,https://taxonomy.openquake.org/?q=lfbr
# lateral load resisting system,lpb,Post and beam ,https://taxonomy.openquake.org/?q=lpb
# lateral load resisting system,lwal,Wall ,https://taxonomy.openquake.org/?q=lwal
# lateral load resisting system,ldual,Dual frame-wall system,https://taxonomy.openquake.org/?q=ldual
# lateral load resisting system,lfls,Flat slab/plate or waffle slab,https://taxonomy.openquake.org/?q=lfls
# lateral load resisting system,lflsinf,Infilled flat slab/plate or infilled waffle slab,https://taxonomy.openquake.org/?q=lflsinf
# lateral load resisting system,lh,Hybrid lateral load-resisting system ,https://taxonomy.openquake.org/?q=lh
# lateral load resisting system,lo,Other lateral load-resisting system,https://taxonomy.openquake.org/?q=lo
# material of lateral load resisting system,cu,"Concrete, unreinforced",https://taxonomy.openquake.org/?q=cu
# material of lateral load resisting system,cr,"Concrete, reinforced",https://taxonomy.openquake.org/?q=cr
# material of lateral load resisting system,src,"Concrete, composite with steel section",https://taxonomy.openquake.org/?q=src
# material of lateral load resisting system,s,Steel,https://taxonomy.openquake.org/?q=s
# material of lateral load resisting system,me,Metal (except steel),https://taxonomy.openquake.org/?q=me
# material of lateral load resisting system,mur,"Masonry, unreinforced",https://taxonomy.openquake.org/?q=mur
# material of lateral load resisting system,mcf,"Masonry, confined",https://taxonomy.openquake.org/?q=mcf
# material of lateral load resisting system,mr,"Masonry, reinforced",https://taxonomy.openquake.org/?q=mr
# material of lateral load resisting system,eu,"Earth, unreinforced",https://taxonomy.openquake.org/?q=eu
# material of lateral load resisting system,er,"Earth, reinforced",https://taxonomy.openquake.org/?q=er
# material of lateral load resisting system,w,Wood,https://taxonomy.openquake.org/?q=w
# material of lateral load resisting system,mato,Other material,https://taxonomy.openquake.org/?q=mato
# purpose,res,Residential,https://taxonomy.openquake.org/?q=res
# purpose,com,Commercial and public,https://taxonomy.openquake.org/?q=com
# purpose,mix,Mixed use,https://taxonomy.openquake.org/?q=mix
# purpose,ind,Industrial,https://taxonomy.openquake.org/?q=ind
# purpose,agr,Agriculture,https://taxonomy.openquake.org/?q=agr
# purpose,ass,Assembly,https://taxonomy.openquake.org/?q=ass
# purpose,gov,Government,https://taxonomy.openquake.org/?q=gov
# purpose,edu,Education,https://taxonomy.openquake.org/?q=edu
# purpose,oco,Other occupancy type,https://taxonomy.openquake.org/?q=oco
# roof_material,rmn,Concrete roof without additional covering,https://taxonomy.openquake.org/?q=rmn
# roof_material,rmt1,Clay or concrete tile,https://taxonomy.openquake.org/?q=rmt1
# roof_material,rmt2,Fibre cement or metal tile,https://taxonomy.openquake.org/?q=rmt2
# roof_material,rmt3,Membrane roofing,https://taxonomy.openquake.org/?q=rmt3
# roof_material,rmt4,Slate,https://taxonomy.openquake.org/?q=rmt4
# roof_material,rmt5,Stone slab,https://taxonomy.openquake.org/?q=rmt5
# roof_material,rmt6,Metal or asbestos sheets,https://taxonomy.openquake.org/?q=rmt6
# roof_material,rmt7,Wooden and asphalt shingles,https://taxonomy.openquake.org/?q=rmt7
# roof_material,rmt8,Vegetative,https://taxonomy.openquake.org/?q=rmt8
# roof_material,rmt9,Earthen,https://taxonomy.openquake.org/?q=rmt9
# roof_material,rmt10,Solar panelled roofs,https://taxonomy.openquake.org/?q=rmt10
# roof_material,rmt11,Tensile membrane or fabric roof,https://taxonomy.openquake.org/?q=rmt11
# roof_material,rmto,"Roof covering, other",https://taxonomy.openquake.org/?q=rmto
# roofshape,rsh1,Flat,https://taxonomy.openquake.org/?q=rsh1
# roofshape,rsh2,Pitched with gable ends,https://taxonomy.openquake.org/?q=rsh2
# roofshape,rsh3,Pitched and hipped,https://taxonomy.openquake.org/?q=rsh3
# roofshape,rsh4,Pitched with dormers,https://taxonomy.openquake.org/?q=rsh4
# roofshape,rsh5,Monopitch,https://taxonomy.openquake.org/?q=rsh5
# roofshape,rsh6,Sawtooth,https://taxonomy.openquake.org/?q=rsh6
# roofshape,rsh7,Curved,https://taxonomy.openquake.org/?q=rsh7
# roofshape,rsh8,Complex regular,https://taxonomy.openquake.org/?q=rsh8
# roofshape,rsh9,Complex irregular,https://taxonomy.openquake.org/?q=rsh9
# roofshape,rsho,"Roof shape, other",https://taxonomy.openquake.org/?q=rsho
# year built,yr1,< 1880,
# year built,yr2,1880 - 1919,
# year built,yr3,1920 - 1944,
# year built,yr4,1945 - 1968,
# year built,yr5,1969 - 1988,
# year built,yr6,1990 - 2008,
# year built,yr7,> 2008,