Image Segmentation Models Compared

from sam2.build_sam import build_sam2
from sam2.sam2_image_predictor import SAM2ImagePredictor
from PIL import Image
import numpy as np

# Load model
checkpoint = "sam2.1_hiera_large.pt"
model_cfg = "configs/sam2.1/sam2.1_hiera_l.yaml"
predictor = SAM2ImagePredictor(build_sam2(model_cfg, checkpoint))

# Load image
image = np.array(Image.open("photo.jpg"))
predictor.set_image(image)

# Segment with point prompt (x, y)
point_coords = np.array([[500, 375]])
point_labels = np.array([1])  # 1 = foreground

masks, scores, logits = predictor.predict(
    point_coords=point_coords,
    point_labels=point_labels,
    multimask_output=True,
)

# Best mask
best_mask = masks[np.argmax(scores)]
print(f"Mask shape: {best_mask.shape}, Score: {scores.max():.3f}")

from sam2.build_sam import build_sam2_video_predictor
import numpy as np

predictor = build_sam2_video_predictor(
    "configs/sam2.1/sam2.1_hiera_l.yaml",
    "sam2.1_hiera_large.pt",
)

# Initialize with a video directory of JPEG frames
state = predictor.init_state(video_path="./video_frames/")

# Add prompt on frame 0
_, _, masks = predictor.add_new_points_or_box(
    inference_state=state,
    frame_idx=0,
    obj_id=1,
    points=np.array([[210, 350]], dtype=np.float32),
    labels=np.array([1], dtype=np.int32),
)

# Propagate through entire video
for frame_idx, obj_ids, masks in predictor.propagate_in_video(state):
    for obj_id, mask in zip(obj_ids, masks):
        binary = (mask[0] > 0.0).cpu().numpy()
        print(f"Frame {frame_idx}, Object {obj_id}: {binary.sum()} pixels")

from transformers import (
    Mask2FormerForUniversalSegmentation,
    Mask2FormerImageProcessor,
)
from PIL import Image

# Load panoptic segmentation model
processor = Mask2FormerImageProcessor.from_pretrained(
    "facebook/mask2former-swin-large-coco-panoptic"
)
model = Mask2FormerForUniversalSegmentation.from_pretrained(
    "facebook/mask2former-swin-large-coco-panoptic"
)

image = Image.open("street.jpg")
inputs = processor(images=image, return_tensors="pt")

with torch.no_grad():
    outputs = model(**inputs)

# Panoptic segmentation
result = processor.post_process_panoptic_segmentation(
    outputs, target_sizes=[image.size[::-1]]
)[0]

seg_map = result["segmentation"]  # H x W tensor of segment IDs
segments = result["segments_info"]

for seg in segments:
    print(f"ID: {seg['id']}, Label: {seg['label_id']}, "
          f"Score: {seg['score']:.3f}, Area: {(seg_map == seg['id']).sum()}")

from autodistill_grounded_sam_2 import GroundedSAM2
from autodistill.detection import CaptionOntology
from PIL import Image

# Define what to segment via text
ontology = CaptionOntology({
    "car": "car",
    "person": "person",
    "traffic light": "traffic light",
})

model = GroundedSAM2(ontology=ontology)

# Run on an image
results = model.predict("street_scene.jpg")

for detection in results:
    print(f"Class: {detection.class_name}")
    print(f"Confidence: {detection.confidence:.3f}")
    print(f"Mask pixels: {detection.mask.sum()}")