yuvansharma/caip-vitl256

CAIP (Contrastive Action-Image Pre-training)

Project page

A CLIP/SigLIP-style encoder that aligns images, text, and egocentric hand-pose actions in a shared embedding space.

from transformers import AutoModel, AutoProcessor
from PIL import Image
import torch

REPO = "yuvansharma/caip-vitl256"
model = AutoModel.from_pretrained(REPO, trust_remote_code=True).eval()
processor = AutoProcessor.from_pretrained(REPO, trust_remote_code=True)

image = Image.open("example.png").convert("RGB")
inputs = processor(images=image, text="pick up the red cup", return_tensors="pt")

with torch.no_grad():
    out = model(**inputs)

# out.image_pooled    [B, 1024]       text-conditioned pooled image embedding
# out.patch_features  [B, 256, 1024]  patch tokens
# out.text_tokens     [B, 64, 1024]   text token embeddings
# out.text_pooled     [B, 1024]       pooled text embedding

# Encode an action chunk [B, T=64, 378] (42 joints x 9D) into the same space -> [B, 1024].
# Actions are delta hand-pose targets and must be normalized with the training
# statistics (mean-std, delta) before encoding -- see action_stats.json in this repo.
import json
from huggingface_hub import hf_hub_download

stats = json.load(open(hf_hub_download(REPO, "action_stats.json")))["delta"]
mean = torch.tensor(stats["mean"]).view(1, 64, 378)
std  = torch.tensor(stats["std"]).view(1, 64, 378)

actions = torch.randn(1, 64, 378)               # raw delta hand-pose chunk [B, 64, 378]
actions = (actions - mean) / std                # normalize (per-timestep, per-dim)
with torch.no_grad():
    action_emb = model.encode_action(actions)   # [B, 1024]

For a smaller download (~1.75 GB), load the bf16 weights with revision="bf16".

Citation

@misc{sharma2026contrastiveactionimagepretrainingvisuomotor,
      title={Contrastive Action-Image Pre-training for Visuomotor Control},
      author={Yuvan Sharma and Dantong Niu and Anirudh Pai and Zekai Wang and Zhuoyang Liu and Baifeng Shi and Stefano Saravalle and Boning Shao and Ruijie Zheng and Jing Wang and Konstantinos Kallidromitis and Yusuke Kato and Fabio Galasso and Yuke Zhu and Danfei Xu and Linxi "Jim" Fan and Jitendra Malik and Trevor Darrell and Roei Herzig},
      year={2026},
      eprint={2606.17256},
      archivePrefix={arXiv},
      primaryClass={cs.RO},
      url={https://arxiv.org/abs/2606.17256},
}