allenai

/

Molmo-7B-D-0924

like 437

Follow

Ai2 1,401

yeah! is ok!

from transformers import AutoModelForCausalLM, AutoProcessor, GenerationConfig,TextIteratorStreamer
from PIL import Image
import requests
from time import perf_counter

model_id = 'allenai/MolmoE-1B-0924'
model_id = 'allenai/Molmo-7B-D-0924'

# load the processor
processor = AutoProcessor.from_pretrained(
    model_id,
    trust_remote_code=True,
    torch_dtype='auto',
    device_map='auto'
)

# load the model
model = AutoModelForCausalLM.from_pretrained(
    model_id,
    trust_remote_code=True,
    torch_dtype='auto',
    device_map='auto'
)

times=[]
start_time = perf_counter()

# process the image and text
inputs = processor.process(
    images=[Image.open(requests.get("https://picsum.photos/id/237/536/354", stream=True).raw)],
    text="Describe this image.",
    #padding=True,
    #return_tensors="pt",
)

# move inputs to the correct device and make a batch of size 1
inputs = {k: v.to(model.device).unsqueeze(0) for k, v in inputs.items()}


# Inference: Generation of the output with TextStreamer
streamer = TextIteratorStreamer(processor.tokenizer, skip_prompt=True, skip_special_tokens=True)
# Use Thread to run generation in background
# Otherwise, the process is blocked until generation is complete
# and no streaming effect can be observed.
from threading import Thread
generation_kwargs = dict(streamer=streamer,tokenizer=processor.tokenizer)
thread = Thread(target=model.generate_from_batch, args=(inputs,GenerationConfig(max_new_tokens=200, stop_strings="<|endoftext|>")), kwargs=generation_kwargs)
thread.start()

generated_text = ""
for new_text in streamer:
    print(new_text,end="")
    times.append(perf_counter() - start_time)
    generated_text += new_text
    start_time = perf_counter()
print(generated_text)

print("times",times)
print("total cost:",sum(times))

run A100 40G on colab

 This image captures a young black Labrador puppy, likely around six months old, sitting on a weathered wooden deck. The puppy's sleek, short fur is entirely black, including its nose, eyes, and ears, which are slightly floppy. The dog is positioned in the center of the frame, looking up directly at the camera with a curious and attentive expression. Its front paws are visible, with one slightly tucked under its body, while its back paws are hidden from view. The wooden deck beneath the puppy is made of light brown planks with visible knots and signs of wear, adding a rustic charm to the scene. The overall composition is simple yet striking, with the puppy's glossy black coat contrasting beautifully against the light wooden background. This image captures a young black Labrador puppy, likely around six months old, sitting on a weathered wooden deck. The puppy's sleek, short fur is entirely black, including its nose, eyes, and ears, which are slightly floppy. The dog is positioned in the center of the frame, looking up directly at the camera with a curious and attentive expression. Its front paws are visible, with one slightly tucked under its body, while its back paws are hidden from view. The wooden deck beneath the puppy is made of light brown planks with visible knots and signs of wear, adding a rustic charm to the scene. The overall composition is simple yet striking, with the puppy's glossy black coat contrasting beautifully against the light wooden background.
times [4.1184672450000335, 0.1259763909999947, 0.04029189400000632, 0.04143443999998908, 0.0393988299999819, 0.039091404000032526, 0.038795281000034265, 0.03873853400000371, 0.03956079600004614, 0.04036988600000768, 0.03858942300007584, 0.03953495199993995, 0.03905083100005413, 0.03855302900001334, 0.0398302880000756, 0.04022023699997135, 0.039297477000104664, 0.040675269999951524, 0.04099108700006582, 0.039244918999997935, 0.03885962599997583, 0.04036941399999705, 0.03898160699998243, 0.04287940599999729, 0.03891989900000681, 0.0390602619999072, 0.04026413600001888, 0.04081131899999946, 0.03898506799998813, 0.04068021300008695, 0.038749141000039344, 0.038894211000069845, 0.04486314300004324, 0.04033141299998988, 0.03950026699999398, 0.03931632000001173, 0.03940672400005951, 0.039351677000013296, 0.0400197849999131, 0.038800519000005806, 0.038625899000066966, 0.03879732400002922, 0.03844953299994813, 0.038615720000052534, 0.046914319000052274, 0.04079339000008986, 0.039373402000023816, 0.03913330300008511, 0.040475439000033475, 0.04238029600003301, 0.04038120399991385, 0.03919793299996854, 0.03985229199997775, 0.03862109700003202, 0.039071728999942934, 0.03986469099993428, 0.0394440919999397, 0.03875089600001047, 0.03882110200004263, 0.039257496999994146, 0.0389652589999514, 0.04003696300003412, 0.03978470599997763, 0.03901883500009262, 0.03892853799993645, 0.038981267999929514, 0.03908283099997334, 0.03991614699998536, 0.03905695899993589, 0.03920949099995141, 0.03937203300006331, 0.038842425999973784, 0.03936440400002539, 0.03967475799993281, 0.039289498000016465, 0.039127414000063254, 0.039001273000053516, 0.039219174999971074, 0.03897438499996042, 0.04117561600003228, 0.03959038599998621, 0.03918464799994581, 0.039056994000020495, 0.039536552000072334, 0.03944160499997906, 0.040302065999981096, 0.039465673000108836, 0.03936466300001484, 0.03982358300004307, 0.039110337000010986, 0.03901152099990668, 0.0401586010000301, 0.03921912300006625, 0.03947212100001707, 0.04689344999997047, 0.04010179099998368, 0.03985182299993539, 0.039740199999982906, 0.03975475099991854, 0.039022434999992583, 0.039878007999959664, 0.040033833000052255, 0.04040963999989344, 0.039808446000051845, 0.039250757999980124, 0.03930387200000496, 0.039651883000033195, 0.03944345299998986, 0.04004076600006101, 0.04078536500003338, 0.040400193999971634, 0.03942205599992121, 0.039750957000023845, 0.04085607100000743, 0.039851016000056916, 0.039828095999951074, 0.03928377599993382, 0.03984419200003231, 0.03994267600000967, 0.040280590000065786, 0.03968490499994459, 0.03918449299999338, 0.04073014799996599, 0.03968114200006312, 0.040680010999949445, 0.04028002400002606, 0.0399213809999992, 0.03987532500002544, 0.039965294000012364, 0.039911655000082646, 0.042511125000032735, 0.04208850199995595, 0.04139655199992376, 0.03982710599996153, 0.04076774899999691, 0.03946689299993977, 0.04186883799991392, 0.03983607899999697, 0.039960805000077926, 0.03999595899995256, 0.04062519199999315, 0.03939121600001272, 0.040155736999963665, 0.03996564400006264, 0.03962665899996409, 0.040226764000067305, 0.04065318400000706, 0.04200165700001435, 0.03946603099996082, 0.040370285999983935, 0.0024612040000420166]
total cost: 10.149745032000396