import requests
from PIL import Imagefrom transformers import AutoProcessor, AutoModelForCausalLMfrom modelscope import snapshot_download

model_dir = snapshot_download("AI-ModelScope/Florence-2-base")model = AutoModelForCausalLM.from_pretrained(model_dir, trust_remote_code=True)processor = AutoProcessor.from_pretrained(model_dir, trust_remote_code=True)
prompt = "<OD>"
url = "https://qinglite-1253448069.cos.ap-shanghai.myqcloud.com/web/31fc426c1575e80b6a4790e320e1f9bec9d7dfa6"image = Image.open(requests.get(url, stream=True).raw)
inputs = processor(text=prompt, images=image, return_tensors="pt")
generated_ids = model.generate(    input_ids=inputs["input_ids"],    pixel_values=inputs["pixel_values"],    max_new_tokens=1024,    do_sample=False,    num_beams=3,)generated_text = processor.batch_decode(generated_ids, skip_special_tokens=False)[0]
parsed_answer = processor.post_process_generation(generated_text, task="<OD>", image_size=(image.width, image.height))
print(parsed_answer)

# 设置pip全局镜像 (加速下载)pip config set global.index-url https://mirrors.aliyun.com/pypi/simple/# 安装ms-swiftgit clone https://github.com/modelscope/swift.gitcd swiftpip install -e '.[llm]'

def process_boundingbox(image, bbox):    """    Process bounding box coordinates relative to the image dimensions.
    Args:        image (PIL.Image): Image object.        bbox (list): List of length four [x1, y1, x2, y2], representing the coordinates of the bounding box's bottom-left (x1, y1) and top-right (x2, y2) corners in pixel coordinates.
    Returns:        list: A list containing a formatted string representing the processed bounding box. The string format is '<loc_x1><loc_y1><loc_x2><loc_y2>'.    """    width = image.width    height = image.height    x1, y1, x2, y2 = [        int(coord / dim * 999) for coord, dim in zip(bbox, [width, height, width, height])    ]    return [f'<loc_{x1}><loc_{y1}><loc_{x2}><loc_{y2}>']

def process_query(object):    """    Process the query for the model to search for the target.        Args:        object (str): The target to be searched by the model.            Returns:        str: The model's input prompt formatted as "<OPEN_VOCABULARY_DETECTION>{object}".    """    return f"<OPEN_VOCABULARY_DETECTION>{object}"

def process_response(object, processed_bbox):    """    Combine the object and processed bounding box into a unified response.        Args:        object (str): The object or target related to the response.        processed_bbox (str): The processed bounding box information.            Returns:        str: A unified response string combining the object and processed bounding box.    """    return object + processed_bbox

from PIL import Imageimage_path = "/coco2014/train2014/COCO_train2014_000000009231.jpg"object = "top left suitcase"bbox = [3,8,380,284]image = Image.open(image_path)processed_bbox = process_boundingbox(image, bbox)query = process_query(object) # <OPEN_VOCABULARY_DETECTION>top left suitcaseresponse = process_response(object, processed_bbox) # top left suitcase<loc_4><loc_18><loc_593>,<loc_666>

```jsonl{"query": "Find <bbox>", "response": "<ref-object>", "images": ["/coco2014/train2014/COCO_train2014_000000001507.jpg"], "objects": "[[\"bottom right sandwich\", [331, 266, 612, 530]]]" }{"query": "Find <ref-object>", "response": "<bbox>", "images": ["/coco2014/train2014/COCO_train2014_000000001507.jpg"], "objects": "[[\"bottom right sandwich\", [331, 266, 612, 530]]]" }```

# Experimental environment: 4090# 7GB GPU memoryCUDA_VISIBLE_DEVICES=0 swift sft \    --model_type florence-2-large-ft \    --dataset refcoco-unofficial-grounding#2000 \    --lora_target_modules ALL
# 2.3GB GPU memoryCUDA_VISIBLE_DEVICES=0 swift sft \    --model_type florence-2-large-ft \    --dataset refcoco-unofficial-grounding#2000 \    --lora_target_modules DEFAULT

# Experimental environment: 4090CUDA_VISIBLE_DEVICES=0 swift infer \    --ckpt_dir output/florence-2-large-ft/vx-xxx/checkpoint-xxx \    --stream false \    --max_new_tokens 1024

<<< <OPEN_VOCABULARY_DETECTION>catInput a media path or URL <<< /coco2014/train2014/COCO_train2014_000000009231.jpg{'Locate cat in the image.': 'cat<loc_643><loc_290><loc_998><loc_773>'}--------------------------------------------------<<< <OPEN_VOCABULARY_DETECTION>dog laying closest to laptopInput a media path or URL <<< /coco2014/train2014/COCO_train2014_000000171435.jpg{'Locate dog laying closest to laptop in the image.': 'dog laying closest to laptop<loc_106><loc_449><loc_660>,<loc_627>'}

from PIL import Image, ImageDrawimport re
def visualize_bounding_box(img, bbox):    img_width, img_height = img.size    numbers = re.findall(r'\d+', bbox)    x1, y1, x2, y2 = [int(num) for num in numbers]
    x1 = int((x1 / 999) * img_width)    y1 = int((y1 / 999) * img_height)    x2 = int((x2 / 999) * img_width)    y2 = int((y2 / 999) * img_height)        draw = ImageDraw.Draw(img)        # 绘制bounding box    draw.rectangle([x1, y1, x2, y2], outline="red", width=2)    img.show()    # img.save("output_image.jpg")
img_path = "/coco2014/train2014/COCO_train2014_000000171435.jpg"img = Image.open(img_path)  bbox = '<loc_643><loc_290><loc_998><loc_773>'visualize_bounding_box(img, bbox)