Image Captioning with Azure and Chrome

Although images can be readily tagged by AI, the task of tagging is far more niche.

To that end we built a chrome extension that links to Azure functions and Azure

Chrome extension:

Azure code:

import logging
import azure.functions as func
import requests

subscription_key = # Your sub key here
endpoint = "" # Your endpoint here
analyze_url = endpoint + "vision/v3.1/describe" # Access only a description

def caption_image(image): # Takes a variable that is a byte array
    headers = {'Ocp-Apim-Subscription-Key': subscription_key,
            'Content-Type': 'application/octet-stream'}
    params = {'visualFeatures': 'Categories,Description,Color'}
    response = requests.post(
        analyze_url, headers=headers, params=params, data=image).json()
    logging.info(response) # Log response
    return response

def main(req: func.HttpRequest) -> func.HttpResponse:
    name = req.get_body()
    if not name:
        try:
            req_body = req.get_json()
            logging.info(req_body)
        except ValueError:
            pass
        else:
            name = req_body.get('data')

    if name:
        caption = caption_image(name)
        return func.HttpResponse(caption["description"]["captions"][0]["text"].capitalize(),
             status_code=200)
    else:
        logging.info('Error, no data found')
        return func.HttpResponse(
             "There was no data passed to the api.",
             status_code=500
        )