LLMOpsツールを色々使ってみる

import weave
import json
from openai import OpenAI

@weave.op()
def extract_fruit(sentence: str) -> dict:
    client = OpenAI()

    response = client.chat.completions.create(
    model="gpt-3.5-turbo-1106",
    messages=[
        {
            "role": "system",
            "content": "You will be provided with unstructured data, and your task is to parse it one JSON dictionary with fruit, color and flavor as keys."
        },
        {
            "role": "user",
            "content": sentence
        }
        ],
        temperature=0.7,
        response_format={ "type": "json_object" }
    )
    extracted = response.choices[0].message.content
    return json.loads(extracted)

weave.init('intro-example')

sentence = "There are many fruits that were found on the recently discovered planet Goocrux. There are neoskizzles that grow there, which are purple and taste like candy."

extract_fruit(sentence)

from weave.flow.scorer import MultiTaskBinaryClassificationF1
sentences = [
    "最近発見されたグークラックス惑星では、多くの果物が見つかりました。そこにはネオスキズルという果物が育っており、紫色でキャンディのような味がします。",
    "パウニッツは鮮やかな緑色で、甘いというよりも風味豊かです。",
    "最後に、グロウルという果物があり、非常に酸っぱくて苦い味わいで酸性と刺激性があり、淡いオレンジ色を帯びています。"
]

labels = [
    {'fruit': 'ネオスキズル', 'color': '紫色', 'flavor': 'キャンディのような'},
    {'fruit': 'パウニッツ', 'color': '鮮やかな緑色', 'flavor': '風味豊か'},
    {'fruit': 'グロウル', 'color': '淡いオレンジ色', 'flavor': '酸っぱくて苦い'}
]

examples = [
    {'id': '0', 'sentence': sentences[0], 'target': labels[0]},
    {'id': '1', 'sentence': sentences[1], 'target': labels[1]},
    {'id': '2', 'sentence': sentences[2], 'target': labels[2]}
]

@weave.op()
def fruit_name_score(target: dict, model_output: dict) -> dict:
    return {'correct': target['fruit'] == model_output['fruit']}


model = ExtractFruitsModel(model_name='gpt-3.5-turbo-1106',
                          prompt_template='以下の文章から（"fruit": <文字列>、"color": <文字列>、"flavor": <文字列>）の情報をJSON形式で抽出してください: {sentence}')

# Finally, we run an evaluation of this model.
# This will generate a prediction for each input example, and then score it with each scoring function.
evaluation = weave.Evaluation(
    name='fruit_eval',
    dataset=examples, scorers=[MultiTaskBinaryClassificationF1(class_names=["fruit", "color", "flavor"]), fruit_name_score],
)
# if you're in a Jupyter Notebook, run:
await evaluation.evaluate(model)

Evaluation summary
{
    'MultiTaskBinaryClassificationF1': {
        'fruit': {'f1': 1.0, 'precision': 1.0, 'recall': 1.0},
        'color': {'f1': 1.0, 'precision': 1.0, 'recall': 1.0},
        'flavor': {'f1': 0.5, 'precision': 0.3333333333333333, 'recall': 1.0}
    },
    'fruit_name_score': {'correct': {'true_count': 3, 'true_fraction': 1.0}},
    'model_latency': {'mean': 5.273736238479614}
}

npx promptfoo@latest init