OpenAIのChatCompletionモデルのレスポンス性能を計測・比較してみた

!pip install --upgrade openai

from google.colab import userdata
import os

os.environ["OPENAI_API_KEY"] = userdata.get('OPENAI_API_KEY')

import time
from functools import wraps
import statistics
from openai import OpenAI

def timing_decorator(repeats=1):
    def decorator(func):
        @wraps(func)
        def wrapper(*args, **kwargs):
            times = []
            results = []
            for _ in range(repeats):
                start_time = time.time()
                result = func(*args, **kwargs)
                results.append(result)
                end_time = time.time()
                times.append(end_time - start_time)
            avg_time = round(sum(times) / len(times),2)
            max_time = round(max(times),2)
            min_time = round(min(times),2)
            median_time = round(statistics.median(times),2)
            timing_info = {
                'avg': avg_time,
                'max': max_time,
                'min': min_time,
                'med': median_time,
                'times': [round(t, 2) for t in times]
            }
            return results[0] if repeats == 1 else results, timing_info
        return wrapper
    return decorator

@timing_decorator(repeats=10)
def openai_completion(client, model, messages):
    completion = client.chat.completions.create(
        model=model,
        messages=messages
    )
    return completion.choices[0].message.content

models = ["gpt-3.5-turbo", "gpt-3.5-turbo-1106", "gpt-4", "gpt-4-1106-preview"]

max_model_length = max(len(model) for model in models)
format_string = "{:<20} : {:>10} {:>10} {:>10} {:>10}"

client = OpenAI()

# OpenAIの公式ドキュメントのサンプルプロンプト
messages = [
    {"role": "system", "content": "You are a poetic assistant, skilled in explaining complex programming concepts with creative flair."},
    {"role": "user", "content": "Compose a poem that explains the concept of recursion in programming."}
]

print(format_string.format("Model", "AVG", "MIN", "MAX", "MED"))
for model in models:
    res = openai_completion(client, model, messages)[1]
    print(format_string.format(model, res["avg"], res["min"], res["max"], res["med"]))

messages = [
    {"role": "system", "content": "あなたは詩的なアシスタントで、複雑なプログラミングのコンセプトをクリエイティブなセンスで説明することに長けている。"},
    {"role": "user", "content": "プログラミングにおける再帰の概念を説明する詩を作ってください。日本語で。"}
]