メモ: GPT-5のプロンプト移行と新オプティマイザーを用いた改善

!git clone https://github.com/openai/openai-cookbook

!cp -pir openai-cookbook/examples/gpt-5/prompt-optimization-cookbook/scripts .
!cp -pir openai-cookbook/examples/gpt-5/prompt-optimization-cookbook/{run_FailSafeQA.py,llm_as_judge.txt} .

openai
matplotlib 
seaborn
datasets

!pip freeze | egrep -i "^(openai|matplotlib|seaborn|datasets)"

datasets==4.0.0
matplotlib==3.10.0
matplotlib-inline==0.1.7
matplotlib-venn==1.1.2
openai==1.99.1
seaborn==0.13.2

!pip install -U openai

(snip)
Successfully installed openai-1.99.6

from google.colab import userdata
import os

os.environ['OPENAI_API_KEY'] = userdata.get('OPENAI_API_KEY')

baseline_prompt = """
Linuxでタスクを解決するPythonコードを書いてください。高速かつ軽量にしてください。

- 標準ライブラリを優先してください。ただし、外部パッケージの方が簡単な場合は使っても構いません。
- メモリ消費を抑えるため、入力は一度だけストリーム処理してください。ただし、解決が明確になるなら再読込やキャッシュも許可します。
- 結果は正確を目指してください。ただし、実際の結果に影響しない場合は近似的な手法でも構いません。
- グローバル変数の使用は避けてください。ただし、 top_k のような便利なグローバル変数は使っても構いません。
- コメントは最小限にしてください。必要な場合は簡単な説明を加えてください。
- 結果は自然で人間にとって分かりやすい順に並べてください。必要に応じて厳密な同点ルールに従ってください。

すべてのimportを含めた、1つのPythonコードブロック内に収まる自己完結型のスクリプトのみを出力してください。
"""

from scripts.gen_baseline import generate_baseline_topk

MODEL = "gpt-5"
N_RUNS = 30
CONCURRENCY = 10
OUTPUT_DIR = "results_topk_baseline"

USER_PROMPT = """
タスク:
グローバル変数 text（str型）と k（int型）が与えられたとき、最も頻出する単語Top-Kを抽出してください。

トークン化:
- ASCIIの正規表現を使い、大文字小文字を区別せずにトークン化してください。各トークンを小文字にしてください。文字列全体を一括で小文字にする必要はありません。
- トークンはASCIIの[a-z0-9]+の連続部分文字列です。それ以外の文字は区切り文字として扱ってください。

出力:
- top_k という、(単語, 出現回数) のタプルのリストを定義してください。
- カウントの降順、同数の場合は単語の昇順でソートしてください。
- リストの長さは min(k, ユニークな単語数) としてください。

注意:
- 与えられたグローバル変数でそのまま実行できるようにしてください。ファイルやネットワークI/Oは行わないでください。
"""

generate_baseline_topk(
    model=MODEL,
    n_runs=N_RUNS,
    concurrency=CONCURRENCY,
    output_dir=OUTPUT_DIR,
    dev_prompt=baseline_prompt,
    user_prompt=USER_PROMPT,
)

[1/30] Wrote results_topk_baseline/run_02.py — remaining: 29
[2/30] Wrote results_topk_baseline/run_03.py — remaining: 28
[3/30] Wrote results_topk_baseline/run_09.py — remaining: 27
[4/30] Wrote results_topk_baseline/run_05.py — remaining: 26
[5/30] Wrote results_topk_baseline/run_07.py — remaining: 25
[6/30] Wrote results_topk_baseline/run_06.py — remaining: 24
(snip)
[29/30] Wrote results_topk_baseline/run_28.py — remaining: 1
[30/30] Wrote results_topk_baseline/run_29.py — remaining: 0
Done. Saved 30 files to: /content/results_topk_baseline
PosixPath('results_topk_baseline')

import re
from typing import List, Tuple

def _top_k_words(text: str, k: int) -> List[Tuple[str, int]]:
    if not isinstance(text, str):
        raise TypeError("text must be str")
    if not isinstance(k, int):
        raise TypeError("k must be int")
    if k <= 0:
        return []

    pat = re.compile(r'[A-Za-z0-9]+', flags=re.ASCII)
    counts = {}
    for m in pat.finditer(text):
        w = m.group(0).lower()
        counts[w] = counts.get(w, 0) + 1

    if not counts:
        return []

    items = sorted(counts.items(), key=lambda kv: (-kv[1], kv[0]))
    return items[:k]

# Expect global variables: text (str), k (int)
top_k = _top_k_words(text, k)

import re
from collections import Counter

# グローバル変数 text (str), k (int) が与えられている前提
# 出力: top_k -> List[Tuple[str, int]]

def _top_k_words(text: str, k: int):
    cnt = Counter()
    # ASCIIの[a-z0-9]+を大文字小文字区別せずに抽出し、各トークンを小文字化
    for m in re.finditer(r'[a-z0-9]+', text, flags=re.IGNORECASE):
        cnt[m.group(0).lower()] += 1
    if k <= 0 or not cnt:
        return []
    # カウント降順、同数は単語昇順
    items = sorted(cnt.items(), key=lambda kv: (-kv[1], kv[0]))
    return items[:k]

top_k = _top_k_words(text, k)

from scripts.topk_eval import evaluate_folder

evaluate_folder(
    folder_path="results_topk_baseline",
    k=500,
    scale_tokens=5_000_000,
    csv_path="run_results_topk_baseline.csv",
)

===== SUMMARY =====
Total evaluated runs: 30
Compilation/Execution Success: 30/30 (100.00%)
Violations (static scan): 0
Average Execution Time (successful): 30.336844 s
Average Peak Memory (successful): 710.49 KB
Exact matches: 30/30
Sorted correctly: 30/30

CSV written to: results_topk_baseline/run_results_topk_baseline.csv
Summary JSON written to: results_topk_baseline/run_results_topk_baseline_summary.json
Summary TXT written to: results_topk_baseline/run_results_topk_baseline_summary.txt
{'total_runs': 30,
 'successes': 30,
 'avg_exec_time_s': 30.336843936733324,
 'avg_peak_mem_kb': 710.4897135416667,
 'exact_matches': 30,
 'sorted_correctly': 30,
 'violations': 0,
 'csv': 'results_topk_baseline/run_results_topk_baseline.csv',
 'folder': 'results_topk_baseline',
 'k': 500,
 'scale_tokens': 5000000}

!ls -lt results_topk_baseline

total 136
-rw-r--r-- 1 root root 7203 Aug  9 16:41 run_results_topk_baseline.csv
-rw-r--r-- 1 root root  330 Aug  9 16:41 run_results_topk_baseline_summary.json
-rw-r--r-- 1 root root  326 Aug  9 16:41 run_results_topk_baseline_summary.txt
-rw-r--r-- 1 root root  631 Aug  9 16:25 run_30.py
-rw-r--r-- 1 root root  544 Aug  9 16:24 run_02.py
-rw-r--r-- 1 root root  667 Aug  9 16:24 run_01.py
(snip)

===== SUMMARY =====
Total evaluated runs: 30
Compilation/Execution Success: 30/30 (100.00%)
Violations (static scan): 0
Average Execution Time (successful): 30.336844 s
Average Peak Memory (successful): 710.49 KB
Exact matches: 30/30
Sorted correctly: 30/30
CSV written to: results_topk_baseline/run_results_topk_baseline.csv

{
  "total_runs": 30,
  "successes": 30,
  "avg_exec_time_s": 30.336843936733324,
  "avg_peak_mem_kb": 710.4897135416667,
  "exact_matches": 30,
  "sorted_correctly": 30,
  "violations": 0,
  "csv": "results_topk_baseline/run_results_topk_baseline.csv",
  "folder": "results_topk_baseline",
  "k": 500,
  "scale_tokens": 5000000
}

optimized_prompt = """
# 目的
- Linux環境でタスクを解決するPythonコードを書いてください。高効率かつ軽量であることを重視します。

# 指示
- 標準ライブラリの使用を優先してください。ただし外部パッケージがより簡単な場合は使用可能です。
- メモリ消費抑制の観点から、入力データはストリーム処理で一度だけ厳密に読み込むようにしてください。再度の読み込みやキャッシュは、解決のために明確に必要な場合のみ許容します。
- 結果の正確性を重視してください。ただし、実際の結果に影響しない場合は類似的な手法の利用も認めます。
- グローバル変数の使用は避けてください。ただし、`top_k` など便利なグローバル変数は許容します。
- コメントは最小限に留め、必要時のみ簡単な説明を記載してください。
- 結果は自然で人間にとって分かりやすい順序で並べてください。必要に応じて厳密な同値判定ルールに従ってください。

# 出力形式
- すべてのimportを含め、1つのPythonコードブロック内に収めた自己完結型スクリプトのみを出力してください。

# 実施手順
- 開始時に、コード作成のための簡潔なチェックリスト（3-7項目、概念的な内容）を提示してください。

# その他
- reasoning_effort=medium とし、内容の複雑さに応じて解決・説明の深度を決めてください。ツール呼び出しや作業結果の出力は簡潔に、コード出力は十分な情報が含まれるようにしてください。
"""

from scripts.gen_optimized import generate_optimized_topk

MODEL = "gpt-5"
N_RUNS = 30
CONCURRENCY = 10
OUTPUT_DIR = "results_topk_optimized"

USER_PROMPT = """
タスク:
グローバル変数 text（str型）と k（int型）が与えられたとき、最も頻出する単語Top-Kを抽出してください。

トークン化:
- ASCIIの正規表現を使い、大文字小文字を区別せずにトークン化してください。各トークンを小文字にしてください。文字列全体を一括で小文字にする必要はありません。
- トークンはASCIIの[a-z0-9]+の連続部分文字列です。それ以外の文字は区切り文字として扱ってください。

出力:
- top_k という、(単語, 出現回数) のタプルのリストを定義してください。
- カウントの降順、同数の場合は単語の昇順でソートしてください。
- リストの長さは min(k, ユニークな単語数) としてください。

注意:
- 与えられたグローバル変数でそのまま実行できるようにしてください。ファイルやネットワークI/Oは行わないでください。
"""

generate_optimized_topk(
    model=MODEL,
    n_runs=N_RUNS,
    concurrency=CONCURRENCY,
    output_dir=OUTPUT_DIR,
    dev_prompt=optimized_prompt,
    user_prompt=USER_PROMPT,
)

[1/30] Wrote results_topk_optimized/run_05.py — remaining: 29
[2/30] Wrote results_topk_optimized/run_08.py — remaining: 28
[3/30] Wrote results_topk_optimized/run_03.py — remaining: 27
[4/30] Wrote results_topk_optimized/run_07.py — remaining: 26
[5/30] Wrote results_topk_optimized/run_04.py — remaining: 25
[6/30] Wrote results_topk_optimized/run_02.py — remaining: 24
[7/30] Wrote results_topk_optimized/run_01.py — remaining: 23
[8/30] Wrote results_topk_optimized/run_10.py — remaining: 22
[9/30] Wrote results_topk_optimized/run_06.py — remaining: 21
[10/30] Wrote results_topk_optimized/run_14.py — remaining: 20
[11/30] Wrote results_topk_optimized/run_16.py — remaining: 19
[12/30] Wrote results_topk_optimized/run_13.py — remaining: 18
[13/30] Wrote results_topk_optimized/run_15.py — remaining: 17
[14/30] Wrote results_topk_optimized/run_11.py — remaining: 16
[15/30] Wrote results_topk_optimized/run_18.py — remaining: 15
[16/30] Wrote results_topk_optimized/run_09.py — remaining: 14
[17/30] Wrote results_topk_optimized/run_20.py — remaining: 13
[18/30] Wrote results_topk_optimized/run_12.py — remaining: 12
[19/30] Wrote results_topk_optimized/run_21.py — remaining: 11
[20/30] Wrote results_topk_optimized/run_19.py — remaining: 10
[21/30] Wrote results_topk_optimized/run_24.py — remaining: 9
[22/30] Wrote results_topk_optimized/run_17.py — remaining: 8
[23/30] Wrote results_topk_optimized/run_26.py — remaining: 7
[24/30] Wrote results_topk_optimized/run_25.py — remaining: 6
[25/30] Wrote results_topk_optimized/run_29.py — remaining: 5
[26/30] Wrote results_topk_optimized/run_23.py — remaining: 4
[27/30] Wrote results_topk_optimized/run_22.py — remaining: 3
[28/30] Wrote results_topk_optimized/run_27.py — remaining: 2
[29/30] Wrote results_topk_optimized/run_28.py — remaining: 1
[30/30] Wrote results_topk_optimized/run_30.py — remaining: 0
Done. Saved 30 files to: /content/results_topk_optimized
PosixPath('results_topk_optimized')

from scripts.topk_eval import evaluate_folder

evaluate_folder(
    folder_path="results_topk_optimized",
    k=500,
    scale_tokens=5_000_000,
    csv_path="run_results_topk_optimized.csv",
)

===== SUMMARY =====
Total evaluated runs: 30
Compilation/Execution Success: 30/30 (100.00%)
Violations (static scan): 0
Average Execution Time (successful): 29.024439 s
Average Peak Memory (successful): 668.88 KB
Exact matches: 29/30
Sorted correctly: 30/30

CSV written to: results_topk_optimized/run_results_topk_optimized.csv
Summary JSON written to: results_topk_optimized/run_results_topk_optimized_summary.json
Summary TXT written to: results_topk_optimized/run_results_topk_optimized_summary.txt
{'total_runs': 30,
 'successes': 30,
 'avg_exec_time_s': 29.024438657599905,
 'avg_peak_mem_kb': 668.8825846354167,
 'exact_matches': 29,
 'sorted_correctly': 30,
 'violations': 0,
 'csv': 'results_topk_optimized/run_results_topk_optimized.csv',
 'folder': 'results_topk_optimized',
 'k': 500,
 'scale_tokens': 5000000}

!ls -lt results_topk_optimized

total 136
-rw-r--r-- 1 root root  333 Aug  9 18:24 run_results_topk_optimized_summary.json
-rw-r--r-- 1 root root  328 Aug  9 18:24 run_results_topk_optimized_summary.txt
-rw-r--r-- 1 root root 7203 Aug  9 18:24 run_results_topk_optimized.csv
-rw-r--r-- 1 root root  661 Aug  9 18:06 run_30.py
-rw-r--r-- 1 root root  760 Aug  9 18:06 run_28.py
-rw-r--r-- 1 root root 1340 Aug  9 18:06 run_27.py
-rw-r--r-- 1 root root  799 Aug  9 18:06 run_22.py
(snip)

# SYSTEM PROMPT

You are an expert judge responsible for evaluating the quality of outputs produced by language models, specifically focusing on how well they follow provided task instructions and the overall code quality (if the output is code). Your evaluation must be fair, thorough, and well-reasoned.

First, carefully read and understand:
- The task instructions provided.
- The output (text or code) produced by the model.

**Your tasks:**
1. **Analyze Task Adherence:**  
   - Step-by-step, explain how the output matches or fails to meet each part of the instructions.  
   - Highlight all instances where instructions are fully, partially, or not followed.  
   - Consider any ambiguities and how reasonable the model's choices are.

2. **Evaluate Code Quality (if applicable):**  
   - Step-by-step, assess the clarity, correctness, efficiency, readability, structure, maintainability, and best practices of the code.
   - Identify any bugs, inefficiencies, or stylistic issues, explaining your reasoning for each point.
   - If the output is not code, skip this step and say so.

**Reasoning Process:**  
- Always **reason first**—do not state your final assessment until after you have fully documented your reasoning about task adherence and code quality.
- Structure your findings in two sections: "Reasoning" (step-by-step analysis), followed by "Final Judgement."

**Output Format:**  
Respond ONLY in the following JSON structure (replace bracketed areas with your content):

{
  "reasoning": {
    "task_adherence": "[Step-by-step analysis of how well the output follows all instructions, including any missed or ambiguous points.]",
    "code_quality": "[Step-by-step code quality assessment, or short note if not applicable.]"
  },
  "final_judgement": {
    "adherence_score": [integer 1-5, where 5=perfectly follows instructions, 1=ignores or subverts instructions],
    "code_quality_score": [integer 1-5, where 5=exceptional code quality, 1=severe issues or missing code; use null if not code],
    "comments": "[Short summary of main issues, overall impression, or suggestions for improvement.]"
  }
}

**Scoring Guidelines:**
- 5 = Exceptional; all instructions/code quality criteria met to a high standard.
- 4 = Good; minor issues.
- 3 = Average; some issues or minor omissions.
- 2 = Major issues or omissions.
- 1 = Severe failure to follow task or produce usable code.

**EXAMPLES:**

**Example 1:**
Input Instructions: "Write a function that returns the sum of two numbers."  
Model Output:  
def add(a, b):  
  return a + b

JSON Output:
{
  "reasoning": {
    "task_adherence": "The output defines a function named 'add' with two arguments and returns their sum as instructed.",
    "code_quality": "The code is concise, correct, and follows Python conventions. No issues."
  },
  "final_judgement": {
    "adherence_score": 5,
    "code_quality_score": 5,
    "comments": "Task followed perfectly; code is clean and correct."
  }
}

**Example 2:**
Input Instructions: "Write a function that checks if a string is a palindrome, ignoring case and spaces."  
Model Output:  
def is_palindrome(s):  
  return s == s[::-1]

JSON Output:
{
  "reasoning": {
    "task_adherence": "The output defines a function, but it does not ignore case and spaces, as required.",
    "code_quality": "The code is correct for a basic palindrome check, but it does not implement the extra requirements."
  },
  "final_judgement": {
    "adherence_score": 2,
    "code_quality_score": 4,
    "comments": "Major task requirement (ignoring case/spaces) omitted; otherwise, basic code is clean."
  }
}

**Important reminders:**  
- Always provide reasoning before your ratings and summary.
- Never start with a conclusion.
- Use the JSON schema strictly.
- Use step-by-step analysis, and detailed explanations, and adjust your scores according to the scoring guidelines.

**Reminder:**
Evaluate how well the output follows instructions first, provide detailed reasoning, then give your overall numeric ratings for task adherence and code quality. Output in the specified JSON format only Do not be nice on scoring, be fair.

from scripts.llm_judge import judge_folder

# ベースラインの結果に対して、LLM-as-judge を実行
judge_folder(
    results_dir="results_topk_baseline",
    out_dir=None,  # results_llm_as_judge_baseline に自動マッピング
    model="gpt-5",
    system_prompt_path="llm_as_judge.txt",
    task_text=None,  # デフォルトのタスク説明を使用する
    concurrency=6,
)

# 最適化後の結果に対して、LLM-as-judge を実行
judge_folder(
    results_dir="results_topk_optimized",
    out_dir=None,  # results_llm_as_judge_optimized に自動マッピング
    model="gpt-5",
    system_prompt_path="llm_as_judge.txt",
    task_text=None,
    concurrency=6,
)

(snip)

# Default task text aligned with the Top-K evaluation used in the notebook
DEFAULT_TASK_TEXT = (
    "Your task:\n"
    "Compute the exact Top-K most frequent tokens from a given text.\n\n"
    "Tokenization:\n"
    "- Case-insensitive tokenization using an ASCII regex; produce lowercase tokens. Lowercasing the entire text is NOT required (per-token lowercasing is acceptable).\n"
    "- Tokens are ASCII [a-z0-9]+ sequences; all other characters are separators (use a regex).\n\n"
    "Inputs:\n"
    "- Two globals are provided: text (string) and k (int). Do not reassign them.\n\n"
    "Requirements:\n"
    "1) Compute Top-K sorted by count desc, then token asc (i.e., sort key = (-count, token)).\n"
    "2) Set top_k to a list of (token, count) tuples, length = min(k, number of unique tokens).\n"
    "3) Handle edge cases: if k <= 0, top_k = [].\n"
    "4) Do not use input(), file I/O, or network access. The script must run as-is with the provided globals.\n\n"
    "Output contract:\n"
    "- At the end of execution, top_k must be defined exactly as described.\n"
    "- Optional: if printing, print only top_k on the last line as a Python literal or JSON.\n\n"
    "Note:\n"
    "- Do not rely on Counter.most_common tie ordering; implement the specified sort.\n"
)

(snip)

from pathlib import Path
import importlib
import scripts.results_summarizer as rs
from IPython.display import Markdown, display

importlib.reload(rs)

fig = rs.render_charts(
    quant_baseline=Path("results_topk_baseline")/"run_results_topk_baseline.csv",
    quant_optimized=Path("results_topk_optimized")/"run_results_topk_optimized.csv",
    judge_baseline=Path("results_llm_as_judge_baseline")/"judgement_summary.csv",
    judge_optimized=Path("results_llm_as_judge_optimized")/"judgement_summary.csv",
    auto_display=True,
    close_after=True,
)
md = rs.build_markdown_summary(
    quant_baseline=Path("results_topk_baseline")/"run_results_topk_baseline.csv",
    quant_optimized=Path("results_topk_optimized")/"run_results_topk_optimized.csv",
    judge_baseline=Path("results_llm_as_judge_baseline")/"judgement_summary.csv",
    judge_optimized=Path("results_llm_as_judge_optimized")/"judgement_summary.csv",
)

display(Markdown(md))

print(md)