How to A/B Test AI Models on Your Real User Queries

from openai import OpenAI
import json

client = OpenAI(
    api_key="mb_your_key",
    base_url="https://aibridge-api.com/v1"
)

# Your test queries
test_queries = [
    "Summarize this article...",
    "Write a Python function to...",
    "Translate this to Chinese...",
    # ... 50-100 real queries
]

# Models to test
models_to_test = [
    "deepseek-v4-pro",
    "qwen3-235b-a22b",
    "glm-4-plus",
    "deepseek-v4-flash"
]

# Run A/B test
results = {}
for model in models_to_test:
    scores = []
    for query in test_queries[:10]:  # Test subset first
        response = client.chat.completions.create(
            model=model,
            messages=[{"role": "user", "content": query}]
        )
        output = response.choices[0].message.content
        latency = response.usage.completion_tokens / response.usage.prompt_tokens  # Rough speed metric

        # Score it (your criteria here)
        score = score_output(output, query)  # Your scoring function
        scores.append({"score": score, "latency": latency, "cost": response.usage.total_tokens})

    results[model] = scores

# Compare
for model, scores in results.items():
    avg_score = sum(s["score"] for s in scores) / len(scores)
    avg_latency = sum(s["latency"] for s in scores) / len(scores)
    print(f"{model}: Score={avg_score:.2f}, Speed={avg_latency:.2f}")