Static Optimization

pip install pydantic requests

import os
import requests
import json

os.environ["OPENAI_API_KEY"] = "YOUR OPENAI API KEY"
BASE_URL = "https://orch.zenbase.ai/api"
API_KEY = "YOUR ZENBASE API KEY"

def api_call(method, endpoint, data=None):
    url = f"{BASE_URL}/{endpoint}"
    headers = {
        "Content-Type": "application/json",
        "Authorization": f"Api-Key {API_KEY}"
    }
    response = requests.request(method, url, headers=headers, data=json.dumps(data) if data else None)
    return response

from pydantic import BaseModel, Field
from typing import Literal

class TextualEntailmentInput(BaseModel):
    premise: str = Field(..., description="The sentence serving as the context or background information.")
    hypothesis: str = Field(..., description="The sentence to be evaluated in relation to the premise.")

class TextualEntailmentOutput(BaseModel):
    relationship: Literal["entailment", "contradiction", "neutral"] = Field(..., description="The logical relationship between the premise and hypothesis.")

from textwrap import dedent
function_data = {
    "name": "Textual Entailment Analysis",
    "description": "Analyze the logical relationship between a premise and a hypothesis.",
    "prompt": dedent("""Analyze the logical relationship between the given premise and hypothesis.
                    Determine if the relationship is entailment, contradiction, or neutral.

                    Definitions:
                    - Entailment: The hypothesis logically follows from the premise.
                    - Contradiction: The hypothesis contradicts the premise.
                    - Neutral: The hypothesis is neither entailed by nor contradicts the premise.
                    """),
    "input_schema": TextualEntailmentInput.model_json_schema(),
    "output_schema": TextualEntailmentOutput.model_json_schema(),
    "api_key": os.environ.get("OPENAI_API_KEY"),
    "model": "gpt-4o-mini"
}

function = api_call("POST", "functions/", function_data)
function_id = function.json()['id']

datasets = {}
for dataset_type in ['magic']:
    dataset_data = {
        "name": f"Textual Entailment {dataset_type.capitalize()} Dataset",
        "description": f"{dataset_type.capitalize()} dataset for Textual Entailment"
    }
    dataset = api_call("POST", "datasets/", dataset_data)
    datasets[dataset_type] = dataset.json()['id']

dataset_items = {
    "magic": [
        {
            "inputs": {"premise": "All roses are flowers.", "hypothesis": "Some flowers are roses."},
            "outputs": {"relationship": "entailment"}
        },
        {
            "inputs": {"premise": "The Earth orbits the Sun.", "hypothesis": "The Moon is made of cheese."},
            "outputs": {"relationship": "neutral"}
        },
        {
            "inputs": {"premise": "Water freezes at 0°C.", "hypothesis": "Water boils at 0°C."},
            "outputs": {"relationship": "contradiction"}
        },
        {
            "inputs": {"premise": "All mammals give birth to live young.", "hypothesis": "Whales give birth to live young."},
            "outputs": {"relationship": "entailment"}
        },
        {
            "inputs": {"premise": "The Eiffel Tower is in Paris.", "hypothesis": "Paris is the capital of Italy."},
            "outputs": {"relationship": "contradiction"}
        },
        {
            "inputs": {"premise": "Gold is a precious metal.", "hypothesis": "Silver is used in jewelry."},
            "outputs": {"relationship": "neutral"}
        },
        {
            "inputs": {"premise": "All squares are rectangles.", "hypothesis": "All rectangles are squares."},
            "outputs": {"relationship": "contradiction"}
        },
        {
            "inputs": {"premise": "The Pacific Ocean is the largest ocean.", "hypothesis": "The Atlantic Ocean exists."},
            "outputs": {"relationship": "neutral"}
        },
        {
            "inputs": {"premise": "Oxygen is necessary for human survival.", "hypothesis": "Humans need to breathe to live."},
            "outputs": {"relationship": "entailment"}
        }
    ]
}

for dataset_type, dataset_id in datasets.items():
    for item in dataset_items[dataset_type]:
        item["dataset"] = dataset_id
        response = api_call("POST", "dataset-items/", item)

optimizer_data = {
    "function": function_id,
    "magical_set": datasets['magic'],
    "parameters": {
        "shots": 3,
        "samples": 5
    },
    "schedule": {
        "cron": "" # Or use a crontab like "0 0 * * *"  # Run daily at midnight
    },
    "api_key": os.environ.get("OPENAI_API_KEY"),
    "model": "gpt-4o-mini"
}
optimizer = api_call("POST", "optimizer-configurations/", optimizer_data)
optimizer_id = optimizer.json()['id']

optimization = api_call("POST", f"optimizer-configurations/{optimizer_id}/run_optimization/")
optimization_id = optimization.json()['optimization_task_id']

import time
while True:
    status = api_call("GET", f"optimization-tasks/{optimization_id}/").json()
    if status['status'] == 'COMPLETED':
        print("Optimization completed successfully!")
        break
    elif status['status'] == 'FAILED':
        print("Optimization failed!")
        break
    time.sleep(30)

test_input = {
    "premise": "B: I do not know. I wonder where he gets it? You know, you must, I think TV is bad. Because they, uh, show all sorts of violence on, A: That and I do not think a lot of parents, I mean, I do not know how it is in the Air Force base. But, uh, I just do not think a lot of people, because of the economy, both need to work, you know. I just do not think a lot of parents are that involved any more.",
    "hypothesis": "a lot of parents are that involved"
}

result = api_call("POST", f"optimizer-configurations/{optimizer_id}/run/", {"inputs": test_input})
print(f"Optimized model prediction: {result.json()}")

result = api_call("GET", f"optimization-tasks/{optimization_id}/").json()
print("Base evaluation score (before optimization):", result['base_evaluation'])
print("Best evaluation score (after optimization):", result['best_candidate_evaluation'])
print("Best few-shot examples (after optimization):", print(result['task_demos']))