import dbnl
import numpy as np
import pandas as pd


num_results = 500
test_data = pd.DataFrame({
    "idx": np.arange(num_results),
    "age": np.random.randint(5,95, num_results),
    "loc": np.random.choice(["NY", "CA", "FL"], num_results),
    "churn_gtruth": np.random.choice([True, False], num_results)
})

# example user ml app / model
def churn_predictor(input):
    return 1.0 / (1.0 + np.exp(-(input["age"] / 10.0 - 3.5)))

def evaluate_predicton(data):
    return (data["churn_score"] > 0.5 and data["churn_gtruth"]) or \
            (data["churn_score"] < 0.5 and not data["churn_gtruth"])

test_data["churn_score"] = test_data.apply(churn_predictor, axis=1)
test_data["pred_correct"] = test_data.apply(evaluate_predicton, axis=1)
test_data = test_data.astype({"age": "int", "loc": "category"})

# Use DBNL
dbnl.login(api_token="<COPY_PASTE_DBNL_API_TOKEN>")
proj = dbnl.get_or_create_project(name="example_churn_predictor")
run = dbnl.report_run_with_results(
    project=proj,
    column_results=test_data,
    row_id=["idx"],
)

dbnl.create_project(
    *,
    name: str,
    description: Optional[str] = None,
) -> Project:

import dbnl
dbnl.login()


proj_1 = dbnl.create_project(name="test_p1")

# DBNLConflictingProjectError: A DBNL Project with name test_p1 already exists.
proj_2 = dbnl.create_project(name="test_p1")

dbnl.export_project_as_json(
    *,
    project: Project,
) -> dict[str, Any]:

import dbnl
dbnl.login()


proj = dbnl.get_or_create_project(name="test_proj")
export_json = dbnl.export_project_as_json(project=proj)

assert export_json["project"]["name"] == "test_proj"

dbnl.get_or_create_project(
    *,
    name: str,
    description: Optional[str] = None,
) -> Project:

import dbnl
dbnl.login()


proj_1 = dbnl.create_project(name="test_p1")
proj_2 = dbnl.get_or_create_project(name="test_p1")

# Calling get_or_create_project will yield same Project object
assert proj_1.id == proj_2.id

dbnl.import_project_from_json(
    *,
    params: dict[str, Any],
) -> Project:

import dbnl
dbnl.login()


proj1 = dbnl.get_or_create_project(name="test_proj1")
export_json = dbnl.export_project_as_json(project=proj1)
export_json["project"]["name"] = "test_proj2"
proj2 = dbnl.import_project_from_json(params=export_json)

assert proj2.name == "test_proj2"

dbnl.get_latest_run_config(
    *,
    project: Project,
) -> RunConfig:

import dbnl
dbnl.login()


proj1 = dbnl.get_or_create_project(name="test_p1")
runcfg1 = dbnl.create_run_config(project=proj1, columns=[{"name": "error", "type": "float"}])

# Retrieving the latest RunConfig
runcfg2 = dbnl.get_latest_run_config(project=proj1)
assert runcfg1.id == runcfg2.id

dbnl.get_run_config(
    *,
    project: Project,
) -> RunConfig:

import dbnl
dbnl.login()


proj1 = dbnl.get_or_create_project(name="test_p1")
runcfg1 = dbnl.create_run_config(project=proj1, columns=[{"name": "error", "type": "float"}])
run1 = dbnl.create_run(
    project=proj1, 
    run_config=runcfg1, 
)
# Retrieving the RunConfig by ID
runcfg2 = dbnl.get_run_config_from_latest_run(project=proj1)
assert runcfg1.id == runcfg2.id

dbnl.get_run_config(
    *,
    run_config_id: str,
) -> RunConfig:

import dbnl
dbnl.login()


proj1 = dbnl.get_or_create_project(name="test_p1")
runcfg1 = dbnl.create_run_config(project=proj1, columns=[{"name": "error", "type": "float"}])

# Retrieving the RunConfig by ID
runcfg2 = dbnl.get_run_config(run_config_id=runcfg1.id)
assert runcfg1.id == runcfg2.id

# DBNLRunConfigNotFoundError: A DBNL RunConfig with id not_exist does not exist
run_config3 = dbnl.get_run_config(run_config_id="runcfg_not_exist")

dbnl.create_run_config(
    *,
    project: Project,
    columns: list[dict[str, Any]],
    scalars: Optional[list[dict[str, Any]]] = None,
    description: Optional[str] = None,
    display_name: Optional[str] = None,
    row_id: Optional[list[str]] = None,
    components_dag: Optional[dict[str, list[str]]] = None,
) -> RunConfig:

components_dags={
    "TweetSource": ["EntityExtractor", "SentimentClassifier"],
    "EntityExtractor": ["TradeRecommender"],
    "SentimentClassifier": ["TradeRecommender"],
    "TradeRecommender": [],
    "Global": [],
}

import dbnl
dbnl.login()


proj = dbnl.get_or_create_project(name="test_p1")
# create a new RunConfig
runcfg1 = dbnl.create_run_config(
    project=proj,
    columns=[
        {"name": "error_type", "type": "category"},
        {"name": "email", "type": "string", "description": "raw email text content from source"},
        {"name": "spam-pred", "type": "boolean"},
    ],
    display_name="Basic RunConfig for spam prediction",
)

import dbnl
dbnl.login()


proj = dbnl.get_or_create_project(name="test_p1")
# create a new RunConfig with DAG
runcfg1 = dbnl.create_run_config(
    project=proj,
    columns=[
        {"name": "error_type", "type": "category"},
        {"name": "email", "type": "string", "component": "data_source", "description": "raw email text content from source"},
        {"name": "spam-pred", "type": "boolean", "component": "spam_classifier"},
    ],
    display_name="Basic RunConfig for spam prediction",
    components_dag={
        "data_source": ["spam_classifier"]
        "spam_classifier": []
)

import dbnl
dbnl.login()


proj = dbnl.get_or_create_project(name="test_p1")
# create a new RunConfig
runcfg1 = dbnl.create_run_config(
    project=proj,
    columns=[
        {"name": "error_type", "type": "category"},
        {"name": "email", "type": "string", "description": "raw email text content from source"},
        {"name": "spam-pred", "type": "boolean"},
        {"name": "email_id", "type": "string", "description": "unique id for each email"},
    ],
    display_name="Basic RunConfig for spam prediction",
    row_id=["email_id"],
)

import dbnl
dbnl.login()


proj = dbnl.get_or_create_project(name="test_p1")
# create a new RunConfig
runcfg1 = dbnl.create_run_config(
    project=proj,
    columns=[
        {"name": "error_type", "type": "category"},
        {"name": "email", "type": "string", "description": "raw email text content from source"},
        {"name": "spam-pred", "type": "boolean"},
        {"name": "email_id", "type": "string", "description": "unique id for each email"},
    ],
    scalars=[
        {"name": "model_F1", "type": "float"},
        {"name": "model_recall", "type": "float"},
    ],
    display_name="Basic RunConfig for spam prediction",
)

dbnl.login(
    *,
    api_token: Optional[str] = None,
    namespace_id: Optional[str] = None,    
    api_url: Optional[str] = None,
    app_url: Optional[str] = None,  
) -> None

import dbnl
# when login() is called without specifying a token, 
# it will use the `DBNL_API_TOKEN` env var
dbnl.login()

# login() can be called with a specific API Token
dbnl.login(api_token="YOUR_TOKEN_AAAA_BBBB_CCCC_DDDD")

dbnl.get_project(
    *,
    name: str,
) -> Project:

import dbnl
dbnl.login()


proj_1 = dbnl.create_project(name="test_p1")
proj_2 = dbnl.get_project(name="test_p1")

# Calling get_project will yield same Project object
assert proj_1.id == proj_2.id

# DBNLProjectNotFoundError: A dnnl Project with name not_exist does not exist
proj_3 = dbnl.get_project(name="not_exist")

dbnl.get_column_results(
    *,
    run: Run,
) -> pandas.DataFrame:

import dbnl
import pandas as pd
dbnl.login()


proj = dbnl.get_or_create_project(name="test_p1")
uploaded_data = pd.DataFrame({"error": [0.11, 0.33, 0.52, 0.24]})
run = dbnl.report_run_with_results(
    project=proj,
    column_results=test_data,
)

downloaded_data = dbnl.get_column_results(run=run)
assert downloaded_data.equals(uploaded_data)

dbnl.get_results(
    *,
    run: Run,
) -> ResultData:

import dbnl
import pandas as pd
dbnl.login()


proj = dbnl.get_or_create_project(name="test_p1")

uploaded_data = pd.DataFrame({"error": [0.11, 0.33, 0.52, 0.24]})
run = dbnl.report_run_with_results(
    project=proj,
    column_results=uploaded_data,
)

downloaded_data = dbnl.get_results(run=run)
assert downloaded_data.columns.equals(uploaded_data)

dbnl.report_scalar_results(
    *,
    run: Run,
    Union[dict[str, Any], pd.DataFrame]
) -> None:

import dbnl
import pandas as pd
dbnl.login()


proj1 = dbnl.get_or_create_project(name="test_p1")
runcfg1 = dbnl.create_run_config(
    project=proj1, 
    columns=[{"name": "error", "type": "float"}],
    scalars=[{"name": "rmse": "type": "float"}],
)
run1 = dbnl.create_run(project=proj1, run_config=runcfg1)
dbnl.report_scalar_results(run=run1, data={"rmse": 0.37})

dbnl.get_scalar_results(
    *,
    run: Run,
) -> pandas.DataFrame:

import dbnl
import pandas as pd
dbnl.login()

proj1 = dbnl.get_or_create_project(name="test_p1")

data = pd.DataFrame({"error": [0.11, 0.33, 0.52, 0.24]})
run = dbnl.report_run_with_results(
    project=proj,
    column_results=data,
    scalar_results={"rmse": 0.37}
)

downloaded_scalars = dbnl.get_scalar_results(run=run)

dbnl.report_results(
    *,
    run: Run,
    data: pandas.DataFrame,
) -> None:

import dbnl
import pandas as pd
dbnl.login()


proj1 = dbnl.get_or_create_project(name="test_p1")
runcfg1 = dbnl.create_run_config(project=proj1, columns=[{"name": "error", "type": "float"}])
run1 = dbnl.create_run(project=proj1, run_config=runcfg1)

data = pd.DataFrame({"error": [0.11, 0.33, 0.52, 0.24]})
dbnl.report_column_results(run=run1, data=data)

dbnl.report_results(
    *,
    run: Run,
    column_data: pandas.DataFrame,
    scalar_data: dict[str, Any] | pandas.DataFrame | None = None
) -> None:

import dbnl
import pandas as pd
dbnl.login()


proj1 = dbnl.get_or_create_project(name="test_p1")
runcfg1 = dbnl.create_run_config(project=proj1, columns=[{"name": "error", "type": "float"}])
run1 = dbnl.create_run(project=proj1, run_config=runcfg1)

data = pd.DataFrame({"error": [0.11, 0.33, 0.52, 0.24]})
dbnl.report_results(run=run1, column_data=data)


import dbnl
import pandas as pd
dbnl.login()


proj1 = dbnl.get_or_create_project(name="test_p1")
runcfg1 = dbnl.create_run_config(
    project=proj1, 
    columns=[{"name": "error", "type": "float"}],
    scalars=[{"name": "rmse": "type": "float"}],
)
run1 = dbnl.create_run(project=proj1, run_config=runcfg1)
data = pd.DataFrame({"error": [0.11, 0.33, 0.52, 0.24]})
dbnl.report_results(run=run1, column_data=data, scalar_data={"rmse": 0.37})

dbnl.copy_project(
    *,
    project: Project,
    name: str,
    description: Optional[str] = None,
) -> Project:

import dbnl
dbnl.login()


proj1 = dbnl.get_or_create_project(name="test_proj1")
proj2 = dbnl.copy_project(project=proj1, name="test_proj2")

assert proj2.name == "test_proj2"

dbnl.close_run(
    *,
    run: Run,
) -> None:

import dbnl
dbnl.login()

dbnl.close_run(run=my_run)

dbnl.get_run(
    *,
    run_id: str,
) -> Run:

import dbnl
dbnl.login()


proj1 = dbnl.get_or_create_project(name="test_p1")
runcfg1 = dbnl.create_run_config(project=proj1, columns=[{"name": "error", "type": "float"}])
run1 = dbnl.create_run(project=proj1, run_config=runcfg1)

# Retrieving the Run by ID
run2 = dbnl.get_run(run_id=run1.id)
assert run1.id == run2.id

# DBNLRunNotFoundError: A DBNL Run with id run_0000000 does not exist.
run3 = dbnl.get_run(run_id="run_0000000")

dbnl.report_run_with_results(
    project: Project,
    column_data: pd.DataFrame,
    scalar_data: Optional[Union[dict[str, Any], pd.DataFrame]] = None
    display_name: Optional[str] = None,
    row_id: Optional[list[str]] = None,
    run_config_id: Optional[str] = None,
    metadata: Optional[dict[str, str]] = None,
) -> Run:

import dbnl
import pandas as pd
dbnl.login()


proj = dbnl.get_or_create_project(name="test_p1")
test_data = pd.DataFrame({"error": [0.11, 0.33, 0.52, 0.24]})

run = dbnl.report_run_with_results(
    project=proj,
    column_data=test_data,
    row_id=["idx"],
)

dbnl.create_run(
    *,
    project: Project,
    run_config: RunConfig,
    display_name: Optional[str] = None,
    metadata: Optional[Dict[str, str]] = None,
) -> Run:

import dbnl
dbnl.login()


proj1 = dbnl.get_or_create_project(name="test_p1")
runcfg1 = dbnl.create_run_config(project=proj1, columns=[{"name": "error", "type": "float"}])

run1 = dbnl.create_run(
    project=proj1, 
    run_config=runcfg1, 
    metadata={"mode": "dev"},
)

dbnl.set_run_as_baseline(
    *,
    run: Run,
) -> None:

dbnl.create_run_query(
    *,
    project: Project,
    name: str,
    query: dict[str, Any]
) -> RunQuery:

dbnl.create_run_query(
  project=project,
  name="look back 3",
  query={
    "offset_from_now": 3,
  },
)

class TestSession(DBNLObject):
    id: str
    project_id: str
    inputs: list[TestSessionInput]
    status: Literal["PENDING", "RUNNING", "PASSED", "FAILED"]
    failure: Optional[str] = None
    num_tests_passed: Optional[int] = None
    num_tests_failed: Optional[int] = None
    num_tests_errored: Optional[int] = None
    include_tag_ids: Optional[list[str]] = None
    exclude_tag_ids: Optional[list[str]] = None
    require_tag_ids: Optional[list[str]] = None

class TestSessionInput(DBNLObject):
    run_alias: str
    run_id: Optional[str] = None
    run_query_id: Optional[str] = None

dbnl.get_run_query(
    project: Project,
    name: str,
) -> RunQuery:

query = dbnl.get_run_query(
  project=project,
  name="look back 3"
)

dbnl.set_run_query_as_baseline(
    *,
    run_query: RunQuery,
) -> None:

class RunConfig:
    id: str
    project_id: str
    columns: list[ColumnSchema]
    scalars: Optional[list[ScalarSchema]] = None
    description: Optional[str] = None
    display_name: Optional[str] = None
    row_id: Optional[list[str]] = None
    components_dag: Optional[dict[str, list[str]]] = None

class Run:
    id: str
    project_id: str
    run_config_id: str
    display_name: Optional[str] = None
    metadata: Optional[dict[str, str]] = None
    run_config: Optional[RunConfig] = None

class Project:
    id: str
    name: str
    description: Optional[str] = None

class RunQuery(DBNLObject):
    id: str
    project_id: str
    name: str
    query: dict[str, Any]

dbnl.create_test_session(
    *,
    experiment_run: Run,
    baseline: Optional[Union[Run, RunQuery]] = None,
    include_tags: Optional[List[str]] = None,
    exclude_tags: Optional[List[str]] = None,
    require_tags: Optional[List[str]] = None,
) -> TestSession:

dbnl.create_test_session(
  experiment_run=new_run,
  baseline=baseline_run,
)

dbnl.create_test_session(
  experiment_run=new_run,
  baseline=baseline_run_query,
)

dbnl.create_test_session(
  experiment_run=new_run,
)

class TestRecalibrationSession(DBNLObject):
    id: str
    project_id: str
    test_session_id: str
    feedback: str
    status: Literal["PENDING", "RUNNING", "COMPLETED", "FAILED"]
    test_ids: Optional[list[str]] = None
    failure: Optional[str] = None

dbnl.experimental.create_test(
    *,
    test_spec_dict: Dict[str, Any]
) -> Dict[str, Any]:

{
  "project_id": string,

  // Test data
  "name": string (project-unique),
  "description": string?,
  "statistic_name": string,
  "statistic_params": map[string, any],
  "statistic_inputs": list[
    {
      "select_query_template": {
        "select": string // a column or a function on column(s)
      }
    }
  ],
  "assertion": {
    "name": string,
    "params": map[string, any]
  },
  "tag_ids": string[]?
}

class ResultData(NamedTuple):
    columns: pd.DataFrame
    scalars: Union[pd.DataFrame, None] = None

class TestGenerationSession(DBNLObject):
    id: str
    project_id: str
    run_id: str
    status: Literal["PENDING", "RUNNING", "COMPLETED", "FAILED"]
    columns: Optional[list[dict[Literal["name"], str]]] = None
    failure: Optional[str] = None
    num_generated_tests: Optional[int] = None

dbnl.experimental.get_tests(
    *,
    test_session_id: str,
) -> List[Dict[str, Any]]:

{
  // Test metadata
  "id": string,
  "org_id": string,
  "created_at": timestamp,
  "updated_at": timestamp,

  "test_session_id": string,

  // Test data
  "author_id": string,
  "value": any?,
  "failure": string?,
  "status": enum(PENDING, RUNNING, PASSED, FAILED),
  "started_at": timestamp?,
  "completed_at": timestamp?,

  // Test Spec data
  "test_spec_id": id,
  "name": string,
  "description": string?,
  "statistic_name": string,
  "statistic_params": map[string, any],
  "assertion": {
    "name": string,
    "params": map[string, any]
    "status": enum(...),
    "failure": string?
  },
  "statistic_inputs": list[
    {
      "select_query_template": {
        "select": string 
      }
    }
  ],
  "tag_ids": string[]?,
}

def create_test_recalibration_session(
  *,
  test_session: TestSession,
  feedback: str,
  test_ids: Optional[list[str]] = None,
) -> TestRecalibrationSession

dbnl.experimental.get_test_sessions(
    *,
    project: Project,
) -> list[TestSession]:

dbnl.experimental.wait_for_test_session(
    *,
    test_session: TestSession
    timeout_s: int = 180,
) -> TestSession:

def wait_for_test_recalibration_session(
    *,
    test_recalibration_session: TestRecalibrationSession,
    timeout_s: int = 180,
) -> TestRecalibrationSession

dbnl.experimental.prepare_incomplete_test_spec_payload(
    *,
    test_spec_dict: IncompleteTestSpecDict,
    project_id: Optional[str] = None,
) -> Dict[str, Any]:

def create_test_generation_session(
    *,
    run: Run,
    columns: Optional[list[str | dict[Literal["name"], str]]] = None,
) -> TestGenerationSession

run = dbnl.get_run(...)
dbnl.experimental.create_test_generation_session(
  run=run,
  columns=["col1", "col4"],
)

run = dbnl.get_run(...)
dbnl.experimental.create_test_generation_session(
  run=run,
  columns=[
    {"name": "col1"},
    {"name": "col4"},
  ],
)

run = dbnl.run(...)
dbnl.experimental.create_test_generation_session(
  run=run,
)

import dbnl
import os
import pandas as pd
from openai import OpenAI
from dbnl.eval.llm import OpenAILLMClient
from dbnl.eval import evaluate

# 1. create client to power LLM-as-judge metrics
base_oai_client = OpenAI(api_key=os.environ["OPENAI_API_KEY"])
oai_client = OpenAILLMClient.from_existing_client(base_oai_client, llm_model="gpt-3.5-turbo-0125")

eval_df = pd.DataFrame(
    [
        { "prediction":"France has no capital",
          "ground_truth": "The capital of France is Paris",},
        { "prediction":"The capital of France is Toronto",
          "ground_truth": "The capital of France is Paris",},
        { "prediction":"Paris is the capital",
          "ground_truth": "The capital of France is Paris",},
    ] * 4
)

# 2. get text metrics that use target (ground_truth) and LLM-as-judge metrics
text_metrics = dbnl.eval.metrics.text_metrics(
    prediction="prediction", target="ground_truth", eval_llm_client=oai_client
)
# 3. run text metrics that use target (ground_truth) and LLM-as-judge metrics
aug_eval_df = evaluate(eval_df, text_metrics)

# 4. publish to DBNL
dbnl.login(api_token=os.environ["DBNL_API_TOKEN"])
project = dbnl.get_or_create_project(name="DEAL_testing")
cols = dbnl.experimental.get_column_schemas_from_dataframe(aug_eval_df)
run_config = dbnl.create_run_config(project=project, columns=cols)
run = dbnl.create_run(project=project, run_config=run_config)
dbnl.report_results(run=run, data=aug_eval_df)
dbnl.close_run(run=run)

def evaluate(df: pd.DataFrame, metrics: Sequence[Metric], inplace: bool = False) -> pd.DataFrame:
    """
    Evaluates a set of metrics on a dataframe, returning an augmented dataframe.

    :param df: input dataframe
    :param metrics: metrics to compute
    :param inplace: whether to modify the input dataframe in place
    :return: input dataframe augmented with metrics
    """

def text_metrics(
    prediction: str,
    target: Optional[str] = None,
    eval_llm_client: Optional[LLMClient] = None,
    eval_embedding_client: Optional[EmbeddingClient] = None,
) -> list[Metric]:
    """
    Returns a set of metrics relevant for a generic text application

    :param prediction: prediction column name (i.e. generated text)
    :param target: target column name (i.e. expected text)
    :return: list of metrics
    """

dbnl.experimental.get_or_create_tag(
    *,
    project_id: str,
    name: str,
    description: Optional[str] = None,
) -> Dict[str, Any]:

{
  // Tag metadata
  "id": string,
  "org_id": string,
  "created_at": timestamp,
  "updated_at": timestamp,
  
  // Tag data
  "name": string,
  "author_id": string,
  "description": string?,
  "project_id": string,
}

dbnl.eval.evaluate(df: DataFrame, metrics: Sequence[Metric], inplace: bool = False) → DataFrame

dbnl.eval.get_column_schemas_from_dataframe_and_metrics(df: DataFrame, metrics: list[Metric]) → list[ColumnSchema]

dbnl.eval.get_column_schemas_from_metrics(metrics: list[Metric]) -> list[ColumnSchema]

# BEFORE : default text metrics including those requiring target (ground_truth) and LLM-as-judge
text_metrics = dbnl.eval.metrics.text_metrics(
    prediction="prediction", target="ground_truth", eval_llm_client=oai_client
)

# AFTER : remove the eval_llm_client to exclude LLM-as-judge metrics
text_metrics = dbnl.eval.metrics.text_metrics(
    prediction="prediction", target="ground_truth"
)

aug_eval_df = evaluate(eval_df, text_metrics)

# BEFORE : default text metrics, including those requiring target (ground_truth) and LLM-as-judge
text_metrics = dbnl.eval.metrics.text_metrics(
    prediction="prediction", target="ground_truth", eval_llm_client=oai_client
)

# AFTER : remove the target to remove metrics that depend on that value being specified
text_metrics = dbnl.eval.metrics.text_metrics(
    prediction="prediction", eval_llm_client=oai_client
)

aug_eval_df = evaluate(eval_df, text_metrics)

# get text metrics for each column in list
monitor_metrics = dbnl.eval.metrics.text_monitor_metrics(
  ["prediction", "input"], eval_llm_client=oai_client
)

aug_eval_df = evaluate(eval_df, monitor_metrics)

import dbnl
import os
import pandas as pd
from openai import OpenAI
from dbnl.eval.llm import OpenAILLMClient
from dbnl.eval import evaluate
from dbnl.eval.metrics.mlflow import MLFlowGenAIFromPromptEvaluationMetric
from dbnl.eval.metrics.metric import Metric
from dbnl.eval.llm.client import LLMClient

# 1. create client to power LLM-as-judge metrics
base_oai_client = OpenAI(api_key=os.environ["OPENAI_API_KEY"])
oai_client = OpenAILLMClient.from_existing_client(base_oai_client, llm_model="gpt-3.5-turbo-0125")

eval_df = pd.DataFrame(
    [
        { "prediction":"France has no capital",
          "ground_truth": "The capital of France is Paris",},
        { "prediction":"The capital of France is Toronto",
          "ground_truth": "The capital of France is Paris",},
        { "prediction":"Paris is the capital",
          "ground_truth": "The capital of France is Paris",},
    ] * 4
)

# 2. define a custom LLM-as-judge metric
def custom_text_similarity(prediction: str, target: str, eval_llm_client: LLMClient) -> Metric:
    custom_prompt_v0 = """
      Given the generated text : {prediction}, score the semantic similarity to the reference text : {target}. 

      Rate the semantic similarity from 1 (completely different meaning and facts between the generated and reference texts) to 5 (nearly the exact same semantic meaning and facts present in the generated and reference texts).

      Example output, make certain that 'score:' and 'justification:' text is present in output:
      score: 4
      justification: XYZ
    """
    
    return MLFlowGenAIFromPromptEvaluationMetric(
        name="custom_text_similarity",
        judge_prompt=custom_prompt_v0,
        prediction=prediction,
        target=target,
        eval_llm_client=eval_llm_client,
        version="v0",
    )

# 3. instantiate the custom LLM-as-judge metric
c_metric = custom_text_similarity(
  prediction='prediction', target='ground_truth', eval_llm_client=oai_client
)
# 4. run only the custom LLM-as-judge metric
aug_eval_df = evaluate(eval_df, [c_metric])

def custom_text_simplicity(prediction: str, target: str, eval_llm_client: LLMClient) -> Metric:
    custom_prompt_v0 = """
      Given the generated text : {prediction}, score the text from 1 to 5 based on whether it is written in simple, easy to understand english 

      Rate the generated text from 5 (completely simple english, very commonly used words, easy to explain vocabulary) to 1 (complex english, uncommon words, difficult to explain vocabulary).

      Example output, make certain that 'score:' and 'justification:' text is present in output:
      score: 4
      justification: XYZ
    """
    
    return MLFlowGenAIFromPromptEvaluationMetric(
        name="custom_text_simplicity",
        judge_prompt=custom_prompt_v0,
        prediction=prediction,
        target=target,
        eval_llm_client=eval_llm_client,
        version="v0",
    )

import dbnl
import os
import pandas as pd
from openai import OpenAI
from dbnl.eval.llm import OpenAILLMClient
from dbnl.eval.embedding_clients import OpenAIEmbeddingClient
from dbnl.eval import evaluate

# 1. create client to power LLM-as-judge and embedding metrics [optional]
base_oai_client = OpenAI(api_key=os.environ["OPENAI_API_KEY"])
eval_llm_client = OpenAILLMClient.from_existing_client(base_oai_client, llm_model="gpt-3.5-turbo-0125")
eval_embd_client = OpenAIEmbeddingClient.from_existing_client(base_oai_client, embedding_model="text-embedding-ada-002")

eval_df = pd.DataFrame(
    [
        {
         "question_text": "Is the protein Cathepsin secreted?",
         "top_k_retrieved_doc_texts": ["Some irrelevant document that the rag system retrieved"],    
         "top_k_retrieved_doc_ids":   ["4123"],
         "top_retrieved_doc_text":    "Some irrelevant document that the rag system retrieved",
         "gt_reference_doc_id": "1099",
         "gt_reference_doc_text": "The protein Cathepsin is known to be secreted",
         "ground_truth_answer": "Yes, Cathepsin is a secreted protein",
         "generated_answer":"I have no relevant knowledge",},
        {
         "question_text": "Is the protein Cathepsin secreted?",
         "top_k_retrieved_doc_texts": ["Some irrelevant document that the rag system retrieved", 
                                       "Many proteins are secreted such as hormones, enzymes, toxins",
                                       "The protein Cathepsin is known to be secreted"],    
         "top_k_retrieved_doc_ids":   ["4123","21","1099"],
         "top_retrieved_doc_text":    "Some irrelevant document that the rag system retrieved",
         "gt_reference_doc_id": "1099",
         "gt_reference_doc_text": "The protein Capilin is known to be secreted",
         "ground_truth_answer": "Yes, Cathepsin is a secreted protein",
         "generated_answer":"Many proteins are known to be secreted",},
        {
         "question_text": "Is the protein Cathepsin secreted?",
         "top_k_retrieved_doc_texts": ["The protein Cathepsin is known to be secreted", 
                                       "Some irrelevant document that the rag system retrieved"],    
         "top_k_retrieved_doc_ids":   ["1099","4123"],
         "top_retrieved_doc_text":    "The protein Cathepsin is known to be secreted",
         "gt_reference_doc_id": "1099",
         "gt_reference_doc_text": "The protein Cathepsin is known to be secreted",
         "ground_truth_answer": "Yes, Cathepsin is a secreted protein",
         "generated_answer":"Yes, cathepsin is a secreted protein",},
    ] * 4
)

# 2. get text metrics appropriate for RAG / QA systems
qa_text_metrics = dbnl.eval.metrics.question_and_answer_metrics(
  prediction="generated_answer", input="question_text", target="ground_truth_answer",
  context="top_k_retrieved_doc_texts", top_retrieved_document_text="top_retrieved_doc_text",
  retrieved_document_ids="top_k_retrieved_doc_ids", ground_truth_document_id="gt_reference_doc_id",
  eval_llm_client=eval_llm_client, eval_embedding_client=eval_embd_client
)
# 3. run qa text metrics
aug_eval_df = evaluate(eval_df, qa_text_metrics)

# 4. publish to DBNL
dbnl.login()
project = dbnl.get_or_create_project(name="RAG_demo")
cols = dbnl.experimental.get_column_schemas_from_dataframe(aug_eval_df)
run_config = dbnl.create_run_config(project=project, columns=cols)
run = dbnl.create_run(project=project, run_config=run_config)
dbnl.report_results(run=run, data=aug_eval_df)
dbnl.close_run(run=run)

def question_and_answer_metrics(
    prediction: str,
    target: Optional[str] = None,
    input: Optional[str] = None,
    context: Optional[str] = None,
    ground_truth_document_id: Optional[str] = None,
    retrieved_document_ids: Optional[str] = None,
    ground_truth_document_text: Optional[str] = None,
    top_retrieved_document_text: Optional[str] = None,
    eval_llm_client: Optional[LLMClient] = None,
    eval_embedding_client: Optional[EmbeddingClient] = None,
) -> list[Metric]:
    """
    Returns a set of metrics relevant for a question and answer task.

    :param prediction: prediction column name (i.e. generated answer)
    :param target: target column name (i.e. expected answer)
    :param input: input column name (i.e. question)
    :param context: context column name (i.e. document or set of documents retrieved)
    :param ground_truth_document_id: ground_truth_document_id containing the information in the target
    :param retrieved_document_ids: retrieved_document_ids containing the full context
    :param ground_truth_document_text: text containing the information in the target 
                                       (ideal is for this to be the top retrieved document)
    :param top_retrieved_document_text: text of the top retrieved document
    :param eval_llm_client: eval_llm_client
    :param eval_embedding_client: eval_embedding_client
    :return: list of metrics
    """

def wait_for_test_generation_session(
    *,
    test_generation_session: TestGenerationSession,
    timeout_s: int = 180,
) -> TestGenerationSession

from openai import OpenAI
from dbnl.eval.llm import OpenAILLMClient
from dbnl.eval.embedding_clients import OpenAIEmbeddingClient

# create client for LLM-as-judge metrics
base_oai_client = OpenAI(api_key=os.environ["OPENAI_API_KEY"])
eval_llm_client = OpenAILLMClient.from_existing_client(
    base_oai_client, llm_model="gpt-3.5-turbo-0125"
)

embd_client = OpenAIEmbeddingClient.from_existing_client(
    base_oai_client, embedding_model="text-embedding-ada-002"
)

from openai import AzureOpenAI
from dbnl.eval.llm import AzureOpenAILLMClient
from dbnl.eval.embedding_clients import AzureOpenAIEmbeddingClient

base_azure_oai_client = AzureOpenAI(
    api_key=os.environ["AZURE_OPENAI_API_KEY"],
    api_version=os.environ["OPENAI_API_VERSION"], # eg 2023-12-01-preview
    azure_endpoint=os.environ["AZURE_OPENAI_ENDPOINT"] # eg https://resource-name.openai.azure.com
)
eval_llm_client = AzureOpenAILLMClient.from_existing_client(
    base_azure_oai_client, llm_model="gpt-35-turbo-16k"
)
embd_client = AzureOpenAIEmbeddingClient.from_existing_client(
    base_azure_oai_client, embedding_model="text-embedding-ada-002"
)

from openai import OpenAI
from dbnl.eval.llm import OpenAILLMClient
base_oai_client = OpenAI(
    api_key=os.environ["TOGETHERAI_API_KEY"],
    base_url="https://api.together.xyz/v1",
)

eval_llm_client = OpenAILLMClient.from_existing_client(
    base_oai_client, llm_model='meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo'
)

{'code': '429', 'message': 'Requests to the Embeddings_Create Operation under 
  Azure OpenAI API version XXXX have exceeded call rate limit of your current 
  OpenAI pricing tier. Please retry after 86400 seconds. 
  Please go here: https://aka.ms/oai/quotaincrease if you would 
  like to further increase the default rate limit.'}

{'message': 'You have been rate limited. Your rate limit is YYY queries per
minute. Please navigate to https://www.together.ai/forms/rate-limit-increase 
to request a rate limit increase.', 'type': 'credit_limit', 
'param': None, 'code': None}

{'message': 'Rate limit reached for gpt-4 in organization XXXX on 
tokens per min (TPM): Limit WWWWW, Used YYYY, Requested ZZZZ. 
Please try again in 1.866s. Visit https://platform.openai.com/account/rate-limits 
to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}

class dbnl.eval.metrics.Metric

column_schema() → ColumnSchema

description() → str | None

abstract evaluate(df: pd.DataFrame) → pd.Series[Any]

abstract expression() → str

greater_is_better() → bool | None

abstract metric() → str

abstract name() → str

abstract type() → Type

class dbnl.eval.metrics.RougeScoreType(value, 
                                       names=None, *, 
                                       module=None, 
                                       qualname=None, 
                                       type=None, 
                                       start=1, 
                                       boundary=None)

dbnl.eval.metrics.answer_quality_llm_accuracy(input: str, context: str, prediction: str, eval_llm_client: LLMClient) → Metric

dbnl.eval.metrics.answer_quality_llm_answer_correctness(input: str, prediction: str, target: str, eval_llm_client: LLMClient) → Metric

dbnl.eval.metrics.answer_quality_llm_answer_similarity(input: str, prediction: str, target: str, eval_llm_client: LLMClient) → Metric

dbnl.eval.metrics.answer_quality_llm_coherence(prediction: str, eval_llm_client: LLMClient) → Metric

dbnl.eval.metrics.answer_quality_llm_commital(prediction: str, eval_llm_client: LLMClient) → Metric

dbnl.eval.metrics.answer_quality_llm_completeness(input: str, prediction: str, eval_llm_client: LLMClient) → Metrics

dbnl.eval.metrics.answer_quality_llm_contextual_relevance(input: str, context: str, eval_llm_client: LLMClient) → Metric

dbnl.eval.metrics.answer_quality_llm_faithfulness(input: str, context: str, prediction: str, eval_llm_client: LLMClient) → Metric

dbnl.eval.metrics.answer_quality_llm_grammar_accuracy(prediction: str, eval_llm_client: LLMClient) → Metric

dbnl.eval.metrics.answer_quality_llm_metrics(input: str | None, prediction: str, context: str | None, target: str | None, eval_llm_client: LLMClient) → list[Metric]

dbnl.eval.metrics.answer_quality_llm_originality(prediction: str, eval_llm_client: LLMClient) → Metric

dbnl.eval.metrics.answer_quality_llm_relevance(input: str, context: str, prediction: str, eval_llm_client: LLMClient) → Metric

dbnl.eval.metrics.answer_viability_llm_metrics(prediction: str, eval_llm_client: LLMClient) → list[Metric]

dbnl.eval.metrics.answer_viability_llm_reading_complexity(prediction: str, eval_llm_client: LLMClient) → Metric

dbnl.eval.metrics.answer_viability_llm_sentiment_assessment(prediction: str, eval_llm_client: LLMClient) → Metric