Index A | C | D | E | F | G | H | I | L | M | O | P | Q | R | S | T | V | W A aggregate() (prompttools.experiment.Experiment method), [1] (prompttools.experiment.experiments.experiment.Experiment method) all_args (prompttools.experiment.Experiment attribute) (prompttools.experiment.experiments.chromadb_experiment.ChromaDBExperiment attribute) (prompttools.experiment.experiments.experiment.Experiment attribute) (prompttools.experiment.experiments.huggingface_hub_experiment.HuggingFaceHubExperiment attribute) (prompttools.experiment.experiments.llama_cpp_experiment.LlamaCppExperiment attribute) anthropic_completion_fn() (prompttools.experiment.AnthropicCompletionExperiment method) AnthropicCompletionExperiment (class in prompttools.experiment), [1] argument_combos (prompttools.experiment.experiments.chromadb_experiment.ChromaDBExperiment attribute) (prompttools.experiment.experiments.experiment.Experiment attribute) (prompttools.experiment.experiments.huggingface_hub_experiment.HuggingFaceHubExperiment attribute) (prompttools.experiment.experiments.llama_cpp_experiment.LlamaCppExperiment attribute) autoeval_binary_scoring() (in module prompttools.utils), [1] (in module prompttools.utils.autoeval) autoeval_from_expected_response() (in module prompttools.utils) autoeval_scoring() (in module prompttools.utils), [1] autoeval_with_documents() (in module prompttools.utils), [1] C CALL_PARAMETERS (prompttools.experiment.experiments.huggingface_hub_experiment.HuggingFaceHubExperiment attribute) (prompttools.experiment.experiments.llama_cpp_experiment.LlamaCppExperiment attribute) (prompttools.experiment.HuggingFaceHubExperiment attribute) (prompttools.experiment.LlamaCppExperiment attribute) (prompttools.experiment.RouterChainExperiment attribute) (prompttools.experiment.SequentialChainExperiment attribute) (prompttools.experiment.StableDiffusionExperiment attribute) ChatHistoryExperimentationHarness (class in prompttools.harness), [1] (class in prompttools.harness.chat_history_harness) ChatModelComparisonHarness (class in prompttools.harness), [1] (class in prompttools.harness.chat_model_comparison_harness) ChatPromptTemplateExperimentationHarness (class in prompttools.harness) chroma_client (prompttools.experiment.experiments.chromadb_experiment.ChromaDBExperiment attribute) chromadb_completion_fn() (prompttools.experiment.ChromaDBExperiment method) (prompttools.experiment.experiments.chromadb_experiment.ChromaDBExperiment method) ChromaDBExperiment (class in prompttools.experiment), [1] (class in prompttools.experiment.experiments.chromadb_experiment) chunk_text() (in module prompttools.utils), [1] compare() (prompttools.harness.chat_model_comparison_harness.ChatModelComparisonHarness method) (prompttools.harness.ChatModelComparisonHarness method) ComparisonWidgetProvider (class in prompttools.experiment.widgets.comparison) completion_fn (prompttools.experiment.Experiment attribute) (prompttools.experiment.experiments.chromadb_experiment.ChromaDBExperiment attribute) (prompttools.experiment.experiments.experiment.Experiment attribute) (prompttools.experiment.experiments.huggingface_hub_experiment.HuggingFaceHubExperiment attribute) (prompttools.experiment.experiments.llama_cpp_experiment.LlamaCppExperiment attribute) compute() (in module prompttools.utils.autoeval) (in module prompttools.utils.expected) (in module prompttools.utils.similarity), [1] compute_similarity_against_model() (in module prompttools.utils), [1] (in module prompttools.utils.expected) cv2_image_to_base64() (prompttools.experiment.Experiment method) (prompttools.experiment.experiments.experiment.Experiment method) D DEFAULT (prompttools.experiment.experiments.llama_cpp_experiment.LlamaCppExperiment attribute) (prompttools.experiment.LlamaCppExperiment attribute) DEFAULT_DISTANCE (prompttools.experiment.QdrantExperiment attribute) display() (prompttools.experiment.widgets.comparison.ComparisonWidgetProvider method) (prompttools.experiment.widgets.feedback.FeedbackWidgetProvider method) display_image_html() (prompttools.experiment.Experiment method) (prompttools.experiment.experiments.experiment.Experiment method) DotDict (class in prompttools.mock.mock) E enqueue() (prompttools.requests.request_queue.RequestQueue method) evaluate() (in module prompttools.utils.autoeval) (in module prompttools.utils.expected) (in module prompttools.utils.similarity) (prompttools.experiment.Experiment method), [1] (prompttools.experiment.experiments.experiment.Experiment method) (prompttools.harness.ExperimentationHarness method), [1] (prompttools.harness.harness.ExperimentationHarness method) (prompttools.harness.ModelComparisonHarness method) (prompttools.harness.multi_experiment_harness.MultiExperimentHarness method) (prompttools.harness.MultiExperimentHarness method) (prompttools.prompttest.runner.runner.PromptTestRunner method) Experiment (class in prompttools.experiment), [1] (class in prompttools.experiment.experiments.experiment) experiment (prompttools.harness.ChatModelComparisonHarness attribute) (prompttools.harness.ExperimentationHarness attribute) (prompttools.harness.harness.ExperimentationHarness attribute) (prompttools.harness.PromptTemplateExperimentationHarness attribute) (prompttools.harness.SystemPromptExperimentationHarness attribute) ExperimentationHarness (class in prompttools.harness), [1] (class in prompttools.harness.harness) F FeedbackWidgetProvider (class in prompttools.experiment.widgets.feedback) full_df (prompttools.harness.ExperimentationHarness property) (prompttools.harness.harness.ExperimentationHarness property) (prompttools.harness.ModelComparisonHarness property) G gather_feedback() (prompttools.harness.multi_experiment_harness.MultiExperimentHarness method) (prompttools.harness.MultiExperimentHarness method) generate_retry_decorator() (in module prompttools.requests.retries) get_footer_widgets() (prompttools.experiment.widgets.comparison.ComparisonWidgetProvider method) (prompttools.experiment.widgets.feedback.FeedbackWidgetProvider method) get_header_widgets() (prompttools.experiment.widgets.comparison.ComparisonWidgetProvider method) (prompttools.experiment.widgets.feedback.FeedbackWidgetProvider method) get_input_args() (prompttools.requests.request_queue.RequestQueue method) get_latencies() (prompttools.requests.request_queue.RequestQueue method) get_results() (prompttools.requests.request_queue.RequestQueue method) get_row_widgets() (prompttools.experiment.widgets.comparison.ComparisonWidgetProvider method) (prompttools.experiment.widgets.feedback.FeedbackWidgetProvider method) get_table() (prompttools.experiment.Experiment method), [1] (prompttools.experiment.experiments.experiment.Experiment method) (prompttools.experiment.experiments.openai_chat_experiment.OpenAIChatExperiment method) (prompttools.experiment.OpenAIChatExperiment method) (prompttools.harness.ChatPromptTemplateExperimentationHarness method) (prompttools.harness.ModelComparisonHarness method) (prompttools.harness.system_prompt_harness.SystemPromptExperimentationHarness method) (prompttools.harness.SystemPromptExperimentationHarness method) GooglePaLMCompletionExperiment (class in prompttools.experiment), [1] (class in prompttools.experiment.experiments.google_palm_experiment) GoogleVertexChatCompletionExperiment (class in prompttools.experiment), [1] H hf_completion_fn() (prompttools.experiment.experiments.huggingface_hub_experiment.HuggingFaceHubExperiment method) (prompttools.experiment.HuggingFaceHubExperiment method) HuggingFaceHubExperiment (class in prompttools.experiment), [1] (class in prompttools.experiment.experiments.huggingface_hub_experiment) I initialize() (prompttools.experiment.ChromaDBExperiment class method) (prompttools.experiment.Experiment class method), [1] (prompttools.experiment.experiments.chromadb_experiment.ChromaDBExperiment class method) (prompttools.experiment.experiments.experiment.Experiment class method) (prompttools.experiment.experiments.llama_cpp_experiment.LlamaCppExperiment class method) (prompttools.experiment.LlamaCppExperiment class method) (prompttools.experiment.QdrantExperiment class method) (prompttools.experiment.WeaviateExperiment class method) is_interactive() (in module prompttools.experiment.widgets.utility) L lancedb_completion_fn() (prompttools.experiment.LanceDBExperiment method) LanceDBExperiment (class in prompttools.experiment), [1] lc_completion_fn() (prompttools.experiment.RouterChainExperiment method) (prompttools.experiment.SequentialChainExperiment method) llama_completion_fn() (prompttools.experiment.experiments.llama_cpp_experiment.LlamaCppExperiment method) (prompttools.experiment.LlamaCppExperiment method) LlamaCppExperiment (class in prompttools.experiment), [1] (class in prompttools.experiment.experiments.llama_cpp_experiment) load_experiment() (prompttools.experiment.experiments.openai_chat_experiment.OpenAIChatExperiment class method) (prompttools.experiment.OpenAIChatExperiment class method) (prompttools.harness.ExperimentationHarness class method), [1] (prompttools.harness.harness.ExperimentationHarness class method) load_revision() (prompttools.experiment.experiments.openai_chat_experiment.OpenAIChatExperiment class method) (prompttools.experiment.OpenAIChatExperiment class method) (prompttools.harness.ExperimentationHarness class method), [1] (prompttools.harness.harness.ExperimentationHarness class method) log_failure() (in module prompttools.prompttest.error.failure) M main() (in module prompttools.prompttest.prompttest) MAXIMUM (prompttools.prompttest.threshold_type.ThresholdType attribute) mindsdb_completion_fn() (prompttools.experiment.MindsDBExperiment method) MindsDBExperiment (class in prompttools.experiment), [1] MINIMUM (prompttools.prompttest.threshold_type.ThresholdType attribute) mock_anthropic_completion_fn() (in module prompttools.mock.mock) mock_chromadb_fn() (in module prompttools.mock.mock) mock_hf_completion_fn() (in module prompttools.mock.mock) mock_lc_completion_fn() (in module prompttools.mock.mock) mock_mindsdb_completion_fn() (in module prompttools.mock.mock) mock_openai_chat_completion_fn() (in module prompttools.mock.mock) mock_openai_chat_function_completion_fn() (in module prompttools.mock.mock) mock_openai_completion_fn() (in module prompttools.mock.mock) mock_palm_completion_fn() (in module prompttools.mock.mock) mock_qdrant_fn() (in module prompttools.mock.mock) mock_replicate_stable_diffusion_completion_fn() (in module prompttools.mock.mock) mock_stable_diffusion() (in module prompttools.mock.mock) MODEL_PARAMETERS (prompttools.experiment.experiments.huggingface_hub_experiment.HuggingFaceHubExperiment attribute) (prompttools.experiment.experiments.llama_cpp_experiment.LlamaCppExperiment attribute) (prompttools.experiment.HuggingFaceHubExperiment attribute) (prompttools.experiment.LlamaCppExperiment attribute) (prompttools.experiment.RouterChainExperiment attribute) (prompttools.experiment.SequentialChainExperiment attribute) (prompttools.experiment.StableDiffusionExperiment attribute) ModelComparisonHarness (class in prompttools.harness) module prompttools prompttools.experiment prompttools.experiment.experiments prompttools.experiment.experiments.chromadb_experiment prompttools.experiment.experiments.error prompttools.experiment.experiments.experiment prompttools.experiment.experiments.google_palm_experiment prompttools.experiment.experiments.huggingface_endpoint_experiment prompttools.experiment.experiments.huggingface_hub_experiment prompttools.experiment.experiments.llama_cpp_experiment prompttools.experiment.experiments.openai_chat_experiment prompttools.experiment.experiments.openai_completion_experiment prompttools.experiment.widgets prompttools.experiment.widgets.comparison prompttools.experiment.widgets.feedback prompttools.experiment.widgets.utility prompttools.harness prompttools.harness.chat_history_harness prompttools.harness.chat_model_comparison_harness prompttools.harness.document_retrieval_harness prompttools.harness.function_call_harness prompttools.harness.harness prompttools.harness.multi_experiment_harness prompttools.harness.prompt_template_harness prompttools.harness.system_prompt_harness prompttools.mock prompttools.mock.mock prompttools.prompttest prompttools.prompttest.error prompttools.prompttest.error.failure prompttools.prompttest.prompttest prompttools.prompttest.runner prompttools.prompttest.runner.runner prompttools.prompttest.threshold_type prompttools.requests prompttools.requests.request_queue prompttools.requests.retries prompttools.utils prompttools.utils.autoeval prompttools.utils.error prompttools.utils.expected prompttools.utils.similarity prompttools.version MultiExperimentHarness (class in prompttools.harness), [1] (class in prompttools.harness.multi_experiment_harness) O OpenAIChatExperiment (class in prompttools.experiment), [1] (class in prompttools.experiment.experiments.openai_chat_experiment) OpenAICompletionExperiment (class in prompttools.experiment), [1] (class in prompttools.experiment.experiments.openai_completion_experiment) P palm_completion_fn() (prompttools.experiment.experiments.google_palm_experiment.GooglePaLMCompletionExperiment method) (prompttools.experiment.GooglePaLMCompletionExperiment method) PARAMETER_NAMES (prompttools.experiment.ChromaDBExperiment attribute) (prompttools.experiment.experiments.chromadb_experiment.ChromaDBExperiment attribute) partial_df (prompttools.harness.ExperimentationHarness property) (prompttools.harness.harness.ExperimentationHarness property) (prompttools.harness.ModelComparisonHarness property) pinecone_completion_fn() (prompttools.experiment.PineconeExperiment method) PineconeExperiment (class in prompttools.experiment), [1] PIVOT_COLUMNS (prompttools.harness.chat_model_comparison_harness.ChatModelComparisonHarness attribute) (prompttools.harness.ChatModelComparisonHarness attribute) (prompttools.harness.ChatPromptTemplateExperimentationHarness attribute) (prompttools.harness.ExperimentationHarness attribute) (prompttools.harness.harness.ExperimentationHarness attribute) (prompttools.harness.ModelComparisonHarness attribute) (prompttools.harness.prompt_template_harness.PromptTemplateExperimentationHarness attribute) (prompttools.harness.PromptTemplateExperimentationHarness attribute) (prompttools.harness.system_prompt_harness.SystemPromptExperimentationHarness attribute) (prompttools.harness.SystemPromptExperimentationHarness attribute) pivot_table() (prompttools.experiment.Experiment method), [1] (prompttools.experiment.experiments.experiment.Experiment method) prepare() (prompttools.experiment.ChromaDBExperiment method) (prompttools.experiment.Experiment method), [1] (prompttools.experiment.experiments.chromadb_experiment.ChromaDBExperiment method) (prompttools.experiment.experiments.experiment.Experiment method) (prompttools.experiment.experiments.huggingface_hub_experiment.HuggingFaceHubExperiment method) (prompttools.experiment.experiments.llama_cpp_experiment.LlamaCppExperiment method) (prompttools.experiment.HuggingFaceHubExperiment method) (prompttools.experiment.LanceDBExperiment method) (prompttools.experiment.LlamaCppExperiment method) (prompttools.experiment.MindsDBExperiment method) (prompttools.experiment.PineconeExperiment method) (prompttools.experiment.QdrantExperiment method) (prompttools.experiment.ReplicateExperiment method) (prompttools.experiment.RouterChainExperiment method) (prompttools.experiment.SequentialChainExperiment method) (prompttools.experiment.StableDiffusionExperiment method) (prompttools.experiment.WeaviateExperiment method) (prompttools.harness.chat_history_harness.ChatHistoryExperimentationHarness method) (prompttools.harness.chat_model_comparison_harness.ChatModelComparisonHarness method) (prompttools.harness.ChatHistoryExperimentationHarness method) (prompttools.harness.ChatModelComparisonHarness method) (prompttools.harness.ChatPromptTemplateExperimentationHarness method) (prompttools.harness.ExperimentationHarness method), [1] (prompttools.harness.harness.ExperimentationHarness method) (prompttools.harness.ModelComparisonHarness method) (prompttools.harness.multi_experiment_harness.MultiExperimentHarness method) (prompttools.harness.MultiExperimentHarness method) (prompttools.harness.prompt_template_harness.PromptTemplateExperimentationHarness method) (prompttools.harness.PromptTemplateExperimentationHarness method) (prompttools.harness.system_prompt_harness.SystemPromptExperimentationHarness method) (prompttools.harness.SystemPromptExperimentationHarness method) PromptExperimentException PromptTemplateExperimentationHarness (class in prompttools.harness), [1] (class in prompttools.harness.prompt_template_harness) prompttest() (in module prompttools.prompttest.prompttest) PromptTestRunner (class in prompttools.prompttest.runner.runner) PromptTestSetupException prompttools module prompttools.experiment module prompttools.experiment.experiments module prompttools.experiment.experiments.chromadb_experiment module prompttools.experiment.experiments.error module prompttools.experiment.experiments.experiment module prompttools.experiment.experiments.google_palm_experiment module prompttools.experiment.experiments.huggingface_endpoint_experiment module prompttools.experiment.experiments.huggingface_hub_experiment module prompttools.experiment.experiments.llama_cpp_experiment module prompttools.experiment.experiments.openai_chat_experiment module prompttools.experiment.experiments.openai_completion_experiment module prompttools.experiment.widgets module prompttools.experiment.widgets.comparison module prompttools.experiment.widgets.feedback module prompttools.experiment.widgets.utility module prompttools.harness module prompttools.harness.chat_history_harness module prompttools.harness.chat_model_comparison_harness module prompttools.harness.document_retrieval_harness module prompttools.harness.function_call_harness module prompttools.harness.harness module prompttools.harness.multi_experiment_harness module prompttools.harness.prompt_template_harness module prompttools.harness.system_prompt_harness module prompttools.mock module prompttools.mock.mock module prompttools.prompttest module prompttools.prompttest.error module prompttools.prompttest.error.failure module prompttools.prompttest.prompttest module prompttools.prompttest.runner module prompttools.prompttest.runner.runner module prompttools.prompttest.threshold_type module prompttools.requests module prompttools.requests.request_queue module prompttools.requests.retries module prompttools.utils module prompttools.utils.autoeval module prompttools.utils.error module prompttools.utils.expected module prompttools.utils.similarity module prompttools.version module PromptToolsUtilityError Q qdrant_completion_fn() (prompttools.experiment.QdrantExperiment method) QdrantExperiment (class in prompttools.experiment), [1] R rank() (prompttools.experiment.Experiment method), [1] (prompttools.experiment.experiments.experiment.Experiment method) (prompttools.harness.ExperimentationHarness method), [1] (prompttools.harness.harness.ExperimentationHarness method) (prompttools.harness.multi_experiment_harness.MultiExperimentHarness method) (prompttools.harness.MultiExperimentHarness method) ranking_correlation() (in module prompttools.utils), [1] replicate_completion_fn() (prompttools.experiment.ReplicateExperiment static method) ReplicateExperiment (class in prompttools.experiment), [1], [2] RequestQueue (class in prompttools.requests.request_queue) RetrievalAugmentedGenerationExperimentationHarness (class in prompttools.harness) RouterChainExperiment (class in prompttools.experiment), [1] run() (prompttools.experiment.ChromaDBExperiment method) (prompttools.experiment.Experiment method), [1] (prompttools.experiment.experiments.chromadb_experiment.ChromaDBExperiment method) (prompttools.experiment.experiments.experiment.Experiment method) (prompttools.experiment.experiments.huggingface_hub_experiment.HuggingFaceHubExperiment method) (prompttools.experiment.experiments.llama_cpp_experiment.LlamaCppExperiment method) (prompttools.experiment.HuggingFaceHubExperiment method) (prompttools.experiment.LanceDBExperiment method) (prompttools.experiment.LlamaCppExperiment method) (prompttools.experiment.MindsDBExperiment method) (prompttools.experiment.PineconeExperiment method) (prompttools.experiment.QdrantExperiment method) (prompttools.experiment.RouterChainExperiment method) (prompttools.experiment.SequentialChainExperiment method) (prompttools.experiment.StableDiffusionExperiment method) (prompttools.experiment.WeaviateExperiment method) (prompttools.harness.chat_history_harness.ChatHistoryExperimentationHarness method) (prompttools.harness.chat_model_comparison_harness.ChatModelComparisonHarness method) (prompttools.harness.ChatHistoryExperimentationHarness method) (prompttools.harness.ChatModelComparisonHarness method) (prompttools.harness.ChatPromptTemplateExperimentationHarness method) (prompttools.harness.ExperimentationHarness method), [1] (prompttools.harness.harness.ExperimentationHarness method) (prompttools.harness.ModelComparisonHarness method) (prompttools.harness.multi_experiment_harness.MultiExperimentHarness method) (prompttools.harness.MultiExperimentHarness method) (prompttools.harness.prompt_template_harness.PromptTemplateExperimentationHarness method) (prompttools.harness.PromptTemplateExperimentationHarness method) (prompttools.harness.RetrievalAugmentedGenerationExperimentationHarness method) (prompttools.harness.system_prompt_harness.SystemPromptExperimentationHarness method) (prompttools.harness.SystemPromptExperimentationHarness method) (prompttools.prompttest.runner.runner.PromptTestRunner method) run_one() (prompttools.experiment.experiments.openai_chat_experiment.OpenAIChatExperiment method) (prompttools.experiment.OpenAIChatExperiment method) run_partial() (prompttools.experiment.experiments.openai_chat_experiment.OpenAIChatExperiment method) (prompttools.experiment.OpenAIChatExperiment method) run_prompttest() (in module prompttools.prompttest.runner.runner) S save_experiment() (prompttools.experiment.experiments.openai_chat_experiment.OpenAIChatExperiment method) (prompttools.experiment.OpenAIChatExperiment method) (prompttools.harness.ExperimentationHarness method), [1] (prompttools.harness.harness.ExperimentationHarness method) score_df (prompttools.harness.ExperimentationHarness property) (prompttools.harness.harness.ExperimentationHarness property) (prompttools.harness.ModelComparisonHarness property) scores() (prompttools.prompttest.runner.runner.PromptTestRunner method) sd_completion_fn() (prompttools.experiment.StableDiffusionExperiment method) semantic_similarity() (in module prompttools.utils), [1] (in module prompttools.utils.similarity) SequentialChainExperiment (class in prompttools.experiment), [1] set_models() (prompttools.experiment.widgets.comparison.ComparisonWidgetProvider method) set_pivot_columns() (prompttools.experiment.widgets.feedback.FeedbackWidgetProvider method) shutdown() (prompttools.requests.request_queue.RequestQueue method) StableDiffusionExperiment (class in prompttools.experiment), [1] structural_similarity() (in module prompttools.utils.similarity) SystemPromptExperimentationHarness (class in prompttools.harness), [1] (class in prompttools.harness.system_prompt_harness) T ThresholdType (class in prompttools.prompttest.threshold_type) to_csv() (prompttools.experiment.Experiment method), [1] (prompttools.experiment.experiments.experiment.Experiment method) to_json() (prompttools.experiment.Experiment method), [1] (prompttools.experiment.experiments.experiment.Experiment method) to_lora_json() (prompttools.experiment.Experiment method), [1] (prompttools.experiment.experiments.experiment.Experiment method) to_markdown() (prompttools.experiment.Experiment method) (prompttools.experiment.experiments.experiment.Experiment method) to_mongo_db() (prompttools.experiment.Experiment method), [1] (prompttools.experiment.experiments.experiment.Experiment method) to_pandas_df() (prompttools.experiment.Experiment method), [1] (prompttools.experiment.experiments.experiment.Experiment method) V validate_json_response() (in module prompttools.utils), [1] validate_keys() (in module prompttools.utils.validate_json) validate_python_response() (in module prompttools.utils), [1] vertex_chat_completion_fn() (prompttools.experiment.GoogleVertexChatCompletionExperiment method) visualize() (prompttools.experiment.Experiment method), [1] (prompttools.experiment.experiments.experiment.Experiment method) (prompttools.experiment.ReplicateExperiment method) (prompttools.harness.ChatPromptTemplateExperimentationHarness method) (prompttools.harness.ExperimentationHarness method), [1] (prompttools.harness.harness.ExperimentationHarness method) (prompttools.harness.ModelComparisonHarness method) (prompttools.harness.multi_experiment_harness.MultiExperimentHarness method) (prompttools.harness.MultiExperimentHarness method) (prompttools.harness.RetrievalAugmentedGenerationExperimentationHarness method) (prompttools.harness.system_prompt_harness.SystemPromptExperimentationHarness method) (prompttools.harness.SystemPromptExperimentationHarness method) (prompttools.prompttest.runner.runner.PromptTestRunner method) W weaviate_completion_fn() (prompttools.experiment.WeaviateExperiment method) WeaviateExperiment (class in prompttools.experiment), [1]