Compare commits
22 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 4baaaa438c | |||
|
363421c61c | ||
|
|
a939a4577c | ||
|
|
15fb8e661d | ||
|
|
2978238318 | ||
|
|
aecfe2292a | ||
|
|
f508aef4fd | ||
|
|
b5a8c960d6 | ||
|
|
a6aee6bb76 | ||
|
|
ac4a35ea9f | ||
|
|
8f1e6ce81d | ||
|
|
01ac301b9f | ||
|
|
80957be75e | ||
|
|
14e05abbe1 | ||
|
|
7441be41ba | ||
|
|
8590e4bd65 | ||
|
|
b41d653f21 | ||
|
|
f6be6c06f2 | ||
|
|
2a32d77be6 | ||
|
|
48f024297c | ||
|
|
ef3ce13d39 | ||
|
|
e22a8dfa89 |
7
LICENSE
7
LICENSE
@ -46,7 +46,8 @@ following citation:
|
||||
----------------------------------------------------------------------
|
||||
Maciej Besta, Nils Blach, Ales Kubicek, Robert Gerstenberger, Lukas
|
||||
Gianinazzi, Joanna Gajda, Tomasz Lehmann, Michał Podstawski, Hubert
|
||||
Niewiadomski, Piotr Nyczyk, Torsten Hoefler: Graph of Thoughts: Solving
|
||||
Elaborate Problems with Large Language Models. In: arXiv preprint
|
||||
arXiv:2308.09687
|
||||
Niewiadomski, Piotr Nyczyk, Torsten Hoefler (2024): Graph of Thoughts:
|
||||
Solving Elaborate Problems with Large Language Models. In: Proceedings
|
||||
of the AAAI Conference on Artificial Intelligence, 38(16),
|
||||
17682-17690. https://doi.org/10.1609/aaai.v38i16.29720
|
||||
----------------------------------------------------------------------
|
||||
|
||||
45
README.md
45
README.md
@ -15,9 +15,9 @@ In order to use this framework, you need to have a working installation of Pytho
|
||||
### Installing GoT
|
||||
|
||||
Before running either of the following two installation methods, make sure to activate your Python environment (if any) beforehand.
|
||||
If you are a user and you just want to use `graph_of_thoughts`, you can install it directly from from PyPI:
|
||||
If you are a user and you just want to use `graph_of_thoughts`, you can install it directly from PyPI:
|
||||
```bash
|
||||
pip install graph-of-thoughts
|
||||
pip install graph_of_thoughts
|
||||
```
|
||||
If you are a developer and you want to modify the code, you can install it in editable mode from source:
|
||||
```bash
|
||||
@ -31,6 +31,15 @@ pip install -e .
|
||||
In order to use the framework, you need to have access to an LLM.
|
||||
Please follow the instructions in the [Controller README](graph_of_thoughts/controller/README.md) to configure the LLM of your choice.
|
||||
|
||||
### OpenRouter and OpenAI-compatible HTTP API
|
||||
|
||||
1. Install API extras: `pip install "graph_of_thoughts[api]"` (or `pip install -e ".[api]"` from a source checkout).
|
||||
2. Copy [`graph_of_thoughts/language_models/config.openrouter.example.yaml`](graph_of_thoughts/language_models/config.openrouter.example.yaml) to `config.openrouter.yaml`, add your [OpenRouter](https://openrouter.ai/) keys and model ids, and either place it in `graph_of_thoughts/language_models/` or point `OPENROUTER_CONFIG` at your file.
|
||||
3. Run the server: `got-openrouter-api` (or `python -m graph_of_thoughts.api`).
|
||||
4. Call `POST /v1/chat/completions` with a standard OpenAI-style JSON body (`messages`, optional `model`, `temperature`, `max_tokens`). The server runs a small Graph of Operations (generate multiple candidates, score, keep the best) via OpenRouter.
|
||||
|
||||
Details: [Language models README](graph_of_thoughts/language_models/README.md).
|
||||
|
||||
## Quick Start
|
||||
|
||||
The following code snippet shows how to use the framework to solve the sorting problem for a list of 32 numbers using a CoT-like approach.
|
||||
@ -38,7 +47,7 @@ Make sure you have followed the [Setup Guide](#setup-guide) before running the c
|
||||
|
||||
```python
|
||||
from examples.sorting.sorting_032 import SortingPrompter, SortingParser, utils
|
||||
from graph_of_thoughts import controller, operations
|
||||
from graph_of_thoughts import controller, language_models, operations
|
||||
|
||||
# Problem input
|
||||
|
||||
@ -51,7 +60,7 @@ gop.append_operation(operations.Score(scoring_function=utils.num_errors))
|
||||
gop.append_operation(operations.GroundTruth(utils.test_sorting))
|
||||
|
||||
# Configure the Language Model (Assumes config.json is in the current directory with OpenAI API key)
|
||||
lm = controller.ChatGPT("config.json", model_name="chatgpt")
|
||||
lm = language_models.ChatGPT("config.json", model_name="chatgpt")
|
||||
|
||||
# Create the Controller
|
||||
ctrl = controller.Controller(
|
||||
@ -59,7 +68,7 @@ ctrl = controller.Controller(
|
||||
gop,
|
||||
SortingPrompter(),
|
||||
SortingParser(),
|
||||
# The following dictionary is used to configure the inital thought state
|
||||
# The following dictionary is used to configure the initial thought state
|
||||
{
|
||||
"original": to_be_sorted,
|
||||
"current": "",
|
||||
@ -76,7 +85,7 @@ To run the more sophisticated GoT approach, you can use the following code snipp
|
||||
|
||||
```python
|
||||
from examples.sorting.sorting_032 import SortingPrompter, SortingParser, got, utils
|
||||
from graph_of_thoughts import controller, operations
|
||||
from graph_of_thoughts import controller, language_models, operations
|
||||
|
||||
# Problem input
|
||||
|
||||
@ -86,7 +95,7 @@ to_be_sorted = "[0, 2, 6, 3, 8, 7, 1, 1, 6, 7, 7, 7, 7, 9, 3, 0, 1, 7, 9, 1, 3,
|
||||
gop = got()
|
||||
|
||||
# Configure the Language Model (Assumes config.json is in the current directory with OpenAI API key)
|
||||
lm = controller.ChatGPT("config.json", model_name="chatgpt")
|
||||
lm = language_models.ChatGPT("config.json", model_name="chatgpt")
|
||||
|
||||
# Create the Controller
|
||||
ctrl = controller.Controller(
|
||||
@ -94,7 +103,7 @@ ctrl = controller.Controller(
|
||||
gop,
|
||||
SortingPrompter(),
|
||||
SortingParser(),
|
||||
# The following dictionary is used to configure the inital thought state
|
||||
# The following dictionary is used to configure the initial thought state
|
||||
{
|
||||
"original": to_be_sorted,
|
||||
"current": "",
|
||||
@ -121,7 +130,13 @@ We took extra care to fully document the code, so that you can easily understand
|
||||
The [examples](examples) directory contains several examples of problems that can be solved using the framework, including the ones presented in the paper.
|
||||
It is a great starting point for learning how to use the framework to solve real problems.
|
||||
Each example contains a `README.md` file with instructions on how to run it and play with it. The code is fully documented and should be easy to follow.
|
||||
You can also run the examples straight from the main directory. Note that the results will be stored in the respective examples sub-directory.
|
||||
|
||||
Try for instance:
|
||||
```bash
|
||||
python -m examples.sorting.sorting_032
|
||||
python -m examples.keyword_counting.keyword_counting
|
||||
```
|
||||
## Paper Results
|
||||
|
||||
You can run the experiments from the paper by following the instructions in the [examples](examples) directory.
|
||||
@ -134,11 +149,17 @@ Got any questions or feedback? Feel free to reach out to [nils.blach@inf.ethz.ch
|
||||
Using this in your work? Please reference us using the provided citation:
|
||||
|
||||
```bibtex
|
||||
@misc{besta2023got,
|
||||
@article{besta2024got,
|
||||
title = {{Graph of Thoughts: Solving Elaborate Problems with Large Language Models}},
|
||||
author = {Besta, Maciej and Blach, Nils and Kubicek, Ales and Gerstenberger, Robert and Gianinazzi, Lukas and Gajda, Joanna and Lehmann, Tomasz and Podstawski, Micha{\l} and Niewiadomski, Hubert and Nyczyk, Piotr and Hoefler, Torsten},
|
||||
year = 2023,
|
||||
eprinttype = {arXiv},
|
||||
eprint = {2308.09687}
|
||||
year = 2024,
|
||||
month = {Mar},
|
||||
journal = {Proceedings of the AAAI Conference on Artificial Intelligence},
|
||||
volume = 38,
|
||||
number = 16,
|
||||
pages = {17682-17690},
|
||||
publisher = {AAAI Press},
|
||||
doi = {10.1609/aaai.v38i16.29720},
|
||||
url = {https://ojs.aaai.org/index.php/AAAI/article/view/29720}
|
||||
}
|
||||
```
|
||||
|
||||
@ -2,4 +2,6 @@
|
||||
|
||||
This directory contains scripts for running various examples using the Graph of Thoughts package. Each script is a standalone Python program that sets up and runs a particular example.
|
||||
|
||||
We further include prompt files for each example that can be used to test prompts manually in a console.
|
||||
|
||||
Please refer to the individual example directories for more information on the specific example.
|
||||
|
||||
@ -14,7 +14,7 @@ import json
|
||||
import csv
|
||||
from statistics import fmean
|
||||
from typing import Dict, List, Callable, Set, Union
|
||||
from graph_of_thoughts import controller, operations, prompter, parser
|
||||
from graph_of_thoughts import controller, language_models, operations, prompter, parser
|
||||
|
||||
|
||||
class DocMergePrompter(prompter.Prompter):
|
||||
@ -65,7 +65,7 @@ Here is the summary NDA <S>:
|
||||
"""
|
||||
|
||||
score_prompt_base = """The following NDA <S> merges NDAs <Doc1> - <Doc{num}>.
|
||||
Please score the merged NDA <S> in terms of how much redundant information is contained, independent of the original NDAs, as well as how much information if retained from the original NDAs.
|
||||
Please score the merged NDA <S> in terms of how much redundant information is contained, independent of the original NDAs, as well as how much information is retained from the original NDAs.
|
||||
A score of 10 for redundancy implies that absolutely no information is redundant, while a score of 0 implies that at least half of the information is redundant (so everything is at least mentioned twice).
|
||||
A score of 10 for retained information implies that all information from the original NDAs is retained, while a score of 0 implies that no information is retained.
|
||||
You may provide reasoning for your scoring, but the final score for redundancy should be between the tags <Redundancy> and </Redundancy>, and the final score for retained information should be between the tags <Retained> and </Retained>, without any additional text within any of those tags.
|
||||
@ -219,6 +219,9 @@ NDA <S{num}>:
|
||||
prompt += self.improve_summary_prompt_end.format(summary=current)
|
||||
return prompt
|
||||
elif method.startswith("got"):
|
||||
parts = (
|
||||
sorted(list(parts)) if len(parts) > 0 else list(range(len(documents)))
|
||||
)
|
||||
if current is None or current == "":
|
||||
prompt += self.merge_doc_prompt_start.format(num=len(parts))
|
||||
for i, part in enumerate(sorted(list(parts))):
|
||||
@ -546,9 +549,7 @@ def got() -> operations.GraphOfOperations:
|
||||
"""
|
||||
operations_graph = operations.GraphOfOperations()
|
||||
|
||||
branch_factor = 10
|
||||
|
||||
operations_graph.append_operation(operations.Generate(1, branch_factor))
|
||||
operations_graph.append_operation(operations.Generate(1, 5))
|
||||
operations_graph.append_operation(operations.Score(3, False))
|
||||
keep_best = operations.KeepBestN(3, True)
|
||||
operations_graph.append_operation(keep_best)
|
||||
@ -557,7 +558,7 @@ def got() -> operations.GraphOfOperations:
|
||||
keep_best2 = operations.KeepBestN(1, True)
|
||||
keep_best2.add_predecessor(keep_best)
|
||||
operations_graph.append_operation(keep_best2)
|
||||
operations_graph.append_operation(operations.Generate(1, branch_factor))
|
||||
operations_graph.append_operation(operations.Generate(1, 10))
|
||||
operations_graph.append_operation(operations.Score(3, False))
|
||||
keep_best3 = operations.KeepBestN(1, True)
|
||||
keep_best3.add_predecessor(keep_best2)
|
||||
@ -655,9 +656,9 @@ def run(
|
||||
"""
|
||||
|
||||
orig_budget = budget
|
||||
path = os.path.join(os.path.dirname(__file__), "documents.csv")
|
||||
data_path = os.path.join(os.path.dirname(__file__), "documents.csv")
|
||||
data = []
|
||||
with open(path, "r", encoding="utf8") as f:
|
||||
with open(data_path, "r", encoding="utf8") as f:
|
||||
reader = csv.reader(f)
|
||||
next(reader)
|
||||
for row in reader:
|
||||
@ -668,12 +669,15 @@ def run(
|
||||
data_ids = list(range(len(data)))
|
||||
selected_data = [data[i] for i in data_ids]
|
||||
|
||||
if not os.path.exists(os.path.join(os.path.dirname(__file__), "results")):
|
||||
os.makedirs(os.path.join(os.path.dirname(__file__), "results"))
|
||||
results_dir = os.path.join(os.path.dirname(__file__), "results")
|
||||
|
||||
if not os.path.exists(results_dir):
|
||||
os.makedirs(results_dir)
|
||||
timestamp = datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
|
||||
extra_info = f"{lm_name}_{'-'.join([method.__name__ for method in methods])}"
|
||||
folder_name = f"results/{extra_info}_{timestamp}"
|
||||
os.makedirs(os.path.join(os.path.dirname(__file__), folder_name))
|
||||
folder_name = f"{extra_info}_{timestamp}"
|
||||
results_folder = os.path.join(results_dir, folder_name)
|
||||
os.makedirs(results_folder)
|
||||
|
||||
config = {
|
||||
"data": selected_data,
|
||||
@ -681,22 +685,18 @@ def run(
|
||||
"lm": lm_name,
|
||||
"budget": budget,
|
||||
}
|
||||
with open(
|
||||
os.path.join(os.path.dirname(__file__), folder_name, "config.json"), "w"
|
||||
) as f:
|
||||
with open(os.path.join(results_folder, "config.json"), "w") as f:
|
||||
json.dump(config, f)
|
||||
|
||||
logging.basicConfig(
|
||||
filename=f"{folder_name}/log.log",
|
||||
filename=os.path.join(results_folder, "log.log"),
|
||||
filemode="w",
|
||||
format="%(name)s - %(levelname)s - %(message)s",
|
||||
level=logging.DEBUG,
|
||||
)
|
||||
|
||||
for method in methods:
|
||||
os.makedirs(
|
||||
os.path.join(os.path.dirname(__file__), folder_name, method.__name__)
|
||||
)
|
||||
os.makedirs(os.path.join(results_folder, method.__name__))
|
||||
|
||||
for data in selected_data:
|
||||
logging.info(f"Running data {data[0]}: {data[1]}")
|
||||
@ -713,8 +713,11 @@ def run(
|
||||
f"Budget has been depleted, stopping. Method {method.__name__} has not been run."
|
||||
)
|
||||
break
|
||||
lm = controller.ChatGPT(
|
||||
"../../graph_of_thoughts/controller/config.json",
|
||||
lm = language_models.ChatGPT(
|
||||
os.path.join(
|
||||
os.path.dirname(__file__),
|
||||
"../../graph_of_thoughts/language_models/config.json",
|
||||
),
|
||||
model_name=lm_name,
|
||||
cache=True,
|
||||
)
|
||||
@ -736,8 +739,7 @@ def run(
|
||||
except Exception as e:
|
||||
logging.error(f"Exception: {e}")
|
||||
path = os.path.join(
|
||||
os.path.dirname(__file__),
|
||||
folder_name,
|
||||
results_folder,
|
||||
method.__name__,
|
||||
f"{data[0]}.json",
|
||||
)
|
||||
|
||||
2626
examples/doc_merge/example_prompts_document_merging.md
Normal file
2626
examples/doc_merge/example_prompts_document_merging.md
Normal file
File diff suppressed because it is too large
Load Diff
998
examples/keyword_counting/example_prompts_keyword_counting.md
Normal file
998
examples/keyword_counting/example_prompts_keyword_counting.md
Normal file
@ -0,0 +1,998 @@
|
||||
# Keyword Counting 4-passage Split (GoT4) - Prompts and Examples
|
||||
## Prompt Templates
|
||||
### GENERATE: split_prompt
|
||||
Replace `{input_text}` with an input text to be split.
|
||||
```
|
||||
<Instruction> Split the following input text into 4 paragraphs of approximately same length.
|
||||
Only output the final 4 paragraphs in the following format without any additional text or thoughts:
|
||||
{
|
||||
"Paragraph 1": "Some paragraph text ...",
|
||||
"Paragraph 2": "Some paragraph text ...",
|
||||
"Paragraph 3": "Some paragraph text ...",
|
||||
"Paragraph 4": "Some paragraph text ..."
|
||||
}
|
||||
</Instruction>
|
||||
|
||||
<Example>
|
||||
Input:
|
||||
Journeying westward, she admired the art in Italy and sipped coffee in France. The music of Spain and the history of Greece deepened her love for Europe. The Nordic beauty of Norway, Sweden, Finland, and Denmark took her breath away. She danced in Ireland, explored castles in Scotland, and marveled at the architecture in Germany and Russia. Italy, Norway, Sweden and Germany will always stay her favourite destinations to visit.
|
||||
Output:
|
||||
{
|
||||
"Paragraph 1": "Journeying westward, she admired the art in Italy and sipped coffee in France.",
|
||||
"Paragraph 2": "The music of Spain and the history of Greece deepened her love for Europe. The Nordic beauty of Norway, Sweden, Finland, and Denmark took her breath away.",
|
||||
"Paragraph 3": "She danced in Ireland, explored castles in Scotland, and marveled at the architecture in Germany and Russia.",
|
||||
"Paragraph 4": "Italy, Norway, Sweden and Germany will always stay her favourite destinations to visit."
|
||||
}
|
||||
</Example>
|
||||
|
||||
Input: {input_text}
|
||||
```
|
||||
|
||||
### GENERATE: count_prompt
|
||||
Replace `{input_text}` with a text where countries should be counted.
|
||||
```
|
||||
<Instruction> Count the frequency of how many times each country is explicitly named in the input text. You can generate any intermedate lists and states, but the final output should only contain the frequency of each country that appears at least once in the following json format, prefixed with "Output: " (make sure to keep the same spelling for each country in the output as in the input text):
|
||||
{
|
||||
"country1": frequency1,
|
||||
"country2": frequency2,
|
||||
...
|
||||
}
|
||||
</Instruction>
|
||||
|
||||
<Approach>
|
||||
To count the frequency for each country follow these steps:
|
||||
1. Split the input passage into four paragraphs of similar length.
|
||||
2. Count the frequency of each country in each paragraph.
|
||||
3. Combine the frequencies of each country from each paragraph by adding them together.
|
||||
</Approach>
|
||||
|
||||
<Examples>
|
||||
Input:
|
||||
Alexandra boarded the first flight of her grand journey, starting from Canada. With a globe-trotting itinerary in hand, she was filled with excitement. Her first stop was Mexico, where she marveled at the Mayan ruins. From there, she explored the rainforests of Brazil and danced the tango in Argentina.
|
||||
Paragraphs:
|
||||
Alexandra boarded the first flight of her grand journey, starting from Canada. With a globe-trotting itinerary in hand, she was filled with excitement.
|
||||
Her first stop was Mexico, where she marveled at the Mayan ruins. From there, she explored the rainforests of Brazil and danced the tango in Argentina.
|
||||
Sublist frequencies:
|
||||
{
|
||||
"Canada": 1
|
||||
}
|
||||
{
|
||||
"Mexico": 1,
|
||||
"Brazil": 1,
|
||||
"Argentina": 1
|
||||
}
|
||||
Output:
|
||||
{
|
||||
"Canada": 1,
|
||||
"Mexico": 1,
|
||||
"Brazil": 1,
|
||||
"Argentina": 1
|
||||
}
|
||||
|
||||
Input:
|
||||
The adventure led him to the peaks of Peru where he trekked to see the mysteries of Machu Picchu. He then headed to Chile to gaze at the vastness of the Atacama Desert. A quick detour to Uruguay and Paraguay allowed him to experience the vibrancy of the local cultures before returning back to Canada through Peru, Brazil and Mexico.
|
||||
Paragraphs:
|
||||
The adventure led him to the peaks of Peru where he trekked to see the mysteries of Machu Picchu. He then headed to Chile to gaze at the vastness of the Atacama Desert.
|
||||
A quick detour to Uruguay and Paraguay allowed him to experience the vibrancy of the local cultures before returning back to Canada through Peru, Brazil and Mexico.
|
||||
Sublists:
|
||||
{
|
||||
"Peru": 1,
|
||||
"Chile": 1
|
||||
}
|
||||
{
|
||||
"Uruguay": 1,
|
||||
"Paraguay": 1,
|
||||
"Canada": 1,
|
||||
"Peru": 1,
|
||||
"Brazil": 1,
|
||||
"Mexico": 1
|
||||
}
|
||||
Output:
|
||||
{
|
||||
"Peru": 2,
|
||||
"Chile": 1,
|
||||
"Uruguay": 1,
|
||||
"Paraguay": 1,
|
||||
"Canada": 1,
|
||||
"Brazil": 1,
|
||||
"Mexico": 1
|
||||
}
|
||||
|
||||
Input:
|
||||
Journeying westward, she admired the art in Italy and sipped coffee in France. The music of Spain and the history of Greece deepened her love for Europe. The Nordic beauty of Norway, Sweden, Finland, and Denmark took her breath away. She danced in Ireland, explored castles in Scotland, and marveled at the architecture in Germany and Russia. Italy, Norway, Sweden and Germany will always stay her favourite destinations to visit.
|
||||
Paragraphs:
|
||||
Journeying westward, she admired the art in Italy and sipped coffee in France.
|
||||
The music of Spain and the history of Greece deepened her love for Europe. The Nordic beauty of Norway, Sweden, Finland, and Denmark took her breath away.
|
||||
She danced in Ireland, explored castles in Scotland, and marveled at the architecture in Germany and Russia.
|
||||
Italy, Norway, Sweden and Germany will always stay her favourite destinations to visit.
|
||||
Sublists:
|
||||
{
|
||||
"Italy": 1,
|
||||
"France": 1
|
||||
}
|
||||
{
|
||||
"Spain": 1,
|
||||
"Greece": 1,
|
||||
"Norway": 1,
|
||||
"Sweden": 1,
|
||||
"Finland": 1,
|
||||
"Denmark": 1
|
||||
}
|
||||
{
|
||||
"Ireland": 1,
|
||||
"Scotland": 1,
|
||||
"Germany": 1,
|
||||
"Russia": 1
|
||||
}
|
||||
{
|
||||
"Italy": 1,
|
||||
"Norway": 1,
|
||||
"Sweden": 1,
|
||||
"Germany": 1
|
||||
}
|
||||
Output:
|
||||
{
|
||||
"Italy": 2,
|
||||
"France": 1,
|
||||
"Spain": 1,
|
||||
"Greece": 1,
|
||||
"Norway": 2,
|
||||
"Sweden": 2,
|
||||
"Finland": 1,
|
||||
"Denmark": 1,
|
||||
"Ireland": 1,
|
||||
"Scotland": 1,
|
||||
"Germany": 2,
|
||||
"Russia": 1
|
||||
}
|
||||
</Examples>
|
||||
|
||||
Input:
|
||||
{input_text}
|
||||
```
|
||||
|
||||
### AGGREGATE: merge_prompt
|
||||
Replace `{dictionary_1}` and `{dictionary_2}` with dictionaries of countries to be merged.
|
||||
```
|
||||
<Instruction> Combine the following 2 dictionaries, each containing the frequency of countries in a text, into a single dictionary.
|
||||
Simply add the frequencies together for each country and if a country is not present in one of the dictionaries, add it to the final dictionary with the frequency from the other dictionary.
|
||||
Only output the final merged dictionary without any additional text or thoughts!
|
||||
</Instruction>
|
||||
|
||||
<Approach>
|
||||
To combine the 2 dictionaries into single one, follow these steps:
|
||||
1. Create a new dictionary to store the combined frequencies.
|
||||
2. Iterate through the keys of the first dictionary and add the frequency of each country to the new dictionary.
|
||||
3. Iterate through the keys of the second dictionary and add the frequency of each country to the new dictionary and if it is already present, add the frequency to the existing value.
|
||||
</Approach>
|
||||
|
||||
Combine the following 2 dictionaries into a single dictionary:
|
||||
{dictionary_1}
|
||||
{dictionary_2}
|
||||
|
||||
Combined Output:
|
||||
```
|
||||
|
||||
### IMPROVE: improve_merge_prompt
|
||||
Replace `{dictionary_1}` and `{dictionary_2}` with dictionaries of countries
|
||||
to be merged and `{dictionary_incorrect}` with incorrectly merged dictionary
|
||||
to be improved.
|
||||
```
|
||||
<Instruction> The following 2 dictionaries were combined into the third dictionary below. However, some mistakes occured and the third dictionary is incorrect. Please fix the third dictionary so that it contains the correct frequencies for each country. The correct frequencies are the sum of the frequencies from the first 2 dictionaries. If a country is not present in one of the dictionaries, add it to the final dictionary with the frequency from the other dictionary.
|
||||
</Instruction>
|
||||
|
||||
<Example>
|
||||
Dictionary 1:
|
||||
{
|
||||
"Peru": 2,
|
||||
"Chile": 1,
|
||||
"Uruguay": 1,
|
||||
"Paraguay": 1
|
||||
}
|
||||
Dictionary 2:
|
||||
{
|
||||
"Peru": 1,
|
||||
"Argentina": 1,
|
||||
"Canada": 1,
|
||||
"Chile": 3,
|
||||
"Germany": 2
|
||||
}
|
||||
Incorrectly Combined Dictionary:
|
||||
{
|
||||
"Peru": 3,
|
||||
"Chile": 2,
|
||||
"Uruguay": 1,
|
||||
"Paraguay": 1,
|
||||
"Argentina": 1,
|
||||
"Chile": 3,
|
||||
"Germany": 2
|
||||
}
|
||||
Output:
|
||||
{
|
||||
"Peru": 3,
|
||||
"Chile": 4,
|
||||
"Uruguay": 1,
|
||||
"Paraguay": 1,
|
||||
"Argentina": 1,
|
||||
"Canada": 1,
|
||||
"Germany": 2
|
||||
}
|
||||
</Example>
|
||||
|
||||
Dictionary 1:
|
||||
{dictionary_1}
|
||||
Dictionary 2:
|
||||
{dictionary_2}
|
||||
Incorrectly Combined Dictionary:
|
||||
{dictionary_incorrect}
|
||||
Output:
|
||||
```
|
||||
|
||||
## Complete Example Prompts
|
||||
### The GoO Summarised
|
||||
1. Split the input text into four paragraphs of roughly equal size (split prompt)
|
||||
2. For each paragraph: Count the occurrences of individual countries (count prompt) 10 times; score each counting attempt; keep the best
|
||||
3. Merge the country counts into one dictionary (merge prompt) 3 times;
|
||||
validate and improve invalid merge attempts (improve merge prompt) up to 3 attempts each; score; keep the best
|
||||
|
||||
|
||||
### Input
|
||||
```
|
||||
My friend, Alex from Peru, once recounted his journey to Argentina where he learned about the strong cultural ties between Argentina and Brazil due to their shared history.
|
||||
He spoke fondly of his time in Argentina and Brazil, marveling at the similar music, dance, and culinary traditions that seamlessly bridged the borders of these countries.
|
||||
It reminded me of a documentary I'd watched about the ancient Silk Road that spanned across Iran and China.
|
||||
It touched upon the shared historical narratives of Iran and China, highlighting how goods, ideas, and culture flowed between these two countries for centuries.
|
||||
Intriguingly, the documentary also brought up some parallels between this eastern route and the vibrant exchange between Italy and France during the Renaissance.
|
||||
The cultural and intellectual bonds between Italy and France were indeed profound, just as the resilient trade relations that existed between the United States and Canada.
|
||||
The United States and Canada, apart from their geographical proximity, shared an economic bond that strongly influenced their policies and international standing.
|
||||
Similarly, the ties between Australia and New Zealand shed light on their gestalt identity in the Pacific region.
|
||||
Despite their unique characteristics, Australia and New Zealand were often viewed as a single entity due to their remarkably similar backgrounds in terms of culture, language, and colonial history.
|
||||
Inspired by these chronicles of interconnectedness, I decided to delve deeper into history and found a fascinating account of how Ukraine and Poland had influenced each other through years of coexistence.
|
||||
Despite their tumultuous past, Ukraine and Poland shared a cultural tapestry that was deeply woven into their histories.
|
||||
It was quite similar to the complex relationship between North Korea and South Korea, which, despite their political differences, shared common heritage and traditions.
|
||||
Thus, from Argentina to South Korea, the world was an intricate web of countries intertwined with each other through shared histories, cultures, and sometimes, shared destinies.
|
||||
```
|
||||
|
||||
### Step 1 - Prompt
|
||||
```
|
||||
<Instruction> Split the following input text into 4 paragraphs of approximately same length.
|
||||
Only output the final 4 paragraphs in the following format without any additional text or thoughts:
|
||||
{
|
||||
"Paragraph 1": "Some paragraph text ...",
|
||||
"Paragraph 2": "Some paragraph text ...",
|
||||
"Paragraph 3": "Some paragraph text ...",
|
||||
"Paragraph 4": "Some paragraph text ..."
|
||||
}
|
||||
</Instruction>
|
||||
|
||||
<Example>
|
||||
Input:
|
||||
Journeying westward, she admired the art in Italy and sipped coffee in France. The music of Spain and the history of Greece deepened her love for Europe. The Nordic beauty of Norway, Sweden, Finland, and Denmark took her breath away. She danced in Ireland, explored castles in Scotland, and marveled at the architecture in Germany and Russia. Italy, Norway, Sweden and Germany will always stay her favourite destinations to visit.
|
||||
Output:
|
||||
{
|
||||
"Paragraph 1": "Journeying westward, she admired the art in Italy and sipped coffee in France.",
|
||||
"Paragraph 2": "The music of Spain and the history of Greece deepened her love for Europe. The Nordic beauty of Norway, Sweden, Finland, and Denmark took her breath away.",
|
||||
"Paragraph 3": "She danced in Ireland, explored castles in Scotland, and marveled at the architecture in Germany and Russia.",
|
||||
"Paragraph 4": "Italy, Norway, Sweden and Germany will always stay her favourite destinations to visit."
|
||||
}
|
||||
</Example>
|
||||
|
||||
Input:
|
||||
My friend, Alex from Peru, once recounted his journey to Argentina where he learned about the strong cultural ties between Argentina and Brazil due to their shared history.
|
||||
He spoke fondly of his time in Argentina and Brazil, marveling at the similar music, dance, and culinary traditions that seamlessly bridged the borders of these countries.
|
||||
It reminded me of a documentary I'd watched about the ancient Silk Road that spanned across Iran and China.
|
||||
It touched upon the shared historical narratives of Iran and China, highlighting how goods, ideas, and culture flowed between these two countries for centuries.
|
||||
Intriguingly, the documentary also brought up some parallels between this eastern route and the vibrant exchange between Italy and France during the Renaissance.
|
||||
The cultural and intellectual bonds between Italy and France were indeed profound, just as the resilient trade relations that existed between the United States and Canada.
|
||||
The United States and Canada, apart from their geographical proximity, shared an economic bond that strongly influenced their policies and international standing.
|
||||
Similarly, the ties between Australia and New Zealand shed light on their gestalt identity in the Pacific region.
|
||||
Despite their unique characteristics, Australia and New Zealand were often viewed as a single entity due to their remarkably similar backgrounds in terms of culture, language, and colonial history.
|
||||
Inspired by these chronicles of interconnectedness, I decided to delve deeper into history and found a fascinating account of how Ukraine and Poland had influenced each other through years of coexistence.
|
||||
Despite their tumultuous past, Ukraine and Poland shared a cultural tapestry that was deeply woven into their histories.
|
||||
It was quite similar to the complex relationship between North Korea and South Korea, which, despite their political differences, shared common heritage and traditions.
|
||||
Thus, from Argentina to South Korea, the world was an intricate web of countries intertwined with each other through shared histories, cultures, and sometimes, shared destinies.
|
||||
```
|
||||
|
||||
### Step 1 - Response
|
||||
```
|
||||
{
|
||||
"Paragraph 1": "My friend, Alex from Peru, once recounted his journey to Argentina where he learned about the strong cultural ties between Argentina and Brazil due to their shared history. He spoke fondly of his time in Argentina and Brazil, marveling at the similar music, dance, and culinary traditions that seamlessly bridged the borders of these countries.",
|
||||
"Paragraph 2": "It reminded me of a documentary I'd watched about the ancient Silk Road that spanned across Iran and China. It touched upon the shared historical narratives of Iran and China, highlighting how goods, ideas, and culture flowed between these two countries for centuries. Intriguingly, the documentary also brought up some parallels between this eastern route and the vibrant exchange between Italy and France during the Renaissance. The cultural and intellectual bonds between Italy and France were indeed profound, just as the resilient trade relations that existed between the United States and Canada.",
|
||||
"Paragraph 3": "The United States and Canada, apart from their geographical proximity, shared an economic bond that strongly influenced their policies and international standing. Similarly, the ties between Australia and New Zealand shed light on their gestalt identity in the Pacific region. Despite their unique characteristics, Australia and New Zealand were often viewed as a single entity due to their remarkably similar backgrounds in terms of culture, language, and colonial history. Inspired by these chronicles of interconnectedness, I decided to delve deeper into history and found a fascinating account of how Ukraine and Poland had influenced each other through years of coexistence.",
|
||||
"Paragraph 4": "Despite their tumultuous past, Ukraine and Poland shared a cultural tapestry that was deeply woven into their histories. It was quite similar to the complex relationship between North Korea and South Korea, which, despite their political differences, shared common heritage and traditions. Thus, from Argentina to South Korea, the world was an intricate web of countries intertwined with each other through shared histories, cultures, and sometimes, shared destinies."
|
||||
}
|
||||
```
|
||||
|
||||
### Step 2a - Prompt
|
||||
```
|
||||
<Instruction> Count the frequency of how many times each country is explicitly named in the input text. You can generate any intermedate lists and states, but the final output should only contain the frequency of each country that appears at least once in the following json format, prefixed with "Output: " (make sure to keep the same spelling for each country in the output as in the input text):
|
||||
{
|
||||
"country1": frequency1,
|
||||
"country2": frequency2,
|
||||
...
|
||||
}
|
||||
</Instruction>
|
||||
|
||||
<Approach>
|
||||
To count the frequency for each country follow these steps:
|
||||
1. Split the input passage into four paragraphs of similar length.
|
||||
2. Count the frequency of each country in each paragraph.
|
||||
3. Combine the frequencies of each country from each paragraph by adding them together.
|
||||
</Approach>
|
||||
|
||||
<Examples>
|
||||
Input:
|
||||
Alexandra boarded the first flight of her grand journey, starting from Canada. With a globe-trotting itinerary in hand, she was filled with excitement. Her first stop was Mexico, where she marveled at the Mayan ruins. From there, she explored the rainforests of Brazil and danced the tango in Argentina.
|
||||
Paragraphs:
|
||||
Alexandra boarded the first flight of her grand journey, starting from Canada. With a globe-trotting itinerary in hand, she was filled with excitement.
|
||||
Her first stop was Mexico, where she marveled at the Mayan ruins. From there, she explored the rainforests of Brazil and danced the tango in Argentina.
|
||||
Sublist frequencies:
|
||||
{
|
||||
"Canada": 1
|
||||
}
|
||||
{
|
||||
"Mexico": 1,
|
||||
"Brazil": 1,
|
||||
"Argentina": 1
|
||||
}
|
||||
Output:
|
||||
{
|
||||
"Canada": 1,
|
||||
"Mexico": 1,
|
||||
"Brazil": 1,
|
||||
"Argentina": 1
|
||||
}
|
||||
|
||||
Input:
|
||||
The adventure led him to the peaks of Peru where he trekked to see the mysteries of Machu Picchu. He then headed to Chile to gaze at the vastness of the Atacama Desert. A quick detour to Uruguay and Paraguay allowed him to experience the vibrancy of the local cultures before returning back to Canada through Peru, Brazil and Mexico.
|
||||
Paragraphs:
|
||||
The adventure led him to the peaks of Peru where he trekked to see the mysteries of Machu Picchu. He then headed to Chile to gaze at the vastness of the Atacama Desert.
|
||||
A quick detour to Uruguay and Paraguay allowed him to experience the vibrancy of the local cultures before returning back to Canada through Peru, Brazil and Mexico.
|
||||
Sublists:
|
||||
{
|
||||
"Peru": 1,
|
||||
"Chile": 1
|
||||
}
|
||||
{
|
||||
"Uruguay": 1,
|
||||
"Paraguay": 1,
|
||||
"Canada": 1,
|
||||
"Peru": 1,
|
||||
"Brazil": 1,
|
||||
"Mexico": 1
|
||||
}
|
||||
Output:
|
||||
{
|
||||
"Peru": 2,
|
||||
"Chile": 1,
|
||||
"Uruguay": 1,
|
||||
"Paraguay": 1,
|
||||
"Canada": 1,
|
||||
"Brazil": 1,
|
||||
"Mexico": 1
|
||||
}
|
||||
|
||||
Input:
|
||||
Journeying westward, she admired the art in Italy and sipped coffee in France. The music of Spain and the history of Greece deepened her love for Europe. The Nordic beauty of Norway, Sweden, Finland, and Denmark took her breath away. She danced in Ireland, explored castles in Scotland, and marveled at the architecture in Germany and Russia. Italy, Norway, Sweden and Germany will always stay her favourite destinations to visit.
|
||||
Paragraphs:
|
||||
Journeying westward, she admired the art in Italy and sipped coffee in France.
|
||||
The music of Spain and the history of Greece deepened her love for Europe. The Nordic beauty of Norway, Sweden, Finland, and Denmark took her breath away.
|
||||
She danced in Ireland, explored castles in Scotland, and marveled at the architecture in Germany and Russia.
|
||||
Italy, Norway, Sweden and Germany will always stay her favourite destinations to visit.
|
||||
Sublists:
|
||||
{
|
||||
"Italy": 1,
|
||||
"France": 1
|
||||
}
|
||||
{
|
||||
"Spain": 1,
|
||||
"Greece": 1,
|
||||
"Norway": 1,
|
||||
"Sweden": 1,
|
||||
"Finland": 1,
|
||||
"Denmark": 1
|
||||
}
|
||||
{
|
||||
"Ireland": 1,
|
||||
"Scotland": 1,
|
||||
"Germany": 1,
|
||||
"Russia": 1
|
||||
}
|
||||
{
|
||||
"Italy": 1,
|
||||
"Norway": 1,
|
||||
"Sweden": 1,
|
||||
"Germany": 1
|
||||
}
|
||||
Output:
|
||||
{
|
||||
"Italy": 2,
|
||||
"France": 1,
|
||||
"Spain": 1,
|
||||
"Greece": 1,
|
||||
"Norway": 2,
|
||||
"Sweden": 2,
|
||||
"Finland": 1,
|
||||
"Denmark": 1,
|
||||
"Ireland": 1,
|
||||
"Scotland": 1,
|
||||
"Germany": 2,
|
||||
"Russia": 1
|
||||
}
|
||||
</Examples>
|
||||
|
||||
Input:
|
||||
My friend, Alex from Peru, once recounted his journey to Argentina where he learned about the strong cultural ties between Argentina and Brazil due to their shared history. He spoke fondly of his time in Argentina and Brazil, marveling at the similar music, dance, and culinary traditions that seamlessly bridged the borders of these countries.
|
||||
```
|
||||
|
||||
### Step 2a - 10 Responses
|
||||
```
|
||||
1. { "Peru": 1, "Argentina": 1, "Brazil": 1 }
|
||||
2. { "Peru": 1, "Argentina": 2, "Brazil": 2 }
|
||||
3. { "Peru": 1, "Argentina": 2, "Brazil": 2 }
|
||||
4. { "Peru": 1, "Argentina": 1, "Brazil": 1 }
|
||||
5. { "Peru": 1, "Argentina": 2, "Brazil": 2 }
|
||||
6. { "Peru": 1, "Argentina": 2, "Brazil": 2 }
|
||||
7. { "Peru": 1, "Argentina": 1, "Brazil": 1 }
|
||||
8. { "Peru": 1, "Argentina": 1, "Brazil": 1 }
|
||||
9. { "Peru": 1, "Argentina": 1, "Brazil": 1 }
|
||||
10. { "Peru": 1, "Argentina": 1, "Brazil": 1 }
|
||||
```
|
||||
|
||||
### Step 2b - Prompt
|
||||
```
|
||||
<Instruction> Count the frequency of how many times each country is explicitly named in the input text. You can generate any intermedate lists and states, but the final output should only contain the frequency of each country that appears at least once in the following json format, prefixed with "Output: " (make sure to keep the same spelling for each country in the output as in the input text):
|
||||
{
|
||||
"country1": frequency1,
|
||||
"country2": frequency2,
|
||||
...
|
||||
}
|
||||
</Instruction>
|
||||
|
||||
<Approach>
|
||||
To count the frequency for each country follow these steps:
|
||||
1. Split the input passage into four paragraphs of similar length.
|
||||
2. Count the frequency of each country in each paragraph.
|
||||
3. Combine the frequencies of each country from each paragraph by adding them together.
|
||||
</Approach>
|
||||
|
||||
<Examples>
|
||||
Input:
|
||||
Alexandra boarded the first flight of her grand journey, starting from Canada. With a globe-trotting itinerary in hand, she was filled with excitement. Her first stop was Mexico, where she marveled at the Mayan ruins. From there, she explored the rainforests of Brazil and danced the tango in Argentina.
|
||||
Paragraphs:
|
||||
Alexandra boarded the first flight of her grand journey, starting from Canada. With a globe-trotting itinerary in hand, she was filled with excitement.
|
||||
Her first stop was Mexico, where she marveled at the Mayan ruins. From there, she explored the rainforests of Brazil and danced the tango in Argentina.
|
||||
Sublist frequencies:
|
||||
{
|
||||
"Canada": 1
|
||||
}
|
||||
{
|
||||
"Mexico": 1,
|
||||
"Brazil": 1,
|
||||
"Argentina": 1
|
||||
}
|
||||
Output:
|
||||
{
|
||||
"Canada": 1,
|
||||
"Mexico": 1,
|
||||
"Brazil": 1,
|
||||
"Argentina": 1
|
||||
}
|
||||
|
||||
Input:
|
||||
The adventure led him to the peaks of Peru where he trekked to see the mysteries of Machu Picchu. He then headed to Chile to gaze at the vastness of the Atacama Desert. A quick detour to Uruguay and Paraguay allowed him to experience the vibrancy of the local cultures before returning back to Canada through Peru, Brazil and Mexico.
|
||||
Paragraphs:
|
||||
The adventure led him to the peaks of Peru where he trekked to see the mysteries of Machu Picchu. He then headed to Chile to gaze at the vastness of the Atacama Desert.
|
||||
A quick detour to Uruguay and Paraguay allowed him to experience the vibrancy of the local cultures before returning back to Canada through Peru, Brazil and Mexico.
|
||||
Sublists:
|
||||
{
|
||||
"Peru": 1,
|
||||
"Chile": 1
|
||||
}
|
||||
{
|
||||
"Uruguay": 1,
|
||||
"Paraguay": 1,
|
||||
"Canada": 1,
|
||||
"Peru": 1,
|
||||
"Brazil": 1,
|
||||
"Mexico": 1
|
||||
}
|
||||
Output:
|
||||
{
|
||||
"Peru": 2,
|
||||
"Chile": 1,
|
||||
"Uruguay": 1,
|
||||
"Paraguay": 1,
|
||||
"Canada": 1,
|
||||
"Brazil": 1,
|
||||
"Mexico": 1
|
||||
}
|
||||
|
||||
Input:
|
||||
Journeying westward, she admired the art in Italy and sipped coffee in France. The music of Spain and the history of Greece deepened her love for Europe. The Nordic beauty of Norway, Sweden, Finland, and Denmark took her breath away. She danced in Ireland, explored castles in Scotland, and marveled at the architecture in Germany and Russia. Italy, Norway, Sweden and Germany will always stay her favourite destinations to visit.
|
||||
Paragraphs:
|
||||
Journeying westward, she admired the art in Italy and sipped coffee in France.
|
||||
The music of Spain and the history of Greece deepened her love for Europe. The Nordic beauty of Norway, Sweden, Finland, and Denmark took her breath away.
|
||||
She danced in Ireland, explored castles in Scotland, and marveled at the architecture in Germany and Russia.
|
||||
Italy, Norway, Sweden and Germany will always stay her favourite destinations to visit.
|
||||
Sublists:
|
||||
{
|
||||
"Italy": 1,
|
||||
"France": 1
|
||||
}
|
||||
{
|
||||
"Spain": 1,
|
||||
"Greece": 1,
|
||||
"Norway": 1,
|
||||
"Sweden": 1,
|
||||
"Finland": 1,
|
||||
"Denmark": 1
|
||||
}
|
||||
{
|
||||
"Ireland": 1,
|
||||
"Scotland": 1,
|
||||
"Germany": 1,
|
||||
"Russia": 1
|
||||
}
|
||||
{
|
||||
"Italy": 1,
|
||||
"Norway": 1,
|
||||
"Sweden": 1,
|
||||
"Germany": 1
|
||||
}
|
||||
Output:
|
||||
{
|
||||
"Italy": 2,
|
||||
"France": 1,
|
||||
"Spain": 1,
|
||||
"Greece": 1,
|
||||
"Norway": 2,
|
||||
"Sweden": 2,
|
||||
"Finland": 1,
|
||||
"Denmark": 1,
|
||||
"Ireland": 1,
|
||||
"Scotland": 1,
|
||||
"Germany": 2,
|
||||
"Russia": 1
|
||||
}
|
||||
</Examples>
|
||||
|
||||
Input:
|
||||
It reminded me of a documentary I'd watched about the ancient Silk Road that spanned across Iran and China. It touched upon the shared historical narratives of Iran and China, highlighting how goods, ideas, and culture flowed between these two countries for centuries. Intriguingly, the documentary also brought up some parallels between this eastern route and the vibrant exchange between Italy and France during the Renaissance. The cultural and intellectual bonds between Italy and France were indeed profound, just as the resilient trade relations that existed between the United States and Canada.
|
||||
```
|
||||
|
||||
### Step 2b - 10 Responses
|
||||
```
|
||||
1. { "Iran": 1, "China": 1, "Italy": 1, "France": 1, "United States": 1, "Canada": 1 }
|
||||
2. { "Iran": 1, "China": 1, "Italy": 1, "France": 1, "United States": 1, "Canada": 1 }
|
||||
3. { "Iran": 3, "China": 2, "Italy": 1, "France": 1, "United States": 1, "Canada": 1 }
|
||||
4. { "Iran": 1, "China": 1, "Italy": 1, "France": 1, "United States": 1, "Canada": 1 }
|
||||
5. { "Iran": 2, "China": 2, "Italy": 1, "France": 1, "United States": 1, "Canada": 1 }
|
||||
6. { "Iran": 1, "China": 1, "Italy": 1, "France": 1, "United States": 1, "Canada": 1 }
|
||||
7. { "Iran": 1, "China": 1, "Italy": 1, "France": 1, "United States": 1, "Canada": 1 }
|
||||
8. { "Iran": 1, "China": 1, "Italy": 1, "France": 1, "United States": 1, "Canada": 1 }
|
||||
9. { "Iran": 1, "China": 1, "Italy": 1, "France": 1, "United States": 1, "Canada": 1 }
|
||||
10. { "Iran": 2, "China": 2, "Italy": 1, "France": 2, "United States": 1, "Canada": 1 }
|
||||
```
|
||||
|
||||
### Step 2c - Prompt
|
||||
```
|
||||
<Instruction> Count the frequency of how many times each country is explicitly named in the input text. You can generate any intermedate lists and states, but the final output should only contain the frequency of each country that appears at least once in the following json format, prefixed with "Output: " (make sure to keep the same spelling for each country in the output as in the input text):
|
||||
{
|
||||
"country1": frequency1,
|
||||
"country2": frequency2,
|
||||
...
|
||||
}
|
||||
</Instruction>
|
||||
|
||||
<Approach>
|
||||
To count the frequency for each country follow these steps:
|
||||
1. Split the input passage into four paragraphs of similar length.
|
||||
2. Count the frequency of each country in each paragraph.
|
||||
3. Combine the frequencies of each country from each paragraph by adding them together.
|
||||
</Approach>
|
||||
|
||||
<Examples>
|
||||
Input:
|
||||
Alexandra boarded the first flight of her grand journey, starting from Canada. With a globe-trotting itinerary in hand, she was filled with excitement. Her first stop was Mexico, where she marveled at the Mayan ruins. From there, she explored the rainforests of Brazil and danced the tango in Argentina.
|
||||
Paragraphs:
|
||||
Alexandra boarded the first flight of her grand journey, starting from Canada. With a globe-trotting itinerary in hand, she was filled with excitement.
|
||||
Her first stop was Mexico, where she marveled at the Mayan ruins. From there, she explored the rainforests of Brazil and danced the tango in Argentina.
|
||||
Sublist frequencies:
|
||||
{
|
||||
"Canada": 1
|
||||
}
|
||||
{
|
||||
"Mexico": 1,
|
||||
"Brazil": 1,
|
||||
"Argentina": 1
|
||||
}
|
||||
Output:
|
||||
{
|
||||
"Canada": 1,
|
||||
"Mexico": 1,
|
||||
"Brazil": 1,
|
||||
"Argentina": 1
|
||||
}
|
||||
|
||||
Input:
|
||||
The adventure led him to the peaks of Peru where he trekked to see the mysteries of Machu Picchu. He then headed to Chile to gaze at the vastness of the Atacama Desert. A quick detour to Uruguay and Paraguay allowed him to experience the vibrancy of the local cultures before returning back to Canada through Peru, Brazil and Mexico.
|
||||
Paragraphs:
|
||||
The adventure led him to the peaks of Peru where he trekked to see the mysteries of Machu Picchu. He then headed to Chile to gaze at the vastness of the Atacama Desert.
|
||||
A quick detour to Uruguay and Paraguay allowed him to experience the vibrancy of the local cultures before returning back to Canada through Peru, Brazil and Mexico.
|
||||
Sublists:
|
||||
{
|
||||
"Peru": 1,
|
||||
"Chile": 1
|
||||
}
|
||||
{
|
||||
"Uruguay": 1,
|
||||
"Paraguay": 1,
|
||||
"Canada": 1,
|
||||
"Peru": 1,
|
||||
"Brazil": 1,
|
||||
"Mexico": 1
|
||||
}
|
||||
Output:
|
||||
{
|
||||
"Peru": 2,
|
||||
"Chile": 1,
|
||||
"Uruguay": 1,
|
||||
"Paraguay": 1,
|
||||
"Canada": 1,
|
||||
"Brazil": 1,
|
||||
"Mexico": 1
|
||||
}
|
||||
|
||||
Input:
|
||||
Journeying westward, she admired the art in Italy and sipped coffee in France. The music of Spain and the history of Greece deepened her love for Europe. The Nordic beauty of Norway, Sweden, Finland, and Denmark took her breath away. She danced in Ireland, explored castles in Scotland, and marveled at the architecture in Germany and Russia. Italy, Norway, Sweden and Germany will always stay her favourite destinations to visit.
|
||||
Paragraphs:
|
||||
Journeying westward, she admired the art in Italy and sipped coffee in France.
|
||||
The music of Spain and the history of Greece deepened her love for Europe. The Nordic beauty of Norway, Sweden, Finland, and Denmark took her breath away.
|
||||
She danced in Ireland, explored castles in Scotland, and marveled at the architecture in Germany and Russia.
|
||||
Italy, Norway, Sweden and Germany will always stay her favourite destinations to visit.
|
||||
Sublists:
|
||||
{
|
||||
"Italy": 1,
|
||||
"France": 1
|
||||
}
|
||||
{
|
||||
"Spain": 1,
|
||||
"Greece": 1,
|
||||
"Norway": 1,
|
||||
"Sweden": 1,
|
||||
"Finland": 1,
|
||||
"Denmark": 1
|
||||
}
|
||||
{
|
||||
"Ireland": 1,
|
||||
"Scotland": 1,
|
||||
"Germany": 1,
|
||||
"Russia": 1
|
||||
}
|
||||
{
|
||||
"Italy": 1,
|
||||
"Norway": 1,
|
||||
"Sweden": 1,
|
||||
"Germany": 1
|
||||
}
|
||||
Output:
|
||||
{
|
||||
"Italy": 2,
|
||||
"France": 1,
|
||||
"Spain": 1,
|
||||
"Greece": 1,
|
||||
"Norway": 2,
|
||||
"Sweden": 2,
|
||||
"Finland": 1,
|
||||
"Denmark": 1,
|
||||
"Ireland": 1,
|
||||
"Scotland": 1,
|
||||
"Germany": 2,
|
||||
"Russia": 1
|
||||
}
|
||||
</Examples>
|
||||
|
||||
Input:
|
||||
The United States and Canada, apart from their geographical proximity, shared an economic bond that strongly influenced their policies and international standing. Similarly, the ties between Australia and New Zealand shed light on their gestalt identity in the Pacific region. Despite their unique characteristics, Australia and New Zealand were often viewed as a single entity due to their remarkably similar backgrounds in terms of culture, language, and colonial history. Inspired by these chronicles of interconnectedness, I decided to delve deeper into history and found a fascinating account of how Ukraine and Poland had influenced each other through years of coexistence.
|
||||
```
|
||||
|
||||
### Step 2c - 10 Responses
|
||||
```
|
||||
1. { "United States": 1, "Canada": 1, "Australia": 2, "New Zealand": 2, "Ukraine": 1, "Poland": 1 }
|
||||
2. { "United States": 1, "Canada": 1, "Australia": 2, "New Zealand": 2, "Ukraine": 1, "Poland": 1 }
|
||||
3. { "United States": 1, "Canada": 1, "Australia": 1, "New Zealand": 1, "Australia and New Zealand": 1, "Ukraine": 1, "Poland": 1 }
|
||||
4. { "United States": 1, "Canada": 1, "Australia": 2, "New Zealand": 2, "Ukraine": 1, "Poland": 1 }
|
||||
5. { "United States": 1, "Canada": 1, "Australia": 1, "New Zealand": 1, "Australia and New Zealand": 1, "Ukraine": 1, "Poland": 1 }
|
||||
6. { "United States": 1, "Canada": 1, "Australia": 1, "New Zealand": 1, "Ukraine": 1, "Poland": 1 }
|
||||
7. { "United States": 1, "Canada": 1, "Australia": 2, "New Zealand": 2, "Ukraine": 1, "Poland": 1 }
|
||||
8. { "United States": 1, "Canada": 1, "Australia": 1, "New Zealand": 1, "Ukraine": 1, "Poland": 1 }
|
||||
9. { "United States": 1, "Canada": 1, "Australia": 2, "New Zealand": 2, "Ukraine": 1, "Poland": 1 }
|
||||
10. { "United States": 1, "Canada": 1, "Australia": 2, "New Zealand": 2, "Ukraine": 1, "Poland": 1 }
|
||||
```
|
||||
|
||||
### Step 2d - Prompt
|
||||
```
|
||||
<Instruction> Count the frequency of how many times each country is explicitly named in the input text. You can generate any intermedate lists and states, but the final output should only contain the frequency of each country that appears at least once in the following json format, prefixed with "Output: " (make sure to keep the same spelling for each country in the output as in the input text):
|
||||
{
|
||||
"country1": frequency1,
|
||||
"country2": frequency2,
|
||||
...
|
||||
}
|
||||
</Instruction>
|
||||
|
||||
<Approach>
|
||||
To count the frequency for each country follow these steps:
|
||||
1. Split the input passage into four paragraphs of similar length.
|
||||
2. Count the frequency of each country in each paragraph.
|
||||
3. Combine the frequencies of each country from each paragraph by adding them together.
|
||||
</Approach>
|
||||
|
||||
<Examples>
|
||||
Input:
|
||||
Alexandra boarded the first flight of her grand journey, starting from Canada. With a globe-trotting itinerary in hand, she was filled with excitement. Her first stop was Mexico, where she marveled at the Mayan ruins. From there, she explored the rainforests of Brazil and danced the tango in Argentina.
|
||||
Paragraphs:
|
||||
Alexandra boarded the first flight of her grand journey, starting from Canada. With a globe-trotting itinerary in hand, she was filled with excitement.
|
||||
Her first stop was Mexico, where she marveled at the Mayan ruins. From there, she explored the rainforests of Brazil and danced the tango in Argentina.
|
||||
Sublist frequencies:
|
||||
{
|
||||
"Canada": 1
|
||||
}
|
||||
{
|
||||
"Mexico": 1,
|
||||
"Brazil": 1,
|
||||
"Argentina": 1
|
||||
}
|
||||
Output:
|
||||
{
|
||||
"Canada": 1,
|
||||
"Mexico": 1,
|
||||
"Brazil": 1,
|
||||
"Argentina": 1
|
||||
}
|
||||
|
||||
Input:
|
||||
The adventure led him to the peaks of Peru where he trekked to see the mysteries of Machu Picchu. He then headed to Chile to gaze at the vastness of the Atacama Desert. A quick detour to Uruguay and Paraguay allowed him to experience the vibrancy of the local cultures before returning back to Canada through Peru, Brazil and Mexico.
|
||||
Paragraphs:
|
||||
The adventure led him to the peaks of Peru where he trekked to see the mysteries of Machu Picchu. He then headed to Chile to gaze at the vastness of the Atacama Desert.
|
||||
A quick detour to Uruguay and Paraguay allowed him to experience the vibrancy of the local cultures before returning back to Canada through Peru, Brazil and Mexico.
|
||||
Sublists:
|
||||
{
|
||||
"Peru": 1,
|
||||
"Chile": 1
|
||||
}
|
||||
{
|
||||
"Uruguay": 1,
|
||||
"Paraguay": 1,
|
||||
"Canada": 1,
|
||||
"Peru": 1,
|
||||
"Brazil": 1,
|
||||
"Mexico": 1
|
||||
}
|
||||
Output:
|
||||
{
|
||||
"Peru": 2,
|
||||
"Chile": 1,
|
||||
"Uruguay": 1,
|
||||
"Paraguay": 1,
|
||||
"Canada": 1,
|
||||
"Brazil": 1,
|
||||
"Mexico": 1
|
||||
}
|
||||
|
||||
Input:
|
||||
Journeying westward, she admired the art in Italy and sipped coffee in France. The music of Spain and the history of Greece deepened her love for Europe. The Nordic beauty of Norway, Sweden, Finland, and Denmark took her breath away. She danced in Ireland, explored castles in Scotland, and marveled at the architecture in Germany and Russia. Italy, Norway, Sweden and Germany will always stay her favourite destinations to visit.
|
||||
Paragraphs:
|
||||
Journeying westward, she admired the art in Italy and sipped coffee in France.
|
||||
The music of Spain and the history of Greece deepened her love for Europe. The Nordic beauty of Norway, Sweden, Finland, and Denmark took her breath away.
|
||||
She danced in Ireland, explored castles in Scotland, and marveled at the architecture in Germany and Russia.
|
||||
Italy, Norway, Sweden and Germany will always stay her favourite destinations to visit.
|
||||
Sublists:
|
||||
{
|
||||
"Italy": 1,
|
||||
"France": 1
|
||||
}
|
||||
{
|
||||
"Spain": 1,
|
||||
"Greece": 1,
|
||||
"Norway": 1,
|
||||
"Sweden": 1,
|
||||
"Finland": 1,
|
||||
"Denmark": 1
|
||||
}
|
||||
{
|
||||
"Ireland": 1,
|
||||
"Scotland": 1,
|
||||
"Germany": 1,
|
||||
"Russia": 1
|
||||
}
|
||||
{
|
||||
"Italy": 1,
|
||||
"Norway": 1,
|
||||
"Sweden": 1,
|
||||
"Germany": 1
|
||||
}
|
||||
Output:
|
||||
{
|
||||
"Italy": 2,
|
||||
"France": 1,
|
||||
"Spain": 1,
|
||||
"Greece": 1,
|
||||
"Norway": 2,
|
||||
"Sweden": 2,
|
||||
"Finland": 1,
|
||||
"Denmark": 1,
|
||||
"Ireland": 1,
|
||||
"Scotland": 1,
|
||||
"Germany": 2,
|
||||
"Russia": 1
|
||||
}
|
||||
</Examples>
|
||||
|
||||
Input:
|
||||
Despite their tumultuous past, Ukraine and Poland shared a cultural tapestry that was deeply woven into their histories. It was quite similar to the complex relationship between North Korea and South Korea, which, despite their political differences, shared common heritage and traditions. Thus, from Argentina to South Korea, the world was an intricate web of countries intertwined with each other through shared histories, cultures, and sometimes, shared destinies.
|
||||
```
|
||||
|
||||
### Step 2d - 10 Responses
|
||||
```
|
||||
1. { "Ukraine": 1, "Poland": 1, "North Korea": 1, "South Korea": 2, "Argentina": 1 }
|
||||
2. { "Ukraine": 1, "Poland": 1, "North Korea": 1, "South Korea": 2, "Argentina": 1 }
|
||||
3. { "Ukraine": 1, "Poland": 1, "North Korea": 1, "South Korea": 2, "Argentina": 1 }
|
||||
4. { "Ukraine": 1, "Poland": 1, "North Korea": 1, "South Korea": 2, "Argentina": 1 }
|
||||
5. { "Ukraine": 1, "Poland": 1, "North Korea": 1, "South Korea": 2, "Argentina": 1 }
|
||||
6. { "Ukraine": 1, "Poland": 1, "North Korea": 1, "South Korea": 2, "Argentina": 1 }
|
||||
7. { "Ukraine": 1, "Poland": 1, "North Korea": 1, "South Korea": 2, "Argentina": 1 }
|
||||
8. { "Ukraine": 1, "Poland": 1, "North Korea": 1, "South Korea": 2, "Argentina": 1 }
|
||||
9. { "Ukraine": 1, "Poland": 1, "North Korea": 1, "South Korea": 2, "Argentina": 1 }
|
||||
10. { "Ukraine": 1, "Poland": 1, "North Korea": 1, "South Korea": 2, "Argentina": 1 }
|
||||
```
|
||||
|
||||
### Step 3a - Prompt
|
||||
```
|
||||
<Instruction> Combine the following 2 dictionaries, each containing the frequency of countries in a text, into a single dictionary.
|
||||
Simply add the frequencies together for each country and if a country is not present in one of the dictionaries, add it to the final dictionary with the frequency from the other dictionary.
|
||||
Only output the final merged dictionary without any additional text or thoughts!
|
||||
</Instruction>
|
||||
|
||||
<Approach>
|
||||
To combine the 2 dictionaries into single one, follow these steps:
|
||||
1. Create a new dictionary to store the combined frequencies.
|
||||
2. Iterate through the keys of the first dictionary and add the frequency of each country to the new dictionary.
|
||||
3. Iterate through the keys of the second dictionary and add the frequency of each country to the new dictionary and if it is already present, add the frequency to the existing value.
|
||||
</Approach>
|
||||
|
||||
Combine the following 2 dictionaries into a single dictionary:
|
||||
{
|
||||
"Peru": 1,
|
||||
"Argentina": 2,
|
||||
"Brazil": 2
|
||||
}
|
||||
{
|
||||
"Iran": 2,
|
||||
"China": 2,
|
||||
"Italy": 1,
|
||||
"France": 2,
|
||||
"United States": 1,
|
||||
"Canada": 1
|
||||
}
|
||||
|
||||
Combined Output:
|
||||
```
|
||||
|
||||
### Step 3a - 3 Responses
|
||||
```
|
||||
1. { "Peru": 1, "Argentina": 2, "Brazil": 2, "Iran": 2, "China": 2, "Italy": 1, "France": 2, "United States": 1, "Canada": 1 }
|
||||
2. { "Peru": 1, "Argentina": 2, "Brazil": 2, "Iran": 2, "China": 2, "Italy": 1, "France": 2, "United States": 1, "Canada": 1 }
|
||||
3. { "Peru": 1, "Argentina": 2, "Brazil": 2, "Iran": 2, "China": 2, "Italy": 1, "France": 2, "United States": 1, "Canada": 1 }
|
||||
```
|
||||
|
||||
### Step 3b - Prompt
|
||||
```
|
||||
<Instruction> Combine the following 2 dictionaries, each containing the frequency of countries in a text, into a single dictionary.
|
||||
Simply add the frequencies together for each country and if a country is not present in one of the dictionaries, add it to the final dictionary with the frequency from the other dictionary.
|
||||
Only output the final merged dictionary without any additional text or thoughts!
|
||||
</Instruction>
|
||||
|
||||
<Approach>
|
||||
To combine the 2 dictionaries into single one, follow these steps:
|
||||
1. Create a new dictionary to store the combined frequencies.
|
||||
2. Iterate through the keys of the first dictionary and add the frequency of each country to the new dictionary.
|
||||
3. Iterate through the keys of the second dictionary and add the frequency of each country to the new dictionary and if it is already present, add the frequency to the existing value.
|
||||
</Approach>
|
||||
|
||||
Combine the following 2 dictionaries into a single dictionary:
|
||||
{
|
||||
"United States": 1,
|
||||
"Canada": 1,
|
||||
"Australia": 2,
|
||||
"New Zealand": 2,
|
||||
"Ukraine": 1,
|
||||
"Poland": 1
|
||||
}
|
||||
{
|
||||
"Ukraine": 1,
|
||||
"Poland": 1,
|
||||
"North Korea": 1,
|
||||
"South Korea": 2,
|
||||
"Argentina": 1
|
||||
}
|
||||
|
||||
Combined Output:
|
||||
```
|
||||
|
||||
### Step 3b - 3 Responses
|
||||
```
|
||||
1. { "United States": 1, "Canada": 1, "Australia": 2, "New Zealand": 2, "Ukraine": 2, "Poland": 2, "North Korea": 1, "South Korea": 2, "Argentina": 1 }
|
||||
2. { "United States": 1, "Canada": 1, "Australia": 2, "New Zealand": 2, "Ukraine": 2, "Poland": 2, "North Korea": 1, "South Korea": 2, "Argentina": 1 }
|
||||
3. { "United States": 1, "Canada": 1, "Australia": 2, "New Zealand": 2, "Ukraine": 2, "Poland": 2, "North Korea": 1, "South Korea": 2, "Argentina": 1 }
|
||||
```
|
||||
|
||||
### Step 3c - Prompt
|
||||
```
|
||||
<Instruction> Combine the following 2 dictionaries, each containing the frequency of countries in a text, into a single dictionary.
|
||||
Simply add the frequencies together for each country and if a country is not present in one of the dictionaries, add it to the final dictionary with the frequency from the other dictionary.
|
||||
Only output the final merged dictionary without any additional text or thoughts!
|
||||
</Instruction>
|
||||
|
||||
<Approach>
|
||||
To combine the 2 dictionaries into single one, follow these steps:
|
||||
1. Create a new dictionary to store the combined frequencies.
|
||||
2. Iterate through the keys of the first dictionary and add the frequency of each country to the new dictionary.
|
||||
3. Iterate through the keys of the second dictionary and add the frequency of each country to the new dictionary and if it is already present, add the frequency to the existing value.
|
||||
</Approach>
|
||||
|
||||
Combine the following 2 dictionaries into a single dictionary:
|
||||
{
|
||||
"Peru": 1,
|
||||
"Argentina": 2,
|
||||
"Brazil": 2,
|
||||
"Iran": 2,
|
||||
"China": 2,
|
||||
"Italy": 1,
|
||||
"France": 2,
|
||||
"United States": 1,
|
||||
"Canada": 1
|
||||
}
|
||||
{
|
||||
"United States": 1,
|
||||
"Canada": 1,
|
||||
"Australia": 2,
|
||||
"New Zealand": 2,
|
||||
"Ukraine": 2,
|
||||
"Poland": 2,
|
||||
"North Korea": 1,
|
||||
"South Korea": 2,
|
||||
"Argentina": 1
|
||||
}
|
||||
|
||||
Combined Output:
|
||||
```
|
||||
|
||||
### Step 3c - 3 Responses
|
||||
```
|
||||
1. { "Peru": 1, "Argentina": 3, "Brazil": 2, "Iran": 2, "China": 2, "Italy": 1, "France": 2, "United States": 2, "Canada": 2, "Australia": 2, "New Zealand": 2, "Ukraine": 2, "Poland": 2, "North Korea": 1, "South Korea": 2 }
|
||||
2. { "Peru": 1, "Argentina": 3, "Brazil": 2, "Iran": 2, "China": 2, "Italy": 1, "France": 2, "United States": 2, "Canada": 2, "Australia": 2, "New Zealand": 2, "Ukraine": 2, "Poland": 2, "North Korea": 1, "South Korea": 2 }
|
||||
3. { "Peru": 1, "Argentina": 3, "Brazil": 2, "Iran": 2, "China": 2, "Italy": 1, "France": 2, "United States": 2, "Canada": 2, "Australia": 2, "New Zealand": 2, "Ukraine": 2, "Poland": 2, "North Korea": 1, "South Korea": 2 }
|
||||
```
|
||||
|
||||
### Final Result
|
||||
2 errors propagated from counting.
|
||||
```
|
||||
{
|
||||
"Peru": 1,
|
||||
"Argentina": 3,
|
||||
"Brazil": 2,
|
||||
"Iran": 2,
|
||||
"China": 2,
|
||||
"Italy": 1,
|
||||
"France": 2,
|
||||
"United States": 2,
|
||||
"Canada": 2,
|
||||
"Australia": 2,
|
||||
"New Zealand": 2,
|
||||
"Ukraine": 2,
|
||||
"Poland": 2,
|
||||
"North Korea": 1,
|
||||
"South Korea": 2
|
||||
}
|
||||
```
|
||||
@ -18,7 +18,7 @@ import csv
|
||||
from collections import Counter
|
||||
from functools import partial
|
||||
from typing import Dict, List, Callable, Union
|
||||
from graph_of_thoughts import controller, operations, prompter, parser
|
||||
from graph_of_thoughts import controller, language_models, operations, prompter, parser
|
||||
|
||||
|
||||
def string_to_list(string: str) -> List[str]:
|
||||
@ -1341,9 +1341,9 @@ def run(
|
||||
"""
|
||||
|
||||
orig_budget = budget
|
||||
path = os.path.join(os.path.dirname(__file__), "countries.csv")
|
||||
data_path = os.path.join(os.path.dirname(__file__), "countries.csv")
|
||||
data = []
|
||||
with open(path, "r") as f:
|
||||
with open(data_path, "r") as f:
|
||||
reader = csv.reader(f)
|
||||
next(reader)
|
||||
for row in reader:
|
||||
@ -1357,12 +1357,15 @@ def run(
|
||||
data_ids = list(range(len(data)))
|
||||
selected_data = [data[i] for i in data_ids]
|
||||
|
||||
if not os.path.exists(os.path.join(os.path.dirname(__file__), "results")):
|
||||
os.makedirs(os.path.join(os.path.dirname(__file__), "results"))
|
||||
results_dir = os.path.join(os.path.dirname(__file__), "results")
|
||||
|
||||
if not os.path.exists(results_dir):
|
||||
os.makedirs(results_dir)
|
||||
timestamp = datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
|
||||
extra_info = f"{lm_name}_{'-'.join([method.__name__ for method in methods])}"
|
||||
folder_name = f"results/{extra_info}_{timestamp}"
|
||||
os.makedirs(os.path.join(os.path.dirname(__file__), folder_name))
|
||||
folder_name = f"{extra_info}_{timestamp}"
|
||||
results_folder = os.path.join(results_dir, folder_name)
|
||||
os.makedirs(results_folder)
|
||||
|
||||
config = {
|
||||
"data": selected_data,
|
||||
@ -1370,13 +1373,11 @@ def run(
|
||||
"lm": lm_name,
|
||||
"budget": budget,
|
||||
}
|
||||
with open(
|
||||
os.path.join(os.path.dirname(__file__), folder_name, "config.json"), "w"
|
||||
) as f:
|
||||
with open(os.path.join(results_folder, "config.json"), "w") as f:
|
||||
json.dump(config, f)
|
||||
|
||||
logging.basicConfig(
|
||||
filename=f"{folder_name}/log.log",
|
||||
filename=os.path.join(results_folder, "log.log"),
|
||||
filemode="w",
|
||||
format="%(name)s - %(levelname)s - %(message)s",
|
||||
level=logging.DEBUG,
|
||||
@ -1384,9 +1385,7 @@ def run(
|
||||
|
||||
for method in methods:
|
||||
# create a results directory for the method
|
||||
os.makedirs(
|
||||
os.path.join(os.path.dirname(__file__), folder_name, method.__name__)
|
||||
)
|
||||
os.makedirs(os.path.join(results_folder, method.__name__))
|
||||
|
||||
for data in selected_data:
|
||||
logging.info(f"Running data {data[0]}: {data[1]}")
|
||||
@ -1403,8 +1402,11 @@ def run(
|
||||
f"Budget has been depleted, stopping. Method {method.__name__} has not been run."
|
||||
)
|
||||
break
|
||||
lm = controller.ChatGPT(
|
||||
"../../graph_of_thoughts/controller/config.json",
|
||||
lm = language_models.ChatGPT(
|
||||
os.path.join(
|
||||
os.path.dirname(__file__),
|
||||
"../../graph_of_thoughts/language_models/config.json",
|
||||
),
|
||||
model_name=lm_name,
|
||||
cache=True,
|
||||
)
|
||||
@ -1427,8 +1429,7 @@ def run(
|
||||
except Exception as e:
|
||||
logging.error(f"Exception: {e}")
|
||||
path = os.path.join(
|
||||
os.path.dirname(__file__),
|
||||
folder_name,
|
||||
results_folder,
|
||||
method.__name__,
|
||||
f"{data[0]}.json",
|
||||
)
|
||||
|
||||
@ -0,0 +1,188 @@
|
||||
# Set Intersection for 32-Element Sets - Prompts and Examples
|
||||
## Prompt Templates
|
||||
### GENERATE: split_prompt
|
||||
Replace `{input}` with the input list/set of numbers to be split.
|
||||
```
|
||||
<Instruction> Split the following list of 32 numbers into 2 lists of 16 numbers each, the first list should contain the first 16 numbers and the second list the second 16 numbers.
|
||||
Only output the 2 lists in the following format without any additional text or thoughts!:
|
||||
{
|
||||
"List 1": [13, 16, 30, 6, 21, 7, 31, ...],
|
||||
"List 2": [25, 24, 10, 4, 27, 0, 14, ...]
|
||||
} </Instruction>
|
||||
|
||||
<Example>
|
||||
Input: [26, 40, 42, 57, 15, 31, 5, 32, 11, 4, 24, 28, 51, 54, 12, 22, 33, 35, 7, 13, 2, 59, 8, 23, 43, 16, 29, 55, 25, 63, 21, 18]
|
||||
Output:
|
||||
{
|
||||
"List 1": [26, 40, 42, 57, 15, 31, 5, 32, 11, 4, 24, 28, 51, 54, 12, 22],
|
||||
"List 2": [33, 35, 7, 13, 2, 59, 8, 23, 43, 16, 29, 55, 25, 63, 21, 18]
|
||||
}
|
||||
</Example>
|
||||
|
||||
Input: {input}
|
||||
```
|
||||
|
||||
### GENERATE: intersect_prompt
|
||||
Replace `{set1}` and `{set2}` with the sets to be intersected.
|
||||
```
|
||||
<Instruction> Find the intersection of two sets of numbers. Output only the set of numbers that are present in both sets, no additional text. </Instruction>
|
||||
|
||||
<Examples>
|
||||
Input Set 1: [13, 16, 30, 6, 21, 7, 31, 15, 11, 1, 24, 10, 9, 3, 20, 8]
|
||||
Input Set 2: [25, 24, 10, 4, 27, 0, 14, 12, 8, 2, 29, 20, 17, 19, 26, 23]
|
||||
Output: [24, 10, 20, 8]
|
||||
|
||||
Input Set 1: [26, 40, 42, 57, 15, 31, 5, 32, 11, 4, 24, 28, 51, 54, 12, 22, 33, 35, 7, 13, 2, 59, 8, 23, 43, 16, 29, 55, 25, 63, 21, 18]
|
||||
Input Set 2: [16, 60, 36, 48, 0, 15, 5, 19, 46, 24, 1, 6, 61, 10, 38, 53, 58, 9, 44, 14, 35, 63, 52, 20, 27, 17, 39, 47, 34, 56, 40, 59]
|
||||
Output: [40, 15, 5, 24, 35, 59, 16, 63]
|
||||
|
||||
Input Set 1: [115, 61, 35, 103, 90, 117, 86, 44, 63, 45, 40, 30, 74, 33, 31, 1, 118, 48, 38, 0, 119, 51, 64, 78, 15, 121, 89, 101, 79, 69, 120, 29, 58, 50, 116, 11, 60, 12, 39, 95, 23, 2, 109, 84, 7, 43, 99, 98, 52, 70, 75, 102, 57, 19, 94, 36, 114, 88, 71, 56, 83, 6, 96, 107]
|
||||
Input Set 2: [13, 35, 20, 96, 34, 18, 47, 127, 126, 9, 21, 16, 77, 22, 111, 122, 85, 73, 42, 105, 123, 15, 33, 59, 67, 57, 104, 8, 30, 89, 76, 12, 65, 84, 32, 40, 7, 100, 108, 50, 14, 28, 24, 53, 90, 17, 91, 81, 124, 63, 5, 46, 125, 93, 49, 66, 117, 37, 115, 113, 2, 106, 41, 72]
|
||||
Output: [115, 35, 90, 117, 63, 40, 30, 33, 15, 89, 50, 12, 2, 84, 7, 57, 96]
|
||||
</Examples>
|
||||
|
||||
Input Set 1: {set1}
|
||||
Input Set 2: {set2}
|
||||
```
|
||||
|
||||
### AGGREGATE: merge_prompt
|
||||
Replace `{input1}` and `{input2}` with the lists/sets to be merged.
|
||||
```
|
||||
<Instruction> Merge the following 2 lists of length 16 each, into one list of length 32 by appending the second list to the first list.
|
||||
Only output the final list without any additional text or thoughts! </Instruction>
|
||||
|
||||
List 1: {input1}
|
||||
List 2: {input2}
|
||||
|
||||
```
|
||||
|
||||
## Complete Example Prompts
|
||||
### The GoO Summarised
|
||||
|
||||
1. Split the second input set into two sub-sets of equal size (split prompt)
|
||||
2. For each sub-set: Intersect the sub-set with the first input set (intersect prompt) five times; score each sort attempt; keep the best
|
||||
3. Merge the resulting intersections into one full intersection set (merge prompt) 10 times; score each merge attempt; keep the best
|
||||
|
||||
### Input
|
||||
```
|
||||
[31, 11, 4, 63, 38, 58, 59, 24, 61, 14, 32, 39, 27, 46, 48, 19, 52, 57, 50, 56, 3, 2, 53, 29, 5, 37, 62, 41, 36, 12, 49, 16]
|
||||
```
|
||||
|
||||
### Step 1 - Prompt
|
||||
```
|
||||
<Instruction> Split the following list of 32 numbers into 2 lists of 16 numbers each, the first list should contain the first 16 numbers and the second list the second 16 numbers.
|
||||
Only output the 2 lists in the following format without any additional text or thoughts!:
|
||||
{
|
||||
"List 1": [13, 16, 30, 6, 21, 7, 31, ...],
|
||||
"List 2": [25, 24, 10, 4, 27, 0, 14, ...]
|
||||
} </Instruction>
|
||||
|
||||
<Example>
|
||||
Input: [26, 40, 42, 57, 15, 31, 5, 32, 11, 4, 24, 28, 51, 54, 12, 22, 33, 35, 7, 13, 2, 59, 8, 23, 43, 16, 29, 55, 25, 63, 21, 18]
|
||||
Output:
|
||||
{
|
||||
"List 1": [26, 40, 42, 57, 15, 31, 5, 32, 11, 4, 24, 28, 51, 54, 12, 22],
|
||||
"List 2": [33, 35, 7, 13, 2, 59, 8, 23, 43, 16, 29, 55, 25, 63, 21, 18]
|
||||
}
|
||||
</Example>
|
||||
|
||||
Input: [31, 11, 4, 63, 38, 58, 59, 24, 61, 14, 32, 39, 27, 46, 48, 19, 52, 57, 50, 56, 3, 2, 53, 29, 5, 37, 62, 41, 36, 12, 49, 16]
|
||||
```
|
||||
|
||||
### Step 1 - Response
|
||||
```
|
||||
{
|
||||
"List 1": [31, 11, 4, 63, 38, 58, 59, 24, 61, 14, 32, 39, 27, 46, 48, 19],
|
||||
"List 2": [52, 57, 50, 56, 3, 2, 53, 29, 5, 37, 62, 41, 36, 12, 49, 16]
|
||||
}
|
||||
```
|
||||
|
||||
### Step 2a - Prompt
|
||||
```
|
||||
<Instruction> Find the intersection of two sets of numbers. Output only the set of numbers that are present in both sets, no additional text. </Instruction>
|
||||
|
||||
<Examples>
|
||||
Input Set 1: [13, 16, 30, 6, 21, 7, 31, 15, 11, 1, 24, 10, 9, 3, 20, 8]
|
||||
Input Set 2: [25, 24, 10, 4, 27, 0, 14, 12, 8, 2, 29, 20, 17, 19, 26, 23]
|
||||
Output: [24, 10, 20, 8]
|
||||
|
||||
Input Set 1: [26, 40, 42, 57, 15, 31, 5, 32, 11, 4, 24, 28, 51, 54, 12, 22, 33, 35, 7, 13, 2, 59, 8, 23, 43, 16, 29, 55, 25, 63, 21, 18]
|
||||
Input Set 2: [16, 60, 36, 48, 0, 15, 5, 19, 46, 24, 1, 6, 61, 10, 38, 53, 58, 9, 44, 14, 35, 63, 52, 20, 27, 17, 39, 47, 34, 56, 40, 59]
|
||||
Output: [40, 15, 5, 24, 35, 59, 16, 63]
|
||||
|
||||
Input Set 1: [115, 61, 35, 103, 90, 117, 86, 44, 63, 45, 40, 30, 74, 33, 31, 1, 118, 48, 38, 0, 119, 51, 64, 78, 15, 121, 89, 101, 79, 69, 120, 29, 58, 50, 116, 11, 60, 12, 39, 95, 23, 2, 109, 84, 7, 43, 99, 98, 52, 70, 75, 102, 57, 19, 94, 36, 114, 88, 71, 56, 83, 6, 96, 107]
|
||||
Input Set 2: [13, 35, 20, 96, 34, 18, 47, 127, 126, 9, 21, 16, 77, 22, 111, 122, 85, 73, 42, 105, 123, 15, 33, 59, 67, 57, 104, 8, 30, 89, 76, 12, 65, 84, 32, 40, 7, 100, 108, 50, 14, 28, 24, 53, 90, 17, 91, 81, 124, 63, 5, 46, 125, 93, 49, 66, 117, 37, 115, 113, 2, 106, 41, 72]
|
||||
Output: [115, 35, 90, 117, 63, 40, 30, 33, 15, 89, 50, 12, 2, 84, 7, 57, 96]
|
||||
</Examples>
|
||||
|
||||
Input Set 1: [11, 60, 1, 49, 21, 33, 14, 56, 54, 15, 23, 40, 45, 22, 7, 28, 20, 46, 51, 6, 34, 37, 3, 50, 17, 8, 25, 0, 35, 47, 18, 19]
|
||||
Input Set 2: [31, 11, 4, 63, 38, 58, 59, 24, 61, 14, 32, 39, 27, 46, 48, 19]
|
||||
```
|
||||
|
||||
### Step 2a - 5 Responses
|
||||
```
|
||||
1. [11, 14, 46, 14, 19]
|
||||
2. Output: [11, 14, 46, 19]
|
||||
3. [11, 14, 46, 14, 19]
|
||||
4. [11, 14, 46, 19]
|
||||
5. [11, 14, 46, 14, 19]
|
||||
```
|
||||
|
||||
### Step 2b - Prompt
|
||||
```
|
||||
<Instruction> Find the intersection of two sets of numbers. Output only the set of numbers that are present in both sets, no additional text. </Instruction>
|
||||
|
||||
<Examples>
|
||||
Input Set 1: [13, 16, 30, 6, 21, 7, 31, 15, 11, 1, 24, 10, 9, 3, 20, 8]
|
||||
Input Set 2: [25, 24, 10, 4, 27, 0, 14, 12, 8, 2, 29, 20, 17, 19, 26, 23]
|
||||
Output: [24, 10, 20, 8]
|
||||
|
||||
Input Set 1: [26, 40, 42, 57, 15, 31, 5, 32, 11, 4, 24, 28, 51, 54, 12, 22, 33, 35, 7, 13, 2, 59, 8, 23, 43, 16, 29, 55, 25, 63, 21, 18]
|
||||
Input Set 2: [16, 60, 36, 48, 0, 15, 5, 19, 46, 24, 1, 6, 61, 10, 38, 53, 58, 9, 44, 14, 35, 63, 52, 20, 27, 17, 39, 47, 34, 56, 40, 59]
|
||||
Output: [40, 15, 5, 24, 35, 59, 16, 63]
|
||||
|
||||
Input Set 1: [115, 61, 35, 103, 90, 117, 86, 44, 63, 45, 40, 30, 74, 33, 31, 1, 118, 48, 38, 0, 119, 51, 64, 78, 15, 121, 89, 101, 79, 69, 120, 29, 58, 50, 116, 11, 60, 12, 39, 95, 23, 2, 109, 84, 7, 43, 99, 98, 52, 70, 75, 102, 57, 19, 94, 36, 114, 88, 71, 56, 83, 6, 96, 107]
|
||||
Input Set 2: [13, 35, 20, 96, 34, 18, 47, 127, 126, 9, 21, 16, 77, 22, 111, 122, 85, 73, 42, 105, 123, 15, 33, 59, 67, 57, 104, 8, 30, 89, 76, 12, 65, 84, 32, 40, 7, 100, 108, 50, 14, 28, 24, 53, 90, 17, 91, 81, 124, 63, 5, 46, 125, 93, 49, 66, 117, 37, 115, 113, 2, 106, 41, 72]
|
||||
Output: [115, 35, 90, 117, 63, 40, 30, 33, 15, 89, 50, 12, 2, 84, 7, 57, 96]
|
||||
</Examples>
|
||||
|
||||
Input Set 1: [11, 60, 1, 49, 21, 33, 14, 56, 54, 15, 23, 40, 45, 22, 7, 28, 20, 46, 51, 6, 34, 37, 3, 50, 17, 8, 25, 0, 35, 47, 18, 19]
|
||||
Input Set 2: [52, 57, 50, 56, 3, 2, 53, 29, 5, 37, 62, 41, 36, 12, 49, 16]
|
||||
```
|
||||
|
||||
### Step 2b - 5 Responses
|
||||
```
|
||||
1. [50, 56, 49]
|
||||
2. [50, 56, 37, 49]
|
||||
3. [56, 49, 37, 3, 50]
|
||||
4. [50, 56, 37, 49]
|
||||
5. [50, 56, 37, 49]
|
||||
```
|
||||
|
||||
### Step 3 - Prompt
|
||||
```
|
||||
<Instruction> Merge the following 2 lists of length 16 each, into one list of length 32 by appending the second list to the first list.
|
||||
Only output the final list without any additional text or thoughts! </Instruction>
|
||||
|
||||
List 1: [11, 14, 46, 19]
|
||||
List 2: [56, 49, 37, 3, 50]
|
||||
```
|
||||
|
||||
### Step 3 - 10 Responses
|
||||
```
|
||||
1. [11, 14, 46, 19, 56, 49, 37, 3, 50]
|
||||
2. [11, 14, 46, 19, 56, 49, 37, 3, 50]
|
||||
3. [11, 14, 46, 19, 56, 49, 37, 3, 50]
|
||||
4. [11, 14, 46, 19, 56, 49, 37, 3, 50]
|
||||
5. [11, 14, 46, 19, 56, 49, 37, 3, 50]
|
||||
6. [11, 14, 46, 19, 56, 49, 37, 3, 50]
|
||||
7. [11, 14, 46, 19, 56, 49, 37, 3, 50]
|
||||
8. [11, 14, 46, 19, 56, 49, 37, 3, 50]
|
||||
9. [11, 14, 46, 19, 56, 49, 37, 3, 50]
|
||||
10. [11, 14, 46, 19, 56, 49, 37, 3, 50]
|
||||
```
|
||||
|
||||
### Final Result
|
||||
```
|
||||
[11, 14, 46, 19, 56, 49, 37, 3, 50]
|
||||
```
|
||||
@ -15,9 +15,13 @@ import datetime
|
||||
import json
|
||||
import csv
|
||||
from typing import Dict, List, Callable, Union
|
||||
from graph_of_thoughts import controller, operations, prompter, parser
|
||||
from graph_of_thoughts import controller, language_models, operations, prompter, parser
|
||||
|
||||
# This is a hack to also allow execution of this file from the examples directory
|
||||
try:
|
||||
from . import utils
|
||||
except ImportError:
|
||||
import utils
|
||||
|
||||
|
||||
class SetIntersectionPrompter(prompter.Prompter):
|
||||
@ -147,7 +151,7 @@ Output:
|
||||
|
||||
Input: {input}"""
|
||||
|
||||
got_merge_prompt = """<Instruction> Merge the following 2 lists of length {length1} each, into one list of length {length2} by appending the second list to the first list.
|
||||
got_merge_prompt = """<Instruction> Merge the following 2 lists into one list by appending the second list to the first list.
|
||||
Only output the final list without any additional text or thoughts! </Instruction>
|
||||
|
||||
List 1: {input1}
|
||||
@ -166,20 +170,10 @@ List 2: {input2}
|
||||
:raise AssertionError: If not exactly two thought states are provided.
|
||||
"""
|
||||
assert len(state_dicts) == 2, "Expected two states for aggregation prompt."
|
||||
len_input1 = len(utils.string_to_list(state_dicts[0]["current"]))
|
||||
len_input2 = len(utils.string_to_list(state_dicts[1]["current"]))
|
||||
if len_input1 == len_input2:
|
||||
length = len_input1
|
||||
elif len_input1 + len_input2 - 32 <= 16:
|
||||
length = 16
|
||||
else:
|
||||
length = 32
|
||||
|
||||
return self.got_merge_prompt.format(
|
||||
input1=state_dicts[0]["current"],
|
||||
input2=state_dicts[1]["current"],
|
||||
length1=length,
|
||||
length2=length * 2,
|
||||
)
|
||||
|
||||
def generate_prompt(
|
||||
@ -614,9 +608,9 @@ def run(
|
||||
"""
|
||||
|
||||
orig_budget = budget
|
||||
path = os.path.join(os.path.dirname(__file__), "set_intersection_032.csv")
|
||||
data_path = os.path.join(os.path.dirname(__file__), "set_intersection_032.csv")
|
||||
data = []
|
||||
with open(path, "r") as f:
|
||||
with open(data_path, "r") as f:
|
||||
reader = csv.reader(f)
|
||||
next(reader)
|
||||
for row in reader:
|
||||
@ -626,12 +620,15 @@ def run(
|
||||
data_ids = list(range(len(data)))
|
||||
selected_data = [data[i] for i in data_ids]
|
||||
|
||||
if not os.path.exists(os.path.join(os.path.dirname(__file__), "results")):
|
||||
os.makedirs(os.path.join(os.path.dirname(__file__), "results"))
|
||||
results_dir = os.path.join(os.path.dirname(__file__), "results")
|
||||
|
||||
if not os.path.exists(results_dir):
|
||||
os.makedirs(results_dir)
|
||||
timestamp = datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
|
||||
extra_info = f"{lm_name}_{'-'.join([method.__name__ for method in methods])}"
|
||||
folder_name = f"results/{extra_info}_{timestamp}"
|
||||
os.makedirs(os.path.join(os.path.dirname(__file__), folder_name))
|
||||
folder_name = f"{extra_info}_{timestamp}"
|
||||
results_folder = os.path.join(results_dir, folder_name)
|
||||
os.makedirs(results_folder)
|
||||
|
||||
config = {
|
||||
"data": selected_data,
|
||||
@ -639,13 +636,11 @@ def run(
|
||||
"lm": lm_name,
|
||||
"budget": budget,
|
||||
}
|
||||
with open(
|
||||
os.path.join(os.path.dirname(__file__), folder_name, "config.json"), "w"
|
||||
) as f:
|
||||
with open(os.path.join(results_folder, "config.json"), "w") as f:
|
||||
json.dump(config, f)
|
||||
|
||||
logging.basicConfig(
|
||||
filename=f"{folder_name}/log.log",
|
||||
filename=os.path.join(results_folder, "log.log"),
|
||||
filemode="w",
|
||||
format="%(name)s - %(levelname)s - %(message)s",
|
||||
level=logging.DEBUG,
|
||||
@ -653,9 +648,7 @@ def run(
|
||||
|
||||
for method in methods:
|
||||
# create a results directory for the method
|
||||
os.makedirs(
|
||||
os.path.join(os.path.dirname(__file__), folder_name, method.__name__)
|
||||
)
|
||||
os.makedirs(os.path.join(results_folder, method.__name__))
|
||||
|
||||
for data in selected_data:
|
||||
logging.info(f"Running data {data[0]}: {data[1]} {data[2]}")
|
||||
@ -672,8 +665,11 @@ def run(
|
||||
f"Budget has been depleted, stopping. Method {method.__name__} has not been run."
|
||||
)
|
||||
break
|
||||
lm = controller.ChatGPT(
|
||||
"../../graph_of_thoughts/controller/config.json",
|
||||
lm = language_models.ChatGPT(
|
||||
os.path.join(
|
||||
os.path.dirname(__file__),
|
||||
"../../graph_of_thoughts/language_models/config.json",
|
||||
),
|
||||
model_name=lm_name,
|
||||
cache=True,
|
||||
)
|
||||
@ -697,8 +693,7 @@ def run(
|
||||
except Exception as e:
|
||||
logging.error(f"Exception: {e}")
|
||||
path = os.path.join(
|
||||
os.path.dirname(__file__),
|
||||
folder_name,
|
||||
results_folder,
|
||||
method.__name__,
|
||||
f"{data[0]}.json",
|
||||
)
|
||||
|
||||
@ -15,9 +15,13 @@ import datetime
|
||||
import json
|
||||
import csv
|
||||
from typing import Dict, List, Callable, Union
|
||||
from graph_of_thoughts import controller, operations, prompter, parser
|
||||
from graph_of_thoughts import controller, language_models, operations, prompter, parser
|
||||
|
||||
# This is a hack to also allow execution of this file from the examples directory
|
||||
try:
|
||||
from . import utils
|
||||
except ImportError:
|
||||
import utils
|
||||
|
||||
|
||||
class SetIntersectionPrompter(prompter.Prompter):
|
||||
@ -151,7 +155,7 @@ Output:
|
||||
|
||||
Input: {input}"""
|
||||
|
||||
got_merge_prompt = """<Instruction> Merge the following 2 lists of length {length1} each, into one list of length {length2} by appending the second list to the first list.
|
||||
got_merge_prompt = """<Instruction> Merge the following 2 lists into one list by appending the second list to the first list.
|
||||
Only output the final list without any additional text or thoughts! </Instruction>
|
||||
|
||||
List 1: {input1}
|
||||
@ -171,20 +175,10 @@ List 2: {input2}
|
||||
"""
|
||||
|
||||
assert len(state_dicts) == 2, "Expected two states for aggregation prompt."
|
||||
len_input1 = len(utils.string_to_list(state_dicts[0]["current"]))
|
||||
len_input2 = len(utils.string_to_list(state_dicts[1]["current"]))
|
||||
if len_input1 == len_input2:
|
||||
length = len_input1
|
||||
elif len_input1 + len_input2 - 32 <= 16:
|
||||
length = 16
|
||||
else:
|
||||
length = 32
|
||||
|
||||
return self.got_merge_prompt.format(
|
||||
input1=state_dicts[0]["current"],
|
||||
input2=state_dicts[1]["current"],
|
||||
length1=length,
|
||||
length2=length * 2,
|
||||
)
|
||||
|
||||
def generate_prompt(
|
||||
@ -644,9 +638,9 @@ def run(
|
||||
"""
|
||||
|
||||
orig_budget = budget
|
||||
path = os.path.join(os.path.dirname(__file__), "set_intersection_064.csv")
|
||||
data_path = os.path.join(os.path.dirname(__file__), "set_intersection_064.csv")
|
||||
data = []
|
||||
with open(path, "r") as f:
|
||||
with open(data_path, "r") as f:
|
||||
reader = csv.reader(f)
|
||||
next(reader)
|
||||
for row in reader:
|
||||
@ -656,12 +650,15 @@ def run(
|
||||
data_ids = list(range(len(data)))
|
||||
selected_data = [data[i] for i in data_ids]
|
||||
|
||||
if not os.path.exists(os.path.join(os.path.dirname(__file__), "results")):
|
||||
os.makedirs(os.path.join(os.path.dirname(__file__), "results"))
|
||||
results_dir = os.path.join(os.path.dirname(__file__), "results")
|
||||
|
||||
if not os.path.exists(results_dir):
|
||||
os.makedirs(results_dir)
|
||||
timestamp = datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
|
||||
extra_info = f"{lm_name}_{'-'.join([method.__name__ for method in methods])}"
|
||||
folder_name = f"results/{extra_info}_{timestamp}"
|
||||
os.makedirs(os.path.join(os.path.dirname(__file__), folder_name))
|
||||
folder_name = f"{extra_info}_{timestamp}"
|
||||
results_folder = os.path.join(results_dir, folder_name)
|
||||
os.makedirs(results_folder)
|
||||
|
||||
config = {
|
||||
"data": selected_data,
|
||||
@ -669,13 +666,11 @@ def run(
|
||||
"lm": lm_name,
|
||||
"budget": budget,
|
||||
}
|
||||
with open(
|
||||
os.path.join(os.path.dirname(__file__), folder_name, "config.json"), "w"
|
||||
) as f:
|
||||
with open(os.path.join(results_folder, "config.json"), "w") as f:
|
||||
json.dump(config, f)
|
||||
|
||||
logging.basicConfig(
|
||||
filename=f"{folder_name}/log.log",
|
||||
filename=os.path.join(results_folder, "log.log"),
|
||||
filemode="w",
|
||||
format="%(name)s - %(levelname)s - %(message)s",
|
||||
level=logging.DEBUG,
|
||||
@ -683,9 +678,7 @@ def run(
|
||||
|
||||
for method in methods:
|
||||
# create a results directory for the method
|
||||
os.makedirs(
|
||||
os.path.join(os.path.dirname(__file__), folder_name, method.__name__)
|
||||
)
|
||||
os.makedirs(os.path.join(results_folder, method.__name__))
|
||||
|
||||
for data in selected_data:
|
||||
logging.info(f"Running data {data[0]}: {data[1]} {data[2]}")
|
||||
@ -702,8 +695,11 @@ def run(
|
||||
f"Budget has been depleted, stopping. Method {method.__name__} has not been run."
|
||||
)
|
||||
break
|
||||
lm = controller.ChatGPT(
|
||||
"../../graph_of_thoughts/controller/config.json",
|
||||
lm = language_models.ChatGPT(
|
||||
os.path.join(
|
||||
os.path.dirname(__file__),
|
||||
"../../graph_of_thoughts/language_models/config.json",
|
||||
),
|
||||
model_name=lm_name,
|
||||
cache=True,
|
||||
)
|
||||
@ -727,8 +723,7 @@ def run(
|
||||
except Exception as e:
|
||||
logging.error(f"Exception: {e}")
|
||||
path = os.path.join(
|
||||
os.path.dirname(__file__),
|
||||
folder_name,
|
||||
results_folder,
|
||||
method.__name__,
|
||||
f"{data[0]}.json",
|
||||
)
|
||||
|
||||
@ -15,9 +15,13 @@ import datetime
|
||||
import json
|
||||
import csv
|
||||
from typing import Dict, List, Callable, Union
|
||||
from graph_of_thoughts import controller, operations, prompter, parser
|
||||
from graph_of_thoughts import controller, language_models, operations, prompter, parser
|
||||
|
||||
# This is a hack to also allow execution of this file from the examples directory
|
||||
try:
|
||||
from . import utils
|
||||
except ImportError:
|
||||
import utils
|
||||
|
||||
|
||||
class SetIntersectionPrompter(prompter.Prompter):
|
||||
@ -159,7 +163,7 @@ Output:
|
||||
|
||||
Input: {input}"""
|
||||
|
||||
got_merge_prompt = """<Instruction> Merge the following 2 lists of length {length1} each, into one list of length {length2} by appending the second list to the first list.
|
||||
got_merge_prompt = """<Instruction> Merge the following 2 lists into one list by appending the second list to the first list.
|
||||
Only output the final list without any additional text or thoughts! </Instruction>
|
||||
|
||||
List 1: {input1}
|
||||
@ -178,22 +182,10 @@ List 2: {input2}
|
||||
:raise AssertionError: If not exactly two thought states are provided.
|
||||
"""
|
||||
assert len(state_dicts) == 2, "Expected two states for aggregation prompt."
|
||||
len_input1 = len(utils.string_to_list(state_dicts[0]["current"]))
|
||||
len_input2 = len(utils.string_to_list(state_dicts[1]["current"]))
|
||||
if len_input1 == len_input2:
|
||||
length = len_input1
|
||||
elif len_input1 + len_input2 - 32 <= 16:
|
||||
length = 16
|
||||
elif len_input1 + len_input2 - 64 <= 32:
|
||||
length = 32
|
||||
else:
|
||||
length = 64
|
||||
|
||||
return self.got_merge_prompt.format(
|
||||
input1=state_dicts[0]["current"],
|
||||
input2=state_dicts[1]["current"],
|
||||
length1=length,
|
||||
length2=length * 2,
|
||||
)
|
||||
|
||||
def generate_prompt(
|
||||
@ -696,9 +688,9 @@ def run(
|
||||
"""
|
||||
|
||||
orig_budget = budget
|
||||
path = os.path.join(os.path.dirname(__file__), "set_intersection_128.csv")
|
||||
data_path = os.path.join(os.path.dirname(__file__), "set_intersection_128.csv")
|
||||
data = []
|
||||
with open(path, "r") as f:
|
||||
with open(data_path, "r") as f:
|
||||
reader = csv.reader(f)
|
||||
next(reader)
|
||||
for row in reader:
|
||||
@ -708,12 +700,15 @@ def run(
|
||||
data_ids = list(range(len(data)))
|
||||
selected_data = [data[i] for i in data_ids]
|
||||
|
||||
if not os.path.exists(os.path.join(os.path.dirname(__file__), "results")):
|
||||
os.makedirs(os.path.join(os.path.dirname(__file__), "results"))
|
||||
results_dir = os.path.join(os.path.dirname(__file__), "results")
|
||||
|
||||
if not os.path.exists(results_dir):
|
||||
os.makedirs(results_dir)
|
||||
timestamp = datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
|
||||
extra_info = f"{lm_name}_{'-'.join([method.__name__ for method in methods])}"
|
||||
folder_name = f"results/{extra_info}_{timestamp}"
|
||||
os.makedirs(os.path.join(os.path.dirname(__file__), folder_name))
|
||||
folder_name = f"{extra_info}_{timestamp}"
|
||||
results_folder = os.path.join(results_dir, folder_name)
|
||||
os.makedirs(results_folder)
|
||||
|
||||
config = {
|
||||
"data": selected_data,
|
||||
@ -721,13 +716,11 @@ def run(
|
||||
"lm": lm_name,
|
||||
"budget": budget,
|
||||
}
|
||||
with open(
|
||||
os.path.join(os.path.dirname(__file__), folder_name, "config.json"), "w"
|
||||
) as f:
|
||||
with open(os.path.join(results_folder, "config.json"), "w") as f:
|
||||
json.dump(config, f)
|
||||
|
||||
logging.basicConfig(
|
||||
filename=f"{folder_name}/log.log",
|
||||
filename=os.path.join(results_folder, "log.log"),
|
||||
filemode="w",
|
||||
format="%(name)s - %(levelname)s - %(message)s",
|
||||
level=logging.DEBUG,
|
||||
@ -735,9 +728,7 @@ def run(
|
||||
|
||||
for method in methods:
|
||||
# create a results directory for the method
|
||||
os.makedirs(
|
||||
os.path.join(os.path.dirname(__file__), folder_name, method.__name__)
|
||||
)
|
||||
os.makedirs(os.path.join(results_folder, method.__name__))
|
||||
|
||||
for data in selected_data:
|
||||
logging.info(f"Running data {data[0]}: {data[1]} {data[2]}")
|
||||
@ -754,8 +745,11 @@ def run(
|
||||
f"Budget has been depleted, stopping. Method {method.__name__} has not been run."
|
||||
)
|
||||
break
|
||||
lm = controller.ChatGPT(
|
||||
"../../graph_of_thoughts/controller/config.json",
|
||||
lm = language_models.ChatGPT(
|
||||
os.path.join(
|
||||
os.path.dirname(__file__),
|
||||
"../../graph_of_thoughts/language_models/config.json",
|
||||
),
|
||||
model_name=lm_name,
|
||||
cache=True,
|
||||
)
|
||||
@ -779,8 +773,7 @@ def run(
|
||||
except Exception as e:
|
||||
logging.error(f"Exception: {e}")
|
||||
path = os.path.join(
|
||||
os.path.dirname(__file__),
|
||||
folder_name,
|
||||
results_folder,
|
||||
method.__name__,
|
||||
f"{data[0]}.json",
|
||||
)
|
||||
|
||||
295
examples/sorting/example_prompts_sorting_032.md
Normal file
295
examples/sorting/example_prompts_sorting_032.md
Normal file
@ -0,0 +1,295 @@
|
||||
# Sorting 32 Numbers - Prompts and Examples
|
||||
## Prompt Templates
|
||||
### GENERATE: split_prompt
|
||||
Replace `{input}` with the input list of numbers to be split.
|
||||
```
|
||||
<Instruction> Split the following list of 32 numbers into 2 lists of 16 numbers each, the first list should contain the first 16 numbers and the second list the second 16 numbers.
|
||||
Only output the final 2 lists in the following format without any additional text or thoughts!:
|
||||
{{
|
||||
"List 1": [3, 4, 3, 5, 7, 8, 1, ...],
|
||||
"List 2": [2, 9, 2, 4, 7, 1, 5, ...]
|
||||
}} </Instruction>
|
||||
|
||||
<Example>
|
||||
Input: [9, 6, 7, 7, 2, 0, 2, 2, 3, 5, 0, 9, 2, 2, 4, 4, 5, 2, 5, 1, 2, 8, 3, 8, 3, 9, 6, 0, 4, 2, 2, 3]
|
||||
Output:
|
||||
{{
|
||||
"List 1": [9, 6, 7, 7, 2, 0, 2, 2, 3, 5, 0, 9, 2, 2, 4, 4],
|
||||
"List 2": [5, 2, 5, 1, 2, 8, 3, 8, 3, 9, 6, 0, 4, 2, 2, 3]
|
||||
}}
|
||||
</Example>
|
||||
|
||||
Input: {input}
|
||||
```
|
||||
|
||||
### GENERATE: sort_prompt
|
||||
Replace `{input}` with the input list of numbers to be sorted.
|
||||
```
|
||||
<Instruction> Sort the following list of numbers in ascending order. Output only the sorted list of numbers, no additional text. </Instruction>
|
||||
|
||||
<Examples>
|
||||
Input: [5, 1, 0, 1, 2, 0, 4, 8, 1, 9, 5, 1, 3, 3, 9, 7]
|
||||
Output: [0, 0, 1, 1, 1, 1, 2, 3, 3, 4, 5, 5, 7, 8, 9, 9]
|
||||
|
||||
Input: [3, 7, 0, 2, 8, 1, 2, 2, 2, 4, 7, 8, 5, 5, 3, 9, 4, 3, 5, 6, 6, 4, 4, 5, 2, 0, 9, 3, 3, 9, 2, 1]
|
||||
Output: [0, 0, 1, 1, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 9]
|
||||
|
||||
Input: [4, 4, 9, 7, 9, 7, 0, 0, 4, 9, 1, 7, 9, 5, 8, 7, 5, 6, 3, 8, 6, 7, 5, 8, 5, 0, 6, 3, 7, 0, 5, 3, 7, 5, 2, 4, 4, 9, 0, 7, 8, 2, 7, 7, 7, 2, 1, 3, 9, 9, 7, 9, 6, 6, 4, 5, 4, 2, 0, 8, 9, 0, 2, 2]
|
||||
Output: [0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9]
|
||||
</Examples>
|
||||
|
||||
Input: {input}
|
||||
```
|
||||
|
||||
### IMPROVE: improve_prompt
|
||||
Replace `{input}` with the input list of numbers to be sorted and `{incorrectly_sorted}` with the incorrectly sorted list of numbers. `{length}` is the length of the input list.
|
||||
```
|
||||
<Instruction> The following two lists represent an unsorted list of numbers and a sorted variant of that list. The sorted variant is not correct. Fix the sorted variant so that it is correct.
|
||||
Make sure that the output list is sorted in ascending order, has the same number of elements as the input list ({length}), and contains the same elements as the input list. </Instruction>
|
||||
|
||||
<Approach>
|
||||
To fix the incorrectly sorted list follow these steps:
|
||||
1. For each number from 0 to 9, compare the frequency of that number in the incorrectly sorted list to the frequency of that number in the input list.
|
||||
2. Iterate through the incorrectly sorted list and add or remove numbers as needed to make the frequency of each number in the incorrectly sorted list match the frequency of that number in the input list.
|
||||
</Approach>
|
||||
|
||||
<Examples>
|
||||
Input: [3, 7, 0, 2, 8, 1, 2, 2, 2, 4, 7, 8, 5, 5, 3, 9]
|
||||
Incorrectly Sorted: [0, 0, 0, 0, 0, 1, 2, 2, 3, 3, 4, 4, 4, 5, 5, 7, 7, 8, 8, 9, 9, 9, 9]
|
||||
Reason: The incorrectly sorted list contains four extra 0s, two extra 4s and three extra 9s and is missing two 2s.
|
||||
Output: [0, 1, 2, 2, 2, 2, 3, 3, 4, 5, 5, 7, 7, 8, 8, 9]
|
||||
|
||||
Input: [6, 4, 5, 7, 5, 6, 9, 7, 6, 9, 4, 6, 9, 8, 1, 9, 2, 4, 9, 0, 7, 6, 5, 6, 6, 2, 8, 3, 9, 5, 6, 1]
|
||||
Incorrectly Sorted: [0, 1, 1, 2, 2, 3, 4, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 7, 7, 7, 8, 8, 9, 9, 9, 9, 9]
|
||||
Reason: The incorrectly sorted list contains two extra 4s and is missing two 6s and one 9.
|
||||
Output: [0, 1, 1, 2, 2, 3, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 8, 8, 9, 9, 9, 9, 9, 9]
|
||||
|
||||
Input: [4, 4, 9, 7, 9, 7, 0, 0, 4, 9, 1, 7, 9, 5, 8, 7, 5, 6, 3, 8, 6, 7, 5, 8, 5, 0, 6, 3, 7, 0, 5, 3, 7, 5, 2, 4, 4, 9, 0, 7, 8, 2, 7, 7, 7, 2, 1, 3, 9, 9, 7, 9, 6, 6, 4, 5, 4, 2, 0, 8, 9, 0, 2, 2]
|
||||
Incorrectly Sorted: [0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 2, 2, 2, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 5, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9]
|
||||
Reason: The incorrectly sorted list contains one extra 8 and is missing two 2s, one 3, three 4s, two 5s, one 6, six 7s and one 9.
|
||||
Output: [0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9]
|
||||
</Examples>
|
||||
|
||||
Input: {input}
|
||||
Incorrectly Sorted: {incorrectly_sorted}
|
||||
```
|
||||
|
||||
### AGGREGATE: aggregate_prompt
|
||||
Replace `{input_list1}` and `{input_list2}` with the input lists of numbers to be merged. `{length}` is the length of each input list and `{length_combined}` is the length of the combined list.
|
||||
```
|
||||
<Instruction> Merge the following 2 sorted lists of length {length} each, into one sorted list of length {length_combined} using a merge sort style approach.
|
||||
Only output the final merged list without any additional text or thoughts!:</Instruction>
|
||||
|
||||
<Approach>
|
||||
To merge the two lists in a merge-sort style approach, foloow these steps:
|
||||
1. Compare the first element of both lists.
|
||||
2. Append the smaller element to the merged list and move to the next element in the list from which the smaller element came.
|
||||
3. Repeat steps 1 and 2 until one of the lists is empty.
|
||||
4. Append the remaining elements of the non-empty list to the merged list.
|
||||
</Approach>
|
||||
|
||||
Merge the following two lists into one sorted list:
|
||||
1: {input_list1}
|
||||
2: {input_list2}
|
||||
|
||||
Merged list:
|
||||
```
|
||||
|
||||
## Complete Example Prompts
|
||||
### The GoO Summarised
|
||||
1. Split the input list into two sub-lists of equal size (split prompt)
|
||||
2. For each sub-list: Sort the sub-list (sort prompt) five times; score each sort attempt; keep the best
|
||||
3. Merge the sorted sub-lists into one fully sorted list (merge prompt) 10 times; score each merge attempt; keep the best
|
||||
4. Fix any potential mistakes in the sorted list (improve prompt) 10 times; score each improvement attempt; keep the best
|
||||
|
||||
### Input
|
||||
```
|
||||
[8, 7, 1, 1, 1, 1, 3, 3, 0, 9, 4, 1, 0, 2, 5, 1, 0, 5, 6, 7, 1, 4, 5, 9, 4, 6, 2, 5, 8, 6, 2, 6]
|
||||
```
|
||||
|
||||
### Step 1 - Prompt
|
||||
```
|
||||
<Instruction> Split the following list of 32 numbers into 2 lists of 16 numbers each, the first list should contain the first 16 numbers and the second list the second 16 numbers.
|
||||
Only output the final 2 lists in the following format without any additional text or thoughts!:
|
||||
{
|
||||
"List 1": [3, 4, 3, 5, 7, 8, 1, ...],
|
||||
"List 2": [2, 9, 2, 4, 7, 1, 5, ...]
|
||||
} </Instruction>
|
||||
|
||||
<Example>
|
||||
Input: [9, 6, 7, 7, 2, 0, 2, 2, 3, 5, 0, 9, 2, 2, 4, 4, 5, 2, 5, 1, 2, 8, 3, 8, 3, 9, 6, 0, 4, 2, 2, 3]
|
||||
Output:
|
||||
{
|
||||
"List 1": [9, 6, 7, 7, 2, 0, 2, 2, 3, 5, 0, 9, 2, 2, 4, 4],
|
||||
"List 2": [5, 2, 5, 1, 2, 8, 3, 8, 3, 9, 6, 0, 4, 2, 2, 3]
|
||||
}
|
||||
</Example>
|
||||
|
||||
Input: [8, 7, 1, 1, 1, 1, 3, 3, 0, 9, 4, 1, 0, 2, 5, 1, 0, 5, 6, 7, 1, 4, 5, 9, 4, 6, 2, 5, 8, 6, 2, 6]
|
||||
```
|
||||
|
||||
### Step 1 - Response
|
||||
```
|
||||
{
|
||||
"List 1": [8, 7, 1, 1, 1, 1, 3, 3, 0, 9, 4, 1, 0, 2, 5, 1],
|
||||
"List 2": [0, 5, 6, 7, 1, 4, 5, 9, 4, 6, 2, 5, 8, 6, 2, 6]
|
||||
}
|
||||
```
|
||||
|
||||
### Step 2a - Prompt
|
||||
```
|
||||
<Instruction> Sort the following list of numbers in ascending order. Output only the sorted list of numbers, no additional text. </Instruction>
|
||||
|
||||
<Examples>
|
||||
Input: [5, 1, 0, 1, 2, 0, 4, 8, 1, 9, 5, 1, 3, 3, 9, 7]
|
||||
Output: [0, 0, 1, 1, 1, 1, 2, 3, 3, 4, 5, 5, 7, 8, 9, 9]
|
||||
|
||||
Input: [3, 7, 0, 2, 8, 1, 2, 2, 2, 4, 7, 8, 5, 5, 3, 9, 4, 3, 5, 6, 6, 4, 4, 5, 2, 0, 9, 3, 3, 9, 2, 1]
|
||||
Output: [0, 0, 1, 1, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 9]
|
||||
|
||||
Input: [4, 4, 9, 7, 9, 7, 0, 0, 4, 9, 1, 7, 9, 5, 8, 7, 5, 6, 3, 8, 6, 7, 5, 8, 5, 0, 6, 3, 7, 0, 5, 3, 7, 5, 2, 4, 4, 9, 0, 7, 8, 2, 7, 7, 7, 2, 1, 3, 9, 9, 7, 9, 6, 6, 4, 5, 4, 2, 0, 8, 9, 0, 2, 2]
|
||||
Output: [0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9]
|
||||
</Examples>
|
||||
|
||||
Input: [8, 7, 1, 1, 1, 1, 3, 3, 0, 9, 4, 1, 0, 2, 5, 1]
|
||||
```
|
||||
|
||||
### Step 2a - 5 Responses
|
||||
```
|
||||
1. [0, 0, 1, 1, 1, 1, 1, 1, 2, 3, 3, 4, 5, 7, 8, 9]
|
||||
2. [0, 0, 1, 1, 1, 1, 1, 2, 3, 3, 4, 5, 7, 8, 9]
|
||||
3. [0, 0, 1, 1, 1, 1, 1, 2, 3, 3, 4, 5, 7, 8, 9]
|
||||
4. [0, 0, 1, 1, 1, 1, 1, 2, 3, 3, 4, 5, 7, 8, 9]
|
||||
5. [0, 0, 1, 1, 1, 1, 1, 2, 3, 3, 4, 5, 7, 8, 9]
|
||||
```
|
||||
|
||||
### Step 2b - Prompt
|
||||
```
|
||||
<Instruction> Sort the following list of numbers in ascending order. Output only the sorted list of numbers, no additional text. </Instruction>
|
||||
|
||||
<Examples>
|
||||
Input: [5, 1, 0, 1, 2, 0, 4, 8, 1, 9, 5, 1, 3, 3, 9, 7]
|
||||
Output: [0, 0, 1, 1, 1, 1, 2, 3, 3, 4, 5, 5, 7, 8, 9, 9]
|
||||
|
||||
Input: [3, 7, 0, 2, 8, 1, 2, 2, 2, 4, 7, 8, 5, 5, 3, 9, 4, 3, 5, 6, 6, 4, 4, 5, 2, 0, 9, 3, 3, 9, 2, 1]
|
||||
Output: [0, 0, 1, 1, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 9]
|
||||
|
||||
Input: [4, 4, 9, 7, 9, 7, 0, 0, 4, 9, 1, 7, 9, 5, 8, 7, 5, 6, 3, 8, 6, 7, 5, 8, 5, 0, 6, 3, 7, 0, 5, 3, 7, 5, 2, 4, 4, 9, 0, 7, 8, 2, 7, 7, 7, 2, 1, 3, 9, 9, 7, 9, 6, 6, 4, 5, 4, 2, 0, 8, 9, 0, 2, 2]
|
||||
Output: [0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9]
|
||||
</Examples>
|
||||
|
||||
Input: [0, 5, 6, 7, 1, 4, 5, 9, 4, 6, 2, 5, 8, 6, 2, 6]
|
||||
```
|
||||
|
||||
### Step 2b - 5 Responses
|
||||
```
|
||||
1. [0, 1, 2, 2, 4, 4, 5, 5, 5, 6, 6, 6, 7, 8, 9]
|
||||
2. [0, 1, 2, 2, 4, 4, 5, 5, 5, 6, 6, 6, 7, 8, 9]
|
||||
3. [0, 1, 2, 2, 4, 4, 5, 5, 5, 6, 6, 6, 7, 8, 9]
|
||||
4. [0, 1, 2, 2, 4, 4, 5, 5, 5, 6, 6, 6, 6, 7, 8, 9]
|
||||
5. [0, 1, 2, 2, 4, 4, 5, 5, 5, 6, 6, 6, 7, 8, 9]
|
||||
```
|
||||
|
||||
### Step 3 - Prompt
|
||||
```
|
||||
<Instruction> Merge the following 2 sorted lists of length 16 each, into one sorted list of length 32 using a merge sort style approach.
|
||||
Only output the final merged list without any additional text or thoughts!:</Instruction>
|
||||
|
||||
<Approach>
|
||||
To merge the two lists in a merge-sort style approach, follow these steps:
|
||||
1. Compare the first element of both lists.
|
||||
2. Append the smaller element to the merged list and move to the next element in the list from which the smaller element came.
|
||||
3. Repeat steps 1 and 2 until one of the lists is empty.
|
||||
4. Append the remaining elements of the non-empty list to the merged list.
|
||||
</Approach>
|
||||
|
||||
Merge the following two lists into one sorted list:
|
||||
1: [0, 0, 1, 1, 1, 1, 1, 1, 2, 3, 3, 4, 5, 7, 8, 9]
|
||||
2: [0, 1, 2, 2, 4, 4, 5, 5, 5, 6, 6, 6, 6, 7, 8, 9]
|
||||
|
||||
Merged list:
|
||||
```
|
||||
|
||||
### Step 3 - 10 Responses
|
||||
```
|
||||
1. [0, 0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 3, 3, 4, 4, 4, 5, 5, 5, 6, 6, 6, 6, 7, 7, 8, 8, 9, 9]
|
||||
2. [0, 0, 0, 1, 1, 1, 1, 1, 2, 2, 2, 3, 3, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 7, 7, 8, 8, 9, 9]
|
||||
3. [0, 0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 3, 3, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 6, 7, 7, 8, 8, 9, 9]
|
||||
4. [0, 0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 3, 3, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 6, 7, 7, 8, 8, 9]
|
||||
5. [0, 0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 3, 3, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 6, 7, 7, 8, 8, 9, 9]
|
||||
6. [0, 0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 3, 3, 4, 4, 4, 5, 5, 5, 6, 6, 6, 6, 7, 7, 8, 8, 9, 9]
|
||||
7. [0, 0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 3, 3, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 6, 7, 7, 8, 8, 9, 9]
|
||||
8. [0, 0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 3, 3, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 7, 7, 8, 8, 9, 9]
|
||||
9. [0, 0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 3, 3, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 7, 7, 8, 8, 9, 9]
|
||||
10. [0, 0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 3, 3, 4, 4, 4, 5, 5, 5, 6, 6, 6, 6, 7, 7, 8, 8, 9, 9]
|
||||
```
|
||||
|
||||
### Step 4 - Prompt
|
||||
```
|
||||
<Instruction> The following two lists represent an unsorted list of numbers and a sorted variant of that list. The sorted variant is not correct. Fix the sorted variant so that it is correct.
|
||||
Make sure that the output list is sorted in ascending order, has the same number of elements as the input list (32), and contains the same elements as the input list. </Instruction>
|
||||
|
||||
<Approach>
|
||||
To fix the incorrectly sorted list follow these steps:
|
||||
1. For each number from 0 to 9, compare the frequency of that number in the incorrectly sorted list to the frequency of that number in the input list.
|
||||
2. Iterate through the incorrectly sorted list and add or remove numbers as needed to make the frequency of each number in the incorrectly sorted list match the frequency of that number in the input list.
|
||||
</Approach>
|
||||
|
||||
<Examples>
|
||||
Input: [3, 7, 0, 2, 8, 1, 2, 2, 2, 4, 7, 8, 5, 5, 3, 9]
|
||||
Incorrectly Sorted: [0, 0, 0, 0, 0, 1, 2, 2, 3, 3, 4, 4, 4, 5, 5, 7, 7, 8, 8, 9, 9, 9, 9]
|
||||
Reason: The incorrectly sorted list contains four extra 0s, two extra 4s and three extra 9s and is missing two 2s.
|
||||
Output: [0, 1, 2, 2, 2, 2, 3, 3, 4, 5, 5, 7, 7, 8, 8, 9]
|
||||
|
||||
Input: [6, 4, 5, 7, 5, 6, 9, 7, 6, 9, 4, 6, 9, 8, 1, 9, 2, 4, 9, 0, 7, 6, 5, 6, 6, 2, 8, 3, 9, 5, 6, 1]
|
||||
Incorrectly Sorted: [0, 1, 1, 2, 2, 3, 4, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 7, 7, 7, 8, 8, 9, 9, 9, 9, 9]
|
||||
Reason: The incorrectly sorted list contains two extra 4s and is missing two 6s and one 9.
|
||||
Output: [0, 1, 1, 2, 2, 3, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 8, 8, 9, 9, 9, 9, 9, 9]
|
||||
|
||||
Input: [4, 4, 9, 7, 9, 7, 0, 0, 4, 9, 1, 7, 9, 5, 8, 7, 5, 6, 3, 8, 6, 7, 5, 8, 5, 0, 6, 3, 7, 0, 5, 3, 7, 5, 2, 4, 4, 9, 0, 7, 8, 2, 7, 7, 7, 2, 1, 3, 9, 9, 7, 9, 6, 6, 4, 5, 4, 2, 0, 8, 9, 0, 2, 2]
|
||||
Incorrectly Sorted: [0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 2, 2, 2, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 5, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9]
|
||||
Reason: The incorrectly sorted list contains one extra 8 and is missing two 2s, one 3, three 4s, two 5s, one 6, six 7s and one 9.
|
||||
Output: [0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9]
|
||||
</Examples>
|
||||
|
||||
Input: [8, 7, 1, 1, 1, 1, 3, 3, 0, 9, 4, 1, 0, 2, 5, 1, 0, 5, 6, 7, 1, 4, 5, 9, 4, 6, 2, 5, 8, 6, 2, 6]
|
||||
Incorrectly Sorted: [0, 0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 3, 3, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 7, 7, 8, 8, 9, 9]
|
||||
```
|
||||
|
||||
### Step 4 - 10 Responses
|
||||
```
|
||||
1. Reason: The incorrectly sorted list is missing two 0s, one 3, one 4, one 6 and one 9. The incorrectly sorted list also contains one extra 1, one extra 2, and one extra 5.
|
||||
|
||||
Output: [0, 0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 3, 3, 4, 5, 5, 5, 6, 6, 6, 6, 7, 7, 8, 8, 9, 9]
|
||||
|
||||
Note: The output list has the same number of elements as the input list, is sorted in ascending order, and contains the same elements as the input list.
|
||||
|
||||
2. Output: [0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 3, 3, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 6, 7, 7, 8, 8, 9, 9]
|
||||
|
||||
3. Output: [0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 4, 4, 4, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 7, 7, 7, 8, 8, 8, 9, 9, 9]
|
||||
|
||||
4. Reason: The incorrectly sorted list is missing two 9s and one 0.
|
||||
Output: [0, 0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 3, 3, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 7, 7, 8, 8, 9, 9, 9, 9]
|
||||
|
||||
5. Reason: The incorrectly sorted list is missing one 0, one 3, one 4, four 6s, and one 9.
|
||||
|
||||
Output: [0, 0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 3, 3, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 8, 8, 8, 9, 9, 9, 9]
|
||||
|
||||
6. Output: [0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 6, 7, 7, 7, 7, 8, 8, 8, 9, 9, 9, 9]
|
||||
|
||||
7. Output: [0, 0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 7, 7, 8, 8, 9, 9]
|
||||
|
||||
8. Reason: The incorrectly sorted list is missing three 0s, two 3s, two 4s, one 5, two 6s, and one 9.
|
||||
Output: [0, 0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 7, 7, 8, 8, 9]
|
||||
|
||||
9. Reason: The incorrectly sorted list is missing one 0, three 2s, two 3s, two 4s, three 5s, two 6s, one 7, and one 9.
|
||||
Output: [0, 0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 7, 8, 8, 9, 9]
|
||||
|
||||
10. Reason: The incorrectly sorted list is missing three 1s, one 3, one 4, two 5s, one 6, one 7, one 8, and one 9.
|
||||
Output: [0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 3, 3, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 7, 7, 8, 8, 9, 9]
|
||||
```
|
||||
|
||||
### Final Result
|
||||
```
|
||||
[0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 3, 3, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 7, 7, 8, 8, 9, 9]
|
||||
```
|
||||
@ -12,8 +12,13 @@ import datetime
|
||||
import json
|
||||
import csv
|
||||
from typing import Dict, List, Callable, Union
|
||||
from graph_of_thoughts import controller, operations, prompter, parser
|
||||
from graph_of_thoughts import controller, language_models, operations, prompter, parser
|
||||
|
||||
# This is a hack to also allow execution of this file from the examples directory
|
||||
try:
|
||||
from . import utils
|
||||
except ImportError:
|
||||
import utils
|
||||
|
||||
|
||||
class SortingPrompter(prompter.Prompter):
|
||||
@ -114,7 +119,7 @@ Incorrectly Sorted: {incorrectly_sorted}
|
||||
"""
|
||||
|
||||
got_split_prompt = """<Instruction> Split the following list of 32 numbers into 2 lists of 16 numbers each, the first list should contain the first 16 numbers and the second list the second 16 numbers.
|
||||
Only output the final 4 lists in the following format without any additional text or thoughts!:
|
||||
Only output the final 2 lists in the following format without any additional text or thoughts!:
|
||||
{{
|
||||
"List 1": [3, 4, 3, 5, 7, 8, 1, ...],
|
||||
"List 2": [2, 9, 2, 4, 7, 1, 5, ...]
|
||||
@ -135,7 +140,7 @@ Input: {input}"""
|
||||
Only output the final merged list without any additional text or thoughts!:</Instruction>
|
||||
|
||||
<Approach>
|
||||
To merge the two lists in a merge-sort style approach, foloow these steps:
|
||||
To merge the two lists in a merge-sort style approach, follow these steps:
|
||||
1. Compare the first element of both lists.
|
||||
2. Append the smaller element to the merged list and move to the next element in the list from which the smaller element came.
|
||||
3. Repeat steps 1 and 2 until one of the lists is empty.
|
||||
@ -555,7 +560,7 @@ def got() -> operations.GraphOfOperations:
|
||||
"""
|
||||
operations_graph = operations.GraphOfOperations()
|
||||
|
||||
plans = operations.Generate(2, 1)
|
||||
plans = operations.Generate(1, 1)
|
||||
operations_graph.append_operation(plans) # generate the sublists
|
||||
for i in range(1, 3):
|
||||
list_id = f"List {i}"
|
||||
@ -616,9 +621,9 @@ def run(
|
||||
"""
|
||||
|
||||
orig_budget = budget
|
||||
path = os.path.join(os.path.dirname(__file__), "sorting_032.csv")
|
||||
data_path = os.path.join(os.path.dirname(__file__), "sorting_032.csv")
|
||||
data = []
|
||||
with open(path, "r") as f:
|
||||
with open(data_path, "r") as f:
|
||||
reader = csv.reader(f)
|
||||
next(reader)
|
||||
for row in reader:
|
||||
@ -628,12 +633,15 @@ def run(
|
||||
data_ids = list(range(len(data)))
|
||||
selected_data = [data[i] for i in data_ids]
|
||||
|
||||
if not os.path.exists(os.path.join(os.path.dirname(__file__), "results")):
|
||||
os.makedirs(os.path.join(os.path.dirname(__file__), "results"))
|
||||
results_dir = os.path.join(os.path.dirname(__file__), "results")
|
||||
|
||||
if not os.path.exists(results_dir):
|
||||
os.makedirs(results_dir)
|
||||
timestamp = datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
|
||||
extra_info = f"{lm_name}_{'-'.join([method.__name__ for method in methods])}"
|
||||
folder_name = f"results/{extra_info}_{timestamp}"
|
||||
os.makedirs(os.path.join(os.path.dirname(__file__), folder_name))
|
||||
folder_name = f"{extra_info}_{timestamp}"
|
||||
results_folder = os.path.join(results_dir, folder_name)
|
||||
os.makedirs(results_folder)
|
||||
|
||||
config = {
|
||||
"data": selected_data,
|
||||
@ -641,22 +649,18 @@ def run(
|
||||
"lm": lm_name,
|
||||
"budget": budget,
|
||||
}
|
||||
with open(
|
||||
os.path.join(os.path.dirname(__file__), folder_name, "config.json"), "w"
|
||||
) as f:
|
||||
with open(os.path.join(results_folder, "config.json"), "w") as f:
|
||||
json.dump(config, f)
|
||||
|
||||
logging.basicConfig(
|
||||
filename=f"{folder_name}/log.log",
|
||||
filename=os.path.join(results_folder, "log.log"),
|
||||
filemode="w",
|
||||
format="%(name)s - %(levelname)s - %(message)s",
|
||||
level=logging.DEBUG,
|
||||
)
|
||||
|
||||
for method in methods:
|
||||
os.makedirs(
|
||||
os.path.join(os.path.dirname(__file__), folder_name, method.__name__)
|
||||
)
|
||||
os.makedirs(os.path.join(results_folder, method.__name__))
|
||||
|
||||
for data in selected_data:
|
||||
logging.info(f"Running data {data[0]}: {data[1]}")
|
||||
@ -673,8 +677,11 @@ def run(
|
||||
f"Budget has been depleted, stopping. Method {method.__name__} has not been run."
|
||||
)
|
||||
break
|
||||
lm = controller.ChatGPT(
|
||||
"../../graph_of_thoughts/controller/config.json",
|
||||
lm = language_models.ChatGPT(
|
||||
os.path.join(
|
||||
os.path.dirname(__file__),
|
||||
"../../graph_of_thoughts/language_models/config.json",
|
||||
),
|
||||
model_name=lm_name,
|
||||
cache=True,
|
||||
)
|
||||
@ -696,8 +703,7 @@ def run(
|
||||
except Exception as e:
|
||||
logging.error(f"Exception: {e}")
|
||||
path = os.path.join(
|
||||
os.path.dirname(__file__),
|
||||
folder_name,
|
||||
results_folder,
|
||||
method.__name__,
|
||||
f"{data[0]}.json",
|
||||
)
|
||||
|
||||
@ -12,9 +12,13 @@ import datetime
|
||||
import json
|
||||
import csv
|
||||
from typing import Dict, List, Callable, Union
|
||||
from graph_of_thoughts import controller, operations, prompter, parser
|
||||
from graph_of_thoughts import controller, language_models, operations, prompter, parser
|
||||
|
||||
# This is a hack to also allow execution of this file from the examples directory
|
||||
try:
|
||||
from . import utils
|
||||
except ImportError:
|
||||
import utils
|
||||
|
||||
|
||||
class SortingPrompter(prompter.Prompter):
|
||||
@ -140,7 +144,7 @@ Input: {input}"""
|
||||
Only output the final merged list without any additional text or thoughts!:</Instruction>
|
||||
|
||||
<Approach>
|
||||
To merge the two lists in a merge-sort style approach, foloow these steps:
|
||||
To merge the two lists in a merge-sort style approach, follow these steps:
|
||||
1. Compare the first element of both lists.
|
||||
2. Append the smaller element to the merged list and move to the next element in the list from which the smaller element came.
|
||||
3. Repeat steps 1 and 2 until one of the lists is empty.
|
||||
@ -674,9 +678,9 @@ def run(
|
||||
"""
|
||||
|
||||
orig_budget = budget
|
||||
path = os.path.join(os.path.dirname(__file__), "sorting_064.csv")
|
||||
data_path = os.path.join(os.path.dirname(__file__), "sorting_064.csv")
|
||||
data = []
|
||||
with open(path, "r") as f:
|
||||
with open(data_path, "r") as f:
|
||||
reader = csv.reader(f)
|
||||
next(reader)
|
||||
for row in reader:
|
||||
@ -686,12 +690,15 @@ def run(
|
||||
data_ids = list(range(len(data)))
|
||||
selected_data = [data[i] for i in data_ids]
|
||||
|
||||
if not os.path.exists(os.path.join(os.path.dirname(__file__), "results")):
|
||||
os.makedirs(os.path.join(os.path.dirname(__file__), "results"))
|
||||
results_dir = os.path.join(os.path.dirname(__file__), "results")
|
||||
|
||||
if not os.path.exists(results_dir):
|
||||
os.makedirs(results_dir)
|
||||
timestamp = datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
|
||||
extra_info = f"{lm_name}_{'-'.join([method.__name__ for method in methods])}"
|
||||
folder_name = f"results/{extra_info}_{timestamp}"
|
||||
os.makedirs(os.path.join(os.path.dirname(__file__), folder_name))
|
||||
folder_name = f"{extra_info}_{timestamp}"
|
||||
results_folder = os.path.join(results_dir, folder_name)
|
||||
os.makedirs(results_folder)
|
||||
|
||||
config = {
|
||||
"data": selected_data,
|
||||
@ -699,13 +706,11 @@ def run(
|
||||
"lm": lm_name,
|
||||
"budget": budget,
|
||||
}
|
||||
with open(
|
||||
os.path.join(os.path.dirname(__file__), folder_name, "config.json"), "w"
|
||||
) as f:
|
||||
with open(os.path.join(results_folder, "config.json"), "w") as f:
|
||||
json.dump(config, f)
|
||||
|
||||
logging.basicConfig(
|
||||
filename=f"{folder_name}/log.log",
|
||||
filename=os.path.join(results_folder, "log.log"),
|
||||
filemode="w",
|
||||
format="%(name)s - %(levelname)s - %(message)s",
|
||||
level=logging.DEBUG,
|
||||
@ -713,9 +718,7 @@ def run(
|
||||
|
||||
for method in methods:
|
||||
# create a results directory for the method
|
||||
os.makedirs(
|
||||
os.path.join(os.path.dirname(__file__), folder_name, method.__name__)
|
||||
)
|
||||
os.makedirs(os.path.join(results_folder, method.__name__))
|
||||
|
||||
for data in selected_data:
|
||||
logging.info(f"Running data {data[0]}: {data[1]}")
|
||||
@ -732,8 +735,11 @@ def run(
|
||||
f"Budget has been depleted, stopping. Method {method.__name__} has not been run."
|
||||
)
|
||||
break
|
||||
lm = controller.ChatGPT(
|
||||
"../../graph_of_thoughts/controller/config.json",
|
||||
lm = language_models.ChatGPT(
|
||||
os.path.join(
|
||||
os.path.dirname(__file__),
|
||||
"../../graph_of_thoughts/language_models/config.json",
|
||||
),
|
||||
model_name=lm_name,
|
||||
cache=True,
|
||||
)
|
||||
@ -755,8 +761,7 @@ def run(
|
||||
except Exception as e:
|
||||
logging.error(f"Exception: {e}")
|
||||
path = os.path.join(
|
||||
os.path.dirname(__file__),
|
||||
folder_name,
|
||||
results_folder,
|
||||
method.__name__,
|
||||
f"{data[0]}.json",
|
||||
)
|
||||
|
||||
@ -12,9 +12,13 @@ import datetime
|
||||
import json
|
||||
import csv
|
||||
from typing import Dict, List, Callable, Union
|
||||
from graph_of_thoughts import controller, operations, prompter, parser
|
||||
from graph_of_thoughts import controller, language_models, operations, prompter, parser
|
||||
|
||||
# This is a hack to also allow execution of this file from the examples directory
|
||||
try:
|
||||
from . import utils
|
||||
except ImportError:
|
||||
import utils
|
||||
|
||||
|
||||
class SortingPrompter(prompter.Prompter):
|
||||
@ -148,7 +152,7 @@ Input: {input}"""
|
||||
Only output the final merged list without any additional text or thoughts!:</Instruction>
|
||||
|
||||
<Approach>
|
||||
To merge the two lists in a merge-sort style approach, foloow these steps:
|
||||
To merge the two lists in a merge-sort style approach, follow these steps:
|
||||
1. Compare the first element of both lists.
|
||||
2. Append the smaller element to the merged list and move to the next element in the list from which the smaller element came.
|
||||
3. Repeat steps 1 and 2 until one of the lists is empty.
|
||||
@ -773,9 +777,9 @@ def run(
|
||||
"""
|
||||
|
||||
orig_budget = budget
|
||||
path = os.path.join(os.path.dirname(__file__), "sorting_128.csv")
|
||||
data_path = os.path.join(os.path.dirname(__file__), "sorting_128.csv")
|
||||
data = []
|
||||
with open(path, "r") as f:
|
||||
with open(data_path, "r") as f:
|
||||
reader = csv.reader(f)
|
||||
next(reader)
|
||||
for row in reader:
|
||||
@ -785,12 +789,15 @@ def run(
|
||||
data_ids = list(range(len(data)))
|
||||
selected_data = [data[i] for i in data_ids]
|
||||
|
||||
if not os.path.exists(os.path.join(os.path.dirname(__file__), "results")):
|
||||
os.makedirs(os.path.join(os.path.dirname(__file__), "results"))
|
||||
results_dir = os.path.join(os.path.dirname(__file__), "results")
|
||||
|
||||
if not os.path.exists(results_dir):
|
||||
os.makedirs(results_dir)
|
||||
timestamp = datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
|
||||
extra_info = f"{lm_name}_{'-'.join([method.__name__ for method in methods])}"
|
||||
folder_name = f"results/{extra_info}_{timestamp}"
|
||||
os.makedirs(os.path.join(os.path.dirname(__file__), folder_name))
|
||||
folder_name = f"{extra_info}_{timestamp}"
|
||||
results_folder = os.path.join(results_dir, folder_name)
|
||||
os.makedirs(results_folder)
|
||||
|
||||
config = {
|
||||
"data": selected_data,
|
||||
@ -798,22 +805,18 @@ def run(
|
||||
"lm": lm_name,
|
||||
"budget": budget,
|
||||
}
|
||||
with open(
|
||||
os.path.join(os.path.dirname(__file__), folder_name, "config.json"), "w"
|
||||
) as f:
|
||||
with open(os.path.join(results_folder, "config.json"), "w") as f:
|
||||
json.dump(config, f)
|
||||
|
||||
logging.basicConfig(
|
||||
filename=f"{folder_name}/log.log",
|
||||
filename=os.path.join(results_folder, "log.log"),
|
||||
filemode="w",
|
||||
format="%(name)s - %(levelname)s - %(message)s",
|
||||
level=logging.DEBUG,
|
||||
)
|
||||
|
||||
for method in methods:
|
||||
os.makedirs(
|
||||
os.path.join(os.path.dirname(__file__), folder_name, method.__name__)
|
||||
)
|
||||
os.makedirs(os.path.join(results_folder, method.__name__))
|
||||
|
||||
for data in selected_data:
|
||||
logging.info(f"Running data {data[0]}: {data[1]}")
|
||||
@ -830,8 +833,11 @@ def run(
|
||||
f"Budget has been depleted, stopping. Method {method.__name__} has not been run."
|
||||
)
|
||||
break
|
||||
lm = controller.ChatGPT(
|
||||
"../../graph_of_thoughts/controller/config.json",
|
||||
lm = language_models.ChatGPT(
|
||||
os.path.join(
|
||||
os.path.dirname(__file__),
|
||||
"../../graph_of_thoughts/language_models/config.json",
|
||||
),
|
||||
model_name=lm_name,
|
||||
cache=True,
|
||||
)
|
||||
@ -853,8 +859,7 @@ def run(
|
||||
except Exception as e:
|
||||
logging.error(f"Exception: {e}")
|
||||
path = os.path.join(
|
||||
os.path.dirname(__file__),
|
||||
folder_name,
|
||||
results_folder,
|
||||
method.__name__,
|
||||
f"{data[0]}.json",
|
||||
)
|
||||
|
||||
1
graph_of_thoughts/api/__init__.py
Normal file
1
graph_of_thoughts/api/__init__.py
Normal file
@ -0,0 +1 @@
|
||||
"""HTTP API helpers (FastAPI) for running Graph of Thoughts with OpenRouter."""
|
||||
4
graph_of_thoughts/api/__main__.py
Normal file
4
graph_of_thoughts/api/__main__.py
Normal file
@ -0,0 +1,4 @@
|
||||
from graph_of_thoughts.api.app import run
|
||||
|
||||
if __name__ == "__main__":
|
||||
run()
|
||||
192
graph_of_thoughts/api/app.py
Normal file
192
graph_of_thoughts/api/app.py
Normal file
@ -0,0 +1,192 @@
|
||||
# Copyright (c) 2023 ETH Zurich.
|
||||
# All rights reserved.
|
||||
#
|
||||
# Use of this source code is governed by a BSD-style license that can be
|
||||
# found in the LICENSE file.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
import uuid
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from graph_of_thoughts.api.got_openai_pipeline import (
|
||||
ChatCompletionParser,
|
||||
ChatCompletionPrompter,
|
||||
build_default_chat_graph,
|
||||
extract_assistant_text,
|
||||
format_chat_messages,
|
||||
)
|
||||
from graph_of_thoughts.controller import Controller
|
||||
from graph_of_thoughts.language_models.openrouter import (
|
||||
OpenRouter,
|
||||
OpenRouterBadRequestError,
|
||||
OpenRouterRateLimitError,
|
||||
)
|
||||
|
||||
try:
|
||||
from fastapi import FastAPI, HTTPException
|
||||
from fastapi.responses import JSONResponse
|
||||
from pydantic import BaseModel, Field
|
||||
except ImportError as e:
|
||||
raise ImportError(
|
||||
"FastAPI and Pydantic are required for the HTTP API. "
|
||||
'Install with: pip install "graph_of_thoughts[api]"'
|
||||
) from e
|
||||
|
||||
|
||||
class ChatMessage(BaseModel):
|
||||
role: str
|
||||
content: str
|
||||
|
||||
|
||||
class ChatCompletionRequest(BaseModel):
|
||||
model: Optional[str] = None
|
||||
messages: List[ChatMessage]
|
||||
temperature: Optional[float] = None
|
||||
max_tokens: Optional[int] = None
|
||||
stream: Optional[bool] = False
|
||||
n: Optional[int] = Field(default=1, ge=1, le=1)
|
||||
|
||||
|
||||
def _get_config_path() -> str:
|
||||
return os.environ.get(
|
||||
"OPENROUTER_CONFIG",
|
||||
os.path.join(
|
||||
os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
|
||||
"language_models",
|
||||
"config.openrouter.yaml",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _run_controller(lm: OpenRouter, user_text: str) -> str:
|
||||
graph = build_default_chat_graph(num_candidates=3)
|
||||
ctrl = Controller(
|
||||
lm,
|
||||
graph,
|
||||
ChatCompletionPrompter(),
|
||||
ChatCompletionParser(),
|
||||
{"input": user_text},
|
||||
)
|
||||
ctrl.run()
|
||||
return extract_assistant_text(ctrl.get_final_thoughts())
|
||||
|
||||
|
||||
app = FastAPI(
|
||||
title="Graph of Thoughts (OpenRouter)",
|
||||
version="0.1.0",
|
||||
description="OpenAI-compatible chat completions backed by Graph of Operations + OpenRouter.",
|
||||
)
|
||||
|
||||
|
||||
@app.on_event("startup")
|
||||
def _startup() -> None:
|
||||
logging.basicConfig(level=os.environ.get("LOG_LEVEL", "INFO"))
|
||||
|
||||
|
||||
@app.get("/v1/models")
|
||||
def list_models() -> Dict[str, Any]:
|
||||
path = _get_config_path()
|
||||
if not os.path.isfile(path):
|
||||
return {"object": "list", "data": []}
|
||||
from graph_of_thoughts.language_models.openrouter import load_openrouter_config
|
||||
|
||||
cfg = load_openrouter_config(path)
|
||||
models = cfg.get("models") or []
|
||||
if isinstance(models, str):
|
||||
models = [models]
|
||||
data = [
|
||||
{
|
||||
"id": m,
|
||||
"object": "model",
|
||||
"created": int(time.time()),
|
||||
"owned_by": "openrouter",
|
||||
}
|
||||
for m in models
|
||||
]
|
||||
return {"object": "list", "data": data}
|
||||
|
||||
|
||||
@app.post("/v1/chat/completions")
|
||||
def chat_completions(body: ChatCompletionRequest) -> JSONResponse:
|
||||
if body.stream:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="stream=true is not supported; use stream=false.",
|
||||
)
|
||||
if body.n != 1:
|
||||
raise HTTPException(status_code=400, detail="Only n=1 is supported.")
|
||||
|
||||
path = _get_config_path()
|
||||
if not os.path.isfile(path):
|
||||
raise HTTPException(
|
||||
status_code=500,
|
||||
detail=f"OpenRouter config not found at {path}. Set OPENROUTER_CONFIG.",
|
||||
)
|
||||
|
||||
lm = OpenRouter(config_path=path)
|
||||
user_text = format_chat_messages(
|
||||
[{"role": m.role, "content": m.content} for m in body.messages]
|
||||
)
|
||||
try:
|
||||
lm.set_request_overrides(
|
||||
model=body.model,
|
||||
temperature=body.temperature,
|
||||
max_tokens=body.max_tokens,
|
||||
)
|
||||
try:
|
||||
answer = _run_controller(lm, user_text)
|
||||
finally:
|
||||
lm.clear_request_overrides()
|
||||
except OpenRouterRateLimitError as e:
|
||||
raise HTTPException(status_code=429, detail=str(e)) from e
|
||||
except OpenRouterBadRequestError as e:
|
||||
raise HTTPException(status_code=400, detail=str(e)) from e
|
||||
|
||||
model_id = (
|
||||
body.model
|
||||
or lm.generation_model_id
|
||||
or lm.last_model_id
|
||||
or (lm.models[0] if lm.models else "openrouter")
|
||||
)
|
||||
resp_id = f"chatcmpl-{uuid.uuid4().hex}"
|
||||
now = int(time.time())
|
||||
payload = {
|
||||
"id": resp_id,
|
||||
"object": "chat.completion",
|
||||
"created": now,
|
||||
"model": model_id,
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"message": {"role": "assistant", "content": answer},
|
||||
"finish_reason": "stop",
|
||||
}
|
||||
],
|
||||
"usage": {
|
||||
"prompt_tokens": lm.prompt_tokens,
|
||||
"completion_tokens": lm.completion_tokens,
|
||||
"total_tokens": lm.prompt_tokens + lm.completion_tokens,
|
||||
},
|
||||
}
|
||||
return JSONResponse(content=payload)
|
||||
|
||||
|
||||
def run() -> None:
|
||||
import uvicorn
|
||||
|
||||
host = os.environ.get("HOST", "0.0.0.0")
|
||||
port = int(os.environ.get("PORT", "8000"))
|
||||
uvicorn.run(
|
||||
"graph_of_thoughts.api.app:app",
|
||||
host=host,
|
||||
port=port,
|
||||
reload=os.environ.get("RELOAD", "").lower() in ("1", "true", "yes"),
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run()
|
||||
123
graph_of_thoughts/api/got_openai_pipeline.py
Normal file
123
graph_of_thoughts/api/got_openai_pipeline.py
Normal file
@ -0,0 +1,123 @@
|
||||
# Copyright (c) 2023 ETH Zurich.
|
||||
# All rights reserved.
|
||||
#
|
||||
# Use of this source code is governed by a BSD-style license that can be
|
||||
# found in the LICENSE file.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from typing import Any, Dict, List, Union
|
||||
|
||||
from graph_of_thoughts.operations import GraphOfOperations, operations
|
||||
from graph_of_thoughts.operations.thought import Thought
|
||||
from graph_of_thoughts.parser import Parser
|
||||
from graph_of_thoughts.prompter import Prompter
|
||||
|
||||
|
||||
def format_chat_messages(messages: List[Dict[str, str]]) -> str:
|
||||
parts: List[str] = []
|
||||
for m in messages:
|
||||
role = m.get("role", "user")
|
||||
content = m.get("content", "")
|
||||
if not isinstance(content, str):
|
||||
content = str(content)
|
||||
parts.append(f"{role.upper()}:\n{content}")
|
||||
return "\n\n".join(parts)
|
||||
|
||||
|
||||
class ChatCompletionPrompter(Prompter):
|
||||
"""Prompter for a small generate → score → keep-best Graph of Operations."""
|
||||
|
||||
def generate_prompt(self, num_branches: int, **kwargs: Any) -> str:
|
||||
problem = kwargs.get("input", "")
|
||||
return (
|
||||
"You are a careful assistant. Read the conversation below and produce "
|
||||
"one candidate answer for the USER's latest needs.\n\n"
|
||||
f"{problem}\n\n"
|
||||
"Reply with your answer only, no preamble."
|
||||
)
|
||||
|
||||
def score_prompt(self, state_dicts: List[Dict], **kwargs: Any) -> str:
|
||||
lines = [
|
||||
"You evaluate candidate answers for the same problem. "
|
||||
"Score each candidate from 0 (worst) to 10 (best) on correctness, "
|
||||
"completeness, and relevance.",
|
||||
"",
|
||||
"Return ONLY a JSON array of numbers, one score per candidate in order, e.g. [7, 5, 9].",
|
||||
"",
|
||||
]
|
||||
for i, st in enumerate(state_dicts):
|
||||
cand = st.get("candidate", "")
|
||||
lines.append(f"Candidate {i}:\n{cand}\n")
|
||||
return "\n".join(lines)
|
||||
|
||||
def aggregation_prompt(self, state_dicts: List[Dict], **kwargs: Any) -> str:
|
||||
raise RuntimeError("aggregation_prompt is not used by the chat completion pipeline")
|
||||
|
||||
def improve_prompt(self, **kwargs: Any) -> str:
|
||||
raise RuntimeError("improve_prompt is not used by the chat completion pipeline")
|
||||
|
||||
def validation_prompt(self, **kwargs: Any) -> str:
|
||||
raise RuntimeError("validation_prompt is not used by the chat completion pipeline")
|
||||
|
||||
|
||||
class ChatCompletionParser(Parser):
|
||||
def parse_generate_answer(self, state: Dict, texts: List[str]) -> List[Dict]:
|
||||
out: List[Dict] = []
|
||||
for i, t in enumerate(texts):
|
||||
out.append({"candidate": (t or "").strip(), "branch_index": i})
|
||||
return out
|
||||
|
||||
def parse_score_answer(self, states: List[Dict], texts: List[str]) -> List[float]:
|
||||
raw = texts[0] if texts else ""
|
||||
scores = self._scores_from_text(raw, len(states))
|
||||
if len(scores) < len(states):
|
||||
scores.extend([0.0] * (len(states) - len(scores)))
|
||||
return scores[: len(states)]
|
||||
|
||||
def _scores_from_text(self, raw: str, n: int) -> List[float]:
|
||||
raw = raw.strip()
|
||||
try:
|
||||
data = json.loads(raw)
|
||||
if isinstance(data, list):
|
||||
return [float(x) for x in data]
|
||||
except (json.JSONDecodeError, ValueError, TypeError):
|
||||
pass
|
||||
nums = re.findall(r"-?\d+(?:\.\d+)?", raw)
|
||||
return [float(x) for x in nums[:n]]
|
||||
|
||||
def parse_aggregation_answer(
|
||||
self, states: List[Dict], texts: List[str]
|
||||
) -> Union[Dict, List[Dict]]:
|
||||
raise RuntimeError("parse_aggregation_answer is not used")
|
||||
|
||||
def parse_improve_answer(self, state: Dict, texts: List[str]) -> Dict:
|
||||
raise RuntimeError("parse_improve_answer is not used")
|
||||
|
||||
def parse_validation_answer(self, state: Dict, texts: List[str]) -> bool:
|
||||
raise RuntimeError("parse_validation_answer is not used")
|
||||
|
||||
|
||||
def build_default_chat_graph(num_candidates: int = 3) -> GraphOfOperations:
|
||||
g = GraphOfOperations()
|
||||
g.append_operation(
|
||||
operations.Generate(
|
||||
num_branches_prompt=1, num_branches_response=num_candidates
|
||||
)
|
||||
)
|
||||
g.append_operation(operations.Score(combined_scoring=True))
|
||||
g.append_operation(operations.KeepBestN(1))
|
||||
return g
|
||||
|
||||
|
||||
def extract_assistant_text(final_thoughts_list: List[List[Thought]]) -> str:
|
||||
"""``get_final_thoughts`` returns one list per leaf operation; we take the first leaf's first thought."""
|
||||
if not final_thoughts_list:
|
||||
return ""
|
||||
thoughts = final_thoughts_list[0]
|
||||
if not thoughts:
|
||||
return ""
|
||||
state = thoughts[0].state or {}
|
||||
return str(state.get("candidate", ""))
|
||||
@ -3,73 +3,16 @@
|
||||
The Controller class is responsible for traversing the Graph of Operations (GoO), which is a static structure that is constructed once, before the execution starts.
|
||||
GoO prescribes the execution plan of thought operations and the Controller invokes their execution, generating the Graph Reasoning State (GRS).
|
||||
|
||||
In order for a GoO to be executed, an instance of Large Language Model (LLM) must be supplied to the controller.
|
||||
Currently, the framework supports the following LLMs:
|
||||
- GPT-4 / GPT-3.5 (Remote - OpenAI API)
|
||||
- Llama-2 (Local - HuggingFace Transformers)
|
||||
In order for a GoO to be executed, an instance of Large Language Model (LLM) must be supplied to the controller (along with other required objects).
|
||||
Please refer to the [Language Models](../language_models/README.md) section for more information about LLMs.
|
||||
|
||||
The following section describes how to instantiate individual LLMs and the Controller to run a defined GoO.
|
||||
Furthermore, the process of adding new LLMs into the framework is outlined at the end.
|
||||
|
||||
## LLM Instantiation
|
||||
- Create a copy of `config_template.json` named `config.json`.
|
||||
- Fill configuration details based on the used model (below).
|
||||
|
||||
### GPT-4 / GPT-3.5
|
||||
- Adjust predefined `chatgpt`, `chatgpt4` or create new configuration with an unique key.
|
||||
|
||||
| Key | Value |
|
||||
|---------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
|
||||
| model_id | Model name based on [OpenAI model overview](https://platform.openai.com/docs/models/overview). |
|
||||
| prompt_token_cost | Price per 1000 prompt tokens based on [OpenAI pricing](https://openai.com/pricing), used for calculating cumulative price per LLM instance. |
|
||||
| response_token_cost | Price per 1000 response tokens based on [OpenAI pricing](https://openai.com/pricing), used for calculating cumulative price per LLM instance. |
|
||||
| temperature | Parameter of OpenAI models that controls randomness and the creativity of the responses (higher temperature = more diverse and unexpected responses). Value between 0.0 and 2.0, default is 1.0. More information can be found in the [OpenAI API reference](https://platform.openai.com/docs/api-reference/completions/create#completions/create-temperature). |
|
||||
| max_tokens | The maximum number of tokens to generate in the chat completion. Value depends on the maximum context size of the model specified in the [OpenAI model overview](https://platform.openai.com/docs/models/overview). More information can be found in the [OpenAI API reference](https://platform.openai.com/docs/api-reference/chat/create#chat/create-max_tokens). |
|
||||
| stop | String or array of strings specifying sequence of characters which if detected, stops further generation of tokens. More information can be found in the [OpenAI API reference](https://platform.openai.com/docs/api-reference/chat/create#chat/create-stop). |
|
||||
| organization | Organization to use for the API requests (may be empty). |
|
||||
| api_key | Personal API key that will be used to access OpenAI API. |
|
||||
|
||||
- Instantiate the language model based on the selected configuration key (predefined / custom).
|
||||
```
|
||||
lm = controller.ChatGPT(
|
||||
"path/to/config.json",
|
||||
model_name=<configuration key>
|
||||
)
|
||||
```
|
||||
|
||||
### Llama-2
|
||||
- Requires local hardware to run inference and a HuggingFace account.
|
||||
- Adjust predefined `llama7b-hf`, `llama13b-hf`, `llama70b-hf` or create a new configuration with an unique key.
|
||||
|
||||
| Key | Value |
|
||||
|---------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
|
||||
| model_id | Specifies HuggingFace Llama 2 model identifier (`meta-llama/<model_id>`). |
|
||||
| cache_dir | Local directory where model will be downloaded and accessed. |
|
||||
| prompt_token_cost | Price per 1000 prompt tokens (currently not used - local model = no cost). |
|
||||
| response_token_cost | Price per 1000 response tokens (currently not used - local model = no cost). |
|
||||
| temperature | Parameter that controls randomness and the creativity of the responses (higher temperature = more diverse and unexpected responses). Value between 0.0 and 1.0, default is 0.6. |
|
||||
| top_k | Top-K sampling method described in [Transformers tutorial](https://huggingface.co/blog/how-to-generate). Default value is set to 10. |
|
||||
| max_tokens | The maximum number of tokens to generate in the chat completion. More tokens require more memory. |
|
||||
|
||||
- Instantiate the language model based on the selected configuration key (predefined / custom).
|
||||
```
|
||||
lm = controller.Llama2HF(
|
||||
"path/to/config.json",
|
||||
model_name=<configuration key>
|
||||
)
|
||||
```
|
||||
- Request access to Llama-2 via the [Meta form](https://ai.meta.com/resources/models-and-libraries/llama-downloads/) using the same email address as for the HuggingFace account.
|
||||
- After the access is granted, go to [HuggingFace Llama-2 model card](https://huggingface.co/meta-llama/Llama-2-7b-chat-hf), log in and accept the license (_"You have been granted access to this model"_ message should appear).
|
||||
- Generate HuggingFace access token.
|
||||
- Log in from CLI with: `huggingface-cli login --token <your token>`.
|
||||
|
||||
Note: 4-bit quantization is used to reduce the model size for inference. During instantiation, the model is downloaded from HuggingFace into the cache directory specified in the `config.json`. Running queries using larger models will require multiple GPUs (splitting across many GPUs is done automatically by the Transformers library).
|
||||
The following section describes how to instantiate the Controller to run a defined GoO.
|
||||
|
||||
## Controller Instantiation
|
||||
- Requires custom `Prompter`, `Parser` and instantiated `GraphOfOperations` - creation of these is described separately.
|
||||
- Use instantiated `lm` from above.
|
||||
- Requires custom `Prompter`, `Parser`, as well as instantiated `GraphOfOperations` and `AbstractLanguageModel` - creation of these is described separately.
|
||||
- Prepare initial state (thought) as dictionary - this can be used in the initial prompts by the operations.
|
||||
```
|
||||
lm = ...create
|
||||
graph_of_operations = ...create
|
||||
|
||||
executor = controller.Controller(
|
||||
@ -83,35 +26,3 @@ executor.run()
|
||||
executor.output_graph("path/to/output.json")
|
||||
```
|
||||
- After the run the graph is written to an output file, which contains individual operations, their thoughts, information about scores and validity and total amount of used tokens / cost.
|
||||
|
||||
## Adding LLMs
|
||||
More LLMs can be added by following these steps:
|
||||
- Create new class as a subclass of `AbstractLanguageModel`.
|
||||
- Use the constructor for loading configuration and instantiating the language model (if needed).
|
||||
```
|
||||
class CustomLanguageModel(AbstractLanguageModel):
|
||||
def __init__(
|
||||
self,
|
||||
config_path: str = "",
|
||||
model_name: str = "llama7b-hf",
|
||||
cache: bool = False
|
||||
) -> None:
|
||||
super().__init__(config_path, model_name, cache)
|
||||
self.config: Dict = self.config[model_name]
|
||||
|
||||
# Load data from configuration into variables if needed
|
||||
|
||||
# Instantiate LLM if needed
|
||||
```
|
||||
- Implement `query` abstract method that is used to get a list of responses from the LLM (call to remote API or local model inference).
|
||||
```
|
||||
def query(self, query: str, num_responses: int = 1) -> Any:
|
||||
# Support caching
|
||||
# Call LLM and retrieve list of responses - based on num_responses
|
||||
# Return LLM response structure (not only raw strings)
|
||||
```
|
||||
- Implement `get_response_texts` abstract method that is used to get a list of raw texts from the LLM response structure produced by `query`.
|
||||
```
|
||||
def get_response_texts(self, query_response: Union[List[Dict], Dict]) -> List[str]:
|
||||
# Retrieve list of raw strings from the LLM response structure
|
||||
```
|
||||
|
||||
@ -1,4 +1 @@
|
||||
from .chatgpt import ChatGPT
|
||||
from .llamachat_hf import Llama2HF
|
||||
from .abstract_language_model import AbstractLanguageModel
|
||||
from .controller import Controller
|
||||
|
||||
@ -9,7 +9,7 @@
|
||||
import json
|
||||
import logging
|
||||
from typing import List
|
||||
from .abstract_language_model import AbstractLanguageModel
|
||||
from graph_of_thoughts.language_models import AbstractLanguageModel
|
||||
from graph_of_thoughts.operations import GraphOfOperations, Thought
|
||||
from graph_of_thoughts.prompter import Prompter
|
||||
from graph_of_thoughts.parser import Parser
|
||||
|
||||
112
graph_of_thoughts/language_models/README.md
Normal file
112
graph_of_thoughts/language_models/README.md
Normal file
@ -0,0 +1,112 @@
|
||||
# Language Models
|
||||
|
||||
The Language Models module is responsible for managing the large language models (LLMs) used by the Controller.
|
||||
|
||||
Currently, the framework supports the following LLMs:
|
||||
- GPT-4 / GPT-3.5 (Remote - OpenAI API)
|
||||
- OpenRouter (Remote - [OpenRouter](https://openrouter.ai/) OpenAI-compatible API, multi-key / multi-model rotation)
|
||||
- LLaMA-2 (Local - HuggingFace Transformers)
|
||||
|
||||
The following sections describe how to instantiate individual LLMs and how to add new LLMs to the framework.
|
||||
|
||||
## LLM Instantiation
|
||||
- Create a copy of `config_template.json` named `config.json`.
|
||||
- Fill configuration details based on the used model (below).
|
||||
|
||||
### GPT-4 / GPT-3.5
|
||||
- Adjust the predefined `chatgpt` or `chatgpt4` configurations or create a new configuration with an unique key.
|
||||
|
||||
| Key | Value |
|
||||
|---------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
|
||||
| model_id | Model name based on [OpenAI model overview](https://platform.openai.com/docs/models/overview). |
|
||||
| prompt_token_cost | Price per 1000 prompt tokens based on [OpenAI pricing](https://openai.com/pricing), used for calculating cumulative price per LLM instance. |
|
||||
| response_token_cost | Price per 1000 response tokens based on [OpenAI pricing](https://openai.com/pricing), used for calculating cumulative price per LLM instance. |
|
||||
| temperature | Parameter of OpenAI models that controls the randomness and the creativity of the responses (higher temperature = more diverse and unexpected responses). Value between 0.0 and 2.0, default is 1.0. More information can be found in the [OpenAI API reference](https://platform.openai.com/docs/api-reference/completions/create#completions/create-temperature). |
|
||||
| max_tokens | The maximum number of tokens to generate in the chat completion. Value depends on the maximum context size of the model specified in the [OpenAI model overview](https://platform.openai.com/docs/models/overview). More information can be found in the [OpenAI API reference](https://platform.openai.com/docs/api-reference/chat/create#chat/create-max_tokens). |
|
||||
| stop | String or array of strings specifying sequences of characters which if detected, stops further generation of tokens. More information can be found in the [OpenAI API reference](https://platform.openai.com/docs/api-reference/chat/create#chat/create-stop). |
|
||||
| organization | Organization to use for the API requests (may be empty). |
|
||||
| api_key | Personal API key that will be used to access OpenAI API. |
|
||||
|
||||
- Instantiate the language model based on the selected configuration key (predefined / custom).
|
||||
```python
|
||||
from graph_of_thoughts.language_models import ChatGPT
|
||||
|
||||
lm = ChatGPT(
|
||||
"path/to/config.json",
|
||||
model_name=<configuration key>
|
||||
)
|
||||
```
|
||||
|
||||
### OpenRouter
|
||||
- Copy `config.openrouter.example.yaml` (or `.json`) to `config.openrouter.yaml` next to this module, or pass an explicit path.
|
||||
- Set `api_keys` (list) and `models` (list). Each request picks a **random** key and a **random** model (uniform over the lists). If the HTTP API passes a `model` field, that model id is used for that request instead of a random one.
|
||||
- Optional: `http_referer` and `x_title` for OpenRouter attribution headers (see [OpenRouter docs](https://openrouter.ai/docs)).
|
||||
- HTTP **429** responses trigger exponential backoff and further rotation; **400** responses are retried a limited number of times with a new key/model pair, then surfaced as an error.
|
||||
|
||||
```python
|
||||
from graph_of_thoughts.language_models import OpenRouter
|
||||
|
||||
lm = OpenRouter("/path/to/config.openrouter.yaml")
|
||||
```
|
||||
|
||||
### LLaMA-2
|
||||
- Requires local hardware to run inference and a HuggingFace account.
|
||||
- Adjust the predefined `llama7b-hf`, `llama13b-hf` or `llama70b-hf` configurations or create a new configuration with an unique key.
|
||||
|
||||
| Key | Value |
|
||||
|---------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
|
||||
| model_id | Specifies HuggingFace LLaMA-2 model identifier (`meta-llama/<model_id>`). |
|
||||
| cache_dir | Local directory where the model will be downloaded and accessed. |
|
||||
| prompt_token_cost | Price per 1000 prompt tokens (currently not used - local model = no cost). |
|
||||
| response_token_cost | Price per 1000 response tokens (currently not used - local model = no cost). |
|
||||
| temperature | Parameter that controls the randomness and the creativity of the responses (higher temperature = more diverse and unexpected responses). Value between 0.0 and 1.0, default is 0.6. |
|
||||
| top_k | Top-K sampling method described in [Transformers tutorial](https://huggingface.co/blog/how-to-generate). Default value is set to 10. |
|
||||
| max_tokens | The maximum number of tokens to generate in the chat completion. More tokens require more memory. |
|
||||
|
||||
- Instantiate the language model based on the selected configuration key (predefined / custom).
|
||||
```python
|
||||
from graph_of_thoughts.language_models import Llama2HF
|
||||
|
||||
lm = Llama2HF(
|
||||
"path/to/config.json",
|
||||
model_name=<configuration key>
|
||||
)
|
||||
```
|
||||
- Request access to LLaMA-2 via the [Meta form](https://ai.meta.com/resources/models-and-libraries/llama-downloads/) using the same email address as for the HuggingFace account.
|
||||
- After the access is granted, go to [HuggingFace LLaMA-2 model card](https://huggingface.co/meta-llama/Llama-2-7b-chat-hf), log in and accept the license (a _"You have been granted access to this model"_ message should appear).
|
||||
- Generate HuggingFace access token.
|
||||
- Log in from CLI with: `huggingface-cli login --token <your token>`.
|
||||
|
||||
Note: 4-bit quantization is used to reduce the model size for inference. During instantiation, the model is downloaded from HuggingFace into the cache directory specified in the `config.json`. Running queries using larger models will require multiple GPUs (splitting across many GPUs is done automatically by the Transformers library).
|
||||
|
||||
## Adding LLMs
|
||||
More LLMs can be added by following these steps:
|
||||
- Create a new class as a subclass of `AbstractLanguageModel`.
|
||||
- Use the constructor for loading the configuration and instantiating the language model (if needed).
|
||||
```python
|
||||
class CustomLanguageModel(AbstractLanguageModel):
|
||||
def __init__(
|
||||
self,
|
||||
config_path: str = "",
|
||||
model_name: str = "llama7b-hf",
|
||||
cache: bool = False
|
||||
) -> None:
|
||||
super().__init__(config_path, model_name, cache)
|
||||
self.config: Dict = self.config[model_name]
|
||||
|
||||
# Load data from configuration into variables if needed
|
||||
|
||||
# Instantiate LLM if needed
|
||||
```
|
||||
- Implement the `query` abstract method that is used to get a list of responses from the LLM (remote API call or local model inference).
|
||||
```python
|
||||
def query(self, query: str, num_responses: int = 1) -> Any:
|
||||
# Support caching
|
||||
# Call LLM and retrieve list of responses - based on num_responses
|
||||
# Return LLM response structure (not only raw strings)
|
||||
```
|
||||
- Implement the `get_response_texts` abstract method that is used to get a list of raw texts from the LLM response structure produced by `query`.
|
||||
```python
|
||||
def get_response_texts(self, query_response: Union[List[Any], Any]) -> List[str]:
|
||||
# Retrieve list of raw strings from the LLM response structure
|
||||
```
|
||||
10
graph_of_thoughts/language_models/__init__.py
Normal file
10
graph_of_thoughts/language_models/__init__.py
Normal file
@ -0,0 +1,10 @@
|
||||
from .abstract_language_model import AbstractLanguageModel
|
||||
from .chatgpt import ChatGPT
|
||||
from .llamachat_hf import Llama2HF
|
||||
from .openrouter import (
|
||||
OpenRouter,
|
||||
OpenRouterBadRequestError,
|
||||
OpenRouterError,
|
||||
OpenRouterRateLimitError,
|
||||
load_openrouter_config,
|
||||
)
|
||||
@ -36,7 +36,7 @@ class AbstractLanguageModel(ABC):
|
||||
self.model_name: str = model_name
|
||||
self.cache = cache
|
||||
if self.cache:
|
||||
self.respone_cache: Dict[str, List[Any]] = {}
|
||||
self.response_cache: Dict[str, List[Any]] = {}
|
||||
self.load_config(config_path)
|
||||
self.prompt_tokens: int = 0
|
||||
self.completion_tokens: int = 0
|
||||
@ -63,7 +63,7 @@ class AbstractLanguageModel(ABC):
|
||||
"""
|
||||
Clear the response cache.
|
||||
"""
|
||||
self.respone_cache.clear()
|
||||
self.response_cache.clear()
|
||||
|
||||
@abstractmethod
|
||||
def query(self, query: str, num_responses: int = 1) -> Any:
|
||||
@ -80,12 +80,12 @@ class AbstractLanguageModel(ABC):
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def get_response_texts(self, query_responses: Union[List[Dict], Dict]) -> List[str]:
|
||||
def get_response_texts(self, query_responses: Union[List[Any], Any]) -> List[str]:
|
||||
"""
|
||||
Abstract method to extract response texts from the language model's response(s).
|
||||
|
||||
:param query_responses: The responses returned from the language model.
|
||||
:type query_responses: Union[List[Dict], Dict]
|
||||
:type query_responses: Union[List[Any], Any]
|
||||
:return: List of textual responses.
|
||||
:rtype: List[str]
|
||||
"""
|
||||
@ -7,11 +7,12 @@
|
||||
# main author: Nils Blach
|
||||
|
||||
import backoff
|
||||
import openai
|
||||
import os
|
||||
import random
|
||||
import time
|
||||
from typing import List, Dict, Union
|
||||
from openai import OpenAI, OpenAIError
|
||||
from openai.types.chat.chat_completion import ChatCompletion
|
||||
|
||||
from .abstract_language_model import AbstractLanguageModel
|
||||
|
||||
@ -53,15 +54,15 @@ class ChatGPT(AbstractLanguageModel):
|
||||
self.organization: str = self.config["organization"]
|
||||
if self.organization == "":
|
||||
self.logger.warning("OPENAI_ORGANIZATION is not set")
|
||||
else:
|
||||
openai.organization = self.organization
|
||||
# The api key is the api key that is used for chatgpt. Env variable OPENAI_API_KEY takes precedence over config.
|
||||
self.api_key: str = os.getenv("OPENAI_API_KEY", self.config["api_key"])
|
||||
if self.api_key == "":
|
||||
raise ValueError("OPENAI_API_KEY is not set")
|
||||
openai.api_key = self.api_key
|
||||
# Initialize the OpenAI Client
|
||||
self.client = OpenAI(api_key=self.api_key, organization=self.organization)
|
||||
|
||||
def query(self, query: str, num_responses: int = 1) -> Dict:
|
||||
def query(
|
||||
self, query: str, num_responses: int = 1
|
||||
) -> Union[List[ChatCompletion], ChatCompletion]:
|
||||
"""
|
||||
Query the OpenAI model for responses.
|
||||
|
||||
@ -72,8 +73,8 @@ class ChatGPT(AbstractLanguageModel):
|
||||
:return: Response(s) from the OpenAI model.
|
||||
:rtype: Dict
|
||||
"""
|
||||
if self.cache and query in self.respone_cache:
|
||||
return self.respone_cache[query]
|
||||
if self.cache and query in self.response_cache:
|
||||
return self.response_cache[query]
|
||||
|
||||
if num_responses == 1:
|
||||
response = self.chat([{"role": "user", "content": query}], num_responses)
|
||||
@ -97,13 +98,11 @@ class ChatGPT(AbstractLanguageModel):
|
||||
total_num_attempts -= 1
|
||||
|
||||
if self.cache:
|
||||
self.respone_cache[query] = response
|
||||
self.response_cache[query] = response
|
||||
return response
|
||||
|
||||
@backoff.on_exception(
|
||||
backoff.expo, openai.error.OpenAIError, max_time=10, max_tries=6
|
||||
)
|
||||
def chat(self, messages: List[Dict], num_responses: int = 1) -> Dict:
|
||||
@backoff.on_exception(backoff.expo, OpenAIError, max_time=10, max_tries=6)
|
||||
def chat(self, messages: List[Dict], num_responses: int = 1) -> ChatCompletion:
|
||||
"""
|
||||
Send chat messages to the OpenAI model and retrieves the model's response.
|
||||
Implements backoff on OpenAI error.
|
||||
@ -113,9 +112,9 @@ class ChatGPT(AbstractLanguageModel):
|
||||
:param num_responses: Number of desired responses, default is 1.
|
||||
:type num_responses: int
|
||||
:return: The OpenAI model's response.
|
||||
:rtype: Dict
|
||||
:rtype: ChatCompletion
|
||||
"""
|
||||
response = openai.ChatCompletion.create(
|
||||
response = self.client.chat.completions.create(
|
||||
model=self.model_id,
|
||||
messages=messages,
|
||||
temperature=self.temperature,
|
||||
@ -124,8 +123,8 @@ class ChatGPT(AbstractLanguageModel):
|
||||
stop=self.stop,
|
||||
)
|
||||
|
||||
self.prompt_tokens += response["usage"]["prompt_tokens"]
|
||||
self.completion_tokens += response["usage"]["completion_tokens"]
|
||||
self.prompt_tokens += response.usage.prompt_tokens
|
||||
self.completion_tokens += response.usage.completion_tokens
|
||||
prompt_tokens_k = float(self.prompt_tokens) / 1000.0
|
||||
completion_tokens_k = float(self.completion_tokens) / 1000.0
|
||||
self.cost = (
|
||||
@ -138,19 +137,21 @@ class ChatGPT(AbstractLanguageModel):
|
||||
)
|
||||
return response
|
||||
|
||||
def get_response_texts(self, query_response: Union[List[Dict], Dict]) -> List[str]:
|
||||
def get_response_texts(
|
||||
self, query_response: Union[List[ChatCompletion], ChatCompletion]
|
||||
) -> List[str]:
|
||||
"""
|
||||
Extract the response texts from the query response.
|
||||
|
||||
:param query_response: The response dictionary (or list of dictionaries) from the OpenAI model.
|
||||
:type query_response: Union[List[Dict], Dict]
|
||||
:type query_response: Union[List[ChatCompletion], ChatCompletion]
|
||||
:return: List of response strings.
|
||||
:rtype: List[str]
|
||||
"""
|
||||
if isinstance(query_response, Dict):
|
||||
if not isinstance(query_response, List):
|
||||
query_response = [query_response]
|
||||
return [
|
||||
choice["message"]["content"]
|
||||
choice.message.content
|
||||
for response in query_response
|
||||
for choice in response["choices"]
|
||||
for choice in response.choices
|
||||
]
|
||||
@ -0,0 +1,21 @@
|
||||
{
|
||||
"base_url": "https://openrouter.ai/api/v1",
|
||||
"api_keys": [
|
||||
"sk-or-v1-replace-me-1",
|
||||
"sk-or-v1-replace-me-2"
|
||||
],
|
||||
"models": [
|
||||
"openai/gpt-4o-mini",
|
||||
"anthropic/claude-3.5-haiku"
|
||||
],
|
||||
"temperature": 0.7,
|
||||
"max_tokens": 4096,
|
||||
"stop": null,
|
||||
"prompt_token_cost": 0.0,
|
||||
"response_token_cost": 0.0,
|
||||
"max_retries_429": 8,
|
||||
"max_retries_400": 3,
|
||||
"base_backoff_seconds": 1.0,
|
||||
"http_referer": "",
|
||||
"x_title": ""
|
||||
}
|
||||
@ -0,0 +1,29 @@
|
||||
# Copy to config.openrouter.yaml (or set path explicitly) and fill in keys.
|
||||
# Per chat request, an API key and model are chosen at random (uniform) from the lists.
|
||||
|
||||
base_url: https://openrouter.ai/api/v1
|
||||
|
||||
api_keys:
|
||||
- sk-or-v1-replace-me-1
|
||||
- sk-or-v1-replace-me-2
|
||||
|
||||
models:
|
||||
- openai/gpt-4o-mini
|
||||
- anthropic/claude-3.5-haiku
|
||||
|
||||
temperature: 0.7
|
||||
max_tokens: 4096
|
||||
stop: null
|
||||
|
||||
# Optional cost accounting (set to 0 if unknown)
|
||||
prompt_token_cost: 0.0
|
||||
response_token_cost: 0.0
|
||||
|
||||
# Retries after HTTP 429 / 400 (each retry uses a fresh random key + model)
|
||||
max_retries_429: 8
|
||||
max_retries_400: 3
|
||||
base_backoff_seconds: 1.0
|
||||
|
||||
# Optional OpenRouter attribution headers (recommended by OpenRouter)
|
||||
http_referer: ""
|
||||
x_title: ""
|
||||
@ -8,7 +8,6 @@
|
||||
|
||||
import os
|
||||
import torch
|
||||
import transformers
|
||||
from typing import List, Dict, Union
|
||||
from .abstract_language_model import AbstractLanguageModel
|
||||
|
||||
@ -48,6 +47,7 @@ class Llama2HF(AbstractLanguageModel):
|
||||
|
||||
# Important: must be done before importing transformers
|
||||
os.environ["TRANSFORMERS_CACHE"] = self.config["cache_dir"]
|
||||
import transformers
|
||||
|
||||
hf_model_id = f"meta-llama/{self.model_id}"
|
||||
model_config = transformers.AutoConfig.from_pretrained(hf_model_id)
|
||||
@ -84,8 +84,8 @@ class Llama2HF(AbstractLanguageModel):
|
||||
:return: Response(s) from the LLaMA 2 model.
|
||||
:rtype: List[Dict]
|
||||
"""
|
||||
if self.cache and query in self.respone_cache:
|
||||
return self.respone_cache[query]
|
||||
if self.cache and query in self.response_cache:
|
||||
return self.response_cache[query]
|
||||
sequences = []
|
||||
query = f"<s><<SYS>>You are a helpful assistant. Always follow the intstructions precisely and output the response exactly in the requested format.<</SYS>>\n\n[INST] {query} [/INST]"
|
||||
for _ in range(num_responses):
|
||||
@ -104,7 +104,7 @@ class Llama2HF(AbstractLanguageModel):
|
||||
for sequence in sequences
|
||||
]
|
||||
if self.cache:
|
||||
self.respone_cache[query] = response
|
||||
self.response_cache[query] = response
|
||||
return response
|
||||
|
||||
def get_response_texts(self, query_responses: List[Dict]) -> List[str]:
|
||||
287
graph_of_thoughts/language_models/openrouter.py
Normal file
287
graph_of_thoughts/language_models/openrouter.py
Normal file
@ -0,0 +1,287 @@
|
||||
# Copyright (c) 2023 ETH Zurich.
|
||||
# All rights reserved.
|
||||
#
|
||||
# Use of this source code is governed by a BSD-style license that can be
|
||||
# found in the LICENSE file.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import random
|
||||
import time
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
|
||||
import yaml
|
||||
from openai import APIStatusError, OpenAI
|
||||
from openai.types.chat.chat_completion import ChatCompletion
|
||||
|
||||
from .abstract_language_model import AbstractLanguageModel
|
||||
|
||||
|
||||
class OpenRouterError(Exception):
|
||||
"""Base error for OpenRouter integration."""
|
||||
|
||||
|
||||
class OpenRouterBadRequestError(OpenRouterError):
|
||||
"""Raised when OpenRouter returns HTTP 400 after retries."""
|
||||
|
||||
|
||||
class OpenRouterRateLimitError(OpenRouterError):
|
||||
"""Raised when OpenRouter returns HTTP 429 after retries."""
|
||||
|
||||
|
||||
def load_openrouter_config(path: str) -> Dict[str, Any]:
|
||||
"""Load a YAML or JSON OpenRouter configuration file."""
|
||||
return _load_config_file(path)
|
||||
|
||||
|
||||
def _load_config_file(path: str) -> Dict[str, Any]:
|
||||
ext = os.path.splitext(path)[1].lower()
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
if ext in (".yaml", ".yml"):
|
||||
data = yaml.safe_load(f)
|
||||
else:
|
||||
data = json.load(f)
|
||||
if not isinstance(data, dict):
|
||||
raise ValueError(f"Config at {path} must be a JSON/YAML object")
|
||||
return data
|
||||
|
||||
|
||||
class OpenRouter(AbstractLanguageModel):
|
||||
"""
|
||||
OpenRouter-backed language model with per-request rotation of API keys and models.
|
||||
|
||||
Configuration is loaded from YAML or JSON (see ``config.openrouter.example.yaml``).
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config_path: str = "",
|
||||
model_name: str = "openrouter",
|
||||
cache: bool = False,
|
||||
) -> None:
|
||||
self._rotation_model_name = model_name
|
||||
self._request_overrides: Dict[str, Any] = {}
|
||||
super().__init__(config_path, model_name, cache)
|
||||
self._apply_openrouter_config()
|
||||
|
||||
def load_config(self, path: str) -> None:
|
||||
if path == "":
|
||||
path = os.path.join(
|
||||
os.path.dirname(os.path.abspath(__file__)),
|
||||
"config.openrouter.yaml",
|
||||
)
|
||||
self.config_path = path
|
||||
self.config = _load_config_file(path)
|
||||
self.logger.debug("Loaded OpenRouter config from %s", path)
|
||||
|
||||
def _apply_openrouter_config(self) -> None:
|
||||
cfg = self.config
|
||||
self.base_url: str = cfg.get("base_url", "https://openrouter.ai/api/v1")
|
||||
keys = cfg.get("api_keys") or []
|
||||
if isinstance(keys, str):
|
||||
keys = [keys]
|
||||
self.api_keys: List[str] = [k for k in keys if k]
|
||||
if not self.api_keys:
|
||||
raise ValueError("OpenRouter config must define non-empty 'api_keys'")
|
||||
|
||||
models = cfg.get("models") or []
|
||||
if isinstance(models, str):
|
||||
models = [models]
|
||||
self.models: List[str] = [m for m in models if m]
|
||||
if not self.models:
|
||||
raise ValueError("OpenRouter config must define non-empty 'models'")
|
||||
|
||||
self.temperature: float = float(cfg.get("temperature", 1.0))
|
||||
self.max_tokens: int = int(cfg.get("max_tokens", 4096))
|
||||
self.stop: Union[str, List[str], None] = cfg.get("stop")
|
||||
self.prompt_token_cost: float = float(cfg.get("prompt_token_cost", 0.0))
|
||||
self.response_token_cost: float = float(cfg.get("response_token_cost", 0.0))
|
||||
|
||||
self.max_retries_429: int = int(cfg.get("max_retries_429", 8))
|
||||
self.max_retries_400: int = int(cfg.get("max_retries_400", 3))
|
||||
self.base_backoff_seconds: float = float(cfg.get("base_backoff_seconds", 1.0))
|
||||
|
||||
self.http_referer: str = cfg.get("http_referer", "") or os.getenv(
|
||||
"OPENROUTER_HTTP_REFERER", ""
|
||||
)
|
||||
self.x_title: str = cfg.get("x_title", "") or os.getenv("OPENROUTER_X_TITLE", "")
|
||||
|
||||
self.model_name = self._rotation_model_name
|
||||
self.last_model_id: Optional[str] = None
|
||||
self.generation_model_id: Optional[str] = None
|
||||
|
||||
def set_request_overrides(self, **kwargs: Any) -> None:
|
||||
"""Optional per-request parameters (used by the HTTP API). Cleared with :meth:`clear_request_overrides`."""
|
||||
self._request_overrides = {k: v for k, v in kwargs.items() if v is not None}
|
||||
|
||||
def clear_request_overrides(self) -> None:
|
||||
self._request_overrides = {}
|
||||
|
||||
def _pick_key(self) -> str:
|
||||
return random.choice(self.api_keys)
|
||||
|
||||
def _pick_model(self, override: Optional[str]) -> str:
|
||||
if override:
|
||||
return override
|
||||
o = self._request_overrides.get("model")
|
||||
if o:
|
||||
return str(o)
|
||||
return random.choice(self.models)
|
||||
|
||||
def _effective_temperature(self) -> float:
|
||||
t = self._request_overrides.get("temperature")
|
||||
return float(t) if t is not None else self.temperature
|
||||
|
||||
def _effective_max_tokens(self) -> int:
|
||||
m = self._request_overrides.get("max_tokens")
|
||||
return int(m) if m is not None else self.max_tokens
|
||||
|
||||
def _client_for_key(self, api_key: str) -> OpenAI:
|
||||
headers: Dict[str, str] = {}
|
||||
if self.http_referer:
|
||||
headers["HTTP-Referer"] = self.http_referer
|
||||
if self.x_title:
|
||||
headers["X-Title"] = self.x_title
|
||||
return OpenAI(
|
||||
base_url=self.base_url,
|
||||
api_key=api_key,
|
||||
default_headers=headers or None,
|
||||
)
|
||||
|
||||
def _sleep_backoff(self, attempt: int) -> None:
|
||||
cap = 60.0
|
||||
delay = min(
|
||||
self.base_backoff_seconds * (2**attempt) + random.random(),
|
||||
cap,
|
||||
)
|
||||
self.logger.warning("Backing off %.2fs (attempt %d)", delay, attempt + 1)
|
||||
time.sleep(delay)
|
||||
|
||||
def chat(
|
||||
self,
|
||||
messages: List[Dict[str, str]],
|
||||
num_responses: int = 1,
|
||||
model_override: Optional[str] = None,
|
||||
) -> ChatCompletion:
|
||||
"""
|
||||
Call OpenRouter chat completions with rotation and retries for 429/400.
|
||||
"""
|
||||
attempts_429 = 0
|
||||
attempts_400 = 0
|
||||
attempt = 0
|
||||
last_exc: Optional[Exception] = None
|
||||
|
||||
while True:
|
||||
api_key = self._pick_key()
|
||||
model_id = self._pick_model(model_override)
|
||||
client = self._client_for_key(api_key)
|
||||
try:
|
||||
response = client.chat.completions.create(
|
||||
model=model_id,
|
||||
messages=messages,
|
||||
temperature=self._effective_temperature(),
|
||||
max_tokens=self._effective_max_tokens(),
|
||||
n=num_responses,
|
||||
stop=self.stop,
|
||||
)
|
||||
if response.usage is not None:
|
||||
self.prompt_tokens += response.usage.prompt_tokens or 0
|
||||
self.completion_tokens += response.usage.completion_tokens or 0
|
||||
pt_k = float(self.prompt_tokens) / 1000.0
|
||||
ct_k = float(self.completion_tokens) / 1000.0
|
||||
self.cost = (
|
||||
self.prompt_token_cost * pt_k
|
||||
+ self.response_token_cost * ct_k
|
||||
)
|
||||
self.last_model_id = model_id
|
||||
if self.generation_model_id is None:
|
||||
self.generation_model_id = model_id
|
||||
self.logger.info(
|
||||
"OpenRouter response model=%s id=%s", model_id, response.id
|
||||
)
|
||||
return response
|
||||
except APIStatusError as e:
|
||||
last_exc = e
|
||||
code = e.status_code
|
||||
if code == 429:
|
||||
if attempts_429 >= self.max_retries_429:
|
||||
raise OpenRouterRateLimitError(
|
||||
f"OpenRouter rate limited after {attempts_429} retries: {e.message}"
|
||||
) from e
|
||||
attempts_429 += 1
|
||||
self._sleep_backoff(attempt)
|
||||
attempt += 1
|
||||
continue
|
||||
if code == 400:
|
||||
self.logger.warning(
|
||||
"OpenRouter HTTP 400 (will retry with rotated key/model if allowed): %s body=%s",
|
||||
e.message,
|
||||
e.body,
|
||||
)
|
||||
if attempts_400 >= self.max_retries_400:
|
||||
raise OpenRouterBadRequestError(
|
||||
f"OpenRouter bad request after {attempts_400} retries: {e.message}"
|
||||
) from e
|
||||
attempts_400 += 1
|
||||
attempt += 1
|
||||
time.sleep(random.uniform(0.2, 0.8))
|
||||
continue
|
||||
raise
|
||||
except Exception:
|
||||
self.logger.exception("Unexpected error calling OpenRouter")
|
||||
raise
|
||||
|
||||
def query(
|
||||
self, query: str, num_responses: int = 1
|
||||
) -> Union[List[ChatCompletion], ChatCompletion]:
|
||||
if self.cache and query in self.response_cache:
|
||||
return self.response_cache[query]
|
||||
|
||||
messages = [{"role": "user", "content": query}]
|
||||
model_ov = self._request_overrides.get("model")
|
||||
model_override = str(model_ov) if model_ov else None
|
||||
|
||||
if num_responses == 1:
|
||||
response = self.chat(messages, 1, model_override=model_override)
|
||||
else:
|
||||
response = []
|
||||
next_try = num_responses
|
||||
total_num_attempts = num_responses
|
||||
remaining = num_responses
|
||||
while remaining > 0 and total_num_attempts > 0:
|
||||
try:
|
||||
assert next_try > 0
|
||||
res = self.chat(
|
||||
messages, next_try, model_override=model_override
|
||||
)
|
||||
response.append(res)
|
||||
remaining -= next_try
|
||||
next_try = min(remaining, next_try)
|
||||
except Exception as e:
|
||||
next_try = max(1, (next_try + 1) // 2)
|
||||
self.logger.warning(
|
||||
"Error in OpenRouter query: %s, retrying with n=%s",
|
||||
e,
|
||||
next_try,
|
||||
)
|
||||
time.sleep(random.uniform(0.5, 2.0))
|
||||
total_num_attempts -= 1
|
||||
|
||||
if self.cache:
|
||||
self.response_cache[query] = response
|
||||
return response
|
||||
|
||||
def get_response_texts(
|
||||
self, query_response: Union[List[ChatCompletion], ChatCompletion]
|
||||
) -> List[str]:
|
||||
if not isinstance(query_response, list):
|
||||
query_response = [query_response]
|
||||
texts: List[str] = []
|
||||
for response in query_response:
|
||||
for choice in response.choices:
|
||||
c = choice.message.content
|
||||
texts.append(c if c is not None else "")
|
||||
return texts
|
||||
@ -14,7 +14,7 @@ from abc import ABC, abstractmethod
|
||||
import itertools
|
||||
|
||||
from graph_of_thoughts.operations.thought import Thought
|
||||
from graph_of_thoughts.controller.abstract_language_model import AbstractLanguageModel
|
||||
from graph_of_thoughts.language_models import AbstractLanguageModel
|
||||
from graph_of_thoughts.prompter import Prompter
|
||||
from graph_of_thoughts.parser import Parser
|
||||
|
||||
|
||||
@ -1,5 +1,12 @@
|
||||
<p align="center">
|
||||
<img src="poster.png" width="40%">
|
||||
</p>
|
||||
|
||||
## Poster
|
||||
The poster presented at the 2024 Association for the Advancement of Artificial Intelligence (AAAI'24) conference is available as [PDF](poster.pdf) and [PNG](poster.png).
|
||||
|
||||
## Plot Data
|
||||
|
||||
The data used to create the figure of the arXiv preprint article can be
|
||||
The data used to create the figures of the paper can be
|
||||
found in the `final_results_gpt35.tar.bz2` archive. Unpack the archive
|
||||
and run the file `plots.py`.
|
||||
|
||||
Binary file not shown.
BIN
paper/poster.pdf
Normal file
BIN
paper/poster.pdf
Normal file
Binary file not shown.
BIN
paper/poster.png
Normal file
BIN
paper/poster.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 1.5 MiB |
@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
||||
|
||||
[project]
|
||||
name = "graph_of_thoughts"
|
||||
version = "0.0.1"
|
||||
version = "0.0.3"
|
||||
authors = [
|
||||
{ name="Maciej Besta", email="maciej.besta@inf.ethz.ch" },
|
||||
{ name="Nils Blach", email="nils.blach@inf.ethz.ch" },
|
||||
@ -20,17 +20,28 @@ classifiers = [
|
||||
"Operating System :: OS Independent",
|
||||
]
|
||||
dependencies = [
|
||||
"backoff>=2.2.1",
|
||||
"openai>=0.27.7",
|
||||
"matplotlib>=3.7.1",
|
||||
"numpy>=1.24.3",
|
||||
"pandas>=2.0.3",
|
||||
"sympy>=1.12",
|
||||
"torch>=2.0.1",
|
||||
"transformers>=4.31.0",
|
||||
"backoff>=2.2.1,<3.0.0",
|
||||
"openai>=1.0.0,<2.0.0",
|
||||
"pyyaml>=6.0.1,<7.0.0",
|
||||
"matplotlib>=3.7.1,<4.0.0",
|
||||
"numpy>=1.24.3,<2.0.0",
|
||||
"pandas>=2.0.3,<3.0.0",
|
||||
"sympy>=1.12,<2.0",
|
||||
"torch>=2.0.1,<3.0.0",
|
||||
"transformers>=4.31.0,<5.0.0",
|
||||
"accelerate>=0.21.0,<1.0.0",
|
||||
"bitsandbytes>=0.41.0,<1.0.0",
|
||||
"scipy>=1.10.1,<2.0.0",
|
||||
]
|
||||
|
||||
[project.optional-dependencies]
|
||||
api = [
|
||||
"fastapi>=0.109.0,<1.0.0",
|
||||
"uvicorn[standard]>=0.27.0,<1.0.0",
|
||||
]
|
||||
|
||||
[project.urls]
|
||||
Homepage = "https://github.com/spcl/graph-of-thoughts"
|
||||
|
||||
[project.scripts]
|
||||
got-openrouter-api = "graph_of_thoughts.api.app:run"
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user