Compare commits
No commits in common. "4baaaa438c450ef3d806303caeae13769f25717d" and "4ca00e9c1c2ac63353cee7fbc577698ff4667752" have entirely different histories.
4baaaa438c
...
4ca00e9c1c
32
.gitignore
vendored
32
.gitignore
vendored
@ -1,32 +0,0 @@
|
|||||||
# Byte-compiled / optimized / DLL files
|
|
||||||
__pycache__/
|
|
||||||
*.py[cod]
|
|
||||||
*$py.class
|
|
||||||
|
|
||||||
# C extensions
|
|
||||||
*.so
|
|
||||||
|
|
||||||
# Distribution / packaging
|
|
||||||
dist/
|
|
||||||
build/
|
|
||||||
|
|
||||||
# Installer logs
|
|
||||||
pip-log.txt
|
|
||||||
|
|
||||||
# IDEs
|
|
||||||
.idea/
|
|
||||||
*.vscode/
|
|
||||||
*.pycproj
|
|
||||||
*.user
|
|
||||||
*.pyproj.user
|
|
||||||
|
|
||||||
# Data
|
|
||||||
results/
|
|
||||||
*.out
|
|
||||||
*.err
|
|
||||||
|
|
||||||
# Environments
|
|
||||||
env/
|
|
||||||
|
|
||||||
# Config File
|
|
||||||
**/config.json
|
|
||||||
53
LICENSE
53
LICENSE
@ -1,53 +0,0 @@
|
|||||||
Copyright (c) 2023 ETH Zurich.
|
|
||||||
All rights reserved.
|
|
||||||
|
|
||||||
Redistribution and use in source and binary forms, with or without
|
|
||||||
modification, are permitted provided that the following conditions are
|
|
||||||
met:
|
|
||||||
|
|
||||||
- Redistributions of source code must retain the above copyright
|
|
||||||
notice, this list of conditions and the following disclaimer.
|
|
||||||
|
|
||||||
- Redistributions in binary form must reproduce the above copyright
|
|
||||||
notice, this list of conditions and the following disclaimer listed
|
|
||||||
in this license in the documentation and/or other materials
|
|
||||||
provided with the distribution.
|
|
||||||
|
|
||||||
- Neither the name of the copyright holders nor the names of its
|
|
||||||
contributors may be used to endorse or promote products derived from
|
|
||||||
this software without specific prior written permission.
|
|
||||||
|
|
||||||
The copyright holders provide no reassurances that the source code
|
|
||||||
provided does not infringe any patent, copyright, or any other
|
|
||||||
intellectual property rights of third parties. The copyright holders
|
|
||||||
disclaim any liability to any recipient for claims brought against
|
|
||||||
recipient by any third party for infringement of that parties
|
|
||||||
intellectual property rights.
|
|
||||||
|
|
||||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
|
||||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
|
||||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
|
||||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
|
||||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
|
||||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
|
||||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
|
||||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
|
||||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
|
||||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
||||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
||||||
|
|
||||||
|
|
||||||
Citation
|
|
||||||
========
|
|
||||||
|
|
||||||
Any published work which uses this software should include the
|
|
||||||
following citation:
|
|
||||||
|
|
||||||
----------------------------------------------------------------------
|
|
||||||
Maciej Besta, Nils Blach, Ales Kubicek, Robert Gerstenberger, Lukas
|
|
||||||
Gianinazzi, Joanna Gajda, Tomasz Lehmann, Michał Podstawski, Hubert
|
|
||||||
Niewiadomski, Piotr Nyczyk, Torsten Hoefler (2024): Graph of Thoughts:
|
|
||||||
Solving Elaborate Problems with Large Language Models. In: Proceedings
|
|
||||||
of the AAAI Conference on Artificial Intelligence, 38(16),
|
|
||||||
17682-17690. https://doi.org/10.1609/aaai.v38i16.29720
|
|
||||||
----------------------------------------------------------------------
|
|
||||||
165
README.md
165
README.md
@ -1,165 +1,2 @@
|
|||||||
# Graph of Thoughts (GoT)
|
# graph-of-thoughts
|
||||||
|
|
||||||
<p align="center">
|
|
||||||
<img src="paper/pics/preview.svg">
|
|
||||||
</p>
|
|
||||||
|
|
||||||
This is the official implementation of [Graph of Thoughts: Solving Elaborate Problems with Large Language Models](https://arxiv.org/pdf/2308.09687.pdf).
|
|
||||||
This framework gives you the ability to solve complex problems by modeling them as a Graph of Operations (GoO), which is automatically executed with a Large Language Model (LLM) as the engine.
|
|
||||||
This framework is designed to be flexible and extensible, allowing you to not only solve problems using the new GoT approach, but also to implement GoOs resembling previous approaches like CoT or ToT.
|
|
||||||
|
|
||||||
## Setup Guide
|
|
||||||
|
|
||||||
In order to use this framework, you need to have a working installation of Python 3.8 or newer.
|
|
||||||
|
|
||||||
### Installing GoT
|
|
||||||
|
|
||||||
Before running either of the following two installation methods, make sure to activate your Python environment (if any) beforehand.
|
|
||||||
If you are a user and you just want to use `graph_of_thoughts`, you can install it directly from PyPI:
|
|
||||||
```bash
|
|
||||||
pip install graph_of_thoughts
|
|
||||||
```
|
|
||||||
If you are a developer and you want to modify the code, you can install it in editable mode from source:
|
|
||||||
```bash
|
|
||||||
git clone https://github.com/spcl/graph-of-thoughts.git
|
|
||||||
cd graph-of-thoughts
|
|
||||||
pip install -e .
|
|
||||||
```
|
|
||||||
|
|
||||||
### Configuring the LLM
|
|
||||||
|
|
||||||
In order to use the framework, you need to have access to an LLM.
|
|
||||||
Please follow the instructions in the [Controller README](graph_of_thoughts/controller/README.md) to configure the LLM of your choice.
|
|
||||||
|
|
||||||
### OpenRouter and OpenAI-compatible HTTP API
|
|
||||||
|
|
||||||
1. Install API extras: `pip install "graph_of_thoughts[api]"` (or `pip install -e ".[api]"` from a source checkout).
|
|
||||||
2. Copy [`graph_of_thoughts/language_models/config.openrouter.example.yaml`](graph_of_thoughts/language_models/config.openrouter.example.yaml) to `config.openrouter.yaml`, add your [OpenRouter](https://openrouter.ai/) keys and model ids, and either place it in `graph_of_thoughts/language_models/` or point `OPENROUTER_CONFIG` at your file.
|
|
||||||
3. Run the server: `got-openrouter-api` (or `python -m graph_of_thoughts.api`).
|
|
||||||
4. Call `POST /v1/chat/completions` with a standard OpenAI-style JSON body (`messages`, optional `model`, `temperature`, `max_tokens`). The server runs a small Graph of Operations (generate multiple candidates, score, keep the best) via OpenRouter.
|
|
||||||
|
|
||||||
Details: [Language models README](graph_of_thoughts/language_models/README.md).
|
|
||||||
|
|
||||||
## Quick Start
|
|
||||||
|
|
||||||
The following code snippet shows how to use the framework to solve the sorting problem for a list of 32 numbers using a CoT-like approach.
|
|
||||||
Make sure you have followed the [Setup Guide](#setup-guide) before running the code.
|
|
||||||
|
|
||||||
```python
|
|
||||||
from examples.sorting.sorting_032 import SortingPrompter, SortingParser, utils
|
|
||||||
from graph_of_thoughts import controller, language_models, operations
|
|
||||||
|
|
||||||
# Problem input
|
|
||||||
|
|
||||||
to_be_sorted = "[0, 2, 6, 3, 8, 7, 1, 1, 6, 7, 7, 7, 7, 9, 3, 0, 1, 7, 9, 1, 3, 5, 1, 3, 6, 4, 5, 4, 7, 3, 5, 7]"
|
|
||||||
|
|
||||||
# Create the Graph of Operations
|
|
||||||
gop = operations.GraphOfOperations()
|
|
||||||
gop.append_operation(operations.Generate())
|
|
||||||
gop.append_operation(operations.Score(scoring_function=utils.num_errors))
|
|
||||||
gop.append_operation(operations.GroundTruth(utils.test_sorting))
|
|
||||||
|
|
||||||
# Configure the Language Model (Assumes config.json is in the current directory with OpenAI API key)
|
|
||||||
lm = language_models.ChatGPT("config.json", model_name="chatgpt")
|
|
||||||
|
|
||||||
# Create the Controller
|
|
||||||
ctrl = controller.Controller(
|
|
||||||
lm,
|
|
||||||
gop,
|
|
||||||
SortingPrompter(),
|
|
||||||
SortingParser(),
|
|
||||||
# The following dictionary is used to configure the initial thought state
|
|
||||||
{
|
|
||||||
"original": to_be_sorted,
|
|
||||||
"current": "",
|
|
||||||
"method": "cot"
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
# Run the Controller and generate the output graph
|
|
||||||
ctrl.run()
|
|
||||||
ctrl.output_graph("output_cot.json")
|
|
||||||
```
|
|
||||||
|
|
||||||
To run the more sophisticated GoT approach, you can use the following code snippet.
|
|
||||||
|
|
||||||
```python
|
|
||||||
from examples.sorting.sorting_032 import SortingPrompter, SortingParser, got, utils
|
|
||||||
from graph_of_thoughts import controller, language_models, operations
|
|
||||||
|
|
||||||
# Problem input
|
|
||||||
|
|
||||||
to_be_sorted = "[0, 2, 6, 3, 8, 7, 1, 1, 6, 7, 7, 7, 7, 9, 3, 0, 1, 7, 9, 1, 3, 5, 1, 3, 6, 4, 5, 4, 7, 3, 5, 7]"
|
|
||||||
|
|
||||||
# Retrieve the Graph of Operations
|
|
||||||
gop = got()
|
|
||||||
|
|
||||||
# Configure the Language Model (Assumes config.json is in the current directory with OpenAI API key)
|
|
||||||
lm = language_models.ChatGPT("config.json", model_name="chatgpt")
|
|
||||||
|
|
||||||
# Create the Controller
|
|
||||||
ctrl = controller.Controller(
|
|
||||||
lm,
|
|
||||||
gop,
|
|
||||||
SortingPrompter(),
|
|
||||||
SortingParser(),
|
|
||||||
# The following dictionary is used to configure the initial thought state
|
|
||||||
{
|
|
||||||
"original": to_be_sorted,
|
|
||||||
"current": "",
|
|
||||||
"phase": 0,
|
|
||||||
"method": "got"
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
# Run the Controller and generate the output graph
|
|
||||||
ctrl.run()
|
|
||||||
ctrl.output_graph("output_got.json")
|
|
||||||
```
|
|
||||||
You can compare the two results by inspecting the output graphs `output_cot.json` and `output_got.json`.
|
|
||||||
The final thought states' scores indicate the number of errors in the sorted list.
|
|
||||||
|
|
||||||
## Documentation
|
|
||||||
The paper gives a high-level overview of the framework and its components.
|
|
||||||
In order to understand the framework in more detail, you can read the documentation of the individual modules.
|
|
||||||
Especially the [Controller](graph_of_thoughts/controller/README.md) and [Operations](graph_of_thoughts/operations/README.md) modules are important for understanding how to make the most out of the framework.
|
|
||||||
We took extra care to fully document the code, so that you can easily understand how it works and how to extend it.
|
|
||||||
|
|
||||||
## Examples
|
|
||||||
|
|
||||||
The [examples](examples) directory contains several examples of problems that can be solved using the framework, including the ones presented in the paper.
|
|
||||||
It is a great starting point for learning how to use the framework to solve real problems.
|
|
||||||
Each example contains a `README.md` file with instructions on how to run it and play with it. The code is fully documented and should be easy to follow.
|
|
||||||
You can also run the examples straight from the main directory. Note that the results will be stored in the respective examples sub-directory.
|
|
||||||
|
|
||||||
Try for instance:
|
|
||||||
```bash
|
|
||||||
python -m examples.sorting.sorting_032
|
|
||||||
python -m examples.keyword_counting.keyword_counting
|
|
||||||
```
|
|
||||||
## Paper Results
|
|
||||||
|
|
||||||
You can run the experiments from the paper by following the instructions in the [examples](examples) directory.
|
|
||||||
However, if you just want to inspect and replot the results, you can use the [paper](paper) directory.
|
|
||||||
|
|
||||||
## Citations
|
|
||||||
|
|
||||||
If you find this repository valuable, please give it a star!
|
|
||||||
Got any questions or feedback? Feel free to reach out to [nils.blach@inf.ethz.ch](mailto:nils.blach@inf.ethz.ch) or open an issue.
|
|
||||||
Using this in your work? Please reference us using the provided citation:
|
|
||||||
|
|
||||||
```bibtex
|
|
||||||
@article{besta2024got,
|
|
||||||
title = {{Graph of Thoughts: Solving Elaborate Problems with Large Language Models}},
|
|
||||||
author = {Besta, Maciej and Blach, Nils and Kubicek, Ales and Gerstenberger, Robert and Gianinazzi, Lukas and Gajda, Joanna and Lehmann, Tomasz and Podstawski, Micha{\l} and Niewiadomski, Hubert and Nyczyk, Piotr and Hoefler, Torsten},
|
|
||||||
year = 2024,
|
|
||||||
month = {Mar},
|
|
||||||
journal = {Proceedings of the AAAI Conference on Artificial Intelligence},
|
|
||||||
volume = 38,
|
|
||||||
number = 16,
|
|
||||||
pages = {17682-17690},
|
|
||||||
publisher = {AAAI Press},
|
|
||||||
doi = {10.1609/aaai.v38i16.29720},
|
|
||||||
url = {https://ojs.aaai.org/index.php/AAAI/article/view/29720}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|||||||
@ -1,7 +0,0 @@
|
|||||||
# Examples
|
|
||||||
|
|
||||||
This directory contains scripts for running various examples using the Graph of Thoughts package. Each script is a standalone Python program that sets up and runs a particular example.
|
|
||||||
|
|
||||||
We further include prompt files for each example that can be used to test prompts manually in a console.
|
|
||||||
|
|
||||||
Please refer to the individual example directories for more information on the specific example.
|
|
||||||
@ -1,38 +0,0 @@
|
|||||||
# Document Merging
|
|
||||||
|
|
||||||
The use case in this directory generates new Non-Disclosure Agreement (NDA) based on several input ones that partially overlap in terms of their contents.
|
|
||||||
We provide implementations of five different approaches:
|
|
||||||
- IO
|
|
||||||
- Chain-of-Thought (CoT)
|
|
||||||
- Tree of Thought (ToT)
|
|
||||||
- Graph of Thoughts (GoT):
|
|
||||||
- GoT: aggregation of fully merged NDAs
|
|
||||||
- GoT2: aggregation of partially merged NDAs
|
|
||||||
|
|
||||||
## Data
|
|
||||||
|
|
||||||
We provide an input file with 50 samples: `documents.csv`.
|
|
||||||
|
|
||||||
## Execution
|
|
||||||
|
|
||||||
The file to execute the use case is called
|
|
||||||
`doc_merge.py`. In the main body, one can
|
|
||||||
select the specific samples to be run (variable samples) and the
|
|
||||||
approaches (variable approaches). It is also possible to set a budget in
|
|
||||||
dollars (variable budget).
|
|
||||||
|
|
||||||
The Python scripts will create the directory `result`, if it is not
|
|
||||||
already present. In the `result` directory, another directory is created
|
|
||||||
for each run: `{name of LLM}_{list of approaches}_{day}_{start time}`.
|
|
||||||
Inside each execution specific directory two files (`config.json`,
|
|
||||||
`log.log`) and a separate directory for each selected approach are
|
|
||||||
created. `config.json` contains the configuration of the run: input data,
|
|
||||||
selected approaches, name of the LLM, and the budget. `log.log` contains
|
|
||||||
the prompts and responses of the LLM as well as additional debug data.
|
|
||||||
The approach directories contain a separate json file for every sample
|
|
||||||
and the file contains the Graph Reasoning State (GRS) for that sample.
|
|
||||||
|
|
||||||
## Plot Data
|
|
||||||
|
|
||||||
Change the results directory in line 158 of `plot.py` and run `python3
|
|
||||||
plot.py` to plot your data.
|
|
||||||
@ -1,767 +0,0 @@
|
|||||||
# Copyright (c) 2023 ETH Zurich.
|
|
||||||
# All rights reserved.
|
|
||||||
#
|
|
||||||
# Use of this source code is governed by a BSD-style license that can be
|
|
||||||
# found in the LICENSE file.
|
|
||||||
#
|
|
||||||
# main author: Nils Blach
|
|
||||||
|
|
||||||
import os
|
|
||||||
import re
|
|
||||||
import logging
|
|
||||||
import datetime
|
|
||||||
import json
|
|
||||||
import csv
|
|
||||||
from statistics import fmean
|
|
||||||
from typing import Dict, List, Callable, Set, Union
|
|
||||||
from graph_of_thoughts import controller, language_models, operations, prompter, parser
|
|
||||||
|
|
||||||
|
|
||||||
class DocMergePrompter(prompter.Prompter):
|
|
||||||
"""
|
|
||||||
DocMergePrompter provides the generation of prompts specific to the document
|
|
||||||
merge example for the language models.
|
|
||||||
|
|
||||||
Inherits from the Prompter class and implements its abstract methods.
|
|
||||||
"""
|
|
||||||
|
|
||||||
merge_doc_prompt_start = """Merge the following {num} NDA documents <Doc1> - <Doc{num}> into a single NDA, maximizing retained information and minimizing redundancy. Output only the created NDA between the tags <Merged> and </Merged>, without any additional text.
|
|
||||||
Here are NDAs <Doc1> - <Doc{num}>
|
|
||||||
"""
|
|
||||||
merge_doc_prompt_block = """
|
|
||||||
<Doc{num}>
|
|
||||||
{document}
|
|
||||||
</Doc{num}>
|
|
||||||
"""
|
|
||||||
|
|
||||||
merge_doc_prompt_cot_start = """Merge the following {num} NDA documents <Doc1> - <Doc{num}> into a single NDA, maximizing retained information and minimizing redundancy.
|
|
||||||
You can generate any intermediate thoughts and documents you want, but the final output should be the merged NDA, placed between the two tags <Merged> and </Merged>.
|
|
||||||
For instance you might want to follow this approach:
|
|
||||||
1. Split each NDA into their logical subparts.
|
|
||||||
2. Merge the subparts of the {num} NDAs.
|
|
||||||
3. Combine the merged subparts into a single NDA.
|
|
||||||
4. Place the merged NDA between the tags <Merged> and </Merged>.
|
|
||||||
|
|
||||||
Here are NDAs <Doc1> - <Doc{num}>:
|
|
||||||
"""
|
|
||||||
|
|
||||||
improve_summary_prompt_start = """The following NDA <S> merges initial NDAs <Doc1> - <Doc{num}>.
|
|
||||||
Please improve the summary NDA <S> by adding more information and removing redundancy. Output only the improved NDA, placed between the two tags <Merged> and </Merged>, without any additional text.
|
|
||||||
|
|
||||||
Here are NDAs <Doc1> - <Doc{num}>:
|
|
||||||
"""
|
|
||||||
|
|
||||||
improve_summary_prompt_block = """
|
|
||||||
<Doc{num}>
|
|
||||||
{document}
|
|
||||||
</Doc{num}>
|
|
||||||
"""
|
|
||||||
|
|
||||||
improve_summary_prompt_end = """
|
|
||||||
Here is the summary NDA <S>:
|
|
||||||
<S>
|
|
||||||
{summary}
|
|
||||||
</S>
|
|
||||||
"""
|
|
||||||
|
|
||||||
score_prompt_base = """The following NDA <S> merges NDAs <Doc1> - <Doc{num}>.
|
|
||||||
Please score the merged NDA <S> in terms of how much redundant information is contained, independent of the original NDAs, as well as how much information is retained from the original NDAs.
|
|
||||||
A score of 10 for redundancy implies that absolutely no information is redundant, while a score of 0 implies that at least half of the information is redundant (so everything is at least mentioned twice).
|
|
||||||
A score of 10 for retained information implies that all information from the original NDAs is retained, while a score of 0 implies that no information is retained.
|
|
||||||
You may provide reasoning for your scoring, but the final score for redundancy should be between the tags <Redundancy> and </Redundancy>, and the final score for retained information should be between the tags <Retained> and </Retained>, without any additional text within any of those tags.
|
|
||||||
|
|
||||||
Here are NDAs <Doc1> - <Doc{num}>:
|
|
||||||
"""
|
|
||||||
|
|
||||||
score_prompt_block = """
|
|
||||||
<Doc{num}>
|
|
||||||
{document}
|
|
||||||
</Doc{num}>
|
|
||||||
"""
|
|
||||||
|
|
||||||
score_prompt_end = """
|
|
||||||
Here is the summary NDA <S>:
|
|
||||||
<S>
|
|
||||||
{summary}
|
|
||||||
</S>
|
|
||||||
"""
|
|
||||||
|
|
||||||
aggregate_full_prompt_base = """The following NDAs <S1> - <S{num_ndas_summary}> each merge the initial NDAs <Doc1> - <Doc{num_ndas}>.
|
|
||||||
Combine the merged NDAs <S1> - <S{num_ndas_summary}> into a new one, maximizing their advantages and overall information retention, while minimizing redundancy.
|
|
||||||
Output only the new NDA between the tags <Merged> and </Merged>, without any additional text.
|
|
||||||
|
|
||||||
Here are the original NDAs <Doc1> - <Doc{num_ndas}>:
|
|
||||||
"""
|
|
||||||
|
|
||||||
aggregate_full_prompt_block1 = """
|
|
||||||
<Doc{num}>
|
|
||||||
{document}
|
|
||||||
</Doc{num}>
|
|
||||||
"""
|
|
||||||
aggregate_full_prompt_mid = """
|
|
||||||
Here are the summary NDAs <S1> - <S{num_ndas_summary}>:
|
|
||||||
"""
|
|
||||||
|
|
||||||
aggregate_full_prompt_block2 = """
|
|
||||||
<S{num}>
|
|
||||||
{summary}
|
|
||||||
</S{num}>
|
|
||||||
"""
|
|
||||||
|
|
||||||
aggregate_sub_prompt_base = """The following NDAs <S1> - <S{num_ndas}> are summaries of some other NDAs.
|
|
||||||
Combine them into a new one, make sure to maximize their advantages and overall information retention, while minimizing redundancy.
|
|
||||||
Output only the new NDA between the tags <Merged> and </Merged>, without any additional text.
|
|
||||||
|
|
||||||
Here are NDAs <S1> - <S{num_ndas}>:
|
|
||||||
"""
|
|
||||||
|
|
||||||
aggregate_sub_prompt_generate = """
|
|
||||||
NDA <S{num}>:
|
|
||||||
{nda}
|
|
||||||
</S{num}>
|
|
||||||
"""
|
|
||||||
|
|
||||||
def aggregation_prompt(self, state_dicts: List[Dict], **kwargs) -> str:
|
|
||||||
"""
|
|
||||||
Generate an aggregation prompt for the language model.
|
|
||||||
|
|
||||||
:param state_dicts: The thought states that should be aggregated.
|
|
||||||
:type state_dicts: List[Dict]
|
|
||||||
:param kwargs: Additional keyword arguments.
|
|
||||||
:return: The aggregation prompt.
|
|
||||||
:rtype: str
|
|
||||||
"""
|
|
||||||
|
|
||||||
if len(state_dicts[0]["parts"]) > 0 and len(state_dicts[0]["parts"]) < len(
|
|
||||||
state_dicts[0]["documents"]
|
|
||||||
):
|
|
||||||
prompt = self.aggregate_sub_prompt_base.format(
|
|
||||||
num_ndas=len(state_dicts),
|
|
||||||
)
|
|
||||||
for i, state_dict in enumerate(state_dicts):
|
|
||||||
prompt += self.aggregate_sub_prompt_generate.format(
|
|
||||||
nda=state_dict["current"], num=i + 1
|
|
||||||
)
|
|
||||||
return prompt
|
|
||||||
else:
|
|
||||||
prompt = self.aggregate_full_prompt_base.format(
|
|
||||||
num_ndas=len(state_dicts[0]["documents"]),
|
|
||||||
num_ndas_summary=len(state_dicts),
|
|
||||||
)
|
|
||||||
for i, document in enumerate(state_dicts[0]["documents"]):
|
|
||||||
prompt += self.aggregate_full_prompt_block1.format(
|
|
||||||
document=document, num=i + 1
|
|
||||||
)
|
|
||||||
prompt += self.aggregate_full_prompt_mid.format(
|
|
||||||
num_ndas_summary=len(state_dicts),
|
|
||||||
)
|
|
||||||
for i, state_dict in enumerate(state_dicts):
|
|
||||||
prompt += self.aggregate_full_prompt_block2.format(
|
|
||||||
summary=state_dict["current"], num=i + 1
|
|
||||||
)
|
|
||||||
return prompt
|
|
||||||
|
|
||||||
def generate_prompt(
|
|
||||||
self,
|
|
||||||
num_branches: int,
|
|
||||||
documents: List[str],
|
|
||||||
method: str,
|
|
||||||
parts: Set[str],
|
|
||||||
current: str,
|
|
||||||
**kwargs,
|
|
||||||
) -> str:
|
|
||||||
"""
|
|
||||||
Generate a generate prompt for the language model.
|
|
||||||
|
|
||||||
:param num_branches: The number of responses the prompt should ask the LM to generate.
|
|
||||||
:type num_branches: int
|
|
||||||
:param documents: The list of documents to be merged.
|
|
||||||
:type documents: List[str]
|
|
||||||
:param method: Method for which the generate prompt is generated.
|
|
||||||
:type method: str
|
|
||||||
:param parts: Indices of the already processed document parts.
|
|
||||||
:type parts: Set[str]
|
|
||||||
:param current: The intermediate solution.
|
|
||||||
:type current: str
|
|
||||||
:param kwargs: Additional keyword arguments.
|
|
||||||
:return: The generate prompt.
|
|
||||||
:rtype: str
|
|
||||||
:raise AssertionError: If method is not implemented yet.
|
|
||||||
"""
|
|
||||||
|
|
||||||
prompt = ""
|
|
||||||
if method.startswith("io") or method.startswith("cot"):
|
|
||||||
if method.startswith("io"):
|
|
||||||
prompt += self.merge_doc_prompt_start.format(num=len(documents))
|
|
||||||
else:
|
|
||||||
prompt += self.merge_doc_prompt_cot_start.format(num=len(documents))
|
|
||||||
for i, document in enumerate(documents):
|
|
||||||
prompt += self.merge_doc_prompt_block.format(
|
|
||||||
document=document, num=i + 1
|
|
||||||
)
|
|
||||||
return prompt
|
|
||||||
elif method.startswith("tot"):
|
|
||||||
if current is None or current == "":
|
|
||||||
prompt += self.merge_doc_prompt_start.format(num=len(documents))
|
|
||||||
for i, document in enumerate(documents):
|
|
||||||
prompt += self.merge_doc_prompt_block.format(
|
|
||||||
document=document, num=i + 1
|
|
||||||
)
|
|
||||||
return prompt
|
|
||||||
else:
|
|
||||||
prompt += self.improve_summary_prompt_start.format(
|
|
||||||
num=len(documents),
|
|
||||||
)
|
|
||||||
for i, document in enumerate(documents):
|
|
||||||
prompt += self.improve_summary_prompt_block.format(
|
|
||||||
document=document, num=i + 1
|
|
||||||
)
|
|
||||||
prompt += self.improve_summary_prompt_end.format(summary=current)
|
|
||||||
return prompt
|
|
||||||
elif method.startswith("got"):
|
|
||||||
parts = (
|
|
||||||
sorted(list(parts)) if len(parts) > 0 else list(range(len(documents)))
|
|
||||||
)
|
|
||||||
if current is None or current == "":
|
|
||||||
prompt += self.merge_doc_prompt_start.format(num=len(parts))
|
|
||||||
for i, part in enumerate(sorted(list(parts))):
|
|
||||||
prompt += self.merge_doc_prompt_block.format(
|
|
||||||
document=documents[part], num=i + 1
|
|
||||||
)
|
|
||||||
return prompt
|
|
||||||
else:
|
|
||||||
prompt += self.improve_summary_prompt_start.format(
|
|
||||||
num=len(parts),
|
|
||||||
)
|
|
||||||
for i, part in enumerate(sorted(list(parts))):
|
|
||||||
prompt += self.improve_summary_prompt_block.format(
|
|
||||||
document=documents[part], num=i + 1
|
|
||||||
)
|
|
||||||
prompt += self.improve_summary_prompt_end.format(summary=current)
|
|
||||||
return prompt
|
|
||||||
else:
|
|
||||||
assert False, "Not implemented yet."
|
|
||||||
|
|
||||||
def score_prompt(self, state_dicts: List[Dict], **kwargs) -> str:
|
|
||||||
"""
|
|
||||||
Generate a score prompt for the language model.
|
|
||||||
|
|
||||||
:param state_dicts: The thought states that should be scored,
|
|
||||||
if more than one, they should be scored together.
|
|
||||||
:type state_dicts: List[Dict]
|
|
||||||
:param kwargs: Additional keyword arguments.
|
|
||||||
:return: The score prompt.
|
|
||||||
:rtype: str
|
|
||||||
:raise AssertionError: If more than one thought state is supplied.
|
|
||||||
"""
|
|
||||||
|
|
||||||
if len(state_dicts) > 1:
|
|
||||||
assert False, "Not implemented yet."
|
|
||||||
else:
|
|
||||||
# perform individual scoring
|
|
||||||
parts = (
|
|
||||||
[
|
|
||||||
state_dicts[0]["documents"][part]
|
|
||||||
for part in sorted(list(state_dicts[0]["parts"]))
|
|
||||||
]
|
|
||||||
if len(state_dicts[0]["parts"]) > 0
|
|
||||||
else state_dicts[0]["documents"]
|
|
||||||
)
|
|
||||||
prompt = self.score_prompt_base.format(
|
|
||||||
num=len(parts),
|
|
||||||
)
|
|
||||||
for i, part in enumerate(parts):
|
|
||||||
prompt += self.score_prompt_block.format(document=part, num=i + 1)
|
|
||||||
prompt += self.score_prompt_end.format(
|
|
||||||
summary=state_dicts[0]["current"],
|
|
||||||
)
|
|
||||||
return prompt
|
|
||||||
|
|
||||||
def improve_prompt(self, **kwargs) -> str:
|
|
||||||
"""
|
|
||||||
Generate an improve prompt for the language model.
|
|
||||||
|
|
||||||
:param kwargs: Additional keyword arguments.
|
|
||||||
:return: The improve prompt.
|
|
||||||
:rtype: str
|
|
||||||
"""
|
|
||||||
pass
|
|
||||||
|
|
||||||
def validation_prompt(self, **kwargs) -> str:
|
|
||||||
"""
|
|
||||||
Generate a validation prompt for the language model.
|
|
||||||
|
|
||||||
:param kwargs: Additional keyword arguments.
|
|
||||||
:return: The validation prompt.
|
|
||||||
:rtype: str
|
|
||||||
"""
|
|
||||||
pass
|
|
||||||
|
|
||||||
|
|
||||||
class DocMergeParser(parser.Parser):
|
|
||||||
"""
|
|
||||||
DocMergeParser provides the parsing of language model reponses specific to the
|
|
||||||
document merge example.
|
|
||||||
|
|
||||||
Inherits from the Parser class and implements its abstract methods.
|
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(self) -> None:
|
|
||||||
"""
|
|
||||||
Inits the response cache.
|
|
||||||
"""
|
|
||||||
self.cache = {}
|
|
||||||
|
|
||||||
def strip_answer_helper(self, text: str, tag: str = "") -> str:
|
|
||||||
"""
|
|
||||||
Helper function to remove tags from a text.
|
|
||||||
|
|
||||||
:param text: The input text.
|
|
||||||
:type text: str
|
|
||||||
:param tag: The tag to be stripped. Defaults to "".
|
|
||||||
:type tag: str
|
|
||||||
:return: The stripped text.
|
|
||||||
:rtype: str
|
|
||||||
"""
|
|
||||||
|
|
||||||
text = text.strip()
|
|
||||||
if "Output:" in text:
|
|
||||||
text = text[text.index("Output:") + len("Output:") :].strip()
|
|
||||||
if tag != "":
|
|
||||||
start = text.rfind(f"<{tag}>")
|
|
||||||
end = text.rfind(f"</{tag}>")
|
|
||||||
if start != -1 and end != -1:
|
|
||||||
text = text[start + len(f"<{tag}>") : end].strip()
|
|
||||||
elif start != -1:
|
|
||||||
logging.warning(
|
|
||||||
f"Only found the start tag <{tag}> in answer: {text}. Returning everything after the tag."
|
|
||||||
)
|
|
||||||
text = text[start + len(f"<{tag}>") :].strip()
|
|
||||||
elif end != -1:
|
|
||||||
logging.warning(
|
|
||||||
f"Only found the end tag </{tag}> in answer: {text}. Returning everything before the tag."
|
|
||||||
)
|
|
||||||
text = text[:end].strip()
|
|
||||||
else:
|
|
||||||
logging.warning(
|
|
||||||
f"Could not find any tag {tag} in answer: {text}. Returning the full answer."
|
|
||||||
)
|
|
||||||
return text
|
|
||||||
|
|
||||||
def parse_aggregation_answer(
|
|
||||||
self, states: List[Dict], texts: List[str]
|
|
||||||
) -> Union[Dict, List[Dict]]:
|
|
||||||
"""
|
|
||||||
Parse the response from the language model for an aggregation prompt.
|
|
||||||
|
|
||||||
:param states: The thought states used to generate the prompt.
|
|
||||||
:type states: List[Dict]
|
|
||||||
:param texts: The responses to the prompt from the language model.
|
|
||||||
:type texts: List[str]
|
|
||||||
:return: The new thought states after parsing the respones from the language model.
|
|
||||||
:rtype: Union[Dict, List[Dict]]
|
|
||||||
"""
|
|
||||||
|
|
||||||
new_states = []
|
|
||||||
for text in texts:
|
|
||||||
if len(states[0]["parts"]) < len(states[0]["documents"]):
|
|
||||||
# subpart aggregation
|
|
||||||
text = self.strip_answer_helper(text, "Merged")
|
|
||||||
new_state = states[0].copy()
|
|
||||||
new_state["current"] = text
|
|
||||||
new_state["parts"] = set()
|
|
||||||
for state in states:
|
|
||||||
new_state["parts"] = new_state["parts"] | state["parts"]
|
|
||||||
|
|
||||||
new_states.append(new_state)
|
|
||||||
else:
|
|
||||||
# full NDA aggregation
|
|
||||||
text = self.strip_answer_helper(text, "Merged")
|
|
||||||
new_state = states[0].copy()
|
|
||||||
new_state["current"] = text
|
|
||||||
new_states.append(new_state)
|
|
||||||
return new_states
|
|
||||||
|
|
||||||
def parse_generate_answer(self, state: Dict, texts: List[str]) -> List[Dict]:
|
|
||||||
"""
|
|
||||||
Parse the response from the language model for a generate prompt.
|
|
||||||
|
|
||||||
:param state: The thought state used to generate the prompt.
|
|
||||||
:type state: Dict
|
|
||||||
:param texts: The responses to the prompt from the language model.
|
|
||||||
:type texts: List[str]
|
|
||||||
:return: The new thought states after parsing the respones from the language model.
|
|
||||||
:rtype: List[Dict]
|
|
||||||
"""
|
|
||||||
new_states = []
|
|
||||||
for text in texts:
|
|
||||||
text = self.strip_answer_helper(text, "Merged")
|
|
||||||
new_state = state.copy()
|
|
||||||
new_state["current"] = text
|
|
||||||
new_states.append(new_state)
|
|
||||||
return new_states
|
|
||||||
|
|
||||||
def parse_score_answer(self, states: List[Dict], texts: List[str]) -> List[float]:
|
|
||||||
"""
|
|
||||||
Parse the response from the language model for a score prompt.
|
|
||||||
|
|
||||||
:param states: The thought states used to generate the prompt.
|
|
||||||
:type states: List[Dict]
|
|
||||||
:param texts: The responses to the prompt from the language model.
|
|
||||||
:type texts: List[str]
|
|
||||||
:return: The scores for the thought states.
|
|
||||||
:rtype: List[float]
|
|
||||||
:raise AssertionError: If the number of thought states is not one.
|
|
||||||
"""
|
|
||||||
assert len(states) == 1, "Only one state is allowed for scoring."
|
|
||||||
if len(states) == 1:
|
|
||||||
# individual scoring
|
|
||||||
redundancy_scores = []
|
|
||||||
retain_scores = []
|
|
||||||
for text in texts:
|
|
||||||
answer = self.strip_answer_helper(text, "Redundancy")
|
|
||||||
res = re.findall(r"\d+\.?\d*", answer)
|
|
||||||
if len(res) == 1:
|
|
||||||
redundancy_scores.append(float(res[0]))
|
|
||||||
elif len(res) > 1:
|
|
||||||
logging.warning(
|
|
||||||
f"Found multiple redundancy scores in answer: {text}. Returning the last one."
|
|
||||||
)
|
|
||||||
redundancy_scores.append(float(res[-1]))
|
|
||||||
else:
|
|
||||||
logging.warning(
|
|
||||||
f"Could not find any redundancy score in answer: {text}. Ignoring this answer."
|
|
||||||
)
|
|
||||||
answer = self.strip_answer_helper(text, "Retained")
|
|
||||||
res = re.findall(r"\d+\.?\d*", answer)
|
|
||||||
if len(res) == 1:
|
|
||||||
retain_scores.append(float(res[0]))
|
|
||||||
elif len(res) > 1:
|
|
||||||
logging.warning(
|
|
||||||
f"Found multiple retained scores in answer: {text}. Returning the last one."
|
|
||||||
)
|
|
||||||
retain_scores.append(float(res[-1]))
|
|
||||||
else:
|
|
||||||
logging.warning(
|
|
||||||
f"Could not find any retained score in answer: {text}. Ignoring this answer."
|
|
||||||
)
|
|
||||||
if len(redundancy_scores) == 0 or len(retain_scores) == 0:
|
|
||||||
logging.warning(
|
|
||||||
f"Could not find any valid score in any answer. Returning 0.0."
|
|
||||||
)
|
|
||||||
return [0.0]
|
|
||||||
mean_redundancy = fmean(redundancy_scores)
|
|
||||||
mean_retain = fmean(retain_scores)
|
|
||||||
f1 = 2 * mean_redundancy * mean_retain / (mean_redundancy + mean_retain)
|
|
||||||
return [f1]
|
|
||||||
|
|
||||||
def parse_improve_answer(self, state: Dict, texts: List[str]) -> Dict:
|
|
||||||
"""
|
|
||||||
Parse the response from the language model for an improve prompt.
|
|
||||||
|
|
||||||
:param state: The thought state used to generate the prompt.
|
|
||||||
:type state: Dict
|
|
||||||
:param texts: The responses to the prompt from the language model.
|
|
||||||
:type texts: List[str]
|
|
||||||
:return: The new thought state after parsing the responses from the language model.
|
|
||||||
:rtype: Dict
|
|
||||||
"""
|
|
||||||
pass
|
|
||||||
|
|
||||||
def parse_validation_answer(self, state: Dict, texts: List[str]) -> bool:
|
|
||||||
"""
|
|
||||||
Parse the response from the language model for a validation prompt.
|
|
||||||
|
|
||||||
:param state: The thought state used to generate the prompt.
|
|
||||||
:type state: Dict
|
|
||||||
:param texts: The responses to the prompt from the language model.
|
|
||||||
:type texts: List[str]
|
|
||||||
:return: Whether the thought state is valid or not.
|
|
||||||
:rtype: bool
|
|
||||||
"""
|
|
||||||
pass
|
|
||||||
|
|
||||||
|
|
||||||
def io() -> operations.GraphOfOperations:
|
|
||||||
"""
|
|
||||||
Generates the Graph of Operations for the IO method.
|
|
||||||
|
|
||||||
:return: Graph of Operations
|
|
||||||
:rtype: GraphOfOperations
|
|
||||||
"""
|
|
||||||
operations_graph = operations.GraphOfOperations()
|
|
||||||
|
|
||||||
operations_graph.append_operation(operations.Generate(1, 1))
|
|
||||||
operations_graph.append_operation(operations.Score(3, False))
|
|
||||||
|
|
||||||
return operations_graph
|
|
||||||
|
|
||||||
|
|
||||||
def cot() -> operations.GraphOfOperations:
|
|
||||||
"""
|
|
||||||
Generates the Graph of Operations for the CoT method.
|
|
||||||
|
|
||||||
:return: Graph of Operations
|
|
||||||
:rtype: GraphOfOperations
|
|
||||||
"""
|
|
||||||
operations_graph = operations.GraphOfOperations()
|
|
||||||
|
|
||||||
operations_graph.append_operation(operations.Generate(1, 1))
|
|
||||||
operations_graph.append_operation(operations.Score(3, False))
|
|
||||||
|
|
||||||
return operations_graph
|
|
||||||
|
|
||||||
|
|
||||||
def tot() -> operations.GraphOfOperations:
|
|
||||||
"""
|
|
||||||
Generates the Graph of Operations for the ToT method.
|
|
||||||
|
|
||||||
:return: Graph of Operations
|
|
||||||
:rtype: GraphOfOperations
|
|
||||||
"""
|
|
||||||
operations_graph = operations.GraphOfOperations()
|
|
||||||
|
|
||||||
branch_factor = 10
|
|
||||||
|
|
||||||
operations_graph.append_operation(operations.Generate(1, branch_factor))
|
|
||||||
operations_graph.append_operation(operations.Score(3, False))
|
|
||||||
keep_best_1 = operations.KeepBestN(1, True)
|
|
||||||
operations_graph.append_operation(keep_best_1)
|
|
||||||
|
|
||||||
for _ in range(2):
|
|
||||||
operations_graph.append_operation(operations.Generate(1, branch_factor))
|
|
||||||
operations_graph.append_operation(operations.Score(3, False))
|
|
||||||
keep_best_2 = operations.KeepBestN(1, True)
|
|
||||||
keep_best_2.add_predecessor(keep_best_1)
|
|
||||||
operations_graph.append_operation(keep_best_2)
|
|
||||||
keep_best_1 = keep_best_2
|
|
||||||
|
|
||||||
return operations_graph
|
|
||||||
|
|
||||||
|
|
||||||
def got() -> operations.GraphOfOperations:
|
|
||||||
"""
|
|
||||||
Generates the Graph of Operations for the GoT method, where full documents
|
|
||||||
are merged.
|
|
||||||
|
|
||||||
:return: Graph of Operations
|
|
||||||
:rtype: GraphOfOperations
|
|
||||||
"""
|
|
||||||
operations_graph = operations.GraphOfOperations()
|
|
||||||
|
|
||||||
operations_graph.append_operation(operations.Generate(1, 5))
|
|
||||||
operations_graph.append_operation(operations.Score(3, False))
|
|
||||||
keep_best = operations.KeepBestN(3, True)
|
|
||||||
operations_graph.append_operation(keep_best)
|
|
||||||
operations_graph.append_operation(operations.Aggregate(5))
|
|
||||||
operations_graph.append_operation(operations.Score(3, False))
|
|
||||||
keep_best2 = operations.KeepBestN(1, True)
|
|
||||||
keep_best2.add_predecessor(keep_best)
|
|
||||||
operations_graph.append_operation(keep_best2)
|
|
||||||
operations_graph.append_operation(operations.Generate(1, 10))
|
|
||||||
operations_graph.append_operation(operations.Score(3, False))
|
|
||||||
keep_best3 = operations.KeepBestN(1, True)
|
|
||||||
keep_best3.add_predecessor(keep_best2)
|
|
||||||
operations_graph.append_operation(keep_best3)
|
|
||||||
|
|
||||||
return operations_graph
|
|
||||||
|
|
||||||
|
|
||||||
def got2() -> operations.GraphOfOperations:
|
|
||||||
"""
|
|
||||||
Generates the Graph of Operations for the GoT2 method, where partial
|
|
||||||
documents are merged.
|
|
||||||
|
|
||||||
:return: Graph of Operations
|
|
||||||
:rtype: GraphOfOperations
|
|
||||||
"""
|
|
||||||
operations_graph = operations.GraphOfOperations()
|
|
||||||
|
|
||||||
sub_parts = []
|
|
||||||
for i in range(0, 4, 2): # should be at most 16 parts
|
|
||||||
sub_text = operations.Selector(
|
|
||||||
lambda thoughts, list_id=i: [
|
|
||||||
operations.Thought(
|
|
||||||
state={**thoughts[0].state, "parts": {list_id, list_id + 1}}
|
|
||||||
)
|
|
||||||
]
|
|
||||||
)
|
|
||||||
operations_graph.add_operation(sub_text)
|
|
||||||
gen_nda = operations.Generate(1, 5)
|
|
||||||
gen_nda.add_predecessor(sub_text)
|
|
||||||
operations_graph.add_operation(gen_nda)
|
|
||||||
score_nda = operations.Score(3, False)
|
|
||||||
score_nda.add_predecessor(gen_nda)
|
|
||||||
operations_graph.add_operation(score_nda)
|
|
||||||
keep_best_nda = operations.KeepBestN(1, True)
|
|
||||||
keep_best_nda.add_predecessor(score_nda)
|
|
||||||
operations_graph.add_operation(keep_best_nda)
|
|
||||||
|
|
||||||
sub_parts.append(keep_best_nda)
|
|
||||||
|
|
||||||
while len(sub_parts) > 1:
|
|
||||||
new_sub_parts = []
|
|
||||||
for i in range(0, len(sub_parts), 2):
|
|
||||||
if i + 1 == len(sub_parts):
|
|
||||||
new_sub_parts.append(sub_parts[i])
|
|
||||||
continue
|
|
||||||
aggregate = operations.Aggregate(5)
|
|
||||||
aggregate.add_predecessor(sub_parts[i])
|
|
||||||
aggregate.add_predecessor(sub_parts[i + 1])
|
|
||||||
operations_graph.add_operation(aggregate)
|
|
||||||
score = operations.Score(3, False)
|
|
||||||
score.add_predecessor(aggregate)
|
|
||||||
operations_graph.add_operation(score)
|
|
||||||
keep_best = operations.KeepBestN(1, True)
|
|
||||||
keep_best.add_predecessor(score)
|
|
||||||
operations_graph.add_operation(keep_best)
|
|
||||||
|
|
||||||
gen_nda = operations.Generate(1, 5)
|
|
||||||
gen_nda.add_predecessor(keep_best)
|
|
||||||
operations_graph.add_operation(gen_nda)
|
|
||||||
score_nda = operations.Score(3, False)
|
|
||||||
score_nda.add_predecessor(gen_nda)
|
|
||||||
operations_graph.add_operation(score_nda)
|
|
||||||
keep_best_nda = operations.KeepBestN(1, True)
|
|
||||||
keep_best_nda.add_predecessor(score_nda)
|
|
||||||
keep_best_nda.add_predecessor(keep_best)
|
|
||||||
operations_graph.add_operation(keep_best_nda)
|
|
||||||
|
|
||||||
new_sub_parts.append(keep_best_nda)
|
|
||||||
sub_parts = new_sub_parts
|
|
||||||
|
|
||||||
return operations_graph
|
|
||||||
|
|
||||||
|
|
||||||
def run(
|
|
||||||
data_ids: List[int],
|
|
||||||
methods: List[Callable[[], operations.GraphOfOperations]],
|
|
||||||
budget: float,
|
|
||||||
lm_name: str,
|
|
||||||
) -> float:
|
|
||||||
"""
|
|
||||||
Controller function that executes each specified method for each specified
|
|
||||||
sample while the budget is not exhausted.
|
|
||||||
|
|
||||||
:param data_ids: Indices of the sample to be run.
|
|
||||||
:type data_ids: List[int]
|
|
||||||
:param methods: List of functions to generate Graphs of Operations.
|
|
||||||
:type methods: Each function generates a Graph of Operation.
|
|
||||||
:param budget: Language model budget for the execution in dollars.
|
|
||||||
:type budget: float
|
|
||||||
:param lm_name: Name of the language model to be used.
|
|
||||||
:type lm_name: str
|
|
||||||
:return: Spent budget in dollars.
|
|
||||||
:rtype: float
|
|
||||||
"""
|
|
||||||
|
|
||||||
orig_budget = budget
|
|
||||||
data_path = os.path.join(os.path.dirname(__file__), "documents.csv")
|
|
||||||
data = []
|
|
||||||
with open(data_path, "r", encoding="utf8") as f:
|
|
||||||
reader = csv.reader(f)
|
|
||||||
next(reader)
|
|
||||||
for row in reader:
|
|
||||||
row[0] = int(row[0])
|
|
||||||
data.append(row)
|
|
||||||
|
|
||||||
if data_ids is None or len(data_ids) == 0:
|
|
||||||
data_ids = list(range(len(data)))
|
|
||||||
selected_data = [data[i] for i in data_ids]
|
|
||||||
|
|
||||||
results_dir = os.path.join(os.path.dirname(__file__), "results")
|
|
||||||
|
|
||||||
if not os.path.exists(results_dir):
|
|
||||||
os.makedirs(results_dir)
|
|
||||||
timestamp = datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
|
|
||||||
extra_info = f"{lm_name}_{'-'.join([method.__name__ for method in methods])}"
|
|
||||||
folder_name = f"{extra_info}_{timestamp}"
|
|
||||||
results_folder = os.path.join(results_dir, folder_name)
|
|
||||||
os.makedirs(results_folder)
|
|
||||||
|
|
||||||
config = {
|
|
||||||
"data": selected_data,
|
|
||||||
"methods": [method.__name__ for method in methods],
|
|
||||||
"lm": lm_name,
|
|
||||||
"budget": budget,
|
|
||||||
}
|
|
||||||
with open(os.path.join(results_folder, "config.json"), "w") as f:
|
|
||||||
json.dump(config, f)
|
|
||||||
|
|
||||||
logging.basicConfig(
|
|
||||||
filename=os.path.join(results_folder, "log.log"),
|
|
||||||
filemode="w",
|
|
||||||
format="%(name)s - %(levelname)s - %(message)s",
|
|
||||||
level=logging.DEBUG,
|
|
||||||
)
|
|
||||||
|
|
||||||
for method in methods:
|
|
||||||
os.makedirs(os.path.join(results_folder, method.__name__))
|
|
||||||
|
|
||||||
for data in selected_data:
|
|
||||||
logging.info(f"Running data {data[0]}: {data[1]}")
|
|
||||||
if budget <= 0.0:
|
|
||||||
logging.error(
|
|
||||||
f"Budget has been depleted, stopping. Data {data[0]} has not been run."
|
|
||||||
)
|
|
||||||
break
|
|
||||||
for method in methods:
|
|
||||||
logging.info(f"Running method {method.__name__}")
|
|
||||||
logging.info(f"Budget left: {budget}")
|
|
||||||
if budget <= 0.0:
|
|
||||||
logging.error(
|
|
||||||
f"Budget has been depleted, stopping. Method {method.__name__} has not been run."
|
|
||||||
)
|
|
||||||
break
|
|
||||||
lm = language_models.ChatGPT(
|
|
||||||
os.path.join(
|
|
||||||
os.path.dirname(__file__),
|
|
||||||
"../../graph_of_thoughts/language_models/config.json",
|
|
||||||
),
|
|
||||||
model_name=lm_name,
|
|
||||||
cache=True,
|
|
||||||
)
|
|
||||||
operations_graph = method()
|
|
||||||
executor = controller.Controller(
|
|
||||||
lm,
|
|
||||||
operations_graph,
|
|
||||||
DocMergePrompter(),
|
|
||||||
DocMergeParser(),
|
|
||||||
{
|
|
||||||
"documents": [data[2], data[3], data[4], data[5]],
|
|
||||||
"parts": set(),
|
|
||||||
"current": "",
|
|
||||||
"method": method.__name__,
|
|
||||||
},
|
|
||||||
)
|
|
||||||
try:
|
|
||||||
executor.run()
|
|
||||||
except Exception as e:
|
|
||||||
logging.error(f"Exception: {e}")
|
|
||||||
path = os.path.join(
|
|
||||||
results_folder,
|
|
||||||
method.__name__,
|
|
||||||
f"{data[0]}.json",
|
|
||||||
)
|
|
||||||
for operation in operations_graph.operations:
|
|
||||||
for thought in operation.thoughts:
|
|
||||||
thought.state["parts"] = list(thought.state["parts"])
|
|
||||||
executor.output_graph(path)
|
|
||||||
budget -= lm.cost
|
|
||||||
|
|
||||||
return orig_budget - budget
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
"""
|
|
||||||
Input (x1, x2, x3, x4): Four NDAs
|
|
||||||
Output (y): A new combined NDA
|
|
||||||
Evaluation: According to information coverage without repetition (scored by the LLM)
|
|
||||||
"""
|
|
||||||
budget = 30
|
|
||||||
samples = [item for item in range(0, 50)]
|
|
||||||
approaches = [io, cot, tot, got, got2]
|
|
||||||
|
|
||||||
spent = run(samples, approaches, budget, "chatgpt")
|
|
||||||
|
|
||||||
logging.info(f"Spent {spent} out of {budget} budget.")
|
|
||||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@ -1,170 +0,0 @@
|
|||||||
# Copyright (c) 2023 ETH Zurich.
|
|
||||||
# All rights reserved.
|
|
||||||
#
|
|
||||||
# Use of this source code is governed by a BSD-style license that can be
|
|
||||||
# found in the LICENSE file.
|
|
||||||
#
|
|
||||||
# main author: Nils Blach
|
|
||||||
|
|
||||||
import json
|
|
||||||
import os
|
|
||||||
import matplotlib.pyplot as plt
|
|
||||||
|
|
||||||
|
|
||||||
def get_complete_results(base_directory):
|
|
||||||
results_complete = {}
|
|
||||||
for folder_name in os.listdir(base_directory):
|
|
||||||
folder_path = os.path.join(base_directory, folder_name)
|
|
||||||
if os.path.isdir(folder_path):
|
|
||||||
results_complete[folder_name] = []
|
|
||||||
for file_name in os.listdir(folder_path):
|
|
||||||
if file_name.endswith(".json"):
|
|
||||||
file_path = os.path.join(folder_path, file_name)
|
|
||||||
with open(file_path, "r") as f:
|
|
||||||
data = json.load(f)
|
|
||||||
results_complete[folder_name].append(
|
|
||||||
{"key": int(file_name.split(".")[0]), "data": data}
|
|
||||||
)
|
|
||||||
for key in results_complete.keys():
|
|
||||||
results_complete[key] = sorted(
|
|
||||||
results_complete[key], key=lambda x: x["key"]
|
|
||||||
)
|
|
||||||
return results_complete
|
|
||||||
|
|
||||||
|
|
||||||
def get_final_scores(results_complete):
|
|
||||||
scores = {}
|
|
||||||
for method in results_complete.keys():
|
|
||||||
scores[method] = []
|
|
||||||
for result in results_complete[method]:
|
|
||||||
score = 0
|
|
||||||
solved = False
|
|
||||||
cost = 1
|
|
||||||
prompt_tokens = 0
|
|
||||||
completion_tokens = 0
|
|
||||||
for op in reversed(result["data"]):
|
|
||||||
if "cost" in op:
|
|
||||||
cost = op["cost"]
|
|
||||||
prompt_tokens = op["prompt_tokens"]
|
|
||||||
completion_tokens = op["completion_tokens"]
|
|
||||||
if "operation" in op and op["operation"] == "score":
|
|
||||||
try:
|
|
||||||
score = max(op["scores"])
|
|
||||||
break
|
|
||||||
except:
|
|
||||||
continue
|
|
||||||
scores[method].append(
|
|
||||||
[result["key"], score, solved, prompt_tokens, completion_tokens, cost]
|
|
||||||
)
|
|
||||||
scores[method] = sorted(scores[method], key=lambda x: x[0])
|
|
||||||
return scores
|
|
||||||
|
|
||||||
|
|
||||||
def get_plotting_data(base_directory):
|
|
||||||
results_complete = get_complete_results(base_directory)
|
|
||||||
scores = get_final_scores(results_complete)
|
|
||||||
results_plotting = {
|
|
||||||
method: {
|
|
||||||
"scores": [x[1] for x in scores[method]],
|
|
||||||
"solved": sum([1 for x in scores[method] if x[2]]),
|
|
||||||
"costs": [x[5] for x in scores[method]],
|
|
||||||
}
|
|
||||||
for method in scores.keys()
|
|
||||||
}
|
|
||||||
return results_plotting
|
|
||||||
|
|
||||||
|
|
||||||
def plot_results(
|
|
||||||
results,
|
|
||||||
methods_order=["io", "cot", "tot", "got", "got2"],
|
|
||||||
model="GPT-3.5",
|
|
||||||
num_ndas=4,
|
|
||||||
y_lower=0,
|
|
||||||
y_upper=10,
|
|
||||||
cost_upper=1.8,
|
|
||||||
display_solved=True,
|
|
||||||
annotation_offset=1,
|
|
||||||
display_left_ylabel=False,
|
|
||||||
display_right_ylabel=False,
|
|
||||||
):
|
|
||||||
methods_order = [method for method in methods_order if method in results]
|
|
||||||
scores_ordered = [
|
|
||||||
[score for score in results[method]["scores"]] for method in methods_order
|
|
||||||
]
|
|
||||||
total_costs = [sum(results[method]["costs"]) for method in methods_order]
|
|
||||||
|
|
||||||
# Create figure and axis
|
|
||||||
fig, ax = plt.subplots(dpi=150, figsize=(3.75, 5))
|
|
||||||
|
|
||||||
# Create boxplots
|
|
||||||
positions = range(1, len(methods_order) + 1)
|
|
||||||
ax.boxplot(scores_ordered, positions=positions)
|
|
||||||
|
|
||||||
fig_fontsize = 12
|
|
||||||
|
|
||||||
# Set the ticks and labels
|
|
||||||
methods_labels = ["IO", "CoT", "ToT", "GoT", "GoT2"]
|
|
||||||
ax.set_xticks(range(1, len(methods_order) + 1))
|
|
||||||
ax.set_xticks(range(1, len(methods_order) + 1))
|
|
||||||
ax.set_xticklabels(methods_labels)
|
|
||||||
# ax.set_xlabel("Approach")
|
|
||||||
|
|
||||||
ax.set_ylim(y_lower, 12 if display_solved else 9.75)
|
|
||||||
plt.yticks(fontsize=fig_fontsize)
|
|
||||||
|
|
||||||
if display_left_ylabel:
|
|
||||||
ax.set_ylabel(
|
|
||||||
f"Score (out of 10); the higher the better", fontsize=fig_fontsize
|
|
||||||
)
|
|
||||||
|
|
||||||
# ax.set_title(f"Document Merging")
|
|
||||||
|
|
||||||
ax2 = ax.twinx()
|
|
||||||
ax2.bar(
|
|
||||||
positions,
|
|
||||||
total_costs,
|
|
||||||
alpha=0.5,
|
|
||||||
color="blue",
|
|
||||||
label="Total Cost ($); the lower the better",
|
|
||||||
)
|
|
||||||
ax2.yaxis.set_tick_params(colors="#1919ff", labelsize=fig_fontsize)
|
|
||||||
ax2.set_ylim(0, cost_upper)
|
|
||||||
number_of_ticks = len(ax.get_yticks())
|
|
||||||
tick_interval = cost_upper / (number_of_ticks)
|
|
||||||
ax2_ticks = [tick_interval * i for i in range(number_of_ticks)]
|
|
||||||
|
|
||||||
# Set custom tick positions for ax2
|
|
||||||
ax2.set_yticks(ax2_ticks)
|
|
||||||
|
|
||||||
if display_right_ylabel:
|
|
||||||
ax2.set_ylabel(
|
|
||||||
"Total Cost ($); the lower the better",
|
|
||||||
color="#1919ff",
|
|
||||||
fontsize=fig_fontsize,
|
|
||||||
)
|
|
||||||
|
|
||||||
if display_solved:
|
|
||||||
annotation_height = y_upper + annotation_offset
|
|
||||||
count = 1
|
|
||||||
for method in methods_order:
|
|
||||||
if method not in results:
|
|
||||||
continue
|
|
||||||
solved = results[method]["solved"]
|
|
||||||
ax.text(
|
|
||||||
count, annotation_height, f"Solved: {solved}", ha="center", va="bottom"
|
|
||||||
)
|
|
||||||
count += 1
|
|
||||||
|
|
||||||
model = model.replace(".", "").replace("-", "").lower()
|
|
||||||
fig.savefig(f"doc_merge_{model}_{num_ndas}.pdf", bbox_inches="tight")
|
|
||||||
|
|
||||||
|
|
||||||
plot_results(
|
|
||||||
get_plotting_data("results/"),
|
|
||||||
num_ndas=4,
|
|
||||||
display_solved=False,
|
|
||||||
model="GPT-3.5",
|
|
||||||
y_upper=10,
|
|
||||||
display_left_ylabel=True,
|
|
||||||
cost_upper=15,
|
|
||||||
)
|
|
||||||
@ -1,52 +0,0 @@
|
|||||||
[
|
|
||||||
"NON-DISCLOSURE AGREEMENT (NDA)\n\n1. Agreement between [Your Company Name] and [Recipient Name] on [Date].\n2. Information sharing for the purpose of [specific project or purpose].\n3. \"Confidential Information\" includes all potentially commercially valuable information, specifically software development tactics, processes, and in-house research results.\n4. Receiving party is obligated to protect the Confidential Information, use it solely for the disclosed purpose, and not disclose it without consent.\n5. Breach penalties include injunctive relief, other remedies, and a $200,000 fee per breach.\n6. The Agreement applies to the Parties and their successors and assigns. It contains all related agreements and lack of enforcement doesn't imply waiver.\n7. The Agreement is under the laws of [State].\n8. Signed by [Your Company Name] and [Recipient Name] at the above date.",
|
|
||||||
"NON-DISCLOSURE AGREEMENT (NDA)\n\nEffective from [Effective Date], this NDA involves [Your Company Name] (\"Disclosing Party\"), and [Recipient Name] (\"Receiving Party\").\n\n1. Purpose: The Disclosing Party will disclose confidential information related to [Topic of Research] to the Receiving Party for [Purpose].\n\n2. Confidential Information: Defined as all non-public reports, data, designs, and other materials provided by the Disclosing Party to the Receiving Party.\n\n3. Receiving Party's Obligations:\n a. Use, reproduce, or distribute the confidential information only for the agreed purpose.\n b. Restrict access to the information to necessary parties, ensuring they abide by strict confidentiality.\n c. Return or destroy all confidential information upon request or at the end of the agreement.\n\n4. Exclusions: Information will not be classified as confidential if it is already known to the Receiving Party, publicly known, or independently developed by the Receiving Party.\n\n5. Non-Competition: The Receiving Party will not engage in any competing business against the Disclosing Party during the agreement and one year after its termination.\n\n6. Term and Termination: The agreement is valid for [e.g., \"two years\"], unless terminated earlier with [e.g., \"30 days\"] written notice. The Receiving Party's non-disclosure and non-competition obligations persist post-termination.\n\n7. General Provisions:\n a. Governing Law: [Your State]'s laws apply.\n b. Amendments: Only valid if written and signed by both parties.\n c. Entire Agreement: This contract overrules previous related agreements.\n\nSigned as of the Effective Date by [Your Company Name] - Disclosing Party [Recipient Name] - Receiving Party.",
|
|
||||||
"CONFIDENTIALITY & NON-DISCLOSURE AGREEMENT\n\n Entities Involved:\n Effective [Date], between [AquaBlue Innovations], established in [State], and [PineTree Solutions], a registered entity.\n\n Objective:\n To safeguard classified data during talks of a potential technological alliance.\n\n Specification of Protected Information:\n Particularly:\n\na. System designs and architectural schematics.\nb. Proprietary computational algorithms.\n\n Receiver's Obligations:\n a. Maintain strict non-disclosure using best practices.\n b. Employ solely for the aforementioned aim.\n c. No unveiling without explicit authorization.\n\n Violation Ramifications:\n A charge of $280,000 for every infringement, plus possible legal proceedings.\n\n General Terms:\n Binding for both parties and any successors. This encapsulates the entire accord.\n\n Legal Reference:\n Governed as per [State]'s legal framework.\n\n Attestation:\n Duly signed on [Date].\n\n[AquaBlue Innovations] [PineTree Solutions]",
|
|
||||||
"SECRECY & DISCLOSURE AGREEMENT\n\n Contracting Parties:\n Dated [Date], drawn between [AquaBlue Innovations], a [State]-based corporation, and [PineTree Solutions], a licensed organization.\n\n Aim:\n To protect exclusive insights amidst dialogues for a technological partnership.\n\n Categorization of Sensitive Data:\n Includes:\n\na. Internal software blueprints.\nb. Intellectual property awaiting patents.\n\n Commitments of Recipient:\n a. Uphold confidentiality, ensuring data integrity.\n b. Utilize strictly for collaborative ventures.\n c. No exposure without prior consensus.\n\n Repercussions for Non-Compliance:\n $295,000 fine for each transgression, and the option for legal recourse.\n\n Overall Provisions:\n Legally enforceable for signatories and successors. Complete and sole agreement.\n\n Juridical Standpoint:\n Under the auspices of [State] laws.\n\n Ratification:\n Confirmed and endorsed on [Date].\n\n[AquaBlue Innovations] [PineTree Solutions]",
|
|
||||||
"This Non-Disclosure and Non-Competition Agreement is made between [Your Company Name] and [Contractor Name/Company].\n\n1. Confidentiality: The Contractor acknowledges access to the Company's confidential information during their relationship.\n\n2. Non-Disclosure: The Contractor agrees not to disclose, use, reproduce, or distribute this confidential information unless necessary for their obligations.\n\n3. Non-Competition: The Contractor agrees not to compete with the company or assist others in doing so for one year after the termination of their relationship. They also agree not to solicit the company's clients or customers for the benefit of a competitor for one year.\n\n4. Return of Confidential Information: At the end of the relationship or upon the company's request, the Contractor will return all confidential information and copies thereof.\n\n5. Remedies: For any breach, the Company may seek specific performance and injunctive relief, in addition to other remedies.\n\n6. Governing Law: The Agreement is governed by the laws of [Your State].\n\n7. Entire Agreement: This document replaces all previous agreements and understandings on the subject.\n\nBoth parties acknowledge understanding and voluntarily accepting the Agreement.\n\nSignatures required from [Your Company Name] and [Contractor Name/Company].",
|
|
||||||
"This Loyalty Agreement is between [Company Name] and [Employee Full Name], where the company agrees to provide specialized training at no cost to the employee, who in turn commits to work for the company for a specified period. If the employee leaves the company within two years after completing training, they must pay $50,000 as compensation for training costs, payable within 30 days of termination. Exceptions to this repayment include termination without cause, resignation due to breach of agreement by the company, or other agreed upon circumstances. Any changes to this agreement must be in writing and signed by both parties, and the agreement will be governed by the laws of [State/Country]. This agreement is binding to all involved parties and their successors. Both the company and the employee sign to attest to these terms.",
|
|
||||||
"EMPLOYEE LOYALTY AGREEMENT\n\nThis agreement is entered into by [Company Name] and [Employee Name] to protect the company's business interests, goodwill, and confidential information, and affirm employee's loyalty. \n\n1. Non-disclosure: Employee agrees to not disclose or use company's confidential information, during or post-employment. \n\n2. Non-competition: Employee will not work for or establish a competitor within [e.g., \"50\"] miles from the company for [e.g., \"12\"] months post-employment.\n\n3. Non-solicitation: Employee will not solicit clients or employees of the company for [e.g., \"12\"] months post-employment.\n\n4. Return of Property: Employee will return all company property upon termination.\n\n5. Remedies: Company can seek injunction for a breach or potential breach of this agreement.\n\n6. Severability: If any provision of this agreement is held invalid, the remainder of the Agreement will continue.\n\n7. Governing Law: This agreement will be governed by the laws of [State, e.g., \"California\"].\n\n8. Agreement: This is the entire agreement and supersedes prior negotiations.\n\n9. Amendments: Any changes must be in writing and signed by both parties.\n\nSignatures of both parties indicate agreement to these terms.\n\n[Company Name] - Authorized Signatory [Employee Name]",
|
|
||||||
"This Loyalty Agreement is between [Company Name] and [Contractor Company Name]. The Agreement ensures the Contractor's loyalty and confidentiality towards the Company during and post engagement. Contractor agrees not to use or disclose the Company's confidential information, or engage in competing business or solicitation for a period of [e.g., \"12\"] months post termination. Contractor must return all Company property upon termination. In case of breach, Company can seek legal remedies including injunction. The Agreement remains valid even if a provision is held invalid. The Agreement follows [State, e.g., \"California\"] laws and replaces all previous understandings. It can be amended only in writing with both parties' signature.",
|
|
||||||
"B2B CONTRACTOR LOYALTY AGREEMENT\n\nThis Agreement is made on _____ day of ______, 20, between [Company Name], located at [Company Address] (\"Company\"), and [Contractor Company Name], located at [Contractor Address] (\"Contractor\").\n\n1. CONFIDENTIALITY\n\nContractor agrees not to disclose, use, or allow the use of the Company's confidential information during or after the relationship, except as required for their services to the Company.\n\n2. NON-COMPETITION\n\nFor 12 months post-relationship, the Contractor won't provide similar services to any entity competing with the Company within a 50-mile radius of any Company location.\n\n3. NON-SOLICITATION\n\nFor 12 months post-relationship, the Contractor won't solicit or induce any entity or individual connected to the Company to cease or reduce their relationship with the Company.\n\n4. RETURN OF PROPERTY\n\nUpon relationship termination or on Company's request, the Contractor will immediately return all Company property and data.\n\n5. PENALTY FOR BREACH\n\nIn the event of a breach of this Agreement, the Contractor shall pay the Company a penalty of $50,000.\n\n6. GOVERNING LAW\n\nThis Agreement is governed by [State, e.g., \"California\"] laws.\n\n7. ENTIRE AGREEMENT\n\nThis Agreement supersedes prior discussions and agreements between the parties.\n\nBy signing below, the parties agree to these terms.\n\n[Company Name] - Signatory [Contractor Company Name] - Signatory\nDate: _______________________ Date: _______________________",
|
|
||||||
"B2B CONTRACTOR LOYALTY AGREEMENT\n\nThis Agreement is made on _____ day of ______, 20, between [Company Name], located at [Company Address] (\"Company\"), and [Contractor Company Name], located at [Contractor Address] (\"Contractor\").\n\n1. DEFINITION OF CONFIDENTIAL INFORMATION\n\nFor the purposes of this Agreement, \"confidential information\" shall refer to research results, software created, devices produced by the Company, and any other information deemed proprietary or not generally known to the public.\n\n2. CONFIDENTIALITY\n\nContractor agrees not to disclose, use, or allow the use of the Company's confidential information, as defined herein, during or after the relationship, except as required for their services to the Company.\n\n3. NON-COMPETITION\n\nFor 12 months post-relationship, the Contractor won't provide similar services to any entity competing with the Company within a 50-mile radius of any Company location.\n\n4. NON-SOLICITATION\n\nFor 12 months post-relationship, the Contractor won't solicit or induce any entity or individual connected to the Company to cease or reduce their relationship with the Company.\n\n5. RETURN OF PROPERTY\n\nUpon relationship termination or on Company's request, the Contractor will immediately return all Company property, including all items containing or pertaining to confidential information.\n\n6. PENALTY FOR BREACH\n\nIn the event of a breach of this Agreement, the Contractor shall pay the Company a penalty of $50,000.\n\n7. GOVERNING LAW\n\nThis Agreement is governed by [State, e.g., \"California\"] laws.\n\n8. ENTIRE AGREEMENT\n\nThis Agreement supersedes prior discussions and agreements between the parties.\n\nBy signing below, the parties agree to these terms.\n\n[Company Name] - Signatory [Contractor Company Name] - Signatory",
|
|
||||||
"The Non-Disclosure Agreement (NDA) dated [Date] is between [Company], based in [Country/State], and [Supplier], also incorporated in [Country/State]. The Company intends to disclose confidential information to the Supplier for [purpose]. This confidential data can include business strategies, financial data, customer information, and product designs. The Supplier agrees to refrain from sharing this information, barring any legal requirements. Exceptions to this confidentiality are in cases where the information becomes public or was already known by the Supplier before the Company's disclosure. If the Supplier breaches this agreement, they face a financial penalty of [$]. The NDA is valid for [X years], unless the Company provides written termination. Upon the Company's request, the Supplier must return or destroy all copies of Confidential Information. This agreement supersedes previous agreements and can only be altered by a written document approved by both parties. The NDA is governed by the laws of [specific country/state].",
|
|
||||||
"NON-DISCLOSURE AND NON-COMPETE AGREEMENT\n\nEffective Date: [Date]\n\nPARTIES:\n\n Company: [Full Legal Name of Company], located at [Company Address].\n Supplier: [Full Legal Name of Supplier], located at [Supplier Address].\n\n1. CONFIDENTIALITY:\n\nSupplier shall not disclose Company's confidential information, which includes business strategies, financial data, and customer details, to any third party. This confidentiality obligation lasts for [X years, e.g., \"5 years\"] from the date of disclosure.\n\n2. NON-COMPETITION:\n\nFor [X years, e.g., \"3 years\"] following the termination of their business relationship, Supplier agrees not to engage in or start any business that directly competes with Company within a [X mile/km radius, e.g., \"50-mile radius\"] of Company's primary business location.\n\n3. PENALTY FOR BREACH:\n\nShould Supplier breach this Agreement, they shall pay Company a penalty of [specific amount, e.g., \"$50,000\"], in addition to any other legal remedies available to Company.\n\n4. RETURN OF INFORMATION:\n\nUpon request, Supplier shall return or destroy all of Company's confidential information and confirm its deletion in writing.\n\n5. GOVERNING LAW:\n\nThis Agreement is governed by the laws of [specific country/state, e.g., \"the State of New York\"].\n\nAGREEMENT ACKNOWLEDGEMENT:\n\n__________ [Company] __________ [Supplier]",
|
|
||||||
"DATA ANALYSIS EMPLOYEE AGREEMENT\n\nThis Agreement (\"Agreement\") is made and entered into as of [Date], by and between [Company Name], a [legal structure, e.g., \"corporation\"] incorporated under the laws of [State/Country], with its principal place of business at [Company Address], herein referred to as the \"Company,\" and [Employee Name], an individual residing at [Employee Address], herein referred to as the \"Employee.\"\n\n Position and Duties:\n a. The Company hereby employs Employee in the capacity of Data Analyst.\n b. The Employee's primary duties will be to [specific data analysis tasks, e.g., \"analyze sales data, forecast trends, and produce reports for managerial review\"].\n\n Term: The Employee's engagement will commence on [Start Date] and will terminate on [End Date].\n\n Compensation: For the services rendered by the Employee under this Agreement, the Company will pay Employee a total sum of [specific amount, e.g., \"$5,000\"] payable on [payment schedule, e.g., \"a monthly basis\"].\n\n Confidentiality: The Employee agrees not to disclose or use, either during or after the term of employment, any proprietary or confidential information or data of the Company without the Company's prior written consent, except as necessary in the course of performing their duties for the Company.\n\n Intellectual Property: Any works, developments, or inventions created by the Employee in the course of this employment related to the Company's business will remain the sole property of the Company.\n\n Termination: Either party may terminate this Agreement with [e.g., \"30\"] days written notice. Upon termination, Employee agrees to return all company property and data.\n\n Governing Law: This Agreement shall be governed by and construed under the laws of [State/Country].\n\n Amendments: This Agreement may only be amended in writing and signed by both parties.\n\n Entire Agreement: This Agreement contains the entire agreement between the parties and supersedes all prior negotiations, understandings, and agreements between the parties.\n\nThe parties hereto have executed this Agreement as of the date first above written.\n\n[Company Name or Authorized [Employee Name]\nRepresentative Name, Title]",
|
|
||||||
"DATA ANALYSIS SERVICE AGREEMENT\n\nThis Agreement (\"Agreement\") is made and entered into as of [Date], by and between [Company Name], a [legal structure, e.g., \"corporation\"] incorporated under the laws of [State/Country], with its principal place of business at [Company Address], herein referred to as the \"Company,\" and [Contractor Business Name], a [legal structure, e.g., \"limited liability company\"] organized under the laws of [State/Country], with its principal place of business at [Contractor Business Address], herein referred to as the \"Contractor.\"\n\n Scope of Work:\n a. The Contractor agrees to provide data analysis services to the Company.\n b. The specific services will include [specific data analysis tasks, e.g., \"analyzing sales data, forecasting trends, and producing reports for managerial review\"].\n\n Term: The Contractor's engagement will commence on [Start Date] and will terminate on [End Date].\n\n Compensation: For the services rendered by the Contractor under this Agreement, the Company will pay the Contractor a total sum of [specific amount, e.g., \"$5,000\"] payable on [payment schedule, e.g., \"a monthly basis\"].\n\n Confidentiality: The Contractor agrees not to disclose or use, either during or after the term of this Agreement, any proprietary or confidential information or data of the Company without the Company's prior written consent, except as necessary in the course of providing the services.\n\n Intellectual Property: Any works, developments, or inventions created by the Contractor in the course of providing the services related to the Company's business will remain the sole property of the Company.\n\n Termination: Either party may terminate this Agreement with [e.g., \"30\"] days written notice. Upon termination, Contractor agrees to return all company data and any other proprietary materials.\n\n Governing Law: This Agreement shall be governed by and construed under the laws of [State/Country].\n\n Amendments: This Agreement may only be amended in writing and signed by both parties.\n\n Entire Agreement: This Agreement contains the entire agreement between the parties and supersedes all prior negotiations, understandings, and agreements between the parties.\n\nThe parties hereto have executed this Agreement as of the date first above written.",
|
|
||||||
"NON-DISCLOSURE AGREEMENT (NDA)\n\nDate: [Insert Date]\n\nParties: [University Name], [University Address] (\"Disclosing Party\") and [Researcher's Full Name], [Researcher's Address] (\"Receiving Party\").\n\n1. Purpose: For the research of [Briefly Describe the Research or Project].\n\n2. Confidential Information: Includes data, studies, reports, patents, and other valuable business-related material.\n\n3. Obligations:\na. Confidential Information must remain secret.\nb. It's for the specified purpose only.\nc. No third-party disclosure without consent.\n\n4. Exceptions: Public knowledge, third-party shared info, or independently developed.\n\n5. Duration: Confidentiality lasts [X years, e.g., 2 years] from disclosure.\n\n6. Return: All Confidential Information must be returned or destroyed upon request.\n\n7. No Transfer: This doesn't grant property rights or licenses.\n\n8. Law: Governed by [State/Country] laws.\n\n9. Amendments: Only in writing and signed.\n\nAgreement: By signing, parties agree to the terms.",
|
|
||||||
"UNIVERSITY-BUSINESS COOPERATION AGREEMENT\n\nThis Cooperation Agreement (the \"Agreement\") is made and entered into on [Insert Date], by and between [Business Name], a [business type, e.g., \"corporation\"] located at [Business Address], hereinafter referred to as the \"Business', and [University Name], a higher education institution located at [University Address], hereinafter referred to as the \"University\".\n\n1. Objective:\n\nThe purpose of this Agreement is to define the terms under which the Business and the University will cooperate to [Objective e.g., \"jointly conduct research, promote innovation, and facilitate the exchange of knowledge in the field of _________\"].\n\n2. Scope of Cooperation:\n\na. Research Collaborations: Joint research initiatives, sharing of resources, and publications.\n\nb. Internships and Placements: Facilitation of student internships, projects, and job placements.\n\nc. Seminars and Workshops: Organizing joint seminars, conferences, and workshops.\n\nd. Facilities and Resource Sharing: Providing access to labs, equipment, libraries, etc.\n\n3. Intellectual Property:\n\nAll intellectual property developed jointly will be shared, and a separate agreement will detail the rights, ownership, and any revenue distribution.\n\n4. Funding and Resources:\n\nBoth parties agree to jointly contribute [Specify Amount or Percentage], and additional fund sourcing details will be determined on a project-by-project basis.\n\n5. Confidentiality:\n\nBoth parties agree to maintain the confidentiality of shared proprietary information.\n\n6. Duration and Termination:\n\nThis Agreement will remain in effect for [e.g., \"three years\"] from the date of signing, unless terminated earlier by either party with [e.g., \"30 days\"] written notice.\n\n7. Dispute Resolution:\n\nAny disputes arising from this Agreement will first attempt resolution through mediation. If unresolved, disputes will be subject to the jurisdiction of [State/Country].\n\n8. Amendments:\n\nChanges to this Agreement must be in writing and signed by both parties.\n\n9. Liability:\n\nEach party assumes responsibility for its actions and is not liable for the actions of the other party.\n\n10. Governing Law:\n\nThis Agreement is governed by the laws of [State/Country].\n\nIN WITNESS WHEREOF, both parties have executed this Agreement as of the date first mentioned above.",
|
|
||||||
"SUPPLY AGREEMENT FOR UNIVERSITY LABORATORY\n\nThis Supply Agreement (the \"Agreement\"), made as of [Insert Date], is entered into by and between [Supplier Name], a [business entity type, e.g., \"corporation\"] having its principal office at [Supplier Address], hereinafter referred to as the \"Supplier', and [University Name], a higher education institution located at [University Address], acting through its [specific department or laboratory, e.g., \"Department of Chemistry\"], hereinafter referred to as the \"University\".\n\n1. Purpose:\n\nThe Supplier agrees to provide specific products/materials/equipment, as detailed in Annex A, to the University for use in its laboratory.\n\n2. Terms of Supply:\n\na. Description of Goods: The goods to be supplied are detailed in Annex A attached herewith.\n\nb. Delivery: Goods will be delivered to [University Address or specific lab address] within [specific timeframe].\n\nc. Pricing: The price for the goods is set out in Annex A and includes all packaging, transportation, and delivery costs unless otherwise specified.\n\n3. Payment Terms:\n\nPayments will be made by the University within [e.g., \"30 days\"] of receiving the invoice from the Supplier.\n\n4. Warranty:\n\nThe Supplier warrants that all goods supplied under this Agreement will be free from defects for a period of [specific duration, e.g., \"12 months\"] from the date of delivery.\n\n5. No Disclosure Clause:\n\na. The University agrees not to disclose, reproduce, or distribute any proprietary information, trade secrets, or other confidential details related to the products/materials/equipment provided by the Supplier without the Supplier's prior written consent.\n\nb. This clause remains effective for a period of [e.g., \"5 years\"] from the date of the last delivery of the goods under this Agreement.\n\n6. Termination:\n\nEither party may terminate this Agreement with [e.g., \"30 days\"] written notice if the other party breaches any term of this Agreement and fails to remedy such breach within the notice period.\n\n7. Governing Law:\n\nThis Agreement shall be governed by and interpreted in accordance with the laws of [State/Country].\n\n8. Amendments:\n\nModifications to this Agreement must be in writing and signed by both parties.\n\nIN WITNESS WHEREOF, the parties hereto have executed this Supply Agreement as of the date first above written.",
|
|
||||||
"LABORATORY SUPPLY AGREEMENT\n\nDate: [Insert Date]\n\nParties:\n\n Supplier: [Supplier Name], [Supplier Address]\n University: [University Name], [University Address]\n\n1. Purpose: Supplier will provide goods as listed in Annex A to the University's laboratory.\n\n2. Delivery: Within [specific timeframe, e.g., \"30 days\"] to [specific lab address].\n\n3. Payment: University will pay within [e.g., \"30 days\"] of invoice receipt.\n\n4. Warranty: Goods are defect-free for [e.g., \"12 months\"] from delivery.\n\n5. Non-disclosure: University will not disclose Supplier\u2019s proprietary details for [e.g., \"5 years\"]. Breach will result in a penalty of [specific amount or formula, e.g., \"$5,000 per incident or actual damages, whichever is greater\"].\n\n6. Termination: [e.g., \"30 days\"] notice for breaches unresolved within said period.\n\n7. Law: Governed by [State/Country] laws.\n\n8. Amendments: Both parties must sign written changes.",
|
|
||||||
"FREELANCER AGREEMENT\n\nEffective Date: [Date]\n\nBETWEEN:\n\n Client: [Client Full Name or Company Name], located at [Client Address].\n Freelancer: [Freelancer Full Name], located at [Freelancer Address].\n\n1. SERVICES:\n\nFreelancer agrees to provide the following services: [Brief description of services, e.g., \"web design, content creation, graphic design\"].\n\n2. PAYMENT TERMS:\n\nFor the services rendered, Client agrees to pay Freelancer a total of [Total Amount, e.g., \"$1,000\"]. Payments shall be made as follows: [Payment structure, e.g., \"50% upfront, 50% upon completion\"].\n\n3. DEADLINE:\n\nThe services will be completed by [End Date, e.g., \"December 31, 2023\"].\n\n4. CONFIDENTIALITY:\n\nFreelancer agrees to maintain confidentiality regarding all proprietary information of the Client.\n\n5. TERMINATION:\n\nEither party may terminate this agreement with [X days, e.g., \"14 days\"] written notice. Upon termination, payments will be adjusted for work completed.\n\n6. INDEPENDENT CONTRACTOR:\n\nFreelancer is an independent contractor and not an employee of the Client. No benefits, rights, or obligations of employment are conferred by this agreement.\n\n7. GOVERNING LAW:\n\nThis Agreement is governed by the laws of [specific country/state, e.g., \"the State of New York\"].\n\n8. AMENDMENTS:\n\nAny changes to this agreement must be in writing and signed by both parties.",
|
|
||||||
"FREELANCER AGREEMENT\n\nEffective Date: [Date]\n\nBETWEEN:\n\n Client: [Client Full Name or Company Name], located at [Client Address].\n Freelancer: [Freelancer Full Name], located at [Freelancer Address].\n\n1. SERVICES:\n\nFreelancer agrees to provide the following services: [Brief description of services, e.g., \"web design, content creation, graphic design\"].\n\n2. PAYMENT TERMS:\n\nFor the services rendered, Client agrees to pay Freelancer a total of [Total Amount, e.g., \"$1,000\"]. Payments shall be made as follows: [Payment structure, e.g., \"50% upfront, 50% upon completion\"].\n\n3. DEADLINE:\n\nThe services will be completed by [End Date, e.g., \"December 31, 2023\"].\n\n4. PENALTIES:\n\na. Late Delivery: If Freelancer fails to deliver the completed service by the specified deadline, a penalty of [specific amount, e.g., \"$50\"] per day will be deducted from the final payment until the service is delivered.\n\nb. Confidentiality Breach: Breaching the confidentiality clause will result in a penalty of [specific amount, e.g., \"$2,000\"].\n\n5. CONFIDENTIALITY:\n\nFreelancer agrees to maintain confidentiality regarding all proprietary information of the Client.\n\n6. TERMINATION:\n\nEither party may terminate this agreement with [X days, e.g., \"14 days\"] written notice. Upon termination, payments will be adjusted for work completed.\n\n7. INDEPENDENT CONTRACTOR:\n\nFreelancer is an independent contractor and not an employee of the Client. No benefits, rights, or obligations of employment are conferred by this agreement.\n\n8. GOVERNING LAW:\n\nThis Agreement is governed by the laws of [specific country/state, e.g., \"the State of New York\"].\n\n9. AMENDMENTS:\n\nAny changes to this agreement must be in writing and signed by both parties.",
|
|
||||||
"This document outlines the terms of cooperation between Company A and Company B for a joint research project. The duties of each company are designated, with a detailed financial contribution outlined in Appendix A. Confidentiality is strictly enforced, and any intellectual property created will be jointly owned. All published findings will be reviewed by both parties for protection of proprietary information. Termination of this agreement requires 30 days' written notice, and each party assumes any risks or liabilities during this collaboration. Amendments must be in writing and signed by both parties. The duration of the agreement lasts from the start date to the end date, unless extended. By signing, both parties acknowledge and agree to these terms.",
|
|
||||||
"BUSINESS AGREEMENT\n\nEffective Date: [Date]\n\nBETWEEN:\n\n Business A: [Full Legal Name of Business A], located at [Business A Address].\n Business B: [Full Legal Name of Business B], located at [Business B Address].\n\n1. PURPOSE:\n\nThis Agreement outlines the terms of the collaboration/project/service between Business A and Business B regarding [Brief Description of the Collaboration/Project/Service].\n\n2. TERMS OF SERVICE:\n\n Business A agrees to: [Specific tasks/responsibilities, e.g., \"Supply 500 units of Product X monthly.\"].\n Business B agrees to: [Specific tasks/responsibilities, e.g., \"Pay $50 per unit of Product X within 30 days of delivery.\"].\n\n3. PAYMENT TERMS:\n\nPayments shall be made as follows: [Payment structure, e.g., \"Payment due within 30 days of invoice.\"].\n\n4. CONFIDENTIALITY:\n\nBoth parties commit to maintaining confidentiality regarding all proprietary information exchanged during this agreement.\n\n5. TERMINATION:\n\nEither party may terminate this Agreement with [X days, e.g., \"30 days\"] written notice. If breached, the aggrieved party may seek remedies as per governing laws.\n\n6. GOVERNING LAW:\n\nThis Agreement is governed by the laws of [specific country/state, e.g., \"the State of California\"].\n\n7. AMENDMENTS:\n\nModifications to this Agreement must be in writing and signed by both parties.\n\nACKNOWLEDGEMENT:\n\nBy signing below, both parties affirm their understanding and acceptance of this Business Agreement.",
|
|
||||||
"CONFIDENTIALITY:\n\n4.1. Confidential Information: For the purposes of this Agreement, \"Confidential Information\" refers to any data or information, regardless of its form, proprietary to or maintained as confidential by either party, which is not publicly known and which is disclosed during the term of this Agreement or in relation to the collaboration/project/service.\n\n4.2. Protection and Non-Disclosure: Both parties agree to use the Confidential Information solely for the purposes of the Agreement and will exert reasonable efforts to prevent the unauthorized disclosure or use of the Confidential Information. Neither party shall disclose, reproduce, or distribute any portion of the Confidential Information without the disclosing party's prior written consent.\n\n4.3. Exclusions: Confidential Information shall not include any data or information which:\n\n Is or becomes publicly known through no wrongful act of the receiving party;\n Is independently developed by the receiving party without the use of the Confidential Information;\n Is rightfully received from a third party without any obligation of confidentiality;\n Is disclosed under legal requirement or order.\n\n4.4. Return or Destruction: Upon the termination of this Agreement, or at the request of the disclosing party, the receiving party shall return all copies of the Confidential Information to the disclosing party or certify in writing that it has destroyed all such copies.\n\n4.5. Duration: The obligations set forth in this Confidentiality section shall survive the termination or expiration of this Agreement for a period of [specific time, e.g., \"five years\"].",
|
|
||||||
"LOYALTY AGREEMENT\n\nEffective Date: [Date]\n\nBETWEEN:\n\n Party A: [Full Legal Name of Party A], located at [Party A Address].\n Party B: [Full Legal Name of Party B], located at [Party B Address].\n\n1. LOYALTY COMMITMENT:\n\nBoth parties acknowledge the mutual value of their business relationship. They commit to work in good faith, ensuring a collaborative environment that prioritizes trust, loyalty, and shared objectives.\n\n2. NON-POACHING OF EMPLOYEES:\n\nFor the duration of this Agreement and [specific time after termination, e.g., \"for 12 months following its termination\"], neither Party A nor Party B shall, without the prior written consent of the other party:\n\na. Directly or indirectly solicit, induce, or encourage any employees of the other party to terminate their employment or to engage in employment or other services elsewhere.\nb. Hire, employ, or contract the services of any employee of the other party who has been employed by the said party within the last 12 months.\n\n3. BREACH:\n\nAny violation of the clauses in this Agreement will be deemed a material breach and may result in legal action or other remedies as available by law.\n\n4. GOVERNING LAW:\n\nThis Agreement is governed by the laws of [specific country/state, e.g., \"the State of California\"].\n\n5. AMENDMENTS:\n\nAny modifications to this Agreement must be in writing and signed by both parties.\n\nACKNOWLEDGEMENT:\n\nBy signing below, both parties affirm their understanding and acceptance of this Loyalty Agreement.",
|
|
||||||
"NON-COMPETE AND LOYALTY AGREEMENT\n\nEffective Date: [Date]\n\nBETWEEN:\n\n Business A: [Full Legal Name of Business A], located at [Business A Address].\n Business B: [Full Legal Name of Business B], located at [Business B Address].\n\n1. PURPOSE:\n\nThis Agreement is designed to protect the proprietary and business interests of both parties by ensuring loyalty and preventing competition during and after the period of collaboration or engagement.\n\n2. NON-COMPETE:\n\nFor the duration of this Agreement and [specific time after termination, e.g., \"for 24 months following its termination\"], neither party shall:\n\na. Engage in or support any venture that directly competes with the core business of the other party within [specific geographical region, e.g., \"the State of California\"].\nb. Invest in, partner with, or advise any business entity that competes directly with the other party.\n\n3. LOYALTY AND NON-POACHING:\n\nBoth parties pledge their commitment to a loyal business relationship. Specifically:\n\na. Neither party will, without the prior written consent of the other, solicit, induce, or encourage any employees or contractors of the other party to terminate their engagement or to join another business.\nb. Neither party shall disparage or encourage others to disparage the other party, its products, services, or its employees.\n\n4. CONFIDENTIALITY:\n\nBoth parties agree to maintain confidentiality regarding any proprietary or business-sensitive information exchanged during the course of this Agreement, ensuring that such information isn't disclosed without the explicit consent of the party owning that information.\n\n5. BREACH AND REMEDIES:\n\nA violation of any provision in this Agreement will be deemed a significant breach. The aggrieved party shall be entitled to seek injunctive relief, damages, or any other remedies available under the laws of [specific country/state, e.g., \"the State of California\"].\n\n6. GOVERNING LAW:\n\nThis Agreement shall be governed by and interpreted in accordance with the laws of [specific country/state, e.g., \"the State of California\"].\n\n7. AMENDMENTS:\n\nModifications or amendments to this Agreement must be in writing and duly signed by authorized representatives of both parties.\n\nACKNOWLEDGEMENT:\n\nBy signing below, representatives from both businesses affirm their understanding and acceptance of this Non-Compete and Loyalty Agreement.",
|
|
||||||
"AMENDMENT TO CONTRACT: LENGTH OF ENGAGEMENT\n\nThis Amendment is made on [Date], and amends the Non-Compete and Loyalty Agreement dated [Original Agreement Date] between:\n\n Business A: [Full Legal Name of Business A], located at [Business A Address].\n Business B: [Full Legal Name of Business B], located at [Business B Address].\n\nAMENDMENT:\n\nThe parties hereby agree to amend the Non-Compete and Loyalty Agreement as follows:\n\nSection [Specific Section Number, e.g., \"2\"] - Length of Engagement\n\nThe period of engagement between Business A and Business B as stipulated in the original Agreement is hereby extended/shortened/set to commence from [New Start Date] and conclude on [New End Date].\n\nGENERAL PROVISIONS:\n\n Except as specifically amended herein, all other terms and conditions of the original Agreement remain unchanged and in full effect.\n This Amendment, together with the original Agreement, represents the entire understanding between the parties.\n\nACKNOWLEDGEMENT:\n\nBoth parties, by their signatures below, indicate their acceptance and approval of this Amendment.",
|
|
||||||
"AMENDMENT TO CONTRACT: FEES FOR LATE DELIVERY\n\nThis Amendment is made on [Date], and amends the Agreement dated [Original Agreement Date] between:\n\n Business A: [Full Legal Name of Business A], located at [Business A Address].\n Business B: [Full Legal Name of Business B], located at [Business B Address].\n\nAMENDMENT:\n\nThe parties hereby agree to amend the Agreement as follows:\n\nSection [Specific Section Number, e.g., \"3\"] - Fees for Late Delivery\n\na. If Business A/B fails to deliver the products/services by the agreed-upon deadline, a late fee of [Specific Amount or Percentage, e.g., \"$100\" or \"5% of the total contract value\"] shall be applied for each [time period, e.g., \"day\"] of delay.\n\nb. The total late fees shall not exceed [Specific Maximum Amount or Percentage, e.g., \"$1,000\" or \"20% of the total contract value\"].\n\nc. The fees will be deducted from the final payment or invoiced separately, as deemed appropriate by the non-defaulting party.\n\nGENERAL PROVISIONS:\n\n Except as specifically amended herein, all other terms and conditions of the original Agreement remain unchanged and in full effect.\n This Amendment, together with the original Agreement, represents the entire understanding between the parties.\n\nACKNOWLEDGEMENT:\n\nBoth parties, by their signatures below, indicate their acceptance and approval of this Amendment.",
|
|
||||||
"AMENDMENT TO CONTRACT\n\nThis Amendment (the \"Amendment\") is entered into on [Date], between [Party One Name], hereinafter referred to as the \"First Party', and [Party Two Name], hereinafter referred to as the \"Second Party'', collectively referred to as the \"Parties\".\n\nWHEREAS, the Parties entered into a contract dated [Original Contract Date], hereinafter referred to as the \"Original Contract', for [Brief Description of the Original Contract, e.g., \"provision of IT services to First Party\"];\n\nWHEREAS, the Parties now wish to amend the Original Contract to add additional responsibilities pertaining to the maintenance of existing IT systems;\n\nNOW, THEREFORE, in consideration of the mutual covenants contained herein and for other good and valuable consideration, the receipt and sufficiency of which is hereby acknowledged, the Parties agree as follows:\n\n Additional Responsibility:\n\n The Second Party shall assume the responsibility of maintaining and ensuring the smooth functioning of the existing IT systems of the First Party. This responsibility includes, but is not limited to:\n\n a. Regular monitoring of the IT systems for any anomalies or issues.\n\n b. Prompt troubleshooting and rectification of any issues identified.\n\n c. Routine updates and patches to ensure the systems are secure and up-to-date.\n\n d. Any other related tasks as deemed necessary by the First Party.\n\n Compensation:\n\n As a result of this additional responsibility, the Parties agree to a revised compensation of [New Compensation Details, e.g., \"$XXX per month\"]. All other payment terms as outlined in the Original Contract shall remain unchanged.\n\n Duration and Termination:\n\n The duration and termination clauses from the Original Contract shall remain applicable to this Amendment unless otherwise agreed upon in writing by the Parties.\n\n Miscellaneous:\n\n All other terms and conditions of the Original Contract, which are not specifically amended by this Amendment, shall remain in full force and effect. In the event of a conflict between this Amendment and the Original Contract, the terms of this Amendment shall prevail.\n\n Entire Agreement:\n\n This Amendment, along with the Original Contract, constitutes the entire agreement between the Parties and supersedes any prior understandings, written or oral, relating to the subject matter of this Amendment.\n\n Governing Law:\n\n This Amendment shall be governed by the laws of [Jurisdiction, e.g., \"State of New York\"].",
|
|
||||||
"This appendix, part of the Contract between Party One and Party Two, sets milestones and deadlines for Party Two. \n\nMilestone 1 involves tasks such as gathering requirements, designing user interface etc. with the objective of developing initial prototype of a software application. Delivery Deadline: September 15, 2023. \n\nMilestone 2 involves tasks like incorporating feedback and conducting beta testing, aiming at finalizing and testing the software application. Delivery Deadline: October 15, 2023. \n\nEach milestone's completion will be reviewed by Party One; if requirements aren't met, Party Two may correct and resubmit. Payment terms and penalties are outlined in the main Contract. This appendix is executed adhering to main Contract's terms and conditions.",
|
|
||||||
"APPENDIX B \u2013 CHANGE IN TIME OF DELIVERY\n\nThis Appendix is an addendum to the contract (the \"Contract\") dated [Original Contract Date], entered into between [Party One Name], hereinafter referred to as the \"First Party', and [Party Two Name], hereinafter referred to as the \"Second Party\". The purpose of this Appendix is to amend and modify the delivery time as specified in the original Contract.\n\n Original Delivery Time: As per the terms of the original Contract, the delivery time was set for [Original Delivery Date, e.g., \"September 15, 2023\"].\n\n Revised Delivery Time: The Parties, through mutual agreement, have now decided to amend the delivery time. The new delivery date shall be [Revised Delivery Date, e.g., \"October 10, 2023\"].\n\n Reason for Change: [Provide a brief explanation for the change in delivery time, e.g., \"Due to unforeseen challenges in the production process, additional time is required to ensure that the deliverables meet the agreed-upon quality standards.\"]\n\n Consequences of Delay: Unless otherwise stated in the main body of the Contract:\n\n a. If the Second Party fails to meet the revised delivery time, penalties or consequences as outlined in the original Contract for late delivery will apply from the revised delivery date.\n\n b. All other terms related to late delivery, including but not limited to penalties, refunds, or rights to terminate, remain effective and unchanged by this Appendix.\n\n Prevailing Terms: All other terms and conditions of the original Contract not specifically amended by this Appendix shall remain in full force and effect. In the event of any inconsistency or conflict between the original Contract and this Appendix, the terms of this Appendix shall prevail with respect to the change in the delivery time.\n\n Acknowledgment: By signing this Appendix, the Parties acknowledge and agree to the revised delivery time and any associated consequences of delays.\n\nThis Appendix is executed as an acknowledgment and agreement to the revised delivery time and shall be considered an integral part of the original Contract.",
|
|
||||||
"APPENDIX C \u2013 ADDITIONAL CONFIDENTIAL INFORMATION\n\nThis Appendix is an extension of the contract (the \"Contract\") dated [Original Contract Date], between [Party One Name] (\"First Party\") and [Party Two Name] (\"Second Party\"). It outlines additional categories of confidential information beyond those detailed in the Contract.\n\n Additional Confidential Information Includes:\n\n a. Non-public financial data.\n\n b. Unpublished marketing strategies and materials.\n\n c. Upcoming product or service details.\n\n d. Proprietary software codes and processes.\n\n e. Personnel records.\n\n f. Any data labeled as \"Confidential\" or \"Proprietary\" after the Contract\u2019s execution.\n\n Protection & Exclusions:\n\nBoth Parties shall extend the same protection to this Additional Confidential Information as previously agreed upon in the Contract. Information that becomes public, is received rightfully from a third party, is independently developed, or gets written release authorization is excluded from confidentiality obligations.\n\n Duration:\n\nThe confidentiality obligations for this Appendix shall persist as defined in the Contract or, if unspecified, for [e.g., \"five years\"] from the disclosure date.\n\n Prevailing Terms:\n\nIf there\u2019s any conflict between this Appendix and the Contract concerning confidentiality, this Appendix takes precedence concerning Additional Confidential Information.\n\nExecuted as an integral part of the Contract.",
|
|
||||||
"AMENDMENT TO NON-DISCLOSURE AGREEMENT\n\nThis Amendment (the \u201cAmendment\u201d) is made and entered into as of [Amendment Date], by and between [Party A Name], having an address at [Party A Address] (\u201cParty A\u201d), and [Party B Name], having an address at [Party B Address] (\u201cParty B\u201d), collectively referred to as the \u201cParties.\u201d\n\nRECITALS\n\nWHEREAS, the Parties entered into a Non-Disclosure Agreement dated [Original NDA Date] (the \u201cOriginal Agreement\u201d);\n\nWHEREAS, the Parties desire to amend the Original Agreement to extend the duration of certain restrictions therein;\n\nNOW, THEREFORE, in consideration of the mutual covenants and promises made by the Parties hereto, the Parties agree as follows:\n\n Extension of Time Restrictions: The time restriction set forth in Section [X] of the Original Agreement, currently stating a period of [Original Time, e.g., \"two (2) years\"], is hereby amended and extended to [New Time, e.g., \"five (5) years\"] from the date of disclosure of the Confidential Information.\n\n Full Force and Effect: Except as expressly modified by this Amendment, all terms, conditions, and provisions of the Original Agreement shall remain in full force and effect. In the event of any conflict between the terms of this Amendment and the Original Agreement, the terms of this Amendment shall govern.\n\n Counterparts: This Amendment may be executed in counterparts, each of which shall be deemed an original and all of which together shall constitute one and the same instrument.\n\n Governing Law: This Amendment shall be governed by and construed in accordance with the laws of [Governing State or Country, e.g., \"the State of California\"], without regard to its conflict of laws principles.\n\nIN WITNESS WHEREOF, the Parties hereto have executed this Amendment as of the date first above written.",
|
|
||||||
"BUSINESS COOPERATION AGREEMENT\n\nThis Agreement is between [Business A Name], at [Business A Address] (\"Business A\"), and [Business B Name], at [Business B Address] (\"Business B\"), effective [Day, Month, Year].\n\n1. Purpose:\nBoth businesses will cooperate in [brief description, e.g., \"joint marketing\"].\n\n2. Responsibilities:\n\n Business A will: [Key obligation, e.g., \"Promote Business B in newsletters.\"]\n Business B will: [Key obligation, e.g., \"Display Business A products.\"]\n\n3. Term:\nEffective from the above date for [e.g., \"12 months\"]. Either party can terminate with [e.g., \"30 days\"] notice.\n\n4. Confidentiality:\nConfidential information remains private, during and post-agreement.\n\n5. Governing Law:\nGoverning laws of [State/Country, e.g., \"California\"].\n\n6. Amendments:\nChanges must be written and signed by both parties.",
|
|
||||||
"APPENDIX TO BUSINESS COOPERATION AGREEMENT\n\nEXTENSION OF CONFIDENTIALITY CONDITIONS\n\nThis Appendix is made as of [Day, Month, Year], and is appended to the Business Cooperation Agreement dated [Original Agreement Date] (\"Original Agreement\") between [Business A Name], located at [Business A Address] (\"Business A\") and [Business B Name], located at [Business B Address] (\"Business B\").\n\n1. Extension of Confidentiality Period:\nThe confidentiality period stipulated in Section 4 (or the appropriate section number) of the Original Agreement is hereby extended. Previously set to expire [Original Expiry Date], it will now extend to [New Expiry Date].\n\n2. Continued Obligations:\nAll other confidentiality obligations and conditions outlined in the Original Agreement remain unchanged and in full effect.\n\n3. Entire Agreement:\nThis Appendix, in conjunction with the Original Agreement, constitutes the entire agreement between the parties regarding the subject matter herein.\n\n4. Governing Law:\nThis Appendix shall be governed by the laws of [State/Country, e.g., \"California\"].\n\nIN WITNESS WHEREOF, both parties hereto have executed this Appendix as of the date first above written.",
|
|
||||||
"APPENDIX: LOYALTY CLAUSE\n\nEffective [Day, Month, Year], attached to the Agreement dated [Original Agreement Date] between [Party A Name] (\"Party A\") and [Party B Name] (\"Party B\").\n\n1. Loyalty Commitment:\nFor one year from the Effective Date, both parties pledge loyalty by refraining from activities harmful or competitive to the other within the context of the Agreement.\n\n2. Consequences:\nBreaches may result in Agreement termination and legal action as per the original terms.\n\n3. Governing Law:\nGoverned by the laws of [State/Country, e.g., \"California\"].",
|
|
||||||
"APPENDIX: CONFIDENTIALITY CLAUSE\n\nThis Appendix is appended to the B2B Contractor Agreement (\"Agreement\") dated [Original Agreement Date, e.g., \"August 15, 2023\"] between [Company Name], hereinafter referred to as \"Company', and [Contractor Name], hereinafter referred to as \"Contractor\".\n\n1. Confidentiality:\n\n1.1 Both Company and Contractor acknowledge that they may have access to or receive information during the term of the Agreement which is confidential to the disclosing party (\"Confidential Information\").\n\n1.2 Confidential Information shall not include information that:\n\n is or becomes public knowledge without breach of this clause;\n was known by the receiving party before receipt from the disclosing party;\n is received from a third party without breach of any obligation of confidentiality.\n\n1.3 The receiving party shall:\n\n use the Confidential Information only for performing under the Agreement;\n take all reasonable precautions to prevent any unauthorized disclosure of the Confidential Information;\n not disclose, reproduce, or distribute Confidential Information without the written consent of the disclosing party.\n\n2. Duration:\n\nThe obligations set forth in this Appendix shall continue for a period of [e.g., \"two years\"] from the date of termination or expiration of the Agreement.\n\n3. Return or Destruction:\n\nUpon the expiration or termination of the Agreement, or upon the disclosing party's request, the receiving party shall return or, if directed by the disclosing party, destroy all copies of the Confidential Information.\n\n4. Governing Law:\n\nThis Appendix shall be governed by the laws of [State/Country, e.g., \"California\"], consistent with the Agreement.",
|
|
||||||
"APPENDIX: CONFIDENTIALITY CLAUSE\n\nThis Appendix is part of the Agreement dated [Original Agreement Date, e.g., \"August 15, 2023\"] between [Company Name] (\"Company\") and [Contractor Name] (\"Contractor\").\n\n1. Confidential Information:\nBoth parties may access or receive the other's confidential information (\"Confidential Information\") during the Agreement term. Confidential Information excludes publicly known details, data known prior, or information obtained from third parties without confidentiality obligations.\n\n2. Obligations:\nThe recipient shall:\n\n Use the Confidential Information solely for the Agreement's purpose.\n Prevent unauthorized disclosures.\n Not disclose without prior written consent.\n\n3. Duration:\nObligations persist for [e.g., \"two years\"] post Agreement termination or expiration.\n\n4. Return/Destruction:\nUpon Agreement conclusion, or on request, all Confidential Information copies should be returned or destroyed.\n\n5. Governing Law:\nGoverned by [State/Country, e.g., \"California\"] laws.",
|
|
||||||
"NON-DISCLOSURE AGREEMENT (NDA)\n\nEffective [Effective Date, e.g., \"August 15, 2023\"], between [Tech Company Name], located at [Tech Company Address], (\"Company\") and [Contractor's Full Name], located at [Contractor Address], (\"Contractor\").\n\nPurpose:\nContractor will access Company's confidential information during their engagement.\n\n1. Definition:\n\"Confidential Information\" means proprietary data related to the Company\u2019s business, excluding publicly known details, prior known information, or data from third parties without confidentiality bounds.\n\n2. Obligation:\nContractor shall:\n\n Use Confidential Information solely for engagement purposes.\n Prevent unauthorized disclosure.\n\n3. Duration:\nObligations persist for [e.g., \"two years\"] from disclosure date.\n\n4. Return:\nContractor shall return all Confidential Information items upon engagement completion or Company's request, retaining no copies.\n\n5. Remedies:\nBreach may result in legal actions, damages, and costs.\n\n6. Governing Law:\nGoverned by [State/Country, e.g., \"California\"] laws.",
|
|
||||||
"APPENDIX: EXTENSION OF CONTRACT DURATION\n\nThis Appendix is a part of the Agreement initially dated [Original Agreement Date, e.g., \"August 15, 2021\"], between [Party A Name], located at [Party A Address] (\"Party A\") and [Party B Name], located at [Party B Address] (\"Party B\").\n\n1. Duration Extension:\nThe duration of the Agreement referenced above is hereby extended for an additional two (2) years from the original expiration date. With this extension, the new expiration date of the Agreement will be [New Expiration Date, e.g., \"August 15, 2025\"].\n\n2. All Other Terms Remain Unchanged:\nExcept for the extension of the contract duration as described herein, all other terms and conditions of the Agreement remain unchanged and in full effect.\n\n3. Entire Agreement:\nThis Appendix, in conjunction with the original Agreement, constitutes the entire agreement between Party A and Party B. Any previous understandings, written or oral, relating to the subject of this Appendix are superseded by the terms herein.\n\n4. Governing Law:\nThis Appendix shall be governed by the laws of [State/Country, e.g., \"California\"], consistent with the original Agreement.",
|
|
||||||
"CONFIDENTIALITY AGREEMENT\n\nEffective [Effective Date, e.g., \"August 15, 2023\"], between [Company Name], located at [Company Address] (\"Company\"), and [Supplier Name], located at [Supplier Address] (\"Supplier\").\n\n1. Definition:\n\"Confidential Information\" means proprietary data of the Company, excluding:\n\n Pre-disclosed or publicly known data.\n Info from third parties without confidentiality bounds.\n\n2. Obligations:\nSupplier will:\n\n Use Confidential Information solely for business purposes with the Company.\n Protect its secrecy and prevent unauthorized disclosure.\n Return or destroy all Confidential Information upon request or business completion.\n\n3. Duration:\nObligations last for [e.g., \"two years\"] from disclosure date.\n\n4. Remedies:\nBreaches may result in legal actions, damages, and costs by the Company.\n\n5. Governing Law:\nGoverned by [State/Country, e.g., \"California\"] laws.",
|
|
||||||
"APPENDIX: BREACH CONSEQUENCES\n\nRelated to the Agreement on [Original Agreement Date, e.g., \"August 15, 2023\"], between [Party A Name] (\"Party A\") and [Party B Name] (\"Party B\").\n\n1. Notification:\nSuspected breaches must be reported in writing by the non-breaching party.\n\n2. Rectification:\nThe breaching party has [e.g., \"14 days\"] from notification to rectify, unless irreparable.\n\n3. Fees:\nBreaches incur a penalty of [e.g., \"$10,000\"], aside from claimed damages.\n\n4. Legal Actions:\nUnresolved or damaging breaches may lead to legal actions, including injunctive relief, damages, and legal fees.\n\n5. Termination:\nRepeated or severe breaches can cause Agreement termination by the non-breaching party.\n\n6. Law:\nGoverned by [State/Country, e.g., \"California\"] laws.",
|
|
||||||
"APPENDIX: TERMS OF CONTRACT TERMINATION\n\nRelated to the Agreement on [Original Agreement Date, e.g., \"August 15, 2023\"], between [Party A Name] (\"Party A\") and [Party B Name] (\"Party B\").\n\n1. Termination for Breach:\nIf either party breaches any conditions of the Agreement, the non-breaching party may terminate the Agreement immediately upon written notice to the breaching party.\n\n2. Termination by Notice:\nEither party may terminate the Agreement for any reason by providing a written notice to the other party. The termination will become effective 30 days after the receipt of such notice.\n\n3. Obligations Upon Termination:\nUpon termination, all rights and obligations under the Agreement will cease, except for those which by their nature should survive termination (e.g., confidentiality, liability for prior breaches, etc.).\n\n4. Governing Law:\nThis Appendix, and any disputes arising from it, will be governed by the laws of [State/Country, e.g., \"California\"], consistent with the original Agreement.",
|
|
||||||
"APPENDIX: OBLIGATIONS UPON TERMINATION\n\nPertaining to the Agreement dated [Original Agreement Date, e.g., \"August 15, 2023\"], between [Party A Name] (\"Party A\") and [Party B Name] (\"Party B\").\n\n1. Return of Property:\nUpon termination, each party shall promptly return to the other all property, materials, and assets belonging to the other party, unless otherwise specified in the Agreement.\n\n2. Confidential Information:\nBoth parties shall continue to abide by any confidentiality obligations set forth in the Agreement. Any confidential information must be returned or destroyed, as instructed by the owning party.\n\n3. Outstanding Payments:\nAll due payments must be settled within [e.g., \"14 days\"] of termination, as per the terms of the original Agreement.\n\n4. Non-Disparagement:\nBoth parties agree not to make any derogatory or disparaging statements about the other party post-termination.\n\n5. Survival of Provisions:\nAny provisions in the Agreement that, by their nature, should persist beyond termination (e.g., indemnity, liability, confidentiality) will continue to be in effect.\n\n6. Notifications:\nEach party must inform their respective stakeholders, if necessary, about the termination in a manner that maintains the goodwill and reputation of both parties.\n\n7. Transition Assistance:\nTo ensure a smooth transition, both parties agree to cooperate, as reasonably requested by the other, for a period of [e.g., \"30 days\"] after termination.\n\n8. Governing Law:\nThis Appendix is governed by the laws of [State/Country, e.g., \"California\"], consistent with the original Agreement.",
|
|
||||||
"NON-DISCLOSURE AGREEMENT (NDA)\n\nEffective [Date, e.g., \"August 15, 2023\"], between [Client Name], (\"Client\") and [Business Name], (\"Business\").\n\nPurpose:\nProtection of confidential information exchanged due to potential collaboration.\n\n1. Confidentiality:\nBusiness agrees to keep secret all Confidential Information shared by Client.\n\n2. Definition:\n\"Confidential Information\" is non-public data shared by either party, excluding info that's publicly available, already known, or received without confidentiality constraints.\n\n3. Duration:\nObligations last [e.g., \"two years\"] from the date of disclosure.\n\n4. Return/Destruction:\nUpon Client's request, Business will return or destroy all Confidential Information.\n\n5. Remedies:\nUnauthorized disclosures may lead to legal action by Client, including damages.\n\n6. Law:\nGoverned by [State/Country, e.g., \"California\"] laws.",
|
|
||||||
"IT SERVICES AGREEMENT\n\nEffective Date: [Date, e.g., \"August 15, 2023\"]\n\nParties:\n\n [Client Name], located at [Client Address] (\"Client\")\n [Service Provider Name], located at [Service Provider Address] (\"Provider\")\n\nScope of Work:\nProvider agrees to offer IT services, including [e.g., \"network setup, software installation, and routine maintenance\"], as detailed in Attachment A.\n\nPayment:\nClient shall pay Provider [e.g., \"$1,000\"] per month. Invoices will be sent monthly and are due within [e.g., \"30 days\"].\n\nDuration:\nThis Agreement starts on [Start Date] and ends on [End Date], unless terminated earlier.\n\nTermination:\nEither party may terminate with [e.g., \"30 days\"] written notice. Upon termination, any unpaid fees for services rendered become immediately due.\n\nConfidentiality:\nBoth parties agree to keep all business and technical information confidential.\n\nLimitation of Liability:\nProvider's liability is limited to the amount paid by the Client for the specific service causing damage.\n\nGoverning Law:\nThis Agreement is governed by the laws of [State/Country, e.g., \"California\"].\n\nEntire Agreement:\nThis constitutes the full agreement between both parties.",
|
|
||||||
"CONFIDENTIALITY AMENDMENT TO NDA\n\nThis Amendment, effective [Date, e.g., \"August 15, 2023\"], modifies the NDA dated [Original Agreement Date] between [Party A Name] (\"Party A\") and [Party B Name] (\"Party B\").\n\n1. Responsibilities:\n\na) Protection: Parties must safeguard Confidential Information at least as they do their own.\n\nb) Access: Access is limited to those needing it who are also bound by confidentiality.\n\nc) Breach Notification: Parties must immediately inform the other of any breaches.\n\nd) Return/Destruction: Upon request or agreement end, parties must return or certify the destruction of Confidential Information.\n\ne) No Reverse Engineering: Receiving party shall not reverse engineer any provided items.\n\n2. Remedies:\nUnauthorized disclosures permit injunctive relief and other legal remedies.\n\n3. Original Agreement:\nExcept for this Amendment, the NDA remains unchanged.\n\nGoverning Law:\nAs per the NDA.",
|
|
||||||
"LOYALTY AGREEMENT\n\nThis Agreement (\"Agreement\") is made as of [Date, e.g., \"August 15, 2023\"], between:\n\n [Party A Name], with its principal office at [Party A Address] (\"Party A\"),\n [Party B Name], with its principal office at [Party B Address] (\"Party B\").\n\nPurpose:\nThe parties wish to collaborate and establish a loyal relationship in their joint business endeavors.\n\n1. Loyalty Commitment:\n\na) Both parties commit to act in good faith and refrain from engaging in any activity or partnership that might conflict with the interests of the other party during the term of this Agreement.\n\nb) Neither party shall assist, collaborate, or engage with third parties that may cause harm or disrepute to the other party.\n\nc) Each party shall prioritize the other's interests in situations where opportunities arise from their collaboration.\n\n2. Non-Solicitation:\nDuring the term of this Agreement, and for [e.g., \"one year\"] thereafter, neither party shall solicit or attempt to entice away any clients, customers, or employees of the other party.\n\n3. Duration:\nThis Agreement will begin on the Effective Date and remain in effect for [e.g., \"two years\"] unless terminated earlier by mutual consent.\n\n4. Termination:\nEither party may terminate this Agreement with [e.g., \"30 days\"] written notice if the other party breaches any term herein.\n\n5. Confidentiality:\nBoth parties agree to maintain the confidentiality of all proprietary or non-public information obtained during the collaboration.\n\n6. Governing Law:\nThis Agreement is governed by the laws of [State/Country, e.g., \"California\"].\n\n7. Entire Agreement:\nThis document constitutes the full understanding between both parties, superseding all prior discussions, agreements, or understandings.",
|
|
||||||
"BUSINESS CONSULTING CONTRACT\n\nThis Consulting Contract (\"Contract\") is made as of [Date, e.g., \"August 15, 2023\"], between:\n\n [Client Name], with its principal office at [Client Address] (\"Client\"),\n [Consultant Name], with its principal office at [Consultant Address] (\"Consultant\").\n\nPurpose:\nThe Consultant will provide professional consulting services to the Client as described below.\n\n1. Scope of Services:\nConsultant agrees to offer services including, but not limited to:\na) Business strategy development\nb) Market analysis\nc) [Other services as needed]\nAny additional services will require an amendment to this Contract.\n\n2. Compensation:\nFor services rendered, the Client shall pay the Consultant [e.g., \"$100\"] per hour. Invoices will be issued [e.g., \"monthly\"] and are due within [e.g., \"30 days\"] of receipt.\n\n3. Duration:\nThis Contract begins on [Start Date] and ends on [End Date], unless extended by mutual agreement or terminated earlier.\n\n4. Termination:\nEither party can terminate this Contract with [e.g., \"30 days\"] written notice. In case of termination, the Client will pay for services rendered up to the notice date.\n\n5. Confidentiality:\nThe Consultant shall maintain the confidentiality of all proprietary information received during the engagement, unless obligated by law to disclose.\n\n6. Non-compete:\nFor [e.g., \"six months\"] after Contract termination, the Consultant agrees not to provide similar services to any direct competitor of the Client within [e.g., \"50 miles\"] of the Client's primary location.\n\n7. Independent Contractor:\nThe Consultant is an independent contractor and not an employee of the Client.\n\n8. Governing Law:\nThis Contract shall be governed by and interpreted under the laws of [State/Country, e.g., \"California\"].\n\n9. Entire Agreement:\nThis Contract represents the entire understanding between both parties, superseding all prior negotiations, discussions, or agreements.",
|
|
||||||
"APPENDIX A: CONFIDENTIALITY BREACH FEES\n\nThis Appendix is attached to and made part of the Contract (\"Original Contract\") dated [Original Contract Date], between [Party A Name] (\"Party A\") and [Party B Name] (\"Party B\").\n\n1. Purpose:\nThis Appendix defines the fees and penalties associated with any breach of confidentiality as stipulated in the Original Contract.\n\n2. Confidentiality Breach Fee:\nIn the event of a breach of the confidentiality provisions in the Original Contract by either party:\n\na) The breaching party will be liable for an immediate penalty of [specific amount, e.g., \"$10,000\"].\n\nb) If the breach results in any direct financial loss to the non-breaching party, the breaching party shall additionally reimburse the non-breaching party for the full amount of such loss.\n\nc) The breaching party will also bear all costs, including legal fees, that the non-breaching party incurs while addressing or remedying the breach.\n\n3. Payment Terms:\nPayment of any penalty or reimbursement as defined above shall be made within [e.g., \"30 days\"] of written notification of the breach.\n\n4. Disputes:\nAny disputes related to this Appendix shall be resolved as stipulated in the dispute resolution clause of the Original Contract.\n\n5. Continuation of Original Contract:\nExcept as modified by this Appendix, the Original Contract remains in full force and effect.\n\n6. Governing Law:\nThis Appendix, consistent with the Original Contract, is governed by the laws of [State/Country, e.g., \"California\"].",
|
|
||||||
"APPENDIX A: STRICT CONFIDENTIALITY BREACH PENALTIES\n\nThis Appendix is annexed to the Contract dated [Original Contract Date] between [Party A Name] and [Party B Name].\n\n1. Breach Fees:\nIf a party breaches confidentiality:\n\na) Immediate penalty: [e.g., \"$50,000\"].\n\nb) For reputational harm or business loss: Additional [e.g., \"$100,000\"].\n\nc) Full reimbursement for direct financial losses caused by the breach.\n\nd) All associated legal and remedy costs borne by the breaching party.\n\n2. Remedial Actions:\nThe breaching party must swiftly rectify the breach, potentially including public apologies or recalling disclosed information.\n\n3. Payment:\nDue within [e.g., \"15 days\"] of breach notification.\n\n4. Termination:\nNon-breaching party can immediately terminate the main contract upon a breach.\n\n5. Governing Law:\nThis Appendix adheres to [State/Country, e.g., \"California\"] laws."
|
|
||||||
]
|
|
||||||
@ -1,45 +0,0 @@
|
|||||||
# Keyword Counting
|
|
||||||
|
|
||||||
The use case in this directory computes the frequencies of occurring countries
|
|
||||||
in a long passage of text. We provide implementations of seven different approaches:
|
|
||||||
- IO
|
|
||||||
- Chain-of-Thought (CoT)
|
|
||||||
- Tree of Thought (ToT):
|
|
||||||
- ToT: wider tree, meaning more branches per level
|
|
||||||
- ToT2: tree with more levels, but fewer branches per level
|
|
||||||
- Graph of Thoughts (GoT):
|
|
||||||
- GoT4: split passage into 4 sub-passages
|
|
||||||
- GoT8: split passage into 8 sub-passages
|
|
||||||
- GoTx: split by sentences
|
|
||||||
|
|
||||||
## Data
|
|
||||||
|
|
||||||
We provide an input file with 100 samples: `countries.csv`. It is also possible to use
|
|
||||||
the data generator `dataset_gen_countries.py` to generate additional or
|
|
||||||
different samples (using GPT-4). The parameters can be updated on line 54 (number of samples to be generated).
|
|
||||||
Note that not every generated sample will be included in the dataset, as each sample is
|
|
||||||
additionally tested for validity (observe script output for details).
|
|
||||||
|
|
||||||
## Execution
|
|
||||||
|
|
||||||
The file to execute the use case is called
|
|
||||||
`keyword_counting.py`. In the main body, one can
|
|
||||||
select the specific samples to be run (variable samples) and the
|
|
||||||
approaches (variable approaches). It is also possible to set a budget in
|
|
||||||
dollars (variable budget).
|
|
||||||
|
|
||||||
The Python scripts will create the directory `result`, if it is not
|
|
||||||
already present. In the `result` directory, another directory is created
|
|
||||||
for each run: `{name of LLM}_{list of approaches}_{day}_{start time}`.
|
|
||||||
Inside each execution specific directory two files (`config.json`,
|
|
||||||
`log.log`) and a separate directory for each selected approach are
|
|
||||||
created. `config.json` contains the configuration of the run: input data,
|
|
||||||
selected approaches, name of the LLM, and the budget. `log.log` contains
|
|
||||||
the prompts and responses of the LLM as well as additional debug data.
|
|
||||||
The approach directories contain a separate json file for every sample
|
|
||||||
and the file contains the Graph Reasoning State (GRS) for that sample.
|
|
||||||
|
|
||||||
## Plot Data
|
|
||||||
|
|
||||||
Change the results directory in line 150 of `plot.py` and run `python3
|
|
||||||
plot.py` to plot your data.
|
|
||||||
@ -1,101 +0,0 @@
|
|||||||
ID,Text,Countries,Sentences,Characters
|
|
||||||
0,"One evening, Sarah, an archaeologist from Norway made a surprising discovery about ancient trade routes between Sweden and Norway. As per her research, the artifacts that were found in Norway were identical to those in Sweden, indicating a deep-rooted cultural connection between Sweden and Norway. This piqued the interest of her colleague, James, who was from Canada. He had been researching the indigenous tribes of Canada and found many similarities with tribes from his neighboring country, the United States. James had always been interested in the historical ties between Canada and United States, and his study further confirmed the age-old connections between the two countries. Upon hearing James's story, Sarah shared a fascinating anecdote from her travels in Portugal. She recalled how locals loved to tell the tale of the shared history between Spain and Portugal. Her anecdotes about Spain and Portugal echoed the same sense of shared culture and past, just like in the case of Norway and Sweden, and Canada and United States. Their conversation reminded James of his stay in South Korea, where he had learned about the close relationship between North Korea and South Korea, despite their current political divide. He recalled stories about the shared history of North Korea and South Korea, whose deep-seated cultural ties transcended political boundaries. Sarah, who had been to Australia, reciprocated with her own experiences of the bond between Australia and New Zealand. She described how, despite geographical separation, Australia and New Zealand shared a unique camaraderie and close historical ties. As they exchanged stories, their conversation moved to South Africa and its various connections with its neighbouring country, Zimbabwe. Sarah shared stories she had heard about the intricate bond between South Africa and Zimbabwe, showcasing the age-old interactions between these two nations. It left them both reflecting on the timeless bonds that connect nations across the world, from Norway to Australia, Canada to Zimbabwe, and all the countries in between.","[Norway, Sweden, Norway, Norway, Sweden, Sweden, Norway, Canada, Canada, United States, Canada, United States, Portugal, Spain, Portugal, Spain, Portugal, Norway, Sweden, Canada, United States, South Korea, North Korea, South Korea, North Korea, South Korea, Australia, Australia, New Zealand, Australia, New Zealand, South Africa, Zimbabwe, South Africa, Zimbabwe, Norway, Australia, Canada, Zimbabwe]",15,2090
|
|
||||||
1,"My friend, Alex from Peru, once recounted his journey to Argentina where he learned about the strong cultural ties between Argentina and Brazil due to their shared history. He spoke fondly of his time in Argentina and Brazil, marveling at the similar music, dance, and culinary traditions that seamlessly bridged the borders of these countries. It reminded me of a documentary I'd watched about the ancient Silk Road that spanned across Iran and China. It touched upon the shared historical narratives of Iran and China, highlighting how goods, ideas, and culture flowed between these two countries for centuries. Intriguingly, the documentary also brought up some parallels between this eastern route and the vibrant exchange between Italy and France during the Renaissance. The cultural and intellectual bonds between Italy and France were indeed profound, just as the resilient trade relations that existed between the United States and Canada. The United States and Canada, apart from their geographical proximity, shared an economic bond that strongly influenced their policies and international standing. Similarly, the ties between Australia and New Zealand shed light on their gestalt identity in the Pacific region. Despite their unique characteristics, Australia and New Zealand were often viewed as a single entity due to their remarkably similar backgrounds in terms of culture, language, and colonial history. Inspired by these chronicles of interconnectedness, I decided to delve deeper into history and found a fascinating account of how Ukraine and Poland had influenced each other through years of coexistence. Despite their tumultuous past, Ukraine and Poland shared a cultural tapestry that was deeply woven into their histories. It was quite similar to the complex relationship between North Korea and South Korea, which, despite their political differences, shared common heritage and traditions. Thus, from Argentina to South Korea, the world was an intricate web of countries intertwined with each other through shared histories, cultures, and sometimes, shared destinies.","[Peru, Argentina, Argentina, Brazil, Argentina, Brazil, Iran, China, Iran, China, Italy, France, Italy, France, United States, Canada, United States, Canada, Australia, New Zealand, Australia, New Zealand, Ukraine, Poland, Ukraine, Poland, North Korea, South Korea, Argentina, South Korea]",13,2095
|
|
||||||
2,"Annie, a marine biologist from Ireland, had always been fascinated by the diverse sealife surrounding the coasts of Australia. She often said that the waters of Australia were some of the most biodiverse in the world, enticing researchers like her from faraway lands like Ireland. Through her explorations, she developed a particular interest in the migratory patterns of certain species, noting that they often ventured towards New Zealand. Annie noticed that from Australia, these marine creatures would cross oceans to reach the coasts of New Zealand, proving the interconnectedness of these ecosystems. Driven by this curiosity, she published her research, attracting the attention of a fellow scientist, Greg, from Canada. Inspired by her findings, Greg shared his own studies on migration patterns, noting a similar scenario between Canada and the United States. He'd found that many animal species in Canada would cross the border into the United States during certain seasons, emphasizing the interplay between these two nations. Annie and Greg started to collaborate, which led their research towards Mexico, where they discovered another fascinating migration phenomenon. They found it intriguing that birds from the United States would fly south, bypassing their own habitats to spend the winter in the warmer climates of Mexico. Their research in Mexico led them to another colleague, Maria, from Spain. Maria had been studying the movements of the Iberian Lynx which often roamed between Spain and Portugal. Maria's research had shown that just like the animals in North America, the Lynx in Spain had established territories in Portugal. Intrigued by the similarities, they expanded their study to cover the movement of bird species between Belgium and the Netherlands. Just as they'd seen in the Americas, they noticed that certain species of birds in Belgium would travel to the Netherlands during certain periods of the year. These fascinating findings opened up new dimensions of understanding for the interconnectedness of the world. From the sealife in Australia to the birds in Belgium, the animal kingdom had its own ways of establishing connections between countries.","[Ireland, Australia, Australia, Ireland, New Zealand, Australia, New Zealand, Canada, Canada, United States, Canada, United States, Mexico, United States, Mexico, Mexico, Spain, Spain, Portugal, Spain, Portugal, Belgium, Netherlands, Belgium, Netherlands, Australia, Belgium]",16,2190
|
|
||||||
3,"During their adventurous journey around the world, Jack and Emily first found themselves in the vibrant markets of India, enamoured by the rich traditions from India reflected in every corner. Their journey then led them to the bustling cityscapes of China, filled with modern marvels that somehow coexisted with ancient traditions from China's past. They then travelled to Australia, where they got lost in the wilderness, and shared stories of the rich biodiversity found in Australia under the southern constellations. Their travel then took them to Argentina, where they were captivated by the diversity of landscapes, from the colossal mountains to the serene lakes of Argentina. They couldn't help but notice the stark contrast to Brazil, which boasted its own unique geographical features and diverse environments, mirroring the diversity of lifestyles in Brazil. Jack, recalling his childhood passion, mentioned his fondness for Italy and its rich history. Emily, sharing his sentiment, mentioned the transcendent beauty of Italy's architecture and the remarkable tales from Italy's past. Soon, their discussion shifted to the dynamic nightlife of South Korea, which they said was unlike anything they had experienced before. South Korea, they described, was the perfect blend of the old and the new, where rich traditions met modern influences. They expressed their shared love for Ireland, with its lush green landscapes and the warm hospitality of the people of Ireland. Their tales also touched upon the unparalleled tranquility of Switzerland and the cultural harmony they found there. Switzerland, they both agreed, was a testament to the beauty of diversity, coexistence and peace. Their curiosity then led them to discuss their memorable visit to Mexico, and the unmatched flavors of cuisine from Mexico that left an indelible mark on their taste buds. As the night grew darker, their conversation veered towards the mystical landscapes of Egypt, where ancient history whispered from every corner of Egypt. As they traversed through their memories, they couldn't help but look forward to their upcoming trip to Kenya, hoping that their experiences in Kenya would be just as enriching and unforgettable. As they parted for the night, they went to sleep with dreams of their world journey, from India to Kenya, brimming in their hearts.","[India, India, China, China, Australia, Australia, Argentina, Argentina, Brazil, Brazil, Italy, Italy, Italy, South Korea, South Korea, Ireland, Ireland, Switzerland, Switzerland, Mexico, Mexico, Egypt, Egypt, Kenya, Kenya, India, Kenya]",16,2350
|
|
||||||
4,"The origins of James, an anthropologist from Australia, traced back to Ireland, a land of heritage and folklore. He often marveled at the similarities between the indigenous cultures of Australia and the rich, traditional societies of Ireland. His fascination grew more profound when he learned about the ancient ties between Ireland and the United Kingdom. Both the United Kingdom and Ireland, he discovered, had histories deeply intertwined, from shared rulers to common cultural practices. His friend, Laura, an archaeologist from the United States, shared his interest in tracing historical connections between nations. She had closely studied the cultural exchanges between the United States and Mexico, intrigued by their shared border and the resultant blend of customs and traditions. The bond between the United States and Mexico was a testament to their shared history of migrations, trade, and even conflicts. Laura also recounted her research on Peru and the cultural parallels it held with Brazil. She discovered that both Peru and Brazil shared ties through the Amazon rainforest, with indigenous communities trading goods and knowledge over centuries. James, on his part, spoke about his recent research on the historical relationship between South Africa and Zimbabwe. He found that South Africa and Zimbabwe shared a lot of similarities in terms of language, culture, and history, owing to their geographical proximity. Drawing on her experience from a trip to Asia, Laura mentioned her encounter with the intriguing links between China and North Korea. She noted that China and North Korea, despite their political differences, shared historical and cultural ties that dated back centuries. The conversation ended on a lighter note, with both of them reminiscing about their culinary adventures in Italy and France. Both Italy and France, they agreed, had unparalleled culinary scenes, deeply rooted in their culture and history. Their discussion underlined the beauty of shared narratives and cultural exchanges across the world.","[Australia, Ireland, Australia, Ireland, Ireland, United Kingdom, United Kingdom, Ireland, United States, United States, Mexico, United States, Mexico, Peru, Brazil, Peru, Brazil, South Africa, Zimbabwe, South Africa, Zimbabwe, China, North Korea, China, North Korea, Italy, France, Italy, France]",16,2048
|
|
||||||
5,"On his journey across the globe, Mark found himself fascinated by the cultural blend in Argentina where the tango originated. His time in Argentina sparked his interest in Brazil, and he took a short plane ride to the country known for its vast Amazon rainforest. The allure of Brazil was intoxicating, its vibrant festivals and unique wildlife were unlike anything he'd previously seen. He'd soon learn that the wildlife in Australia, however, was equally captivating. While in Australia, Mark found himself charmed by the affectionate kangaroos and aloof koalas. After Australia, he headed towards Asia, beginning at the vibrant country of India. India's cultural diversity, historical grandeur, and the warmth of the people enticed him to spend more time exploring the country. Another country that captured his heart in the same way was China, with its rich history and splendid landscapes. China amazed him with its ancient monuments and the Great Wall, leaving him yearning for more such experiences. Such yearning led him to Egypt, home to the majestic Pyramids. The history and mysteries that shrouded Egypt kept him intrigued, so much so that he extended his stay. Encouraged by his experiences in Egypt, Mark decided to venture further into the landscapes of Africa and arrived at Kenya. In Kenya, he was awestruck by the vast savannahs and spectacular wildlife. His adventures in Kenya made him realize the untapped beauty of Africa, prompting him to extend his African journey to South Africa. In South Africa, he was spellbound by the natural beauty of the landscapes, making him fall in love with Africa. His next destination was Europe, where he found himself fascinated by the architectural grandeur of Italy and the historical stories that surrounded every alleyway. Italy, with its enchanting cities and delicious food, left an unforgettable impression on him. The final leg of his journey led him to cold but beautiful Norway, where he witnessed the spectacular Northern Lights, ending his journey on a high note.","[Argentina, Argentina, Brazil, Brazil, Australia, Australia, Australia, India, India, China, China, Egypt, Egypt, Egypt, Kenya, Kenya, Kenya, South Africa, South Africa, Italy, Italy, Norway]",18,2032
|
|
||||||
6,"As Hannah walked down the bustling streets of Tokyo, Japan, she felt the energy of the city. Japan, with its blend of tradition and modernity, was nothing like her quiet hometown in Ireland. Upon returning to Ireland, she was greeted with the serene green landscape, a stark contrast to the skyscrapers of Japan, but equally beautiful in its own way. After her time in Japan and Ireland, she couldn't help but feel restless, wanting to explore more of the world. Her next adventure took her to the vibrant city of Rio in Brazil, famed for its lively carnivals. Brazil’s rich culture and friendly people left a deep impression on Hannah, making her yearn for more such experiences. Her friend, Alejandro from Mexico, suggested that she visit his home country next. In Mexico, she was awestruck by the historic architecture and the vibrant, bustling markets. The food of Mexico, in particular, was a revelation to her. After Mexico, she headed to France. France's elegance, be it in their language, fashion, or gastronomy, left her fascinated. She spent hours sitting in cafes in France, watching the world go by, and feeling a sense of peace that she hadn't felt for a long time. After France, her journey took her to the far East, to China. In China, she found a dynamic cityscape intermingling with centuries-old architecture. China’s history fascinated her, and her journey through the country felt like traveling through time. Her last stop was Australia and she spent weeks exploring the diverse landscapes of the country. From the bustling cities to the serene Outback, Australia was a beautiful end to her travels. Her experiences in Japan, Ireland, Brazil, Mexico, France, China, and Australia had given her a new perspective, a deeper appreciation for the beauty in diversity.","[Japan, Japan, Ireland, Ireland, Japan, Japan, Ireland, Brazil, Brazil, Mexico, Mexico, Mexico, Mexico, France, France, France, France, China, China, China, Australia, Australia, Japan, Ireland, Brazil, Mexico, France, China, Australia]",18,1784
|
|
||||||
7,"During his travels in Italy, John discovered a remarkable tale about the intricate trade networks of the past that connected Italy, Greece, and Turkey. Drawn in by the fascinating history, he began researching further about Italy, particularly its ancient maritime links with Greece and Turkey, as well as their shared cultural heritage. This sparked a memory in his friend Anna from the United States, who'd been studying the historical exchanges between the indigenous cultures of the United States and Canada. She mentioned how these groups from both the United States and Canada had a rich history of sharing knowledge, customs and even conflict, demonstrating an age-old connection between the two countries. Fascinated, John shared snippets from his trip to Brazil, where he'd learned about the cultural and historical ties between Brazil and Argentina. He revealed that despite their political differences, Brazil and Argentina share profound cultural links, with their histories deeply intertwined. This reminded Anna of her journey to the United Kingdom. She vividly recalled an exhibition she had visited in the United Kingdom, showcasing the country's historic ties with Ireland. It was intriguing, she narrated, how the United Kingdom and Ireland, despite their tumultuous history, have shared and shaped each other's destinities. John then told Anna about his visit to Egypt, where he encountered the millennia-old bond between Egypt and Israel. Intriguingly, Egypt and Israel, despite their conflicts, have a shared history that can be traced back to the times of antiquity. Finally, Anna ended the conversation with a quirky fact about Australia and New Zealand. She revealed that despite their friendly rivalry on the sports field, Australia and New Zealand share a deeply ingrained kinship, both in terms of history and culture. Their conversation, filled with tales from Italy to Australia, was a testament to the interconnectedness of our world's history through the lens of different countries.","[Italy, Italy, Greece, Turkey, Italy, Greece, Turkey, United States, United States, Canada, United States, Canada, Brazil, Brazil, Argentina, Brazil, Argentina, United Kingdom, United Kingdom, Ireland, United Kingdom, Ireland, Egypt, Egypt, Israel, Egypt, Israel, Australia, New Zealand, Australia, New Zealand, Italy, Australia]",14,2014
|
|
||||||
8,"Upon his visit to Norway, David, an avid history buff, learned that the Vikings, who hailed from both Norway and Sweden, were not just raiders and traders, but also skillful navigators who discovered new lands. Both Norway and Sweden shared a rich Viking heritage, with numerous historic sites and artifacts revealing their past. He later shared these insights with his friend Susan, who lived in Canada and was equally passionate about history. She told him about the first European settlers in Canada, most of whom were from France and United Kingdom. Both France and the United Kingdom had made significant contributions to the early development of Canada, she explained. Her fascination with history also led her to explore the ancient civilizations of Egypt, with its magnificent pyramids and intriguing hieroglyphics. She mentioned that her visit to Egypt opened her eyes to the true grandeur of ancient civilizations, something she felt was often overlooked. David shared her sentiment, recounting his own experiences in Greece, where he had explored the renowned archaeological sites. He revealed that Greece’s rich history, from its mythical heroes to its philosophical legacy, truly captivated him. In response, Susan brought up her travels to Mexico, where she got to witness the remnants of the great Mayan civilization. She also spoke about the complex Aztec culture in Mexico, which she found just as fascinating as the Mayans. This discourse on ancient civilizations prompted David to talk about his trip to Italy, famous for its Roman heritage. He said that visiting Italy was like a journey through time, with landmarks and ruins offering a glimpse into the past. They both agreed that whether it's the Viking heritage of Norway and Sweden or the ancient civilizations of Egypt, Greece, Mexico, or Italy, every country has a rich and captivating history to explore and learn from.","[Norway, Norway, Sweden, Norway, Sweden, Canada, Canada, France, United Kingdom, France, United Kingdom, Canada, Egypt, Egypt, Greece, Greece, Mexico, Mexico, Italy, Italy, Norway, Sweden, Egypt, Greece, Mexico, Italy]",14,1898
|
|
||||||
9,"In the heart of Portugal, Jack, a historian, found aged parchments detailing a fascinating relationship with Spain. The parchments spoke of a time when merchants from Portugal would regularly travel to Spain for trade and cultural exchange, hinting at a deeply entwined history between Portugal and Spain. Jack shared this discovery with his colleague, Laura, an anthropologist from Switzerland who was researching similar cultural exchanges. She had observed that Switzerland historically had a close relationship with Italy. She spoke of trade routes and shared cultural practices that had bridged gaps between Switzerland and Italy for centuries. Intrigued, Jack mentioned an old map he'd found which suggested an age-old connection between the United States and Canada. Apparently, during certain historical periods, the borders between the United States and Canada were fluid, with peoples, culture, and trade often transitioning seamlessly. Laura confessed her interest in Asian histories, particularly those of India and Pakistan. She found it fascinating how India and Pakistan, despite their conflicts, shared a deep-rooted cultural and historical bond. She also mentioned the ancient ties between China and Japan, a relationship marked by both cooperation and conflict, with cultural exchanges and wars shaping the relationship between China and Japan. Jack agreed and shared that he was equally fascinated with the shared history of Australia and New Zealand. The indigenous cultures of Australia and New Zealand, he explained, had rich oral histories that spoke of ancestral spirits crossing the sea between the two countries. As the day ended, they both realized how intertwined the world was, united by shared histories and cultural exchanges from Portugal to New Zealand.","[Portugal, Spain, Portugal, Spain, Portugal, Spain, Switzerland, Switzerland, Italy, Switzerland, Italy, United States, Canada, United States, Canada, India, Pakistan, India, Pakistan, China, Japan, China, Japan, Australia, New Zealand, Australia, New Zealand, Portugal, New Zealand]",13,1786
|
|
||||||
10,"John, an avid traveler from Canada, had spent his summer exploring the heart of Australia, specifically, the Outback. The vast, arid landscapes of Australia presented a stark contrast to the snow-filled winters of his home in Canada, and he reveled in the difference. He then shared stories of his trip to Brazil, where he fell in love with the vibrant rhythms and the people's warm hospitality. Indeed, Brazil left such a strong impression on him that he visited the country again, this time to explore the dense Amazon rainforest. As John recounted his travels, his friend Sarah, a history buff from the United Kingdom, couldn't help but gush about her trips to Italy and Greece. She explained how she had spent weeks soaking up the culture, history, and mythology of Italy and Greece. Intrigued by Sarah's stories, John revealed his fascination for Northern countries, particularly Norway and Sweden. He cherished his memories of hiking through the scenic landscapes of Norway and the breathtaking fjords of Sweden. Sarah, not to be outdone, discussed her recent visit to Mexico and Cuba. Highlighting the unique colonial architecture of Mexico and the vibrant music scene in Cuba, Sarah couldn't conceal her wanderlust. She ended the conversation by expressing her desire to visit South Korea and Japan. She was particularly interested in the modern cities and ancient temples of South Korea, as well as the unique blend of tradition and technology in Japan. As they parted ways, both agreed to continue exploring and understanding the world, one country at a time.","[Canada, Australia, Australia, Canada, Brazil, Brazil, United Kingdom, Italy, Greece, Italy, Greece, Norway, Sweden, Norway, Sweden, Mexico, Cuba, Mexico, Cuba, South Korea, Japan, South Korea, Japan]",13,1569
|
|
||||||
11,"On a balmy day, Thomas from Ireland was engrossed in a book detailing the luscious landscapes of South Africa. The text painted images of South Africa's mesmerizing coastlines and diverse wildlife that had always enchanted Thomas, who had a particular fondness for this unique and captivating land. His fascination for South Africa led him to discuss his interest with his friend Maria, who hailed from Brazil. During their conversation, Maria started reminiscing about her visit to Mexico where she discovered a rich culture steeped in history. There seemed to be a silent agreement that Mexico, with its vibrant traditions and ancient Mayan ruins, was a country that intrigued them both. Maria also revealed her fascination for Argentina, pointing out its dynamic cities and beautiful landscapes. Thomas quickly chimed in, saying he too had always wanted to explore Argentina, enthralled by its history, and was planning a trip soon. The discussion soon veered towards Thomas's recent trip to Belgium, where he was impressed by its medieval towns and gothic architecture. Maria, too, expressed her interest in Belgium, particularly fascinated by the intricate lacework and delicious chocolates. Their conversation then took a turn towards the Far East, with Maria excitedly talking about her unforgettable experiences in Japan. She spoke about the unique mix of tradition and modernity, making Japan a fascinating destination. Thomas, a lover of sushi, agreed and mentioned that Japan, with its rich cultural history and delectable cuisine, was certainly on his bucket list. Filled with a sense of wonder and shared experiences, they parted ways, promising to meet again over a cup of coffee from Colombia.","[Ireland, South Africa, South Africa, South Africa, Brazil, Mexico, Mexico, Argentina, Argentina, Belgium, Belgium, Japan, Japan, Japan, Colombia]",13,1708
|
|
||||||
12,"John, a history professor from the United States, had always been fascinated by the ancient civilizations of Egypt and their influence in Greece. He spent considerable time studying how Egypt had shaped the thought processes, cultural practices, and literary works of Greece. His colleague, Laura from Argentina, invited him to visit Argentina and Brazil, since these countries also had rich historical relations. Indeed, Argentina had been greatly influenced by the immigrants from Brazil who had shaped its local culture and traditions. She also spoke of North Korea and South Korea, two oddly paired countries that had multiple common threads in their history. Despite being seen as adversaries, North Korea and South Korea shared a lot of similarities linguistically and culturally. On the other hand, John shared his recent research on India and China, the two enigmatic giants of Asia. He described how India had been swayed by the powerful cultural and historical waves from China over centuries. Both India and China, he narrated, had a long history of philosophical exchanges and cultural learnings. They further discussed Australia and its unusual connections with New Zealand. Both Australia and New Zealand shared a unique bond in terms of natural heritage and sports. The lively chat shifted to Europe, with Laura speaking about her visit to Poland and Ukraine. Laura shared her experiences in Poland and Ukraine which, despite their complicated historical relations, had strong cultural links. They wrapped up their conversation reflecting how every corner of the world, from the United States to Ukraine, was interwoven with intriguing historical connections.","[United States, Egypt, Greece, Egypt, Greece, Argentina, Argentina, Brazil, Argentina, Brazil, North Korea, South Korea, North Korea, South Korea, India, China, India, China, India, China, Australia, New Zealand, Australia, New Zealand, Poland, Ukraine, Poland, Ukraine, United States, Ukraine]",14,1674
|
|
||||||
13,"Alice, a renowned historian from Australia, had always been fascinated with the intriguing ancient cultures of Egypt. These cultures of Egypt, as she discovered, had a profound influence on the art of Greece, which was a testament to the exchange of thoughts and ideas between the nations of the past. While studying ancient cultures, Alice met Daniel, a scholar from Sweden, who was equally interested in historical exchanges and was researching on the close bonds between the United Kingdom and Ireland. The United Kingdom and Ireland, Daniel explained, shared a tumultuous but enriched history that shaped the cultural landscape of both the countries. Their discussion led Alice to remember her trip to Argentina, where she was told about the shared history of Argentina and Brazil. The history between Argentina and Brazil, Alice recalled, was marked by both rivalry and camaraderie, which shaped their identities in a distinctive way. Much like the ties between Australia and New Zealand which was defined by geographical proximity and cultural exchanges. Australia and New Zealand, she elaborated, had a dynamic relationship characterized by joint endeavors in several fields. In a similar vein, Daniel shared his findings about Portugal and Spain, two countries with a deep shared history. Portugal and Spain, he explained, have a history of battles and alliances that have changed the course of Europe. With a keen interest in Asian history, Alice narrated her understanding of India and Pakistan, two countries divided by a line but bonded by shared history. India and Pakistan, despite their political differences, she said, have a profound shared cultural heritage and history. The dialogue ended with a discussion about North Korea and South Korea, two nations with an inseparable shared past. North Korea and South Korea, as Alice knew from her readings, were divided by political ideology but shared a common history and culture. This discussion reaffirmed their belief in the interconnectedness of countries, from Australia to Korea, all sharing an intricate weave of histories and cultural exchanges.","[Australia, Egypt, Egypt, Greece, Sweden, United Kingdom, Ireland, United Kingdom, Ireland, Argentina, Argentina, Brazil, Argentina, Brazil, Australia, New Zealand, Australia, New Zealand, Portugal, Spain, Portugal, Spain, India, Pakistan, India, Pakistan, North Korea, South Korea, North Korea, South Korea, Australia]",15,2116
|
|
||||||
14,"John, an enthusiastic historian from Norway, recently uncovered a document referencing a series of secret expeditions from Norway to Russia during the time of the Vikings. Intriguingly, these expeditions didn't go to Russia alone but also to Ukraine, suggesting a robust trading network in the region during that historical period. The document also highlighted the close cultural connections between Russia and Ukraine, despite their turbulent history. Excited, John shared his findings with his colleague Sarah, an anthropologist from Canada, who was equally fascinated. Sarah, in turn, introduced John to her recent work on the deep-rooted exchanges between Canada and the United States. This work illustrated the complex interweaving of cultural, political, and economic ties between Canada and the United States, dating back hundreds of years. She also mentioned a paper she had read about the ancient links between ancient civilizations in Peru and Mexico. The research suggested that the civilizations in Peru and Mexico, although separated by geography, shared similar cultural practices and beliefs, likely due to extensive trade networks. John then mentioned his own trip to Australia, where he had uncovered the historical connection between indigenous peoples of Australia and New Zealand. These connections, often overlooked in history, highlighted the shared heritage and extensive trade between Australia and New Zealand. Sarah shared her own travel experience to China, where she had studied the historical relationships between China and Japan. Despite their turbulent history, she explained that China and Japan both had a rich cultural exchange that influenced their arts, philosophy, and customs. To their surprise, they realized that despite being from different continents - Norway and Canada - they both shared a fascination for exploring interconnected global histories.","[Norway, Norway, Russia, Russia, Ukraine, Russia, Ukraine, Canada, Canada, United States, Canada, United States, Peru, Mexico, Peru, Mexico, Australia, Australia, New Zealand, Australia, New Zealand, China, China, Japan, China, Japan, Norway, Canada]",13,1894
|
|
||||||
15,"James, an avid traveler from Australia, had just returned from a trip to Argentina where he had been fascinated by the cultural and historical similarities between Argentina and Brazil. He had found that Argentina and Brazil shared not just physical borders, but a rich and intertwined past as well, bringing about a complex yet beautiful tapestry of shared cultures. He excitedly shared these stories with his friend, Lisa, from Canada, who herself had an interesting experience from her visit to Mexico. Lisa said that, during her travels in Mexico, she had found a deep-rooted connection between Mexico and the United States. Stories of migration, trade, and cultural exchanges between Mexico and the United States were embedded in the very spirit of the two nations. James then brought up his forthcoming trip to Europe, focusing on Spain and Portugal. He hoped to explore the interconnected history of Spain and Portugal, two nations that were not just geographically close but historically linked with an indelible bond of shared kings, wars, and cultural influences. Lisa, intrigued by his plan, shared her own European experiences, particularly her visit to Poland and Ukraine. Lisa shared how she was deeply moved by the shared history and resilient spirit of both Poland and Ukraine, two countries that had seen their share of turmoil yet maintained their unique identities. Finally, they discussed their plans to visit Asia, with China and Japan being at the top of their list. They were both intrigued by the complex relations between China and Japan, two nations with ancient civilizations and tumultuous histories. As they sipped their coffee, they marveled at the fascinating tapestry of global history, from Argentina to Japan.","[Australia, Argentina, Argentina, Brazil, Argentina, Brazil, Canada, Mexico, Mexico, Mexico, United States, Mexico, United States, Spain, Portugal, Spain, Portugal, Poland, Ukraine, Poland, Ukraine, China, Japan, China, Japan, Argentina, Japan]",12,1743
|
|
||||||
16,"Emily was a well-known archaeologist whose work had taken her across the globe, from the ancient pyramids of Egypt to the uncharted jungles of Brazil. Her research had revealed hidden connections between cultures across time, such as the shared aspects of ancient rituals between Egypt and Greece, a connection that remained a mystery. Meanwhile, her expeditions in Brazil had led her to uncover a long-lost civilization that showed astonishing similarities with ancient societies in Argentina. Brazil and Argentina, despite being geographically close, had surprisingly similar, yet distinct, ancient civilizations. Emily's work had also taken her to Ireland, where she examined the ancient Celtic ruins and unearthed links between Ireland and the United Kingdom. The level of similarity between the ancient structures and culture of Ireland and the United Kingdom was fascinating, providing a deeper understanding of their shared history. Emily's travels next took her to India, where she studied ancient Sanskrit manuscripts, uncovering connections between India and China. It was surprising to find so many parallels between the ancient traditions of India and China, hinting at a shared past. Emily's fascination with the ancient past didn't stop at historical sites. She had previously spent time in Japan, where she had delved into the historical links between Japan and South Korea. She discovered that Japan and South Korea had a shared past that predated the modern political ties. Lastly, Emily had traveled to Mexico, where she had explored ancient Aztec structures and found cultural links between Mexico and Peru. Mexico and Peru, although separated by distance, had shared striking similarities in their ancient civilizations. The more Emily traveled and studied these nations, the more she understood that countries from Egypt to Peru had intertwined and complex shared histories.","[Egypt, Brazil, Egypt, Greece, Brazil, Argentina, Brazil, Argentina, Ireland, Ireland, United Kingdom, Ireland, United Kingdom, India, India, China, India, China, Japan, Japan, South Korea, Japan, South Korea, Mexico, Mexico, Peru, Mexico, Peru, Egypt, Peru]",14,1895
|
|
||||||
17,"In the grand museums of Italy, visitors are often amazed by the rich collection of artifacts that tell a story of Italy's vibrant past. Among these artifacts are rare gems and stones that were once believed to have been imported from Egypt and were a sign of Italy's prosperous trade with Egypt. Speaking of gemstones, Jim remembered his visit to South Africa, where he had learnt about diamond mining. As he discovered, diamonds were not only mined in South Africa, but also in neighboring Zimbabwe. The vast diamond mines in both South Africa and Zimbabwe are a testament to the abundant mineral wealth these countries hold. While discussing his travels, Jim's friend Lisa spoke of her fascinating visit to China. She recalled how she was in awe of the Great Wall of China, which demonstrated the country's rich history and the military might of ancient China. While in China, she also learned about the country's historical connections with North Korea. It seems China and North Korea have shared cultural and political ties for centuries. Recalling her travels, Lisa mentioned her trip to Ireland where she had discovered the ancient history of the Celtic people. She also pointed out that the connection between Ireland and the United Kingdom is not just geographical but also deeply historical. The shared history and cultural exchanges between Ireland and the United Kingdom have shaped the identities of both nations. Reflecting on their conversations, they both marveled at the fascinating historical and cultural links between countries like Italy and Egypt, South Africa and Zimbabwe, China and North Korea, and Ireland and the United Kingdom.","[Italy, Italy, Egypt, Italy, Egypt, South Africa, South Africa, Zimbabwe, South Africa, Zimbabwe, China, China, China, China, North Korea, China, North Korea, Ireland, Ireland, United Kingdom, Ireland, United Kingdom, Italy, Egypt, South Africa, Zimbabwe, China, North Korea, Ireland, United Kingdom]",13,1654
|
|
||||||
18,"In a small cafe in Norway, Thomas and Maggie were discussing their travels around the world. Thomas started narrating his adventures in Iran, mentioning how he had been fascinated by the ancient architecture and the rich history of Iran. He spoke of the intricate details of Persian carpets from Iran and the fragrant aroma of the local cuisine. Maggie, in turn, shared her experiences from her travels to Poland. She reminisced about the charm of the old towns in Poland and the friendly locals who generously shared details about the incredible history of Poland. Intrigued, Thomas asked about her journeys to South Korea. He was particularly interested in the blend of modern and traditional cultures in South Korea. Maggie described how the ancient palaces in South Korea coexisted with towering skyscrapers and how the tech-savvy people of South Korea were also fiercely proud of their rich heritage.Thomas then spoke about his time in Zimbabwe. He described the stunning wildlife and the warm hospitality of the people in Zimbabwe. He fondly remembered the local markets in Zimbabwe, vibrant with color and full of life. Maggie then mentioned her trip to Mexico. She had been impressed by the lively music, delicious foods, and colorful festivals of Mexico. She shared that it was in Mexico where she had learned about the Day of the Dead, a unique tradition of Mexico.They ended their conversation on Australia, where they had both traveled. They fondly remembered the serene landscapes and the hospitable locals of Australia. They narrated tales of their adventures in the outback of Australia, reminiscing about the unique flora and fauna that Australia is known for. As they spoke, they marveled at how travel had allowed them to connect with diverse cultures, from Iran to Australia, enriching their lives in unimaginable ways.","[Norway, Iran, Iran, Iran, Poland, Poland, Poland, South Korea, South Korea, South Korea, South Korea, Zimbabwe, Zimbabwe, Zimbabwe, Mexico, Mexico, Mexico, Mexico, Australia, Australia, Australia, Australia, Iran, Australia]",18,1838
|
|
||||||
19,"On a global excursion, John's first stop was at Argentina, where he learned about the country's fascinating history and its profound influence on Brazil. He discovered that Argentina and Brazil had a long-standing relationship, marked by cultural exchanges, shared regional cuisine, and a fervent football rivalry. From South America, he made his way to the Middle East, where he was drawn to the historical sites of Iran. He spent a few days in Iran, exploring the ancient cities and learning about the country's many cultural layers, which he found were equally respected and mirrored in neighboring Iraq. Next, he traveled to Ireland, a country known for its lush landscapes and rich folklore that reminded him of the verdant hills and vibrant culture of its neighbour, the United Kingdom. Struck by the similarities, he found that Ireland and the United Kingdom shared deep historical ties, language influences, and even traditional music. After Ireland, he ventured into the heart of Europe, finding himself in the charming lanes of Belgium. The intricate architecture and delicate cuisine of Belgium enticed him, and they seemed all the more familiar when he moved on to the Netherlands. He learned that Belgium and the Netherlands had shared monarchies and currencies in the past, highlighting a linked history. From there, his journey took him halfway across the globe to South Korea. The bustling cities and fascinating traditions of South Korea captivated him, and he noticed many similarities when he crossed the border into North Korea. Despite the political differences, he found cultural threads that connected South Korea and North Korea, from food to language. Looking back on his travels, John was intrigued by the deep-rooted ties between these pairs of countries: Argentina and Brazil, Iran and Iraq, Ireland and the United Kingdom, Belgium and the Netherlands, and South Korea and North Korea.","[Argentina, Brazil, Argentina, Brazil, Iran, Iran, Iraq, Ireland, United Kingdom, Ireland, United Kingdom, Ireland, Belgium, Belgium, Netherlands, Belgium, Netherlands, South Korea, South Korea, North Korea, South Korea, North Korea, Argentina, Brazil, Iran, Iraq, Ireland, United Kingdom, Belgium, Netherlands, South Korea, North Korea]",13,1913
|
|
||||||
20,"John, an archaeologist from Canada, always had an intrigue for the mystery that surrounded the ancient civilizations of Mexico. From the Aztec temples to Mayan ruins, Mexico had always held a special place in his heart. On one of his travels to Mexico, he stumbled across a significant finding that led to the revelation of a forgotten connection between Peru and Mexico. The ruins unveiled a history of trade and cultural exchange between Peru and Mexico, which had faded away with time. John was intrigued by this deep-rooted connection between Mexico and Peru, which prompted him to delve deeper into other such unexplored historical ties. During his research, he came across references to a trade route between Italy and Greece. As per the ancient texts, Italy and Greece had a deep connection, which was not only reflected in their foods but also in their architectural style. This connection between Italy and Greece brought a new perspective to the way he viewed these countries. Further in his research, he discovered ties between Russia and Ukraine. Ukraine had a prolonged history of being under Russia's dominion, which greatly influenced Ukraine's culture, language, and political structure. The connection between Russia and Ukraine was something that had always been known but was never deeply explored. The relationship between Australia and New Zealand also caught John's attention. The proximity and shared history between Australia and New Zealand, coupled with the similarities in their culture and people, fascinated him. His research on Australia and New Zealand unveiled how these two countries had so much shared history, yet they had maintained their distinct identities. The deeper he delved into the history between these countries, the more he was amazed by the interconnectedness of the world. The relationships between Canada and Mexico, Italy and Greece, Russia and Ukraine, and Australia and New Zealand were examples of how the world was intertwined historically and culturally.","[Canada, Mexico, Mexico, Mexico, Peru, Mexico, Peru, Mexico, Mexico, Peru, Italy, Greece, Italy, Greece, Italy, Greece, Russia, Ukraine, Ukraine, Russia, Ukraine, Russia, Ukraine, Australia, New Zealand, Australia, New Zealand, Australia, New Zealand, Canada, Mexico, Italy, Greece, Russia, Ukraine, Australia, New Zealand]",16,2010
|
|
||||||
21,"Emma, a history teacher, recently visited Egypt for an archeological expedition where she unearthed coins from the ancient Egypt era bearing the emblem of Greece. These coins, she learnt, were a solid proof of a powerful alliance between Egypt and Greece during the period of Ptolemaic Pharaohs. Intrigued by her findings, her colleague David, a history enthusiast from Canada, stressed upon a similar historical connection between Canada and the United States. He explained that despite being two separate nations today, Canada and the United States share a common history of colonial times and have closely influenced each other's cultural development. Emma further shared her experience of visiting the breath-taking ruins of ancient Rome in Italy, where she had found scrolls mentioning strong trade relations between Italy and Spain. These scrolls showed that during the Roman era, Italy and Spain had a strong bond based on mutual respect and trade. David smiled and shared his own experience of visiting China, where he found evidence of a deep-rooted bond between China and Japan. During his tour in China, he saw ancient scrolls that depicted the cultural and philosophical exchanges between China and Japan. These exchanges played a crucial role in the development of their unique, yet interconnected, cultural identities. Intrigued by each other's narratives, Emma and David marvelled at how distant countries such as Greece and Japan, and closer ones like Canada and the United States, or Italy and Spain, were linked. They concluded that despite geographical boundaries, the world has always been a single interwoven tapestry of connected histories and cultures.","[Egypt, Egypt, Greece, Egypt, Greece, Canada, Canada, United States, Canada, United States, Italy, Italy, Spain, Italy, Spain, China, China, Japan, China, China, Japan, Greece, Japan, Canada, United States, Italy, Spain]",11,1675
|
|
||||||
22,"In his travels, John had discovered the rich biodiversity of Australia, with its unique flora and fauna unlike anywhere else on earth. Australia's landscapes, from its vast deserts to lush rainforests, were a testament to the country's geographical diversity. John's next destination was Argentina, a country that greatly contrasted with the landscapes of Australia. Argentina with its sprawling Pampas grasslands, snow-capped Andes, and vibrant cities had left an indelible impression on John. He found himself comparing the cultural richness of Argentina with that of neighboring Brazil. The lively rhythms of Brazil, its stunning architecture, and its multicultural society were a sharp contrast to the quieter, more rural beauty of Argentina. This reminded him of his visit to Ireland and its relationship with the United Kingdom. Ireland, with its breathtaking landscapes and charming towns, stood in marked contrast to the United Kingdom, a country known for its regal history and bustling cities. The cultural difference between Ireland and the United Kingdom was as fascinating to him as the contrast between China and Japan. China, with its vast history and iconic landmarks, fascinated him just as much as Japan, a country blending tradition with modernity in an intricate dance. His mind then wandered to his time spent in Canada and its southern neighbor, the United States. He had marveled at the similarities and differences in culture, geography, and history of Canada and the United States. Engulfed in his memories, John realized how every journey had expanded his understanding and appreciation of the world's diversity.","[Australia, Australia, Argentina, Australia, Argentina, Argentina, Brazil, Brazil, Argentina, Ireland, United Kingdom, Ireland, United Kingdom, Ireland, United Kingdom, China, Japan, China, Japan, Canada, United States, Canada, United States]",13,1638
|
|
||||||
23,"Lucas, an avid traveler from Canada, had recently returned from his excursion in Norway. In Norway, he had heard tales of ancient Vikings, their voyages and their mysterious connection with Ireland. Intriguingly, these legends suggested that some of these Vikings had not only raided but settled in parts of Ireland, influencing its early history. Lucas was fascinated by these stories and shared them with his friend Sofia, who hailed from Brazil. Sofia, fascinated, narrated her own experiences in Mexico where she had learned about the historical links between her native Brazil and Mexico. Both Brazil and Mexico, despite their geographical distance, shared legends of ancient gods and mythical creatures. Intriguingly, she mentioned the tales from New Zealand she had heard during her travels. Similar to Mexico and Brazil, New Zealand had its share of mythologies, but what fascinated her was the more modern link between the two regions - New Zealand and Australia. New Zealand, just like Australia, was known for its wild, untouched landscapes and unique wildlife. Their conversation then turned to the memories of her trip to Egypt. In Egypt, Sofia had learned about the once powerful empire’s relations with its neighbor Israel. The two countries - Egypt and Israel - had an intricate, complex history relationship spanning over thousands of years. Shifting gears, Lucas then shared his experiences from Vietnam, particularly the rich historical connections between Vietnam and its neighbor China. Just like Israel and Egypt, Vietnam and China had their share of ancient histories and conflicts, shaping the countries as they were today. Their exchange of travel tales and cultural learnings proved to be a fascinating journey itself, moving from Canada to China, and witnessing the rich, interwoven histories of the world.","[Canada, Norway, Norway, Ireland, Ireland, Brazil, Mexico, Brazil, Mexico, Brazil, Mexico, New Zealand, Mexico, Brazil, New Zealand, New Zealand, Australia, New Zealand, Australia, Egypt, Egypt, Israel, Egypt, Israel, Vietnam, Vietnam, China, Israel, Egypt, Vietnam, China, Canada, China]",15,1833
|
|
||||||
24,"Joan, a historian from the United States, stumbled upon an ancient journal detailing the age-old trade relations between Canada and the United States. She was intrigued to find how inhabitants from both Canada and the United States took perilous journeys across the border to exchange goods and knowledge. Keen to share this, she contacted her friend Lars, a fellow historian from Sweden, who had also been studying the ties between Norway and Sweden for years. Based on his research, he found that the Vikings from both Norway and Sweden embarked on several maritime expeditions together, revealing the profound connections between these two nations. Their conversation shifted towards the East when Lars shared his fascination with the historical links between China and Japan. He pointed out that despite their conflicting histories, China and Japan had significant cultural influences on each other, dating back to centuries. Joan, intrigued by Asia, mentioned her recent trip to South Korea, where she learned about the age-old diplomatic and cultural links between North Korea and South Korea. Despite the current political divide, North Korea and South Korea share deep cultural roots that could be traced back to ancient dynasties. This reminded Lars of his journey to Israel, where he gained insight into the complex history between Israel and Iran. The journal he had come across in Israel cited instances from the past when trade and diplomatic relations between Israel and Iran were prevalent. They ended their conversation reflecting on the interconnectedness of nations, from Canada to Iran, how despite their distinct identities, their paths had intertwined in the annals of history.","[United States, Canada, United States, Canada, United States, Sweden, Norway, Sweden, Norway, Sweden, China, Japan, China, Japan, South Korea, North Korea, South Korea, North Korea, South Korea, Israel, Israel, Iran, Israel, Israel, Iran, Canada, Iran]",11,1698
|
|
||||||
25,"Robert, an avid bird watcher from Canada, packed his bags for his most awaited trip to Brazil. He aimed to explore the biodiversity of Brazil, especially the bird species endemic to the region. His trip to Brazil was an eye-opener as he marveled at the varied and vibrant birds he saw, some of which were critically endangered. After a successful expedition in Brazil, his next stop was Australia. He was thrilled to observe the distinctive bird species in Australia, such as the Emu and the Kookaburra, that make Australia a paradise for bird watchers. Australia, he discovered, also had a significant number of migratory birds that added to its rich diversity. His next destination was South Africa, a country known for its unique fauna. He spent several days in South Africa, amazed by the incredible diversity and uniqueness of its bird species. Not only were the birds in South Africa unique, but also the landscape was breathtaking. From South Africa, Robert set off for his next adventure in Vietnam. He discovered that Vietnam bore a diverse range of habitats that are home to various bird species. The bird species in Vietnam, he concluded, make the country a bird watcher's paradise. His final stop was France, known for its rich cultural heritage and gastronomy. In France, he observed a range of fascinating birds, each species as unique and captivating as the country itself. France, he noted, is not only a treasure trove of historical and cultural richness but also holds a significant number of bird species. As he reflected on his trips from Canada to France, Robert felt grateful for these experiences that allowed him to appreciate the diverse bird species each country had to offer.","[Canada, Brazil, Brazil, Brazil, Brazil, Australia, Australia, Australia, Australia, South Africa, South Africa, South Africa, South Africa, Vietnam, Vietnam, Vietnam, France, France, France, Canada, France]",16,1702
|
|
||||||
26,"John, an avid traveler from Canada, had recently returned from a remarkable trip to Australia. During his time in Australia, he was introduced to the country's ancient aboriginal culture and traditions, which had a profound impact on him. His fascination grew so much that he decided to share his experiences with his childhood friend, Lucy, who was living in the United States. Lucy, intrigued by John's experiences in Australia, shared stories of her own travels to Mexico. Mexico, she said, was a country rich in history, boasting enigmatic pyramids and ancient civilizations that rivalled even those of Egypt. She recounted her adventures in Mexico, detailing the vibrant colors, the cuisine, and the generosity of the locals she had encountered. This conversation led John to recall his journey to Russia, where he had been especially fascinated by the grand palaces and the resilient spirit of the local people. Russia, he said, was a country that held countless stories in its vast stretches of land. Lucy, listening intently, mentioned her expedition to South Africa. It was in South Africa where she had seen the majestic African wildlife and met the diverse communities that called the nation home. She went on about her time in South Africa, noting how the country's challenges were as immense as its inspiring landscapes. John, in response, shared his experience in Japan. He talked about the harmonic blend of Japan's past and future, its cherry-blossom-lined streets, and its bustling urban areas. Japan, he concluded, was a country that treasured both its history and its future. Sharing their adventures, they reveled in the wonder of exploration and the rich diversity of cultures from Canada to Japan.","[Canada, Australia, Australia, United States, Australia, Mexico, Mexico, Egypt, Mexico, Russia, Russia, South Africa, South Africa, South Africa, Japan, Japan, Japan, Canada, Japan]",15,1719
|
|
||||||
27,"Lucia, an adventurous globetrotter from Argentina, had an insatiable curiosity about the ancient connections between Argentina and Brazil. She had heard tales of a secret path between Argentina and Brazil that the indigenous people used for trade and escape during conflicts. Her interest piqued, she embarked on a journey to South Korea to meet an old historian who specialized in ancient globalization. The historian shared with her intriguing stories about the ancient relationship between South Korea and China, involving wars, trade, and cultural exchanges. Specifically, he highlighted stories of South Korea sending tributary missions to the court of China, deepening her understanding of the intricacies of ancient relations. These stories from South Korea and China fascinated Lucia, prompting her to plan her next adventure to Norway. In Norway, she met with a renowned archaeologist who offered insights into the historical ties between Norway and Sweden. These ranged from the Viking ages, when Norway and Sweden were often in conflict, to the union period when they were under the same monarch. Lucia couldn't help but connect these inter-country relationships to those between Egypt and Israel. She had heard that Egypt and Israel had shared a tumultuous history marked by conflicts and peace treaties. As she delved into the historical records of Egypt and Israel, she found out about the fascinating dynamics of their past relations, painting a complex picture of friendship, rivalry, and everything in between. Reflecting on her global expeditions, Lucia mused about the rich and diverse histories of countries from Argentina to Israel and how their interconnections form a global tapestry of human civilization.","[Argentina, Argentina, Brazil, Argentina, Brazil, South Korea, South Korea, China, South Korea, China, South Korea, China, Norway, Norway, Norway, Sweden, Norway, Sweden, Egypt, Israel, Egypt, Israel, Egypt, Israel, Argentina, Israel]",12,1729
|
|
||||||
28,"It was a bustling afternoon in the local market of South Korea, as Liam, a historian from Australia, was talking to a shopkeeper about the local lore. The shopkeeper shared tales about the history of South Korea and its cultural exchange with Russia, their distant neighbor. He said that it was not uncommon for traders from South Korea to travel to Russia for trade, despite the geographical distance and harsh climatic conditions. Intrigued, Liam shared this story with his friend, Emma, who was in Japan, conducting some anthropological research. Emma narrated her encounters in Japan, about the deep cultural influences from China, which had significantly shaped Japan in the past. Both Japan and China, she said, had a rich history of cultural exchange, ranging from art and literature to philosophy and religion. The conversation reminded Liam of his research in Argentina, where he had discovered similar historic ties with Brazil. He described how Argentina and Brazil, in spite of political differences, have shared a rich cultural and historical connection since ancient times. Emma, in response, mentioned her time in Mexico, reminiscing about the cultural similarities she found with its neighbor, the United States. Both Mexico and the United States, she claimed, had histories strongly intertwined by migrations, trade, and shared cultural influences. It fascinated them both how far apart countries like Australia and the United States, or close neighbors like Japan and China, were all threads in the intricate tapestry of global history.","[South Korea, Australia, South Korea, Russia, South Korea, Russia, Japan, Japan, China, Japan, Japan, China, Argentina, Brazil, Argentina, Brazil, Mexico, United States, Mexico, United States, Australia, United States, Japan, China]",11,1554
|
|
||||||
29,"Eva, an historian from Poland, was working on uncovering the story of a secret society from Ukraine. This society, according to the chronicles she found in Poland, was said to have originated from Ukraine and established a network of operations in various European nations. Intrigued, she decided to collaborate with her colleague, Adam, from Norway who specialized in societies like these. Adam was fascinated with the historical connection between Norway and Sweden, often speaking about the Viking roots that influenced both nations. He believed that the society from Ukraine may have also touched Norway and Sweden, given their close geographical proximity and shared cultural elements. Their research got a surprising twist when they discovered an old map, possibly hinting at the society's reach to the United States. Detailed around the map were accounts of clandestine operations carried out by the society, extending from the United States to Canada. The United States and Canada, owing to their geographical closeness and shared history, seemed like a plausible area of influence for this secret society. Their bond deepened when Eva shared her research about the society's ties with Argentina. She postulated that members from Argentina and Brazil could have been part of the society, due to certain cryptic symbols found in the ruins of both Argentina and Brazil. As their work progressed, they stumbled upon clues pointing towards the society's presence in the Middle East - specifically Iraq and Iran. It seemed, as confirmed from records found in Iraq, that through a series of underground networks, the society managed to infiltrate regions as far as Iran. As they delved deeper, they marveled at the society's widespread reach, from Poland to Iran, their narrative spun across continents and centuries to create an enthralling tale of history.","[Poland, Ukraine, Poland, Ukraine, Norway, Norway, Sweden, Ukraine, Norway, Sweden, United States, United States, Canada, United States, Canada, Argentina, Argentina, Brazil, Argentina, Brazil, Iraq, Iran, Iraq, Iran, Poland, Iran]",13,1860
|
|
||||||
30,"John, an eminent historian from South Africa, was intrigued by the ancient tribes of Australia, whose unique cultural practices presented a rich historical tapestry. He observed a striking similarity between the indigenous cultures of Australia and South Africa, despite being geographically distant. His fascination led him to the United Kingdom, where he dove into the country's vast archives. Here, he unearthed records that illustrated the historical interactions between the United Kingdom and Australia during the colonial era. His research led him to Canada, drawn to the shared colonial histories between Canada and the United Kingdom. Interestingly, both Canada and the United Kingdom had experienced similar cultural transformations post-independence. His next stop was Brazil, a country known for its diverse cultural influences. He found intriguing parallels between the indigenous cultures of Brazil and Australia, opening new vistas for his research. Meanwhile, Maria, a sociologist from Poland, was exploring the unique cultural fabric of Ukraine. Her research highlighted the historic ties between Poland and Ukraine, two countries that shared borders and histories. Her work took her to the United States, a melting pot of cultures, where she discovered tales of immigration from Poland that heralded a new chapter in the history of the United States. She also studied the close political connections between the United States and Ukraine, unravelling the intricate web of international relations. Together, John in South Africa and Maria in Poland, though miles apart, were uncovering the hidden threads of our global tapestry, from Australia to Ukraine, revealing how history weaves us all together.","[South Africa, Australia, Australia, South Africa, United Kingdom, United Kingdom, Australia, Canada, Canada, United Kingdom, Canada, United Kingdom, Brazil, Brazil, Australia, Poland, Ukraine, Poland, Ukraine, United States, Poland, United States, United States, Ukraine, South Africa, Poland, Australia, Ukraine]",13,1718
|
|
||||||
31,"As a renowned anthropologist, Alice had journeyed to every continent, but her heart always returned to India. Within India, she had explored the lively markets, sacred temples, and remote villages, each more captivating than the last. After years of fieldwork in India, she had recently turned her attention to the musical traditions of Iran. Mesmerized by the distinctive melodies and rhythms, she found in Iran an entirely different cultural fabric woven with time. In Iran, she learned about the ancient tradition of storytelling through music, echoing the enchanting tales she had heard in China. The folk tales from China often shared elements with those from Vietnam, a country she was now eager to explore. She recalled her visits to Vietnam where she had discovered a vibrant culture rich in history and tradition. Her journeys took her from India to Vietnam but it was the deep connections between the people that fascinated her the most. Drawn by art and culture, she planned to spend her next summer in France exploring the local museums. In France, she hoped to uncover links to the artistic movements in Italy. While Italy was renowned for its Renaissance art, she found that France and Italy shared a mutual influence in the world of impressionist art. After France, Italy was next on her list. Alice had always been enchanted by the history of Italy, which was steeped in art, architecture, cuisine, and music. Having studied the cultural links between France and Italy, she was also eager to explore the connection between the United Kingdom and Ireland. She had read about the shared history, the shifting boundaries, and intertwined cultures of the United Kingdom and Ireland. As her thoughts wandered from the bustling streets of India, through the rhythmic heart of Iran, and over to the picturesque landscapes of Italy, her mind spun with anticipation for what the future held in Ireland. Each journey fuelled her curiosity and deepened her understanding, as she followed the intricate web of culture and history that spanned from India to Ireland.","[India, India, India, Iran, Iran, Iran, China, China, Vietnam, Vietnam, India, Vietnam, France, France, Italy, Italy, France, Italy, France, Italy, Italy, France, Italy, United Kingdom, Ireland, United Kingdom, Ireland, India, Iran, Italy, Ireland, India, Ireland]",17,2069
|
|
||||||
32,"Tom, a history scholar in the United Kingdom, had always been enraptured by tales of bygone eras, particularly those involving trade and cultural exchange. One day, he stumbled upon a curious old map from Iraq. This map documented ancient trade routes between Iraq and Iran, revealing a deep sense of camaraderie between these historic rivals. Excited by this discovery, he shared these maps with his friend, Carla, in the United States, who was an anthropologist researching the cultural influences in North America. Carla was excited to tell him about her recent findings about the interactions between Canada and the United States, specifically during the times of the fur trade. She mentioned how the United States and Canada had shared not just borders, but histories, economies, and even conflicts. The exchange of ideas triggered Tom's memory about Australia's rapport with New Zealand, as he had once read about their shared history in an old manuscript. He told Carla about this, noting how Australia and New Zealand had fought together in wars, pioneered in the sheep farming industry, and even shared a similar passion for rugby. Carla reciprocated the excitement and brought up her research on South Korea and Japan, detailing the cultural give-and-take between the two East Asian nations. While there were numerous historical conflicts between South Korea and Japan, there was also an undeniable exchange of ideas, trends, and even cuisines. Their conversation took a detour to Europe, specifically France and Italy, as Tom remembered the documented cultural interactions he had studied. Here, France and Italy, despite being known for their unique contributions to art and cuisine, had shared a long history of influences on each other. Their discussion made them reflect on the interconnectedness of nations, from the United Kingdom to Italy, and how cultural exchanges made the world a tapestry of shared histories.","[United Kingdom, Iraq, Iraq, Iran, United States, Canada, United States, United States, Canada, Australia, New Zealand, Australia, New Zealand, South Korea, Japan, South Korea, Japan, France, Italy, France, Italy, United Kingdom, Italy]",13,1931
|
|
||||||
33,"On a summer vacation, Jane, an archeology enthusiast from the United Kingdom, decided to embark on a journey to Egypt to explore the mystical pyramids. This fascination had been ignited by her reading about the ancient civilizations of Egypt, specifically the Pharaohs and their majestic reign. She also held a deep interest in the historical ties between the United Kingdom and India during colonial times, drawn by the cultural exchanges and conflicts that moulded both nations. Her journey to India was transformative, as she discovered the myriad influences from ancient civilizations like China that had left a profound impact on India's rich tapestry of history. Captivated by these transboundary influences, Jane thought of her friend Larry, a historian from Canada, who had done extensive research on trade relations between China and North Korea. According to Larry, China and North Korea, despite their political differences, share a vibrant history of trade and mutual influences, which he discovered during his visits. This reminded Jane of her trip to Argentina, where she learned about the historical relations between Argentina and Brazil. She was intrigued by how Argentina and Brazil, despite their geographical proximity, had distinct cultural identities interwoven with shared histories. This led her to envisage her next destination, the quaint and picturesque country of Norway, known for its ties with Sweden. Norway and Sweden, she knew, had a shared history as well as a unique cultural differentiation which she was excited to experience firsthand. As she wandered through the streets of Norway, Jane couldn't help but think about her childhood trip to Australia and its relations with New Zealand. The way Australia and New Zealand have constantly influenced each other, she reflected, was a testament to the connections forged by sharing a region. Her adventure affirmed to Jane that despite distances, countries from Egypt to Australia are bound by shared histories and cultural reciprocities.","[United Kingdom, Egypt, Egypt, United Kingdom, India, India, China, India, Canada, China, North Korea, China, North Korea, Argentina, Argentina, Brazil, Argentina, Brazil, Norway, Sweden, Norway, Sweden, Norway, Australia, New Zealand, Australia, New Zealand, Egypt, Australia]",13,2021
|
|
||||||
34,"In her recent journey, Lisa, an adventurous traveler from the United States, was fascinated by the similarities she noticed between indigenous art forms from Australia and New Zealand. She found that story-telling through art was a common practice among the native people of both Australia and New Zealand, a captivating way to pass down their rich cultural heritage through generations. Intrigued by this, she contacted her good friend, Robert, who had spent many years studying and exploring both South Africa and Kenya. Robert told her about the strong tribal culture in South Africa and Kenya, where art was not just an expression, but a way of life. He also shared heartwarming stories about his travels through the lesser-known parts of India and Pakistan. Robert explained that despite the political differences, India and Pakistan shared a rich tapestry of history and culture that was evident in their food, music, and traditional wear. Interested in this common cultural thread, Lisa recalled her experiences in Portugal and Spain, how the architecture, music, and food in both Portugal and Spain were influenced by centuries of shared history. Then, she asked Robert about his travels to North Korea and South Korea. Robert shared his observations about North Korea and South Korea, how despite the division, the shared roots reflected in their traditions, language, and festivals. Lisa then shared her memories of her trip to Brazil and Argentina. She marveled at how the indigenous cultures of both Brazil and Argentina had shaped vibrant festivals, music, and dance that were celebrated with great fervor. In the end, they both agreed that the cultures, traditions, and histories of countries from Australia to Argentina were beautifully intertwined, forming an intricate tapestry of shared human experiences.","[United States, Australia, New Zealand, Australia, New Zealand, South Africa, Kenya, South Africa, Kenya, India, Pakistan, India, Pakistan, Portugal, Spain, Portugal, Spain, North Korea, South Korea, North Korea, South Korea, Brazil, Argentina, Brazil, Argentina, Australia, Argentina]",12,1823
|
|
||||||
35,"One evening, Sarah, an archaeologist from Norway made a surprising discovery about ancient trade routes between Sweden and Norway. As per her research, the artifacts that were found in Norway were identical to those in Sweden, indicating a deep-rooted cultural connection between Sweden and Norway. This piqued the interest of her colleague, James, who was from Canada. He had been researching the indigenous tribes of Canada and found many similarities with tribes from his neighboring country, the United States. James had always been interested in the historical ties between Canada and United States, and his study further confirmed the age-old connections between the two countries. Upon hearing James's story, Sarah shared a fascinating anecdote from her travels in Portugal. She recalled how locals loved to tell the tale of the shared history between Spain and Portugal. Her anecdotes about Spain and Portugal echoed the same sense of shared culture and past, just like in the case of Norway and Sweden, and Canada and United States. Their conversation reminded James of his stay in South Korea, where he had learned about the close relationship between North Korea and South Korea, despite their current political divide. He recalled stories about the shared history of North Korea and South Korea, whose deep-seated cultural ties transcended political boundaries. Sarah, who had been to Australia, reciprocated with her own experiences of the bond between Australia and New Zealand. She described how, despite geographical separation, Australia and New Zealand shared a unique camaraderie and close historical ties. As they exchanged stories, their conversation moved to South Africa and its various connections with its neighbouring country, Zimbabwe. Sarah shared stories she had heard about the intricate bond between South Africa and Zimbabwe, showcasing the age-old interactions between these two nations. It left them both reflecting on the timeless bonds that connect nations across the world, from Norway to Australia, Canada to Zimbabwe, and all the countries in between.","[Norway, Sweden, Norway, Norway, Sweden, Sweden, Norway, Canada, Canada, United States, Canada, United States, Portugal, Spain, Portugal, Spain, Portugal, Norway, Sweden, Canada, United States, South Korea, North Korea, South Korea, North Korea, South Korea, Australia, Australia, New Zealand, Australia, New Zealand, South Africa, Zimbabwe, South Africa, Zimbabwe, Norway, Australia, Canada, Zimbabwe]",15,2090
|
|
||||||
36,"My friend, Alex from Peru, once recounted his journey to Argentina where he learned about the strong cultural ties between Argentina and Brazil due to their shared history. He spoke fondly of his time in Argentina and Brazil, marveling at the similar music, dance, and culinary traditions that seamlessly bridged the borders of these countries. It reminded me of a documentary I'd watched about the ancient Silk Road that spanned across Iran and China. It touched upon the shared historical narratives of Iran and China, highlighting how goods, ideas, and culture flowed between these two countries for centuries. Intriguingly, the documentary also brought up some parallels between this eastern route and the vibrant exchange between Italy and France during the Renaissance. The cultural and intellectual bonds between Italy and France were indeed profound, just as the resilient trade relations that existed between the United States and Canada. The United States and Canada, apart from their geographical proximity, shared an economic bond that strongly influenced their policies and international standing. Similarly, the ties between Australia and New Zealand shed light on their gestalt identity in the Pacific region. Despite their unique characteristics, Australia and New Zealand were often viewed as a single entity due to their remarkably similar backgrounds in terms of culture, language, and colonial history. Inspired by these chronicles of interconnectedness, I decided to delve deeper into history and found a fascinating account of how Ukraine and Poland had influenced each other through years of coexistence. Despite their tumultuous past, Ukraine and Poland shared a cultural tapestry that was deeply woven into their histories. It was quite similar to the complex relationship between North Korea and South Korea, which, despite their political differences, shared common heritage and traditions. Thus, from Argentina to South Korea, the world was an intricate web of countries intertwined with each other through shared histories, cultures, and sometimes, shared destinies.","[Peru, Argentina, Argentina, Brazil, Argentina, Brazil, Iran, China, Iran, China, Italy, France, Italy, France, United States, Canada, United States, Canada, Australia, New Zealand, Australia, New Zealand, Ukraine, Poland, Ukraine, Poland, North Korea, South Korea, Argentina, South Korea]",13,2095
|
|
||||||
37,"Annie, a marine biologist from Ireland, had always been fascinated by the diverse sealife surrounding the coasts of Australia. She often said that the waters of Australia were some of the most biodiverse in the world, enticing researchers like her from faraway lands like Ireland. Through her explorations, she developed a particular interest in the migratory patterns of certain species, noting that they often ventured towards New Zealand. Annie noticed that from Australia, these marine creatures would cross oceans to reach the coasts of New Zealand, proving the interconnectedness of these ecosystems. Driven by this curiosity, she published her research, attracting the attention of a fellow scientist, Greg, from Canada. Inspired by her findings, Greg shared his own studies on migration patterns, noting a similar scenario between Canada and the United States. He'd found that many animal species in Canada would cross the border into the United States during certain seasons, emphasizing the interplay between these two nations. Annie and Greg started to collaborate, which led their research towards Mexico, where they discovered another fascinating migration phenomenon. They found it intriguing that birds from the United States would fly south, bypassing their own habitats to spend the winter in the warmer climates of Mexico. Their research in Mexico led them to another colleague, Maria, from Spain. Maria had been studying the movements of the Iberian Lynx which often roamed between Spain and Portugal. Maria's research had shown that just like the animals in North America, the Lynx in Spain had established territories in Portugal. Intrigued by the similarities, they expanded their study to cover the movement of bird species between Belgium and the Netherlands. Just as they'd seen in the Americas, they noticed that certain species of birds in Belgium would travel to the Netherlands during certain periods of the year. These fascinating findings opened up new dimensions of understanding for the interconnectedness of the world. From the sealife in Australia to the birds in Belgium, the animal kingdom had its own ways of establishing connections between countries.","[Ireland, Australia, Australia, Ireland, New Zealand, Australia, New Zealand, Canada, Canada, United States, Canada, United States, Mexico, United States, Mexico, Mexico, Spain, Spain, Portugal, Spain, Portugal, Belgium, Netherlands, Belgium, Netherlands, Australia, Belgium]",16,2190
|
|
||||||
38,"While studying the complex geopolitics of the Cold War, Harry found it fascinating how the United States and Russia had played pivotal roles. He wondered how the United States, a country dedicated to democracy, and Russia, with its communist regime, had become superpowers competing for global influence. On the other hand, his friend Anna from Australia, a political science expert, was studying the lengthy conflict between North Korea and South Korea. Both North Korea and South Korea, despite sharing the same peninsula, developed vastly different political systems causing a taut geopolitical situation. Anna also shared her interest in the Middle East, particularly the tension between Iran and Iraq. The conflict between Iran and Iraq had shaped the political landscape of the Middle East, influencing regional alliances, she explained. Later, Harry discussed his recent project on the relationship between the United Kingdom and Ireland. He highlighted how the political and cultural ties between the United Kingdom and Ireland have significantly influenced each other's history. Meanwhile, Anna brought up a book she'd read about the historical connections between Argentina and Brazil. The book illustrated how Argentina and Brazil, despite their varied history and culture, shared an intricate bond. Harry countered with an interesting trivia about China, pointing out its longstanding relationship with Japan. In the past, China and Japan, despite instances of conflict, had exchanges that enriched both nations' cultures and philosophies. As the conversation took a culinary turn, Harry discussed the famous rivalry between Italy and France over culinary supremacy. The ongoing debate between Italy and France over gastronomic traditions and wine production, he asserted, was a reflection of their vibrant cultures. Their engaging conversation came to a close as they agreed to travel to Canada next summer, to study the influence of its neighbor, the United States on its cultural and political landscape. Clearly, every country, from the United States to France, has a unique story to tell, a story that is intertwined with another's.","[United States, Russia, United States, Russia, Australia, North Korea, South Korea, North Korea, South Korea, Iran, Iraq, Iran, Iraq, United Kingdom, Ireland, United Kingdom, Ireland, Argentina, Brazil, Argentina, Brazil, China, Japan, China, Japan, Italy, France, Italy, France, Canada, United States, United States, France]",16,2149
|
|
||||||
39,"After reading a fascinating book about Australia, John decided to take a journey himself to truly experience the wonders of Australia firsthand. He started his journey from Sydney, exploring the bustling city life before moving towards the quieter parts of Australia, eventually reaching the outback. On returning home to Canada, he shared his incredible experiences of Australia with his childhood friend Sarah, who lived in the United Kingdom. Sarah, in turn, shared her tales of a recent trip to Kenya, where she had interacted with local tribes and learned about their history. She spoke passionately about the vibrant culture of Kenya and how it had touched her deeply. Sarah revealed that her journey to Kenya was inspired by her grandmother, who had a similar experience when she visited Zimbabwe in her youth. She shared stories her grandmother had told her about Zimbabwe, including its rich wildlife and bustling markets. John loved the tales of Zimbabwe and began to think about his next journey. He said that he had been considering a trip to Ireland for a while. Sarah immediately chimed in, saying she had been to Ireland last year and found it to be a beautiful experience. She said that the hospitality and warmth of people in Ireland reminded her of her time spent in New Zealand. She shared that New Zealand, much like Ireland, had a charm that was both invigorating and relaxing at the same time. She particularly remembered the peacefulness of the countryside in both Ireland and New Zealand. After hearing about Ireland and New Zealand, John was even more intrigued and decided to visit them both in his upcoming holidays. As they finished their conversation, they felt grateful for the opportunity to learn about various countries from personal experiences and stories, from Australia to New Zealand.","[Australia, Australia, Australia, Canada, Australia, United Kingdom, Kenya, Kenya, Kenya, Zimbabwe, Zimbabwe, Zimbabwe, Ireland, Ireland, Ireland, New Zealand, New Zealand, Ireland, Ireland, New Zealand, Ireland, New Zealand, Australia, New Zealand]",15,1823
|
|
||||||
40,"During their adventurous journey around the world, Jack and Emily first found themselves in the vibrant markets of India, enamoured by the rich traditions from India reflected in every corner. Their journey then led them to the bustling cityscapes of China, filled with modern marvels that somehow coexisted with ancient traditions from China's past. They then travelled to Australia, where they got lost in the wilderness, and shared stories of the rich biodiversity found in Australia under the southern constellations. Their travel then took them to Argentina, where they were captivated by the diversity of landscapes, from the colossal mountains to the serene lakes of Argentina. They couldn't help but notice the stark contrast to Brazil, which boasted its own unique geographical features and diverse environments, mirroring the diversity of lifestyles in Brazil. Jack, recalling his childhood passion, mentioned his fondness for Italy and its rich history. Emily, sharing his sentiment, mentioned the transcendent beauty of Italy's architecture and the remarkable tales from Italy's past. Soon, their discussion shifted to the dynamic nightlife of South Korea, which they said was unlike anything they had experienced before. South Korea, they described, was the perfect blend of the old and the new, where rich traditions met modern influences. They expressed their shared love for Ireland, with its lush green landscapes and the warm hospitality of the people of Ireland. Their tales also touched upon the unparalleled tranquility of Switzerland and the cultural harmony they found there. Switzerland, they both agreed, was a testament to the beauty of diversity, coexistence and peace. Their curiosity then led them to discuss their memorable visit to Mexico, and the unmatched flavors of cuisine from Mexico that left an indelible mark on their taste buds. As the night grew darker, their conversation veered towards the mystical landscapes of Egypt, where ancient history whispered from every corner of Egypt. As they traversed through their memories, they couldn't help but look forward to their upcoming trip to Kenya, hoping that their experiences in Kenya would be just as enriching and unforgettable. As they parted for the night, they went to sleep with dreams of their world journey, from India to Kenya, brimming in their hearts.","[India, India, China, China, Australia, Australia, Argentina, Argentina, Brazil, Brazil, Italy, Italy, Italy, South Korea, South Korea, Ireland, Ireland, Switzerland, Switzerland, Mexico, Mexico, Egypt, Egypt, Kenya, Kenya, India, Kenya]",16,2350
|
|
||||||
41,"In her travelogue, Emily wrote about the striking cultural similarities between Mexico and Spain. She felt the powerful impact Spain has had on Mexico's language, food, and traditions during her stay in Mexico, which reminded her of the time when she was in Spain. Intriguingly, Emily's friend, Marc, an avid traveler himself, had fascinating stories about the subtle cultural ties between China and North Korea. Having lived in China for a few years, he noted that North Korea's strong cultural and political ties with China were evident in their shared festivals and traditional customs. His stories stirred a memory of Emily's own journey to Australia and New Zealand. The unity between Australia and New Zealand, carved by shared history and geographic proximity, had left a deep impression on her. Their adventurous spirits and friendly rivalry had always amazed her, much like the lively competition between Argentina and Brazil. She was fascinated by the healthy sporting rivalry between Argentina and Brazil, and how it symbolized the love and unity between these two nations. Marc then spoke about his recent travels to Sweden and Norway. His account of the breathtaking landscapes of Sweden and their similarity to the majestic fjords of Norway struck a chord with Emily, who had been longing to visit these countries. She then recalled her trip to Portugal and Spain, where she was captivated by the intricate architectural designs and the warm hospitality of the people. The cultural similarities between Portugal and Spain, deeply entrenched in their shared history and common language, had left a lasting impression on her. Their shared love for music, dance, and festivals was evident in their vibrant local life, much like the shared heritage between Iran and Iraq. It was during her travels in the Middle East where she had discovered a rich tapestry of shared history and traditions between Iran and Iraq, leaving her with an enriched understanding of these ancient lands. Their conversations always ended with a sense of awe for the interconnectedness of countries and their cultures, spanning from the lands of Australia to Iraq.","[Mexico, Spain, Spain, Mexico, Mexico, Spain, China, North Korea, China, North Korea, China, Australia, New Zealand, Australia, New Zealand, Argentina, Brazil, Argentina, Brazil, Sweden, Norway, Sweden, Norway, Portugal, Spain, Portugal, Spain, Iran, Iraq, Iran, Iraq, Australia, Iraq]",15,2149
|
|
||||||
42,"While trekking through the dense forests of Argentina, Rebecca came across a map pointing towards an unexplored cave in Brazil. The map evidently indicated that these caves in Brazil were speckled with ancient drawings that supposedly revealed a deep-seated connection with Argentina. Her curiosity piqued, she decided to delve into this further and bring it to the notice of her friend Jack, who was a historian in Russia. Jack, fascinated by her findings, revealed that he had stumbled upon something similar in Russia that alluded to an ancient bond with Ukraine. The historical artifacts found in Russia had signs pertaining to some age-old customs of Ukraine, suggesting a shared cultural heritage. He also mentioned his recent visit to China where he learned about its intriguing historical ties with North Korea. There were ancient scrolls in China that depicted tales of kings who had traveled to North Korea in search of wisdom and alliances. Intrigued, Rebecca couldn't help but share her adventurous experiences in Mexico and its shared history with its neighboring country, the United States. While in Mexico, she had stumbled upon tales of shared traditions and alliances with the United States that had been created and broken over the centuries. Jack reciprocated with his ventures in Egypt that gave him insights into the country's links with neighboring Israel. In the heart of Egypt, he found stories etched in hieroglyphics and ancient paintings that spoke of a past that was deeply intertwined with the history of Israel. This exchange led to an elaborate discussion about their recent in-depth exploration of Italy and its historical ties with Spain. Both Italy and Spain had a rich and shared history of trade, conquests and alliances that cemented their connection over the centuries. Their discovery drew attention to the intricate tapestry of human history and how borders merely separated lands, not the shared stories woven from Argentina to Spain.","[Argentina, Brazil, Brazil, Argentina, Russia, Russia, Ukraine, Russia, Ukraine, China, North Korea, China, North Korea, Mexico, United States, Mexico, United States, Egypt, Israel, Egypt, Israel, Italy, Spain, Italy, Spain, Argentina, Spain]",14,1975
|
|
||||||
43,"Fredrick, a globe-trotting historian, was working on a project studying the bonds between pairs of countries and their shared histories. He began his research in Sweden, intrigued by the historic ties between it and neighbouring Norway. He discovered that despite their geographical proximity, the histories of Sweden and Norway were both intertwined and fiercely independent. As he delved deeper into this relationship of Sweden and Norway, he found numerous fascinating anecdotes of cooperation and rivalry. Next, he flew to Egypt and immersed himself in the ancient relationship between Egypt and Israel. The dynamic relationship between Egypt and Israel, he noted, was filled with faith, conflict, and a shared ethos of survival amidst desert landscapes. From the warm desert sands of Egypt, Fredrick then ventured to the chilly landscapes of Canada where he explored its historic ties with the United States. He learnt about the deep-rooted kinship and friendly neighbourly associations between Canada and the United States, as well as their conflicts, negotiations and shared ambitions. Intrigued, he then voyaged to South America, where he delved into the shared histories of Colombia and Venezuela. Colombia and Venezuela, he found, had a rich history of shared culture, colonial struggles, political debates, and interwoven economies. Fredrick then travelled to Asia, where he studied the complex historical ties between China and Japan. He unveiled tales of ancient trade, invasions, cultural exchanges and diplomatic negotiations that marked the relationship between China and Japan. His expedition concluded in South Africa, where he delved into the liberating tale of South Africa's journey towards freedom - a story echoed by its neighbour, Zimbabwe. Just like South Africa, Zimbabwe had also battled against colonial rule to embrace freedom. Sitting in a cafe in Cape Town, he marvelled at the world's intricate tapestry of interconnected histories, from Sweden to South Africa, reflecting on how distance and borders mattered so little in the shared narratives of the human spirit.","[Sweden, Norway, Sweden, Norway, Sweden, Norway, Egypt, Egypt, Israel, Egypt, Israel, Egypt, Canada, United States, Canada, United States, Colombia, Venezuela, Colombia, Venezuela, China, Japan, China, Japan, South Africa, South Africa, Zimbabwe, South Africa, Zimbabwe, Sweden, South Africa]",15,2097
|
|
||||||
44,"Growing up in Norway, Erik was always fascinated by tales of the Vikings who had once ruled the lands and seas. He remembered reading about the journeys of the Vikings from Norway to the far-off lands of Ireland, carrying with them a rich cultural exchange and influence. Intrigued, he decided to travel to Ireland, a country rich in lore and legends. He was not disappointed by his visit to Ireland, which was inundated with an array of ancient structures that narrated a tale of a powerful maritime history that was interconnected with Norway. His next destination was Canada, where he had always dreamed of exploring the expansive wilderness and learning more about the indigenous peoples. In his journey across Canada, he uncovered stories of the country's close connection with the United Kingdom, mainly through their shared history and culture. The relationship between Canada and the United Kingdom not only had deep historical roots but was still evident in Canada's modern society. His travels then led him to the United States, known for its diverse landscapes and vibrant cities. While in the United States, he could not help but notice the clear influence of Mexico in several regions due to their shared border and history. The vibrant culture, food, music, and art of Mexico was reflected in various aspects of life in the United States, especially in the southern regions. Intrigued by this, he made his next stop in Mexico, a country filled with rich history, traditions, and culinary delights. His journey through Mexico introduced him to the deep bond between this country and the United States, a relationship molded by shared borders, history, and an ongoing exchange of ideas and cultures. His travels ended in Egypt, inspired by the tales of its ancient civilization. The influence of Greece was also quite evident in the architecture and mythology of Egypt. Both Egypt and Greece, he found, were bound by their shared love for philosophy, arts, and science, deeply entwined through centuries of exchange and influence. Despite the geographical distance, the connection between Egypt and Greece was a testament to the fascinating interconnectivity of our world's cultures and histories.","[Norway, Norway, Ireland, Ireland, Ireland, Norway, Canada, Canada, United Kingdom, Canada, United Kingdom, Canada, United States, United States, Mexico, Mexico, United States, Mexico, Mexico, United States, Egypt, Greece, Egypt, Egypt, Greece, Egypt, Greece]",16,2209
|
|
||||||
45,"On a trip to Norway, Jake stumbled upon an old journal that mentioned a secret Viking expedition from Norway to Canada. It was believed that this expedition had taken place before the known explorations, involving a small group of Vikings from Norway who had discovered and briefly settled in Canada. Intrigued by this revelation, he shared the information with his friend Lucy, who lived in the United Kingdom. Lucy, an avid historian, excitedly recalled a similar story that originated in the United Kingdom, involving a clandestine journey to Australia. The United Kingdom, she explained, had a history of covert voyages to Australia in search of new territories and resources. Shifting the conversation, Lucy spoke about a book she'd recently read which gave detailed accounts of the complex relationship between Japan and China. This book claimed that in the ancient era, Japan and China had significant cultural exchanges and trade relations despite ongoing conflicts. She also mentioned the fascinating history of Iran and Iraq, two countries with intertwined destinities. Iran and Iraq, despite their political turbulence, had a shared historical and cultural legacy that often gets overlooked. Jake then added a story from his recent travels to Spain, where he learned about the country's ancient connections with Portugal. He explained that Spain and Portugal, though culturally distinct, had a shared maritime history that led to significant exchanges between the two. While on the topic of shared histories, Lucy mentioned her visit to Zimbabwe where she discovered the country's profound historical ties with South Africa. Zimbabwe and South Africa, she said, were bound together by a shared struggle for independence and a deep-rooted sense of brotherhood despite geographical boundaries. The conversation fueled their wanderlust, casting a light on the fascinating interconnectedness and shared legacies of countries spanning from Norway to South Africa.","[Norway, Norway, Canada, Norway, Canada, United Kingdom, United Kingdom, Australia, United Kingdom, Australia, Japan, China, Japan, China, Iran, Iraq, Iran, Iraq, Spain, Portugal, Spain, Portugal, Zimbabwe, South Africa, Zimbabwe, South Africa, Norway, South Africa]",14,1969
|
|
||||||
46,"Beginning her journey in France, Jane was mesmerized by the art and culture sensibilities that defined France. Intrigued by the architecture, history, and fashion that was distinctive of France, she was compelled to dig deeper into the historical ties that connected France with neighboring Italy. Italy, with its rich history and cultural heritage, shared an artistic bond with France, both historically and contemporarily. Jane next decided to visit South Africa, where she found a stark contrast to the art and sophistication of Italy, yet an enticing allure in the vibrant traditions and natural wonders that South Africa had to offer. The wildlife in South Africa, she found, was unmatched and the conversations she had with locals revealed tales of connections between South Africa and neighboring Zimbabwe. Zimbabwe, she learned, had been a trading partner with South Africa, their histories intertwining in ways she found fascinating. Next on her list was the United States, where she was taken by the diverse cultures and the sprawling metropolises. The technological advancements of the United States astounded her, and she was interested to learn about the political and economic ties between the United States and Canada. Canada, she found, held its unique charm, with its breathtaking landscapes and welcoming locals. She found out about the historical bond between Canada and the United Kingdom through her conversations. Finally, she ended her journey in the United Kingdom, where she was captivated by the rich history and architectural marvels. The United Kingdom had historic ties with Canada and it was fascinating to see the influence each country had on the other. This journey taught her a lot about the world and its intricate links, promising to revisit these countries and explore more connections.","[France, France, France, France, Italy, Italy, France, South Africa, Italy, South Africa, South Africa, South Africa, Zimbabwe, Zimbabwe, South Africa, United States, United States, United States, Canada, Canada, Canada, United Kingdom, United Kingdom, United Kingdom, Canada]",13,1823
|
|
||||||
47,"James, a historian, found himself engrossed in the medieval stories of knights from the United Kingdom, where armor and chivalry were the symbols of bravery. He was particularly interested in the historical ties between the United Kingdom and Ireland, how the two countries, despite their conflicted history, had shared traditions, myths, and legends. Inspired by these narratives, he wrote to his friend Rosa in Mexico, telling her about his research. Rosa, a linguistics expert, was intrigued and wrote back about the rich history of Mexico and the indigenous cultures that shaped the country. She also spoke about the vibrant traditions indigenous to Mexico and Peru, and how both Mexico and Peru had a shared history of ancient civilizations, with languages, rituals, and architecture that had a profound influence on their respective societies. She then shared about her trip to Russia, which she found so diverse and captivating. The historian in James was excited by the detailed descriptions Rosa gave about Russia and Ukraine, about their shared history, the influence of the Byzantine Empire, and the development of Cyrillic script. He then reminisced about his visit to South Africa where he explored the local folktales. He mentioned how South Africa and Zimbabwe's stories were intertwined, with themes of resilience and liberation echoing through their histories. Rosa responded, sharing her fascination with Japan and its unique cultural heritage. She wrote about Japan and South Korea, their shared past, cultural exchanges, and how they had influenced each other's art and literature. They both concluded that the fascinating journeys of these countries, from the United Kingdom to South Korea, suggested a world much more interconnected than generally perceived.","[United Kingdom, United Kingdom, Ireland, Mexico, Mexico, Mexico, Peru, Mexico, Peru, Russia, Russia, Ukraine, South Africa, South Africa, Zimbabwe, Japan, Japan, South Korea, United Kingdom, South Korea]",12,1780
|
|
||||||
48,"In the heart of Mexico, a historian named Marta unearthed a series of artifacts that were traced back to the ancient civilization of Peru. This led her to believe that during prehistoric times, there might have been a direct route from Mexico to Peru, allowing a fascinating exchange of ideas, cultures, and traditions. Thrilled by the discovery, she shared the information with her friend Leo, an anthropologist from the United States who had recently been studying cultural connections between the United States and Canada. Leo was intrigued as he had discovered similar links between the United States and Canada, suggesting that there might have been prehistoric connections between these seemingly distant regions. Leo then shared his recent study about the intriguing relationship between South Africa and Kenya. The two African nations, South Africa and Kenya, had a rich tapestry of legends suggesting a shared ancestry and rich cultural exchanges. Marta was reminded of a book she read about the historical ties between Russia and Ukraine. This book discussed extensive trade routes and cultural exchanges between Russia and Ukraine, adding another layer to their complicated history. Similarly, her team had recently found artifacts in Australia, which surprisingly pointed towards a connection with New Zealand. It seemed that Australia and New Zealand, despite being known for their unique isolated biodiversity, also shared a deep historical bond. Leo, smiling, told Marta that his recent visit to Italy had been enlightening as he learned of the ancient interactions between Italy and its neighbor, Greece. He explained that Italy and Greece, despite their renowned and distinct ancient cultures, had influential exchanges in art, language, and governance. Finally, they chatted about their shared trip to South Korea and North Korea where they had learned about the deep ties between the two nations. Their conversation was a testament to the intriguing web of connections between countries from Mexico to Greece and beyond, adding another dimension to the study of human history.","[Mexico, Peru, Mexico, Peru, United States, United States, Canada, United States, Canada, South Africa, Kenya, South Africa, Kenya, Russia, Ukraine, Russia, Ukraine, Australia, New Zealand, Australia, New Zealand, Italy, Italy, Greece, Italy, Greece, South Korea, North Korea, Mexico, Greece]",14,2095
|
|
||||||
49,"Alice, an archeologist from France, spent her career uncovering the ancient history between France and its neighboring country, Spain. She specialized in the medieval age, focusing particularly on the cross-cultural influences between France and Spain at the time. These nations shared a border and a rich, intertwined history, filled with wars, alliances, and cultural exchange. Her work in France led her to the involvement of another country, Italy, in this historic web. She launched an expedition to Italy, hoping to uncover clues about Italy's relation with Spain in the past. There, she discovered the influence of Italy on Spain's art and architecture during the Renaissance period. Alice's fascination with history drove her from Europe to the Middle East, focusing on the ancient civilizations of Iraq and Iran. She uncovered that Iraq and Iran, despite their modern political differences, share a vast historical connection dating back thousands of years. The Persian Empire, centered in Iran, and the Mesopotamian civilizations, centered in Iraq, had left indelible marks on each other. Alice's research also took her to East Asia, specifically to China and Japan. Alice was captivated by China's ancient dynasties and their influence on Japan through Buddhism and Confucianism. Her investigation revealed that China and Japan had a complex relationship spanning centuries, shaped by the exchange of ideas, trade, and occasional conflicts. Inspired by her findings, Alice decided to delve into the history of North and South America, focusing particularly on Mexico and the United States. She learned that Mexico and the United States shared a layered history, marked by conflicts, treaties, and profound cultural exchanges. The captivating tales of France, Spain, Italy, Iraq, Iran, China, Japan, Mexico, and the United States sparked an unforgettable journey across the globe for Alice, deepening her appreciation for the interconnectedness of the world's history.","[France, France, Spain, France, Spain, France, Italy, Italy, Italy, Spain, Italy, Spain, Iraq, Iran, Iraq, Iran, Iran, Iraq, China, Japan, China, Japan, China, Japan, Mexico, United States, Mexico, United States, France, Spain, Italy, Iraq, Iran, China, Japan, Mexico, United States]",15,1978
|
|
||||||
50,"Often in his dreams, James would find himself in the enchanting landscapes of Sweden, the country where his grandfather was born, raised, and met his grandmother. Stories of Sweden, of their meetings by the pristine lakes and under the northern lights, would always be his favorites. But his grandmother was from Norway, a land equally captivating with its fjords and vibrant cities. She too, had tales to narrate about Norway, featuring the midnight sun and her expeditions to the icy Svalbard. Curiosity piqued, James decided to visit these lands his grandparents hailed from, starting with a trip to Sweden and then making his way to Norway. In his travels, he befriended a fellow adventurer, Laura, who was from the United States and on a similar journey. Laura, like James, was tracing her roots that lay in the heart of the United States where her parents ran a farm. Yet, she was also fascinated by the history of United Kingdom, the birthplace of her paternal grandparents. She would revisit stories her grandparents spun about United Kingdom, its beautiful countryside and bustling cities, the ancient monuments and the modern marvels. It struck James that while he was drawn towards Sweden and Norway, his grandparents' childhood homes, Laura was drawn towards the United States and United Kingdom, where her family's history was rooted. Sharing stories, they soon found common ground in their love for Canada, a place they had both visited during their respective journeys. They laughed over shared memories of Canada, of the breathtaking wilderness, the cosmopolitan cities, and the kind locals, and made plans for a future visit to Canada together. As they continued their explorations, their respect for the past deepened, and their hearts filled with gratitude for the present, realizing the interconnections of countries from Sweden to Canada.","[Sweden, Sweden, Norway, Norway, Sweden, Norway, United States, United States, United Kingdom, United Kingdom, Sweden, Norway, United States, United Kingdom, Canada, Canada, Canada, Sweden, Canada]",13,1859
|
|
||||||
51,"Sophie, an avid traveler and photographer, had recently visited the naturally diverse landscapes of Canada. She was particularly struck by the wildlife in Canada, where she had the rare opportunity to photograph the grizzly bears and orcas. Inspired by her experiences in Canada, she decided to plan her next adventure to Australia, known for its unique flora and fauna. She had always dreamed of photographing the kangaroos and koalas that Australia is famous for. With the memories of Canada still fresh and the anticipation for Australia building, she started sorting through her photographs from her trip to South Africa. She remembered the breathtaking sunrises she had captured in South Africa and the wild Safari experiences where she got close to the Big Five animals. After South Africa, her journey took her to New Zealand, where she was truly enchanted by the magical landscapes. The mountains and the clear blue waters of New Zealand had left a deep imprint on her soul. While reminiscing about her past travels, she remembered her unforgettable journey to Brazil when she had covered the carnival and the vibrant life of the people there. She loved the music and the street art of Brazil and the country's spirit of celebration. She then recalled her time in Mexico, where she was fascinated by the ancient Maya and Aztec civilizations. The diverse culture and the colorful festivals of Mexico were something she would never forget. With these beautiful memories in mind, she was also looking forward to her upcoming visit to Japan. She was particularly excited to witness the cherry blossom festival in Japan and capture the stunning pink hues that blanket the country during spring. Recalling the thrill she felt every time she stepped into a new country, from Canada to Japan, she started packing for her next adventure to Australia. As she packed, she held close the experiences she had gathered so far and looked forward to creating new ones.","[Canada, Canada, Canada, Australia, Australia, Canada, Australia, South Africa, South Africa, South Africa, New Zealand, New Zealand, Brazil, Brazil, Mexico, Mexico, Japan, Japan, Canada, Japan, Australia]",16,1960
|
|
||||||
52,"Robert, an experienced traveler, decided to narrate tales of his adventures in Brazil and Argentina to his friends at a cozy get-together. He described the vibrant culture in Brazil, the passionate people, and how his journey took him all the way to Argentina, where he was welcomed by the warm locals. His descriptions of Argentina and Brazil made his friends eager to hear more about his travels. Next, he took them on a verbal tour of Japan and China, two countries he had visited repeatedly due to their rich histories and contrasting cultures. Robert described the tranquility he experienced in Japan, followed by the bustling energy he witnessed in China - a country with ancient roots and a bright future. China, he mused, was a country of contrasts much like Japan, with its fusion of tradition and modernity. Struck by his tales, one of his friends, Sarah, shared her experiences from her visit to Russia and Ukraine. Sarah expressed how despite the harsh winters in Russia, the warmth of the people was unforgettable, and how moving across the border to Ukraine, she was amazed by the shared history but distinct identities of the two nations. Unlike Russia, Ukraine had a different atmosphere, yet there were undeniable shared experiences between the two nations. Next, their friend James chimed in about his time in Italy and France. As an art lover, James found Italy to be his paradise, filled with awe-inspiring architecture and art. He then journeyed to France, where he was equally enchanted by the art scene and the shared passion for art between Italy and France. The evening ended with everyone vowing to travel more, from Brazil to France, inspired by the shared stories and the cross-cultural bonds that tie the world together.","[Brazil, Argentina, Brazil, Argentina, Argentina, Brazil, Japan, China, Japan, China, China, Japan, Russia, Ukraine, Russia, Ukraine, Russia, Ukraine, Italy, France, Italy, France, Italy, France, Brazil, France]",13,1749
|
|
||||||
53,"John, an archaeologist based in Australia, recently discovered a connection between artifacts found in his home country and those discovered in South Africa. This fascinating link between Australia and South Africa had never been explored before and offered a new perspective on ancient trade routes. Inspired by this discovery, John reached out to his colleague, Sarah, in Canada. Sarah was a leading expert in ancient civilizations and had been studying long-lost connections between Canada and the United Kingdom. Drawing parallels between the cultures of Canada and the United Kingdom, Sarah explained how shared cultural symbols suggested ancient migratory patterns. Meanwhile, in South Korea, a curious historian named Lee had uncovered records from North Korea, detailing ancient diplomatic ties with China. These documents showed that North Korea and China had shared a significant cross-border exchange in the past, influencing the culture of both nations. Intrigued by these findings, Lee reached out to his colleague in Russia, Elena, who had recently been studying the historical association between Russia and Ukraine. Elena had found evidence that Russia and Ukraine, despite their turbulent history, had once shared a mutually beneficial trade relationship. John, Sarah, and Elena, and Lee decided to publish their findings together, sharing their unique insights into the historical connections between Australia and South Korea, Canada and the United Kingdom, and Russia and Ukraine. A celebrated historian from Sweden, Olaf, upon reading their research, was reminded of the ancient connections between Sweden and Norway. Sweden and Norway, being closely situated, had a history of deep cultural and social exchanges. As they all delved deeper into their research, they realized how interconnected the world truly was, from Australia to Sweden, and how history echoed through the ages in surprising and fascinating ways.","[Australia, South Africa, Australia, South Africa, Canada, Canada, United Kingdom, Canada, United Kingdom, South Korea, North Korea, China, North Korea, China, Russia, Russia, Ukraine, Russia, Ukraine, Australia, South Korea, Canada, United Kingdom, Russia, Ukraine, Sweden, Sweden, Norway, Sweden, Norway, Australia, Sweden]",13,1937
|
|
||||||
54,"When Lara visited Spain, she was drawn to the stories of the local people, their traditions, and history. Spain, with its rich culture, was a treasure trove for Lara, who was an anthropologist from Sweden. She found that the architecture in Spain bore a striking resemblance to the structures she had seen in Italy, leading her to delve into the historical connections between Spain and Italy. While in Italy, she also explored its ties with neighboring France, learning about the shared histories, cultural exchanges, and rivalries that shaped the identities of these two countries. Lara's fascination with cultural connections did not stop there; she traveled to Argentina, where she found it intriguing how the country's history was intertwined with Brazil. It was intriguing how Argentina and Brazil, despite historical conflicts, managed to share music, dance forms, and food traditions. Her journey then took her to the United States where she learned about its historical ties with Canada. The United States and Canada, she found, had a history of peaceful coexistence and cooperation despite the occasional disputes. In the Middle East, she explored the deep-rooted historical connections between Iran and Iraq. She discovered that these countries, Iran and Iraq, despite modern-day conflicts, shared a rich history of cultural and economic exchanges. Lara's last trip was to Australia, where she explored its ties with neighboring New Zealand. The closeness between Australia and New Zealand, she found, went beyond geography and was deeply etched in their shared history and culture. As Lara journeyed from Spain to Australia, she found herself marvelling at the way these countries, despite their differences, were interconnected through history and culture.","[Spain, Spain, Sweden, Spain, Italy, Spain, Italy, Italy, France, Argentina, Brazil, Argentina, Brazil, United States, Canada, United States, Canada, Iran, Iraq, Iran, Iraq, Australia, New Zealand, Australia, New Zealand, Spain, Australia]",13,1769
|
|
||||||
55,"John, an historian from Mexico, always had a fascination for the ancient civilizations of Egypt. His interest in Egypt ignited when he discovered that there were ties between the Mesoamerican cultures of Mexico and the ancient dynasties of Egypt, possibly through a network of trade routes lost to the annals of history. His research led him to Italy, famous for its connection to Greece through shared mythology and cultural exchanges. He was intrigued to learn that both Italy and Greece, despite being separate nations, had histories that were intertwined and filled with shared tales of heroism and mythological gods. His interests soon expanded to the Far East, focusing on South Korea and its neighbor, North Korea, despite the political standoff between the two. He learned that South Korea and North Korea, both with a common past, presented a stark contrast in their modern development paths. On his return from Asia, he made a stopover in Ireland and was captivated by the country's rich folklore and legends. Intriguingly, the historical ties between Ireland and the United Kingdom were far more complex than he had assumed, showcasing a mix of conflict and cooperation that had shaped the destinies of both nations. His next stop was the United States, with its vast melting pot of cultures and its dynamic history. Specifically, he was drawn to the deep-rooted cultural and historical links between the United States and Canada that depict a story of neighborly relations, shared values, and common defense interests. Lastly, he found himself in the Middle East, exploring the intricate relationship between Iran and Iraq that spanned millennia. Despite their modern conflicts, he was surprised to find that both Iran and Iraq shared a long history of cultural exchanges, even during periods of strife. His journey, touching on countries from Mexico to Iraq, was a testament to the intricate web of connections historic and modern societies share.","[Mexico, Egypt, Egypt, Mexico, Egypt, Italy, Greece, Italy, Greece, South Korea, North Korea, South Korea, North Korea, Ireland, Ireland, United Kingdom, United States, United States, Canada, Iran, Iraq, Iran, Iraq, Mexico, Iraq]",13,1960
|
|
||||||
56,"During his adventures in Brazil, Jack discovered an ancient map detailing trade routes between Brazil and Argentina. The map indicated that spices, precious metals, and unique textiles were frequently traded between Brazil and Argentina, highlighting their historical connection. Intrigued, he shared his discovery with his historian buddy, Mike, who lives in Canada. Mike reminded him of the notable similarities between such trade accounts and the historical exchanges between the United Kingdom and Ireland, both of which had profound cultural and economic influences on each other. They also recalled a famous book detailing the fascinating expeditions linking South Africa with Australia. The book portrayed their marine adventures, unraveling the rich biodiversity found in both South Africa and Australia's marine ecosystems. Their discussion shifted to the lively bazaars of Pakistan, remembered by Jack from his past travels. He elaborated on the apparent cultural crossovers between Pakistan and Iran, evident from their cuisine, music, and architectural styles. Mike, moved by this story, reminisced on his trip to Sweden. He narrated his experiences about the historic and cultural ties between Sweden and its neighboring country, Norway, deeply rooted in their shared Viking heritage. The conversation then drifted towards the East, to the land of the rising sun, Japan. They discussed the influence Japan had on South Korea and vice versa, especially in areas such as technology and pop culture. Fascinated with these tales of global connections, they reflected upon how countries, from Brazil to Japan, have shared rich histories, thus knitting a beautiful tapestry of human legacy.","[Brazil, Brazil, Argentina, Brazil, Argentina, Canada, United Kingdom, Ireland, South Africa, Australia, South Africa, Australia, Pakistan, Pakistan, Iran, Sweden, Sweden, Norway, Japan, Japan, South Korea, Brazil, Japan]",13,1697
|
|
||||||
57,"In his quest to decipher the mysteries of the world, Tom began his journey in Iraq, where he uncovered antiquated texts that revealed hidden knowledge from Iran. This knowledge was believed to be carefully guarded secrets of Iran's ancient civilization, preserved for millennia and eventually moved to Iraq for protection. The texts sparked Tom's curiosity about the historical intricacies of other countries and led him to continue his journey to China. The land of China unraveled the mysteries of its neighbor, North Korea, sharing ancient myths and legends that intertwined the histories of China and North Korea. This ignited his interest in the connection between neighboring nations and he traveled next to Italy. There he discovered the intertwined history of Italy and France, with tales of kings and queens, wars and peace, and a shared love for art and culture between Italy and France. His travels then led him to South Africa, where he uncovered the deep historical ties between South Africa and Zimbabwe. Tales of migrations and cultural exchanges between South Africa and Zimbabwe intrigued him, revealing a shared history that transcended modern borders. His fascination with history took him next to the United States, where he discovered the profound influences that the United Kingdom had left behind. The United States and the United Kingdom, he found, were more intertwined than he had initially thought, sharing a common language, legal systems, and cultural trends. The final leg of his voyage took him to South America, to Peru, where he learned about the strong connections between Peru and Argentina. Both Peru and Argentina, he discovered, had a shared history of ancient civilizations and the Incas, creating an incredible bridge between the two nations. This journey had taken him from Iraq to Peru, unraveling the shared history and relationships between nations, highlighting both the uniqueness and similarities of the world's cultures.","[Iraq, Iran, Iran, Iraq, China, China, North Korea, China, North Korea, Italy, Italy, France, Italy, France, South Africa, South Africa, Zimbabwe, South Africa, Zimbabwe, United States, United Kingdom, United States, United Kingdom, Peru, Peru, Argentina, Peru, Argentina, Iraq, Peru]",13,1968
|
|
||||||
58,"Once, on a voyage from Argentina to Australia, Captain Fredericks discovered links between these two countries based on the indigenous art forms. The indigenous people of Argentina and Australia, as he noted, shared similar creation stories depicting a great serpent, which was fascinating. Later, he journeyed to the cold regions of Canada and found evidence of ancient trade links with Russia. He unearthed artifacts in Canada that traced back to Russia, suggesting a deep historical connection between these distant lands. This discovery reminded him of the time he spent in China, where he chanced upon cultural parallels with its neighbor, North Korea. China and North Korea, though politically distinct, seem to share a lot in common in terms of their folklore and traditional practices. This sparked his interest in the connectivity of cultures and he decided to visit Egypt, with its long and rich history. Fredericks found astonishing similarities between the ancient civilizations of Egypt and Iraq. Both Egypt and Iraq, he found, possessed complex structures and writing systems that have baffled historians for centuries. These findings led him to his next destination, the colorful land of India, where he found intriguing links with its neighbour, Pakistan. India and Pakistan, despite their bitter history, had much in common culturally speaking, from their cuisine to their shared history of colonial rule. On his return, he stopped by in Spain, where he met scholars who spoke about the shared history of Spain and Portugal. Both Spain and Portugal, having been great maritime powers, left their mark on many parts of the world. Finally, he landed in the United States, where he decided to pen down his travels, highlighting the surprising bonds countries share, from Argentina to the United States.","[Argentina, Australia, Argentina, Australia, Canada, Russia, Canada, Russia, China, North Korea, China, North Korea, Egypt, Egypt, Iraq, Egypt, Iraq, India, Pakistan, India, Pakistan, Spain, Spain, Portugal, Spain, Portugal, United States, Argentina, United States]",14,1816
|
|
||||||
59,"Cora, a history enthusiast from the United States, found an old journal that belonged to her great-grandfather who was a sailor. The journal had detailed accounts of his travels and adventures, and it particularly highlighted his experiences in Cuba. In Cuba, he had discovered a community that was deeply influenced by traditions from Spain. He wrote extensively about the uncanny similarities between Spain and Cuba, from the vibrant music to the architecture. There were also multiple stories of the shared history between the United States and Canada. He described how the border between the United States and Canada was more than just a political boundary, but a symbol of a robust relationship formed over many years. Intriguingly, he also wrote about his travels to Argentina and Brazil, noting the rich shared culture and rivalry between Argentina and Brazil. His reflections on Argentina were filled with admiration for its landscapes and the warmth of its people, while Brazil filled him with wonder with its rich history and natural beauty. Cora, moved by the stories, shared the journal with her friend Bill who had just returned from Italy. Bill was fascinated and narrated his own experiences from Italy, particularly enchanting were his tales about the ancient links between Italy and Greece. He spoke vividly of how the art, architecture, and philosophy of Greece were deeply engrained in the culture of Italy. Bill, intrigued by her great-grandfather's travels, decided to embark on his own adventure, taking with him the insights from the journal and his own experiences from Italy to Greece, naively hoping to write a sequel to her great-grandfather's journal. They both were left in awe, at how interconnected the world was, despite the distances, borders, and years that separated these nations.","[United States, Cuba, Cuba, Spain, Spain, Cuba, United States, Canada, United States, Canada, Argentina, Brazil, Argentina, Brazil, Argentina, Brazil, Italy, Italy, Italy, Greece, Greece, Italy, Italy, Greece]",13,1816
|
|
||||||
60,"Maria, a renowned historian, stumbled upon a century-old document in Argentina, which pointed out that the Incas from Peru had travelled to Argentina for trading purposes. It illustrated the extensive ties between Argentina and Peru, showcasing an intertwining history of trade and influence that lasted several centuries. The document also suggested intriguing overlaps with similar trade routes between Brazil and Colombia. Traders from Brazil had reportedly journeyed to Colombia in pursuit of valuable resources, creating a significant impact on both Brazil and Colombia's cultural and economic landscapes. Enthralled by her findings, Maria shared them with her friend Jack, a professor from Canada. Jack was deeply engrossed in researching the ancient connections between Canada and the United States. He explained that the indigenous tribes from Canada would often travel to the United States for ceremonies and trade, leading to profound interconnections between Canada and the United States. Jack also mentioned his fascination with the historical ties between Australia and New Zealand. Australia and New Zealand, he explained, had been closely linked due to their geographical proximity and shared roots among the indigenous peoples. Maria, intrigued by Jack's findings, recalled her own research into the ancient connections between Egypt and Israel. She elaborated on the complex and layered history between Egypt and Israel, marked by centuries of warfare, trade, and cultural exchanges. Their fascinating conversation underscored the rich tapestry of interconnected histories that spanned from Argentina to Israel, and the profound influence these connections have had on shaping the world we know today.","[Argentina, Peru, Argentina, Argentina, Peru, Brazil, Colombia, Brazil, Colombia, Brazil, Colombia, Canada, Canada, United States, Canada, United States, Canada, United States, Australia, New Zealand, Australia, New Zealand, Egypt, Israel, Egypt, Israel, Argentina, Israel]",12,1718
|
|
||||||
61,"Studying world history, Alice was fascinated by the parallel developments in science and technology in Japan and South Korea. She observed that Japan and South Korea had both made significant advancements in robotics and electronics in the recent past. This prompted her to dive deeper into the technological revolutions of Canada and the United States. In both Canada and the United States, she discovered, the growth of technology was heavily driven by the rise of Silicon Valley and the numerous tech start-ups it inspired. She shared these findings with her friend Ben who was researching on the political history of Iraq and Iran. Ben said that he'd observed a similar pattern, where Iraq and Iran, despite their long-standing conflicts, exhibited parallel political transformations throughout history. This made Alice curious about the indigenous cultures of Australia and New Zealand. She found that both Australia and New Zealand had rich Maori cultures that had significantly influenced their history and traditions. Ben, in response, mentioned his interest in European history, particularly the intertwined histories of France and Italy. He shared how France and Italy, besides sharing borders, had a shared history of art and cuisine, influencing each other in profound ways. Alice found this intriguing and decided to look into the cultural exchanges between Brazil and Argentina. She learned that Brazil and Argentina, both known for their love of soccer, had a lot of cultural exchanges due to the sport. As they continued to delve deeper into their research, they marveled at how interconnected the world truly is, from Japan to Argentina, learning from each other and growing together.","[Japan, South Korea, Japan, South Korea, Canada, United States, Canada, United States, Iraq, Iran, Iraq, Iran, Australia, New Zealand, Australia, New Zealand, France, Italy, France, Italy, Brazil, Argentina, Brazil, Argentina, Japan, Argentina]",13,1701
|
|
||||||
62,"In a quaint coffee shop in France, a group of friends were engaged in a spirited debate about the finest arts and literary traditions around the world. The discussion began when Adam, an aspiring novelist, mentioned how he always admired the elegance of literature from France, citing famous authors who inspired him. He then passed the conversation onto Lisa, who had spent a few years studying in Italy. She described the history of Italy, where art and architecture stood as a testament to its sprawling cultural legacy. Lisa also mentioned Greece, drawing parallels to Italy in terms of the historical significance of its art and philosophy. She elaborated on her visits to Greece, explaining how she was captivated by the timeless beauty of the ancient ruins and the enduring spirit of its philosophies. The discussion then turned to the East, to India, where Mike, a devout yoga practitioner, extolled the spiritual wisdom inherent in the ancient scriptures of India. He also spoke about China, sharing how his travels there had enhanced his understanding of Eastern philosophies. Mike's vivid descriptions of the grandeur of China's historical architectures, and his recollections of the Great Wall fascinated everyone. Carla, who was known for her love of adventure, spoke about her time in Australia. The wildlife, landscapes, and the stories of indigenous tribes of Australia captured the group's imagination. Conversing about Australia, she couldn't help but mention New Zealand as well, expressing her fond memories of its breathtaking landscapes and rich Maori culture. She detailed her exhilarating experiences of nature in New Zealand, leaving everyone longing for a taste of such adventures. Their discussion evolved into an appreciation of global diversity, linking arts, literature, and cultures from France to New Zealand.","[France, France, Italy, Italy, Greece, Italy, Greece, India, India, China, China, Australia, Australia, Australia, New Zealand, New Zealand, France, New Zealand]",14,1841
|
|
||||||
63,"Steven, an adventurer from Canada, decided to visit the exotic wildlife in Kenya. One could not help but marvel at the vast diversity in Kenya, where everything seemed part of an intricate web of life. As he explored, tales of another country resonated in his mind - Australia. Steven had read about the immense biodiversity in Australia, ranging from its unique marsupials to its thriving marine life, and he dreamt of experiencing it someday. During his stay in Kenya, Steven came across traditional art that he found was remarkably similar to the native art forms from Australia. This prompted him to study further, leading him to the indigenous people of Australia and their distinct culture. Next, he planned a visit to South Korea, intrigued by stories of its advanced technology. He was amazed at the rapid strides South Korea had made in fields like robotics and AI. His fascination with technology then took him to Japan, the birthplace of many technological advancements. As Steven navigated the intricate lanes of Japan, he found himself admiring the delicate balance the country had struck between preserving its traditions and embracing modernity. Inspired by his experiences, he decided to write a book on the cultural exchanges and developments between Canada and the United States. The interconnected histories of Canada and the United States, from the founding of their nations to their shared cultural practices, offered a wealth of material for his book. His travels and his book were a testament to the intricate tapestry of global cultures, showing how countries like Kenya and the United States had narratives that were deeply intertwined with others around the world.","[Canada, Kenya, Kenya, Australia, Australia, Kenya, Australia, Australia, South Korea, South Korea, Japan, Japan, Canada, United States, Canada, United States, Kenya, United States]",13,1690
|
|
||||||
64,"As a professor of ancient civilizations, John spent most of his time studying archaeological sites in Mexico. He cherished every moment he spent in Mexico, exploring its rich history and culture. His most significant discoveries, however, came from his expeditions in Peru, where he unearthed ancient artifacts that provided a glimpse into the life of the Incan civilization. Peru, with its breathtaking landscapes and deep-rooted customs, had always fascinated him. It reminded him of his earlier work in Egypt, where he spent years studying the Great Pyramids and other historical marvels. The allure of Egypt, with its iconic structures and timeless tales, was undeniably captivating. During a symposium in the United Kingdom, he met Lisa, a fellow archaeologist from Israel. She had conducted extensive research in Iran, unearthing the secrets of the Persian Empire. She spoke enthusiastically about the enigmatic ruins and inscriptions she found in Iran. She also showed him photographs of the sites she had explored in Iraq. The artifacts she discovered in Iraq suggested complex trade relations that once existed between the civilizations of the Middle East. Drawn to the mysteries of the Far East, Lisa was currently researching the ancient civilizations of China. The abundance of historical wealth in China was mesmerizing, and only added to her fascination with the region. Intrigued, John suggested they collaborate on an archaeological project in South Korea, a country he had always wanted to explore. South Korea, with its unique blend of tradition and modernity, offered a compelling cultural context for their research. They agreed, looking forward to unraveling the mysteries of South Korea together. As the meeting ended, they marveled at the historical tapestry that stretched from Mexico to South Korea, promising more adventures and discoveries to come.","[Mexico, Mexico, Peru, Peru, Egypt, Egypt, United Kingdom, Israel, Iran, Iran, Iraq, Iraq, China, China, South Korea, South Korea, South Korea, Mexico, South Korea]",17,1875
|
|
||||||
65,"Sarah, an archaeologist from the United Kingdom, had been involved in numerous expeditions in Egypt and was intrigued by the historic similarities between key artifacts in both the United Kingdom and Egypt. It was a fascinating correlation she discovered during her research - that both the United Kingdom and Egypt had a shred of commonality in their ancient cultures. She shared her findings with her colleague John from the United States, who was equally interested in the historical interconnections between different countries. He highlighted an interesting parallel between Russia and Ukraine. He pointed out that Russia and Ukraine, despite their tumultuous history, shared an array of cultural motifs and traditions. This struck a chord with Sarah, who had recently been researching the history of Italy and France. It had seemed to her that Italy and France, despite being distinctive in their cultural styles, had a common thread of influences in their art and architecture. With keen interest, John recalled his expedition to China, where he had noticed an intriguing connection with its neighbor Japan. According to his research, China and Japan, despite their contrasting lifestyles, had a profound influence on each other's ancient practices and philosophies. Sarah then mentioned her recent visit to South Africa, where she spotted a surprising similarity in ancient artifacts with those in Kenya. She explained that Kenya and South Africa, for all their unique cultural variations, appeared to share a historical bond. As their exploration deepened, they realized more profound connections. Sarah brought up her knowledge about the historical links between Canada and the United States. She noted that Canada and the United States had deeply intertwined histories, due to their geographical proximity and shared colonial past. Both archaeologists agreed that these instances of historical interconnections were fascinating, demonstrating how our world is much more interconnected than it might first appear.","[United Kingdom, Egypt, United Kingdom, Egypt, United Kingdom, Egypt, United States, Russia, Ukraine, Russia, Ukraine, Italy, France, Italy, France, China, Japan, China, Japan, South Africa, Kenya, Kenya, South Africa, Canada, United States, Canada, United States]",15,2023
|
|
||||||
66,"On a crisp autumn day, Jack, an esteemed professor from the United States, began sharing with his class tales of hidden architectural wonders of Mexico. Mexico, he explained, was home to ancient civilizations whose remnants were still visible in the form of grand pyramids and intricate carvings. After Mexico, he turned the discussion towards Italy, and the architectural marvels that stand in testament to their centuries-old history. The Roman coliseum and the tower of Pisa were favorite examples when discussing Italy, he said, as they represent different periods and styles prevalent in Italy. He also touched upon Iran, describing how the country's rich history is evident in its architecture, from the elaborate mosques to the ancient city of Persepolis. Iran, he said, was a treasure trove for historians and architects alike, offering insights into the glorious past of Persia. As the lecture advanced, he started talking about Egypt and its awe-inspiring pyramids. Egypt, he said, was a living museum with countless artifacts and structures that had survived thousands of years. Stepping outside architecture, Jack described a trip to South Korea, where he had explored the technological advancements of the country. He explained how South Korea was on the cutting edge of technology, with high-speed internet and tech giants that were globally recognized. South Korea, he held, was a testament to how technology could drive development. Towards the end, Ireland was brought up, as he detailed how the country's landscapes and traditions were equally captivating. Ireland, he stated, was an enchanting blend of history and natural beauty, with its numerous castles and lush green landscapes. The students left the class, minds brimming with stories of Mexico to Ireland, eager to see the world's wonders firsthand.","[United States, Mexico, Mexico, Mexico, Italy, Italy, Italy, Iran, Iran, Egypt, Egypt, South Korea, South Korea, South Korea, Ireland, Ireland, Mexico, Ireland]",14,1825
|
|
||||||
67,"On a recent cultural exchange trip, Mia from South Korea learned fascinating historical links between North and South Korea. Despite the contemporary political divide, she discovered common roots in their shared history, culture, and tradition, and this was true for both North and South Korea. She discussed this with her roommate, Anne, from Australia, who shared a similar story about Australia's relationship with New Zealand. Anne explained that Australia and New Zealand, despite being separate nations, shared a profound bond due to their geographic proximity and shared history. This conversation triggered Mia's memory of a book she had read about the long-standing ties between Venezuela and Colombia. In the book, the author made a compelling argument about the common cultural heritage and intertwined history of Venezuela and Colombia. Inspired by their exchange, Anne brought up her cousin's travels to Zimbabwe and South Africa. She recalled the stories about the cultural exchanges between Zimbabwe and South Africa, the shared struggle against colonial rule, and the enduring bonds of friendship. Mia found the stories fascinating and told Anne about an article she read about the historical connections between Italy and France. This piece detailed the centuries of economic, political, and cultural exchange between Italy and France, pointing out how the Roman Empire's influence stretched across both regions. As they continued to discuss, Anne remembered her teacher's colorful anecdotes about the United Kingdom and Ireland. The United Kingdom and Ireland, she learned, shared complex and layered histories, with influences from literature to politics making their relationship an intriguing study. As they ended their conversation, they realized that whether one talks of South Korea or Ireland, the story of humanity is deeply interconnected and beautifully complex.","[South Korea, South Korea, South Korea, Australia, Australia, New Zealand, Australia, New Zealand, Venezuela, Colombia, Venezuela, Colombia, Zimbabwe, South Africa, Zimbabwe, South Africa, Italy, France, Italy, France, United Kingdom, Ireland, United Kingdom, Ireland, South Korea, Ireland]",13,1890
|
|
||||||
68,"On her journey around the world, Rebecca discovered a fascinating link between the countries of Ireland and the United Kingdom. She had always been intrigued by the rich history shared by the United Kingdom and Ireland, the tales of conflicts, and the long-standing cultural exchanges that have shaped both nations. Her interest in historical bonds didn't end there; while in the United Kingdom, she came across a book that described the past relations of the United States and Canada. The book detailed how the United States and Canada shared a border and a long history of trade, migration, and sometimes contentious diplomacy. This reminded Rebecca of the complex history between Russia and Ukraine. Having lived in Ukraine, she had firsthand experience of the intricate relationship between Russia and Ukraine, a story of shared heritage, conflicts, and divergent paths. It was a stark contrast to the peaceful coexistence she had witnessed between Norway and Sweden. Norway and Sweden, she reminisced, shared a harmonious relationship, full of mutual respect and cooperation, making them a model for neighboring countries. Her thoughts then wandered to her time in South Africa, where she had learned about the country's connections with Zimbabwe. Historically, South Africa and Zimbabwe had a great deal of interaction due to their geographic proximity and similar colonial pasts. She also recollected her visit to Australia and the stories she'd heard about the country's ties with New Zealand. Australia and New Zealand, as she learned, had a fascinating history of competitive friendship, shaped by geographic proximity and shared culture. From there, her mind drifted to Asia, where she had spent time in both China and Japan. Despite their historical tensions, China and Japan had periods of cultural exchanges that had greatly influenced both nations. Finally, her thoughts turned to Italy and France, two countries inextricably linked by art, cuisine, and mutual history. Italy and France, she realized, epitomized the interconnectedness of the world's countries, from Ireland to Japan.","[Ireland, United Kingdom, United Kingdom, Ireland, United Kingdom, United States, Canada, United States, Canada, Russia, Ukraine, Ukraine, Russia, Ukraine, Norway, Sweden, Norway, Sweden, South Africa, Zimbabwe, South Africa, Zimbabwe, Australia, New Zealand, Australia, New Zealand, China, Japan, China, Japan, Italy, France, Italy, France, Ireland, Japan]",16,2099
|
|
||||||
69,"One sunny afternoon, Emily and Robert, an adventurous couple from Spain, decided to visit the beautiful landscapes of Ireland after hearing stories about its mesmerizing beauty. Their friend, who had traveled throughout Ireland, had shared tales of emerald valleys, ancient castles, and warm-hearted locals, compelling them to see it for themselves. Having experienced the charming sites of Ireland, they set their sights on Argentina, inspired by the journey of their close friend Jack, who had raved about his experiences in Argentina and Brazil. He described Argentina as a place of magnificent mountains and vast pampas, while Brazil offered magical rainforests and vibrant city life, creating an irresistible wanderlust in Emily and Robert. After exploring the stunning sights of Argentina and Brazil, they got an unexpected call from their friend in Australia. She told them about a festival in Australia that celebrated the cultural ties between Australia and New Zealand, where the exchange of arts, music, and cuisines took center stage. Intrigued, Emily and Robert decided to witness this amalgamation of cultures, which further solidified their respect for Australia and New Zealand's bond. They later received word from another friend residing in Belgium who spoke of an exciting event commemorating the historical ties between Belgium and the Netherlands. Intriguingly, both Belgium and the Netherlands had histories steeped in similar traditions of craftsmanship, resulting in shared architectural and artistic styles. Finally, Emily and Robert decided to conclude their journey in South Korea, where a friendly local spoke about the influence of China in their country. He explained that South Korea and China had a dynamic relationship, a product of centuries of trade, and sometimes conflict, that shaped their current culture and way of life. Thus, this epic journey took them from Spain to South Korea, with experiences that deepened their understanding of the world's shared histories.","[Spain, Ireland, Ireland, Ireland, Argentina, Argentina, Brazil, Argentina, Brazil, Argentina, Brazil, Australia, Australia, Australia, New Zealand, Australia, New Zealand, Belgium, Belgium, Netherlands, Belgium, Netherlands, South Korea, China, South Korea, China, Spain, South Korea]",12,2005
|
|
||||||
70,"In the heart of Africa, there is a story that connects Kenya to Egypt. The tale suggests that the ancient kingdoms of Kenya and Egypt shared a bond, trading goods and knowledge across the harsh desert. Jack, a traveler from Ireland, always dreamt of retracing the steps of these ancient traders, from the humid coasts of Kenya, through the barren Sahara, to the marvels of Egypt. On his journey, he was accompanied by his friend Hiroshi from Japan, who was fascinated by the ancient cultures and their connections. Japan itself had a vibrant history of exchanges with its neighbor, China. Hiroshi would always recount the tales of how samurais from Japan used to visit China to learn new strategies and martial arts. The mention of martial arts brought back memories to Jack about his trip to South Korea, where he experienced the discipline and beauty of Taekwondo. He learned that Taekwondo, despite being indigenous to South Korea, was influenced by ancient martial arts from China. Just as Jack was talking about martial arts, Hiroshi spoke of his visit to the United States, where he saw how baseball, considered America's pastime, had been embraced by Japan. Interestingly, the United States also had a deep connection with Canada, its historical ally and neighbor. Both the United States and Canada not only shared the world's longest international border, but also a rich history of cultural exchange and friendship. It suddenly dawned on Hiroshi and Jack that they were not just travelers, but links in a chain that stretched generations and across the globe, from Kenya to Canada, and from Japan to the United States.","[Kenya, Egypt, Kenya, Egypt, Ireland, Kenya, Egypt, Japan, Japan, China, Japan, China, South Korea, South Korea, China, United States, Japan, United States, Canada, United States, Canada, Kenya, Canada, Japan, United States]",12,1627
|
|
||||||
71,"John, a renowned historian from the United States, specialized in studying the historical ties among nations. He found himself deeply immersed in researching the bond between North Korea and South Korea, noting that despite their political divide, shared history still tied them together. He discussed his findings with his colleague Lisa, who had an expertise in tracing the historical links between India and Pakistan. Both India and Pakistan share a similar language, culture, and history, suggesting traces of a shared past despite their political differences. These shared elements in culture and history also reminded John of the complex relationship between the United States and Canada. A shared border and common language allowed the United States and Canada to collaborate in various sectors like trade, technology, and arts. Lisa brought up her recent trip to Australia and New Zealand, and their unique relationship, sharing histories of colonialism, indigenous cultures, and a fierce sporting rivalry. She found the bonds Australia and New Zealand shared were reminiscent of the ties between Belgium and the Netherlands, two countries she explored during her sabbatical. Belgium and Netherlands, both located in Western Europe, share a rich history of trade, art, and cultural exchange. They concluded their discussion by drawing parallels with South America, particularly the close alliance and shared cultural history between Argentina and Brazil. They marveled at how interconnected these countries were, from North Korea to Brazil, illustrating the intricate web of shared histories and cultural exchanges across the globe.","[United States, North Korea, South Korea, India, Pakistan, India, Pakistan, United States, Canada, United States, Canada, Australia, New Zealand, Australia, New Zealand, Belgium, Netherlands, Belgium, Netherlands, Argentina, Brazil, North Korea, Brazil]",11,1640
|
|
||||||
72,"In a quaint coffee shop in Portugal, Alice found herself immersed in a riveting book about the deep-rooted historical rivalry between Portugal and Spain. The book elaborated how despite their disagreements, Portugal and Spain had a shared heritage that was hard to ignore. Her friend Ben, who had lived in both Spain and Portugal, often spoke about the shared traditions and culture that overpowered their differences. Their discussion then shifted to the exotic travel stories of Ben who had recently returned from a trip to China. He spoke about the beautiful landscapes, bustling cities, and enchanting history of China, and how it beautifully contrasted with its neighboring country, Vietnam. The elegance of China and the charm of Vietnam had left a profound impact on him. He also mentioned how Vietnam and China, despite their past conflicts, shared a deep cultural bond. Ben's travel stories reminded Alice of her friend Claire who had recently moved to Canada. Claire had spoken at length about the differences between Canada and the United States; how the two countries, while very close geographically and often compared to one another, had very distinct identities. She shared stories about her experiences in both Canada and the United States, highlighting the unique cultural facets of each nation. Intrigued by such narratives, Alice too expressed her desire to explore these countries. After a day filled with tales from Spain to the United States, they walked out of the coffee shop with a newfound appreciation for the world's diverse cultures and histories.","[Portugal, Portugal, Spain, Portugal, Spain, Spain, Portugal, China, China, Vietnam, China, Vietnam, Vietnam, China, Canada, Canada, United States, Canada, United States, Spain, United States]",12,1576
|
|
||||||
73,"John, an avid lover of history, had always been fascinated by the ancient civilizations of Egypt and their influence on Greece. The tales of how the Pharaohs' knowledge shaped the philosophies of Greece had amazed him ever since his childhood. His fascination wasn't limited to Egypt and Greece though; he was equally captivated by the historical exchanges between the Persian empires of Iran and the Indus valley civilization in Pakistan. These two countries, Iran and Pakistan, despite being so geographically apart, had a rich and intertwining history. One day, while reading a book on world history, he stumbled upon the cultural and political exchanges between the prosperous kingdoms of India and China in ancient times. The way India and China, two colossal civilizations, had intermingled their philosophies fascinated him. As he delved deeper, he discovered the lesser-known naval expeditions from the royal fleets of China visiting the archipelago of Indonesia, trading silk for spices. These maritime interactions between China and Indonesia reflected the strategic significance of the latter. His interest deepened when he read about the exploration expeditions between the United Kingdom and Australia, with the latter once seen as the ""Land of Golden Opportunities"" by the UK. Intriguingly, the connection wasn't just limited trading but was rather a complex relationship between the United Kingdom and Australia involving colonization, settlement, and later, independence. This led him to learn about the profound relationships between neighboring nations like Portugal and Spain, with their shared culture, history, and rivalry. The more John read about Portugal and Spain, the more he found himself engrossed in their fascinating intertwining histories. He was equally intrigued about the historical bonds between Belgium and the Netherlands, two countries that despite being distinct, shared a common past that was still evident in their culture. These complex ties between Belgium and the Netherlands made him even more passionate about world history. His extensive readings affirmed that the world was wonderfully intricate and deeply interconnected, right from Egypt to the Netherlands.","[Egypt, Greece, Greece, Egypt, Greece, Iran, Pakistan, Iran, Pakistan, India, China, India, China, China, Indonesia, China, Indonesia, United Kingdom, Australia, United Kingdom, Australia, Portugal, Spain, Portugal, Spain, Belgium, Netherlands, Belgium, Netherlands, Egypt, Netherlands]",15,2207
|
|
||||||
74,"In the peaceful calm of Canada, Bob, a history enthusiast, stumbled across accounts of ancient trading routes between Canada and China. The documents hinted at merchants from China voyaging across the Pacific Ocean to trade with tribes in Canada, creating a bond between these two different worlds. This sparked his curiosity and led him to his friend Maria, a linguist from Brazil who had deep knowledge of ancient languages and cultures. Maria was excited about Bob's discovery as she had recently learned about ancient routes between Brazil and Argentina. These routes were a reflection of the intertwined histories of Brazil and Argentina, supposedly established by indigenous tribes for trade and cultural exchange. Maria then shared an interesting story about the historic relations between Russia and Ukraine, where shared cultural, linguistic, and political experiences have largely influenced their intertwined destinies. This symbiotic relationship between Russia and Ukraine reminded Bob of an ancient story he had heard about the ties between Ireland and the United Kingdom. The story spoke of a time when Ireland and the United Kingdom, despite their tumultuous history, had shared literary and cultural traditions. Maria, further intrigued, shared her travels to the Middle East, specifically to Iran and Iraq, countries inseparable in their shared history and geography. Her tales about Iran and Iraq's common heritage, despite recent conflicts, fascinated him, as they mirrored the similar situation between Israel and Egypt. Israel and Egypt, despite their political differences, have a rich shared history dating back to biblical times. Enraptured by these tales of intertwined histories, both Bob and Maria marveled at how interconnected the world truly is, from Canada's ancient ties with China to Israel's shared history with Egypt.","[Canada, Canada, China, China, Canada, Brazil, Brazil, Argentina, Brazil, Argentina, Russia, Ukraine, Russia, Ukraine, Ireland, United Kingdom, Ireland, United Kingdom, Iran, Iraq, Iran, Iraq, Israel, Egypt, Israel, Egypt, Canada, China, Israel, Egypt]",12,1853
|
|
||||||
75,"In his youth, John had always been fascinated by the heroic tales of the samurais of Japan. He read voraciously about these warriors from Japan, their rigorous discipline, and their peculiar code of honor. Inspired, he decided to learn more about martial arts and ended up in South Korea, enrolling in a Taekwondo academy. Spending years in South Korea, he noticed parallels between the samurai culture of Japan and the martial philosophy of South Korea. His time in Asia also exposed him to the rich spiritual traditions of India. He was particularly intrigued by yoga from India, which he found to be a potent combination of physical discipline and mental concentration. On his return to the western hemisphere, he met Maria in Peru, who had a similar passion for spirituality and mysticism. She enlightened him about the Inca civilization of Peru and its unique spiritual practices. Intrigued by the Inca civilization, John became fascinated with the Mayan civilization of Mexico. He read books about Mexico, about its ancient pyramids and astronomical knowledge. Maria, inspired by John's interest in ancient civilizations, shared her experiences of visiting the pyramids in Egypt. She described Egypt with such passion that John decided to plan his next adventure there. In Egypt, he found connections with Peru and Mexico, each bound by their ancient civilizations. Through their travels, Maria and John realized how interconnected the world is, from the samurais of Japan to the pyramids of Egypt, and it only deepened their passion for exploration and learning.","[Japan, Japan, South Korea, South Korea, Japan, South Korea, India, India, Peru, Peru, Mexico, Mexico, Egypt, Egypt, Egypt, Peru, Mexico, Japan, Egypt]",14,1569
|
|
||||||
76,"John was a seasoned traveler and had been to numerous countries around the world. He began his journey in Canada, exploring the vast wilderness and rich cultural heritage of the country. John spent a year in Canada, initially living in the vibrant city of Toronto, then making his way west to the picturesque Rockies. From Canada, he flew to its southern neighbour, the United States, where he was mesmerized by the diversity and spirit of the country. He spent another year traveling through the United States, visiting the bustling cities like New York and Los Angeles, and the beautiful national parks like Yellowstone and Yosemite. From the United States, he ventured further south to Mexico. He was enamored by the history and food culture in Mexico and spent several months traversing the country. The ruins of ancient civilizations in Mexico fascinated him and he learnt about the country's rich past. The next leg of his journey took him to Australia, a land of stunning landscapes and diverse wildlife. John found himself captivated by Australia's unique flora and fauna, and the friendly nature of its people. From Australia, he made his way to Japan. Japan's unique blend of tradition and modernity captured his heart and he spent a sizable chunk of his journey exploring the country. He admired Japan's commitment to its traditions, its technological prowess, and its obsession with perfection. From Japan, he flew to Norway, a country he had always dreamed of visiting. On his first day in Norway, he was awestruck by the fjords and the northern lights. The more he explored Norway, the more he fell in love with the country, its people, and its culture. As he boarded the flight for his next destination, South Africa, he looked back at his journey from Canada to Norway, and was filled with gratitude for the incredible experiences he had had. South Africa awaited him with its own set of adventures and wonders, and he was ready to embrace them.","[Canada, Canada, Canada, United States, United States, United States, Mexico, Mexico, Mexico, Australia, Australia, Australia, Japan, Japan, Japan, Japan, Norway, Norway, Norway, South Africa, Canada, Norway, South Africa]",18,1961
|
|
||||||
77,"John, an avid traveler from Canada, had recently returned from an exciting trip to Japan, a country he had always wanted to visit. He was mesmerized by Japan's rich history and unique culture and was intrigued by the stories he heard about the ancient samurai warriors of Japan. In his conversations with locals, he heard tales of the historic trade relations between Japan and China, and how both Japan and China had influenced each other's art, culture, and technological progress. Captivated by these historical narratives, John decided to take his next adventure to China. Upon reaching China, he was fascinated by the Great Wall, which served as a symbol of China's strength and resilience. The locals in China also shared stories about the Silk Road and how it connected China with Iran, leading to the exchange of goods, ideas, and culture. Intrigued by this, John penciled in Iran as his next destination. Upon his arrival in Iran, John was amazed by the grandeur of the Persepolis and the many tales of Iran's ancient civilizations. In Iran, he learned of the country's deep-rooted connections with Iraq, from the shared religious beliefs to the common history of the Persian Gulf. Curiosity piqued, John subsequently decided to visit Iraq. In Iraq, he witnessed the remnants of Babylon, illustrating Iraq's significance in world history. Iraq's historical narrative also emphasized its ties with Turkey, a country known for its rich history and diverse culture. John then mapped out his route to Turkey, eager to explore another chapter in the grand tale of interconnected nations. In Turkey, the grandeur of the Hagia Sophia awestruck him, sparking conversations about Turkey's unique position at the crossroads of Europe and Asia. Stories of Turkey's past reminded him of Greece's history, particularly Turkey and Greece's shared heritage and sometimes tumultuous coexistence. This made John eager to continue his historical exploration in Greece. The links that had drawn him from Canada to Japan, from there to China and Iran, and eventually to Iraq, Turkey, and Greece, reminded him of the intricate web of connections that made up human history.","[Canada, Japan, Japan, Japan, Japan, China, Japan, China, China, China, China, China, China, Iran, Iran, Iran, Iran, Iran, Iraq, Iraq, Iraq, Iraq, Iraq, Turkey, Turkey, Turkey, Turkey, Turkey, Greece, Turkey, Greece, Greece, Canada, Japan, China, Iran, Iraq, Turkey, Greece]",17,2161
|
|
||||||
78,"Jenny, a travel enthusiast from the United States, had a particularly fascinating story to tell about her time in Russia. She insisted that the more she traveled around Russia, the more it felt like she was navigating through a book of diverse and complex tales. She further mentioned that the art, culture, and history of Russia left a profound impact on her, making her visit all too memorable. Recently, she met Kazuo, a friend from Japan, who had a completely unique experience in Iran. He claimed Iran to be a nation of warm hospitality and stunning heritage, contrasting drastically with the public perception of Iran. Kazuo explained how Iran, despite controversies, held immense beauty and complex history that would surprise many. He also had a chance to explore Pakistan, and he found it equally diverse and intriguing. Each city in Pakistan, he noted, told a different story and held fascinating secrets that often went unnoticed in the world's eyes. Meanwhile, Jenny also had a fantastic experience during her visit to Australia. The wildlife, the landscapes, and the laid-back lifestyle of Australia were both thrilling and relaxing at the same time. She stated how Australia's natural beauty and its unique flora and fauna were truly captivating. Speaking of natural beauty, Jenny and Kazuo were both eager to visit Canada. Stories of the breathtaking landscapes, wildlife, and multicultural cities of Canada had always intrigued them. Canada, they believed, symbolized a perfect blend of urban life harmoniously co-existing with nature. After exchanging tales of travel and their shared yearning for Canada, they concluded that exploring different nations like Russia, Iran, Australia, and Pakistan opens doors to learning about diverse cultures, histories, and lifestyles. It was the shared understanding of the immense beauty each country held, and the desire to see more of the world that strengthened their bond.","[United States, Russia, Russia, Russia, Japan, Iran, Iran, Iran, Iran, Pakistan, Pakistan, Australia, Australia, Australia, Canada, Canada, Canada, Canada, Russia, Iran, Australia, Pakistan]",16,1931
|
|
||||||
79,"As a historian, Alice was fascinated by the ancient civilizations of Greece and Egypt. She loved reading about how scholars from Greece and Egypt exchanged ideas, contributing to the foundation of philosophical and mathematical thoughts. Alice, who was from the United States, had a longtime friend, Ying, from China. Ying, who studied ancient arts, was particularly interested in the historical links between China and Japan. She explained that the cultural exchanges between China and Japan had fostered a significant development in arts, religions, philosophy, and social organizations. It was evident that Alice and Ying shared a profound interest in the historical connections between countries. Alice, fondly reminiscing about her trip to Australia, mentioned its curious connection with New Zealand. She mentioned how Australia and New Zealand shared a similar culture and history, with both being colonies of the United Kingdom. Ying, reminded of her visit to South Korea, shared how she learned about the deep cultural and historical connections between North Korea and South Korea. Despite the current political divide, there was undeniable shared heritage and history between North Korea and South Korea. Both excitedly shared stories about their visit to Europe, Alice had been to Ireland and Ying had visited Poland. Alice brightened up talking about the shared culture of Ireland and the United Kingdom, while Ying shared her fascination with the rich history and cultural ties between Poland and Ukraine. As they connected these dots, they found themselves growing more passionate about history and decided to plan a trip to explore more connections around the world.","[Greece, Egypt, Greece, Egypt, United States, China, China, Japan, China, Japan, Australia, New Zealand, Australia, New Zealand, United Kingdom, South Korea, North Korea, South Korea, North Korea, South Korea, Ireland, Poland, Ireland, United Kingdom, Poland, Ukraine]",13,1682
|
|
||||||
80,"In the heartland of Argentina, a team of archaeologists unearthed ancient artifacts believed to have links to the indigenous cultures of Brazil. These artifacts were a testament to a historical bond between Argentina and Brazil, bonds that spanned commerce, culture, and shared histories. Intrigued by this, a historian from the United States began researching historical records, asserting there were similar cultural exchanges between United States and Canada. The evidence suggested that despite the political boundaries, the indigenous communities of the United States and Canada were interconnected in their beliefs, customs, and trading routes. This reminded the historian of his colleague from South Korea, who had dedicated her life to unraveling the historical ties between South Korea and North Korea. Despite modern tensions, she explained, the past of South Korea and North Korea was full of shared mythologies, philosophies, and even royal lineages. Recalling her work, the historian was encouraged to delve deeper into European histories, particularly of Poland and Ukraine. He discovered that Poland and Ukraine, despite distinct identities, shared common historical threads; they had alternately been allies or enemies over centuries, shaping each other's history. While exploring this, he stumbled upon an interesting account of Australia's relationship with New Zealand. Both Australia and New Zealand, isolated in the Pacific Ocean, had closely knit bonds since ancient times, with migration, trade, and even sports. The historian's wife, a botanist from Ireland, had her own tales to share; she had unearthed fascinating links between the native flora of Ireland and Scotland. The plants of Ireland and Scotland, she proposed, told a story of ancient land bridges and shared ecosystems. As they shared their findings, they marveled at the complex web of relationships that spanned from Argentina to Scotland, demonstrating the interconnectedness of our world's histories.","[Argentina, Brazil, Argentina, Brazil, United States, United States, Canada, United States, Canada, South Korea, South Korea, North Korea, South Korea, North Korea, Poland, Ukraine, Poland, Ukraine, Australia, New Zealand, Australia, New Zealand, Ireland, Ireland, Ireland, Argentina]",13,1991
|
|
||||||
81,"On a historical expedition in Vietnam, professor Albert uncovered evidence of an ancient civilization that had moved from China. The artifacts had intricate designs and symbols that were distinctly from China's long-lost dynasties, suggesting an unknown migration. Elated from his discovery, Albert decided to share the news with his colleague, professor Catherine, an archaeology expert residing in Canada. She expressed excitement and revealed an interesting fact about the indigenous tribes within Canada and their ancestral connections to the United States. Many tribes in Canada, she revealed, had migrated from the United States, carrying with them rich cultural traditions and histories that continue to thrive today. Catherine had also collaborated with an archaeologist from Australia, who had discovered ties between the indigenous cultures of Australia and New Zealand. Both Australia and New Zealand housed tribes that shared certain hunting techniques, artistic styles, and even linguistic patterns, pointing towards a common ancestor. Albert recalled his recent trip to the exotic landscapes of Peru, where he had learned about its historical connections with Argentina. Peru and Argentina, he learned, had strong historical links due to shared Inca traditions and the movement of people between the two territories in ancient times. Speaking of South America, Catherine shared her intriguing research on the pre-colonial connections between Mexico and Cuba. She explained that Mexico and Cuba had ancient ties, probably linked through trade and the Caribbean sea currents. Albert then reminisced about his experiences in Greece where he had discovered links between Greece and Italy. Both Greece and Italy, it seemed, were part of an ancient network of intellectual and commercial exchanges that shaped the western civilization. As the conversation ended, they realized how interwoven the world truly was - from Vietnam to Italy, human history had created countless connections across continents.","[Vietnam, China, China, Canada, Canada, United States, Canada, United States, Australia, Australia, New Zealand, Australia, New Zealand, Peru, Argentina, Peru, Argentina, Mexico, Cuba, Mexico, Cuba, Greece, Greece, Italy, Greece, Italy, Vietnam, Italy]",14,2011
|
|
||||||
82,"Travelling through Norway, Anna came across ancient tales of Viking expeditions to Ireland. The stories described how the fierce Vikings from Norway had established settlements in many parts of Ireland, transforming its landscape and culture. After her trip to Norway, she journeyed to Ireland, hoping to unearth more about the Viking history hidden in its verdant valleys. In Ireland, the Vikings' influence was still palpable through the myriad of artifacts displayed in the local museums and the intriguing tales spun by the locals. While in Ireland, she met a researcher from Australia who was studying the movements of the Vikings. He shared some fascinating insights about the Vikings' travels to the United Kingdom, and how they had left a profound impact there too, raiding and settling in what are now England, Scotland, and Northern Ireland. This sparked Anna's curiosity about Australia, prompting her next career move to the continent down under. Once in Australia, she discovered an entirely different narrative of indigenous cultures and European colonization. Interestingly, she found parallels to the history of colonisation in New Zealand. Discussions with locals and historians led her to understand that Australia and New Zealand share a complex cultural and political history, their paths shaped by European interventions. Recalling her earlier passion for the Viking history, she decided to travel to Sweden, another country with a rich Viking past. The country's history reminded her of the tales she had heard in Norway, especially the exploits of the Vikings in Sweden and their adventures across the North Sea. In Sweden, she delved deeper into the history of Viking travels to Russia, a country that had been significantly impacted by the Vikings. She found the connections between Sweden and Russia fascinating, the imprint of Viking trade routes still visible through the historical sites scattered across Russia. Her journey through Norway, Ireland, Australia, New Zealand, Sweden and Russia had been a fascinating exploration of historical links and cultural exchanges, leaving her with a deeper understanding and appreciation of the interconnected world.","[Norway, Ireland, Norway, Ireland, Norway, Ireland, Ireland, Ireland, Australia, United Kingdom, Ireland, Australia, Australia, New Zealand, Australia, New Zealand, Sweden, Norway, Sweden, Sweden, Russia, Sweden, Russia, Russia, Norway, Ireland, Australia, New Zealand, Sweden, Russia]",15,2185
|
|
||||||
83,"In his travels, John had the opportunity to visit Vietnam, where he learned about the country's long-rooted cultural ties with China. It seems that ancient dynasties in Vietnam were deeply influenced by the philosophy, art, and culture from China. He then flew to Japan, another country that shared an intricate and complex history with China. The two nations, Japan and China, have shared a heated rivalry and mutual respect over the course of history. After returning home to Canada, he read a book about the historical relationship between Spain and Portugal. The book shed light on the intense competition and cooperation that existed between Spain and Portugal during the era of exploration. On a lunch break one day, he chatted with a friend from Iran who mentioned the ancient trade links between India and Iran. Apparently, merchants would sail from India to Iran, trading spices, textiles, and precious stones. Intrigued, John started researching more about Iran and discovered that the Persians had even established trade connections with Russia. It was arresting to learn that both Iran and Russia were linked through centuries of trade, cultural exchange, and occasional conflicts. His friend from South Africa then told him about a fascinating documentary showcasing the historical ties between the United Kingdom and South Africa. The documentary illustrated how the United Kingdom had a significant influence on South Africa, shaping its institutions and infrastructure. Swapping stories about her own travels, his friend mentioned her unforgettable trip to Argentina and its stunning landscapes that reminded her of neighboring Brazil. Interestingly, Argentina and Brazil share not only borders but also cultural elements, football rivalries, and a history of political alliances and tensions. Their conversation left them both astounded at the interconnectedness and shared histories of countries from Vietnam to Brazil.","[Vietnam, China, Vietnam, China, Japan, China, Japan, China, Canada, Spain, Portugal, Spain, Portugal, Iran, India, Iran, India, Iran, Iran, Russia, Iran, Russia, South Africa, United Kingdom, South Africa, United Kingdom, South Africa, Argentina, Brazil, Argentina, Brazil, Vietnam, Brazil]",15,1937
|
|
||||||
84,"As Alice walked the bustling streets of Mexico, she couldn't help but marvel at the vibrancy and warmth that reminded her of her home country, India. She began comparing the street food markets of Mexico with those in India, realizing that despite the wide geographical and cultural distance, there were uncanny similarities. On the other hand, her friend John, who hailed from Australia, shared stories of his homeland. He spoke fondly of the stunning landscapes that defined Australia, the wildlife, and the serene beaches, drawing a stark contrast to the rugged beauty of Canada. Canada, where Alice had visited once, was indeed a magnificent land boasting snow-capped peaks, lush forests, and vast open space. Subsequently, their conversation shifted towards Europe, and John reminisced about a trip he took some years ago to Spain. Landmark buildings, rich history, and exquisite cuisine were some of the things that made Spain unforgettable for him. However, he noted, there was a unique charm about neighboring Portugal. Both Spain and Portugal, he explained, had a lasting impact on world history due to their exploration and colonization efforts. Alice, on her part, shared fascinating details about her recent trip to Iran. Iran's rich history, she said, was deeply intertwined with that of Iraq, prompting her to visit Iraq next. She recounted tales of the beautiful landscapes, ancient ruins, and friendly locals in Iran and Iraq that mesmerized her. Lastly, both friends agreed that no travel conversation could be complete without mentioning Italy. While Alice had always been enchanted by the architectural grandeur and artistry of Italy, John couldn't stop praising Italy's delectable cuisine and wine culture. They both dreamed about their next adventure, hoping to explore more wonders that lay within the borders of countries like India, Australia, Canada, Spain, Portugal, Iran, Iraq, and Italy.","[Mexico, India, Mexico, India, Australia, Australia, Canada, Canada, Spain, Spain, Portugal, Spain, Portugal, Iran, Iran, Iraq, Iraq, Iran, Iraq, Italy, Italy, Italy, India, Australia, Canada, Spain, Portugal, Iran, Iraq, Italy]",15,1915
|
|
||||||
85,"In his travels, Jake had been fascinated by the natural wonders of Australia, from its towering cliffs and eucalyptus forests to the bustling wildlife of the outback. His next stop was New Zealand, a land equally renowned for its scenic beauty including shimmering lakes and towering mountains. In contrast to Australia, New Zealand was characterized by a milder climate and more hilly terrain. He then traveled to South Africa, intrigued by stories of its diverse wildlife and breathtaking landscapes. South Africa offered him an opportunity to explore the subtropical regions, different from the temperate zones of Australia and New Zealand. On his return, he visited India, drawn by its rich culture and historical significance. The architecture in India, particularly the ancient temples and palaces, held him in awe. The contrast between India's bustling cities and tranquil rural areas was striking. He then decided to explore the United States, home to a broad variety of landscapes and cultures. The United States, with its buzzing metropolises and vast natural parks, presented a unique blend of modernity and wilderness. On his journey to Cuba, he was enchanted by the country's vibrant culture and its love for music and dance. Cuba, with its rich history and distinctive architectural styles, was a photographer's paradise. His final destination was the United Kingdom, a place steeped in history and tradition. The United Kingdom, with its dramatic landscapes and magnificently preserved castles, offered a dive into a fascinating past. As Jake looked back on his travels from Australia to the United Kingdom, he was reminded of the diversity and richness of our planet, a world teeming with history, culture, and natural wonders.","[Australia, New Zealand, Australia, New Zealand, South Africa, South Africa, Australia, New Zealand, India, India, India, United States, United States, Cuba, Cuba, United Kingdom, United Kingdom, Australia, United Kingdom]",15,1743
|
|
||||||
86,"Jane, a seasoned traveler from the United States, had a deep fascination with the cultural ties between countries. She had recently come across some documentaries highlighting the historic threads connecting the United Kingdom to Ireland. In the documentaries about Ireland and the United Kingdom, she learned about the shared history, bloodlines, and contentious political past. Intrigued, she decided to call her friend Carlos, a historian from Mexico, to discuss these findings. Carlos, having spent years studying Latin America, was keen to share his knowledge about the intertwined histories of Mexico and Argentina. He told Jane about the migration patterns between Mexico and Argentina and their shared revolutionary pasts. Meanwhile, Jane remembered her visit to South Africa, where she had learned about the country's deep connections with Zimbabwe. She elaborated on the dynamics between South Africa and Zimbabwe, discussing their shared struggle against colonialism and their ongoing economic interdependence. Feeling adventurous, Jane shared her dream of visiting Egypt, fascinated by the country's ancient ties with Israel. Carlos marveled at the intricate relationship between Egypt and Israel, from their Biblical connections to the modern political landscape. Jane, in return, expressed her longing to experience the cultures of Japan and South Korea firsthand. She had read extensively about Japan and South Korea, their shared history, their wars, and their fragile peace. As Jane and Carlos ended their call, they marveled at the rich tapestry of interconnectedness, stretching from the United States to South Korea, revealing a world that was complex, diverse, and beautifully connected.","[United States, United Kingdom, Ireland, Ireland, United Kingdom, Mexico, Mexico, Argentina, Mexico, Argentina, South Africa, Zimbabwe, South Africa, Zimbabwe, Egypt, Israel, Egypt, Israel, Japan, South Korea, Japan, South Korea, United States, South Korea]",13,1708
|
|
||||||
87,"Ella and John are writers who love to travel, drawing inspiration from their journeys. They first set off to Italy, visiting Rome and Venice, finding hidden tales in every corner of Italy. Fascinated by the romances in Italy, they decided to explore France next, getting absorbed in the rich culture and vibrant arts of France. In Paris, they spun tales about knights of France, which were met with much appreciation from their readers. They decided then to journey to the realms of Spain, leaving behind the streets of France for the majestic castles of Spain. Spain, with its flamenco music beating at its heart, lent vibrant hues to their stories. John, having a deep interest in history, suggested a visit to Egypt after their tour of Spain. They visited the pyramids, the Sphinx, and the ancient temples of Egypt, constantly marveling at the grandeur of Egypt. From Egypt, they traveled to Greece, immersing themselves in the myths and legends that abound in Greece. The mythical creatures of Greece and its ancient gods colored their stories beautifully. Their adventures then took them to Japan, where they learned about the samurai and the geishas of Japan. Japan, they felt, was a unique blend of tradition and modernity. Inspired by the tranquility of Japan, they decided to visit India next, where they explored the mystical temples and tasted the exotic spices of India. They were enchanted by the diverse culture and rich traditions of India. As they journeyed from Italy to India, Ella and John realized how much their stories reflected the diversity of cultures, the essence of the countries they visited, and their own growth as storytellers.","[Italy, Italy, Italy, France, France, France, Spain, France, Spain, Spain, Egypt, Spain, Egypt, Egypt, Egypt, Greece, Greece, Greece, Japan, Japan, Japan, Japan, India, India, India, Italy, India]",15,1658
|
|
||||||
88,"Sarah, an avid traveler from the United States, had spent a considerable amount of time in France and was fascinated by its history and culture. Her experiences in France had given her a deeper understanding of the world, inspiring her to visit more countries to learn about their histories. She shared her experiences with her friend John from Australia, who was interested in the wildlife and ecology of different regions. He mentioned his travels to Kenya and how the local tribes in Kenya had a culture vastly different from Australia. His stories about Kenya intrigued Sarah, piquing her interest in the African continent. Sarah, in turn, told John about her visit to Colombia, where she dived deep into the country's history of emeralds and other precious gems. During her time in Colombia, she had visited the emerald mines and learned how the mining industry had shaped the country's economy. It was the richness of Colombia's history that led her to her next destination, Peru, known for its Incan heritage. She shared tales about Peru, filled with myths, mysteries, and ancient civilizations that fascinated John. Intrigued by Sarah's tales, John spoke about his visit to north, to Russia. He found Russia vastly different from Australia, with its vast landscapes and rich history that dated back many centuries. His fascination for Russia led him to explore more about its neighboring country, Ukraine. He found the shared history of Russia and Ukraine intriguing, with its many cultural, political, and historical overlaps. This led them both to ponder about how interconnected the world is, from the United States to Ukraine, each country with its unique story to tell. They ended the evening with a promise to explore the world more and find the hidden stories within the landscapes of these incredible countries.","[United States, France, France, Australia, Kenya, Kenya, Australia, Kenya, Colombia, Colombia, Colombia, Peru, Peru, Russia, Russia, Australia, Russia, Ukraine, Russia, Ukraine, United States, Ukraine]",15,1827
|
|
||||||
89,"Once upon a time, there was a history professor from the United States who led an epoch-making expedition to the deep jungles of Mexico. The expedition aimed to uncover the lost cities of Mexico, rumored to hold ancient artifacts dating back thousands of years. Just the thought of setting foot on the same soil that had been tread by ancient civilizations filled the professor and his team from the United States with a surge of adrenaline. As they ventured deeper into Mexico, they found themselves coming across remnants of civilizations that hinted at strong ties with Colombia. The artifacts, they believed, symbolized a time when traders from Mexico traveled south to Colombia, exchanging goods and ideas. The professor, fascinated, decided to take his team next to Colombia, where they would attempt to dig deeper into this historical connection. During their time in Colombia, the professor and his team discovered more clues to ancient civilizations that mirrored those they had found in Mexico. They learnt about the great voyages undertaken by seafarers from Colombia to the distant shores of Venezuela. Hearing this, the professor was anxious to return to the United States to share his findings and plan another expedition, this time to Venezuela. His time in Colombia had only heightened his curiosity about the possible links between Venezuela and Colombia in the past. As expected, his voyage to Venezuela opened up another chapter of history that reflected the intertwined past of Venezuela and Colombia. The professor's voyages from the United States to Mexico, then to Colombia, and finally to Venezuela created ripples in the world of archaeology. His journeys brought to light the interconnectedness of these countries, uniting continents and people through their shared pasts. He returned to the United States a changed man, profoundly moved by how, despite the vast distances, the similar threads of human history connected countries as disparate as Colombia and the United States.","[United States, Mexico, Mexico, United States, Mexico, Colombia, Mexico, Colombia, Colombia, Colombia, Mexico, Colombia, Venezuela, United States, Venezuela, Colombia, Venezuela, Colombia, Venezuela, Venezuela, Colombia, United States, Mexico, Colombia, Venezuela, United States, Colombia, United States]",14,2004
|
|
||||||
90,"John, a travel blogger from Australia, had been mesmerized by the vibrant culture and history of Mexico, where he was currently exploring. He had taken a particular interest in the ancient Mayan civilization, which once flourished in Mexico and whose influence could still be seen in the country's architecture and traditions. While in Mexico, he had also discovered a shared history with Peru, where Mayan traders were believed to have ventured for trade. By studying the artifacts and tales in Peru, John was able to see a reflection of Mexico's bygone era. His adventures in Mexico and Peru fueled his interest in indigenous cultures, leading him to plan a trip to Canada. His journey to Canada introduced him to the rich and colourful traditions of the First Nations. In Canada, he found echoes of Australia's own native tribes, reinforcing his belief in the interconnectedness of cultures. His exploration in Canada led him to learn about the country's historical ties with the United Kingdom. Having spent a year in the United Kingdom earlier, he couldn't help but see the shared patterns of history, architecture, and societal structures between Canada and the United Kingdom. The following year, he decided to explore Asia, starting with South Korea. He was taken by the technological advancements in South Korea, a stark contrast to the traditional cultures he had been studying. South Korea fascinated him with its unique blend of tradition and modernity, reminding him of his own hometown in Australia. The urban fabric of South Korea, he thought, was reminiscent of the United Kingdom's bustling cities. His travels didn't stop there, as he ventured to India, drawn by its rich history and diversity. He found that, much like Australia, India was a melting pot of cultures, languages, and traditions, just with a different flavor. He realized that no matter where he went, be it Mexico or South Korea, India or Canada, each country had its unique charm and a story to tell.","[Australia, Mexico, Mexico, Mexico, Peru, Peru, Mexico, Mexico, Peru, Canada, Canada, Canada, Australia, Canada, United Kingdom, United Kingdom, Canada, United Kingdom, South Korea, South Korea, South Korea, Australia, South Korea, United Kingdom, India, Australia, India, Mexico, South Korea, India, Canada]",16,1985
|
|
||||||
91,"Maria, who hailed from Australia, gathered with her friends to discuss their recent travels and discoveries. They began with their visits to the United States, reflecting on the mesmerizing cityscapes and diverse cultures that define the United States. They then moved on to describe Canada, reminiscing about Canada's breath-taking landscapes and friendly locals. The conversation shifted as they recounted their experiences in Mexico, marvelling at the vibrant festivals and centuries-old traditions that shape Mexico. Oliver, the anthropologist, regaled them with tales from Brazil. He spoke about Brazil's wildlife, the Amazon rainforest, its indigenous cultures, and the sense of adventure the country offered. From Brazil, the conversation moved to Argentina, discussing the country's rich history, from its colonial past to a thriving nation today. Argentina, they mused, had as much charm as Brazil, with a unique blend of European influences alongside indigenous traditions. Grace, who studied linguistics, shared her recent journey to China and the deep immersion she had experienced into the Mandarin language there. She also recalled the ancient history of China and how it still influences modern China in many ways. Grace's twin, Hannah, spoke excitedly about her recent trip to Japan. She recounted how Japan was a country that seemed to effortlessly blend traditional cultures with ultra-modern technology. Japan, she said, was a country where one could feel a deep sense of spirituality amidst neon-lit cities. As the night grew older, the friends reflected on their shared trip to India, where they had experienced a stunning array of cultures, cuisines and landscapes. They reminisced about India's architectural marvels, the warmth of its people and the mystic allure of its spirituality. Towards the end of the evening, they found themselves talking about their dream destinations, and Italy was a unanimous choice. They all shared a collective dream of experiencing Italy's rich history, exquisite cuisine, and the romance that the country promised. Italy, they agreed, held a charm that seemed to beckon travellers from across the globe.","[Australia, United States, United States, Canada, Canada, Mexico, Mexico, Brazil, Brazil, Brazil, Argentina, Argentina, Brazil, China, China, China, Japan, Japan, Japan, India, India, Italy, Italy, Italy]",18,2160
|
|
||||||
92,"In the picturesque landscapes of Norway, a budding archaeologist named Peter uncovered a peculiar artifact that hinted at an intricate connection between Norway and Sweden. The artifact insinuated that ages ago, traders from Sweden might have voyaged to Norway, bringing with them unique cultural elements. As he dug deeper, Peter discovered more artifacts resembling those found in Poland, showing evidence of a historical connection between Norway and Poland. It seemed that at one point in history, Norway might have had significant interaction with Poland, possibly through trade or migration. Sharing this discovery with his university professor in Canada, Peter was encouraged to explore this further. Interestingly, the professor also recited legends from Canada, detailing the migration stories and cultural exchanges with the United States. Canada, it seemed, shared a rich and intricate history with the United States, a testament to their geographic proximity and socio-political ties. During this conversation, Peter recalled his time in Argentina, where he came across narratives of exchanges between Argentina and Brazil. Intriguingly, Argentina and Brazil, despite their differences, had a shared heritage rooted in indigenous cultures and European colonization. Thinking of his time in Brazil, Peter remembered how stories of sugar-exporting routes to Portugal were common. Brazil and Portugal, bound by colonial history, had a complex relationship that shaped their identities. The professor, listening intently, shared that he had spent time in India, where he had learned about the nation's historical links with Pakistan. The shared history between India and Pakistan, he explained, was an intricate narrative of culture, conflict, and camaraderie, just like those of Norway with Sweden and Poland, Argentina with Brazil, or Canada with the United States. As they ended their conversation, they both reflected on the complex web of connections that linked countries across continents.","[Norway, Norway, Sweden, Sweden, Norway, Poland, Norway, Poland, Norway, Poland, Canada, Canada, United States, Canada, United States, Argentina, Argentina, Brazil, Argentina, Brazil, Brazil, Portugal, Brazil, Portugal, India, Pakistan, India, Pakistan, Norway, Sweden, Poland, Argentina, Brazil, Canada, United States]",14,2004
|
|
||||||
93,"While Sarah was excited about her upcoming trip to Italy, she fondly remembered her previous journey to Australia and New Zealand. She had made lifelong friends in Australia and had been amazed by the diverse wildlife and stunning landscapes of New Zealand. Back in Italy, she was planning to visit the vineyards, flooded with memories of her visit to France where she had tasted the world's best wines. Her experiences in France had enraptured her with a love for wine that she hoped to find again during her stay in Italy. That's when her friend Mark, from Canada, suggested she try the ice wines there. He boasted about the unique sweetness of the ice wines produced in Canada, explaining that the freezing weather conditions in Canada had a role to play in its incredible taste. This made Sarah think of her trip to Russia, where she had endured the harsh winters, but had been equally mesmerized by the beauty of its snow-covered landscapes and the warm hospitality of the people there. Russia, with its tumultuous history and magnificent architecture, had left a profound impact on her. Mark then recalled his adventures in South Korea and North Korea. While South Korea offered him a blend of modernity and tradition, his visit to North Korea was both mysterious and challenging. South Korea's energetic cities and North Korea's secretive aura had both left an indelible mark on his traveler's soul. They then talked about their combined trip to Mexico and Cuba, where they had experienced vibrant colors, rich history, and spicy foods. From the ancient ruins in Mexico to the stunning beaches of Cuba, they realized how much they had learned about themselves and others during their travels. As they bid goodbye, they promised to meet again, maybe in a new country, where they would write another chapter of their unending travel stories.","[Italy, Australia, New Zealand, Australia, New Zealand, Italy, France, France, Italy, Canada, Canada, Canada, Russia, Russia, South Korea, North Korea, South Korea, North Korea, South Korea, North Korea, Mexico, Cuba, Mexico, Cuba]",14,1846
|
|
||||||
94,"As a diplomat from the United States, Michael had the opportunity to travel widely and learn about the intricacies of global politics. Striking up a conversation with a fellow diplomat from Canada at a summit in New York, he learned about the nuanced relationship between the United States and Canada; how they shared not only a border, but a multitude of cultural and economic ties. He was fascinated by the unique dynamics between Australia and New Zealand as well, as he had recently been posted there. Being neighboring countries, Australia and New Zealand shared a deep bond cultivated over the years through a shared history, culture, and friendly competition. This stirred his interest in similar dynamics between other nations such as South Korea and North Korea. Despite their political differences, the shared history and culture between South Korea and North Korea is undeniably profound. He then recalled his previous posting in Iran, where he had learned a lot about its complex relationship with Iraq. The adversities and struggles between Iran and Iraq had shaped their histories in the most significant of ways. His thoughts shifted towards Europe, particularly Norway and Sweden. He had always been inspired by the peaceful co-existence of Norway and Sweden, two nations known for their progressive societies. When he returned to the United States, he remembered his interactions with diplomats from Mexico and how they often discussed underlying bonds and issues between the United States and Mexico. Both the United States and Mexico had strong economic ties and a shared border, along with a history filled with conflict and cooperation. Little did he know that a career in diplomacy would offer him such a deep understanding of how interconnected the world truly is, from the United States to Sweden, Mexico to Norway, and beyond.","[United States, Canada, United States, Canada, Australia, New Zealand, Australia, New Zealand, South Korea, North Korea, South Korea, North Korea, Iran, Iraq, Iran, Iraq, Norway, Sweden, Norway, Sweden, United States, Mexico, United States, Mexico, United States, Mexico, United States, Sweden, Mexico, Norway]",13,1851
|
|
||||||
95,"In the bustling markets of India, Marie, a trained archaeologist, overheard stories of an ancient wall built to protect against invasions from Pakistan. Intriguingly, it seemed that the wall, although iconic in India, was also a point of historical significance for Pakistan, marking a shared past of conflicts and collaborations. Inspired, she recounted the tales to Thomas, an avid historian from Australia, familiar with tales of interlinked histories between Australia and New Zealand. The Maori folk tales from New Zealand were echoed in Aboriginal stories of Australia, painting a parallel tapestry of rich cultural history that linked Australia and New Zealand. Thomas, in turn, shared the tales from South Africa, where he worked on a dig, unearthing connections between the Zulus of South Africa and the Maasai of Kenya. South Africa and Kenya, despite their geographical distance, seemed to share migration stories that connected their people in extraordinary ways. Moving onto a different continent, Marie spoke of her recent expedition to Poland, where she discovered evidence of ancient trade links with Ukraine. The traditional crafts found in Poland bore striking resemblance to those in Ukraine, indicating a significant crossover of culture and trade. On a similar note, Thomas disclosed his knowledge about the age-old ties between Argentina and Brazil. The indigenous cultures of Argentina shared uncanny similarities with that of Brazil, further exemplifying the interconnectedness of histories. Marie, remembering her visit to Israel, shared stories of its linked history with Egypt, where artifacts identical to those found in Israel were discovered. Both Egypt and Israel, holding a complex shared history, reflected in their art, culture, and archaeological remnants. Through their discussion, from India to Israel, Marie and Thomas marveled at how intricately histories of different countries were intertwined, creating a fascinating web of shared cultures and experiences.","[India, Pakistan, India, Pakistan, Australia, Australia, New Zealand, New Zealand, Australia, Australia, New Zealand, South Africa, South Africa, Kenya, South Africa, Kenya, Poland, Ukraine, Poland, Ukraine, Argentina, Brazil, Argentina, Brazil, Israel, Egypt, Israel, Egypt, Israel, India, Israel]",13,1998
|
|
||||||
96,"In Emily's journey across the globe, she found a peculiar connection between North Korea and South Korea. It was evident that North Korea and South Korea shared deep cultural ties, despite political differences. Emily's travels also led her to the bustling streets of India, where she learned about its longstanding relationship with Iran. Historically, India and Iran worked together to establish trade routes and cultural exchanges, which strengthened their bond over time. She then visited Russia, intrigued by its historical connections with Ukraine. The intertwined histories of Russia and Ukraine often mirrored each other, reflecting their shared past and complex dynamics. Emily's journey then brought her to Italy, where she discovered an ancient route that linked it with Greece. She learned that Italy and Greece, despite their unique cultures, had distinct influences on each other in terms of architecture, philosophy, and cuisine. Emily's path then led her to South Africa, and she was surprised to learn about its connections with Zimbabwe. South Africa and Zimbabwe, despite being geographically close, had surprisingly divergent histories, yet shared certain cultural elements. This prompted her to visit the United States, where she had always been intrigued by the relationship it had with its neighbor, Canada. She found that the United States and Canada, though distinct, share a long history of cooperation and shared culture. Finally, Emily journeyed to Australia, fascinated by its ties with neighboring New Zealand. Here, she discovered that Australia and New Zealand have a unique relationship; they’re distinctly different yet undeniably interconnected. Intriguingly, her journey gave her a unique understanding of how these nations, from North Korea to Australia, even with their contrasting cultures, shared a tapestry of interconnected histories.","[North Korea, South Korea, North Korea, South Korea, India, Iran, India, Iran, Russia, Ukraine, Russia, Ukraine, Italy, Greece, Italy, Greece, South Africa, Zimbabwe, South Africa, Zimbabwe, United States, Canada, United States, Canada, Australia, New Zealand, Australia, New Zealand, North Korea, Australia]",15,1876
|
|
||||||
97,"James, an avid traveler from the United States, found himself in Argentina, fascinated by the rich history and vibrant culture. His curiosity about Argentina's past was sparked by stories about the Indigenous people who had been there long before European settlers arrived. Interestingly, he learned that Argentina had a profound cultural exchange with Brazil, due to their geographical proximity. Argentina and Brazil, despite their political differences, shared many similar cultural practices and traditions. Inspired by the intertwining histories of Argentina and Brazil, James planned his next journey to Italy. While in Italy, stories of how Italy had been profoundly influenced by Greece fascinated him. From Italy's architecture to philosophy, the impact of Greece was truly remarkable. Italy and Greece, in many ways, seemed like siblings separated at birth. After Italy, James was drawn to the unique cultures and histories of Asia. He decided to visit China, where he learned about China's one-time most influential ally, North Korea. Despite the significant changes in their relationship over the years, China and North Korea share a dynamic and complex history that defies simple categorization. Then, the stories of North Korea's estrangement from South Korea intrigued him. Intrigued by Korea's split, he traveled to South Korea to understand the other side of the story. South Korea, he discovered, had a completely different narrative to share about their relationship with North Korea. As he ended his trip in South Korea, he realized how compelling the stories of different nations are, from Argentina to South Korea, and how their histories are interconnected.","[United States, Argentina, Argentina, Argentina, Brazil, Argentina, Brazil, Argentina, Brazil, Italy, Italy, Italy, Greece, Italy, Greece, Italy, Greece, Italy, China, China, North Korea, China, North Korea, North Korea, South Korea, South Korea, South Korea, North Korea, South Korea, Argentina, South Korea]",15,1680
|
|
||||||
98,"John, a history enthusiast from Australia, had always been drawn to the intriguing stories of ancient civilizations and their hidden treasures. During his studies, he found that Australia had been a crossroads of cultures, with influences flowing from the far clubs of Indonesia, enriching the local societies. Indonesia, with its diverse culture and rich history, had a lot more to offer than it had received. Similarly, according to an ancient text John discovered, there was evidence of strong connections between the cultures of Egypt and Iran. The text suggested that Egypt and Iran might have influenced each other's architectural styles, owing to the trades and nomadic migrations between the two. Intrigued by this interplay of influences, John decided to delve deeper, leading him to the complex relationship between Brazil and Argentina. The historical records revealed that Brazil and Argentina, despite their differences, were tied together by a shared history of colonial conquest, rebellion, and periods of peace. To his surprise, he stumbled upon a document about a significant trade agreement between Poland and Sweden in the 15th century. It appeared that Poland and Sweden, despite political tensions, had formed alliances for mutual benefits, resulting in interesting overlaps in their folklores. As he researched further, John discovered that the United Kingdom and Ireland had a similarly complicated yet intriguing shared history. The United Kingdom and Ireland, despite their tumultuous past, have witnessed cultural exchanges and shared literary traditions. Lastly, a short anecdote about the close-knit relationship between South Korea and Japan caught his interest. The story revealed how South Korea and Japan, two nations with a complicated history, have influenced each other's cuisine, art, and music. Reflecting on his research, John marveled at the intricate web of relationships that exists between countries such as Australia and Japan, revealing the fascinating tapestry of human history and cultural evolution.","[Australia, Australia, Indonesia, Indonesia, Egypt, Iran, Egypt, Iran, Brazil, Argentina, Brazil, Argentina, Poland, Sweden, Poland, Sweden, United Kingdom, Ireland, United Kingdom, Ireland, South Korea, Japan, South Korea, Japan, Australia, Japan]",14,2046
|
|
||||||
99,"On a recent trip to Mexico, Adam unearthed a collection of old artifacts that seemed to tell the story of a long-gone civilization. The artifacts gave insights into the ancient history of Mexico, revealing a society that was both culturally sophisticated and technologically advanced. Intrigued by his findings, he reached out to his childhood friend Lily, an archaeologist based in the United States. Lily had previously worked on archaeological sites that dated back to the prehistoric period in the United States and the neighboring country, Canada. Both the United States and Canada were replete with archaeological sites that testified to the existence of advanced indigenous societies. Adam's discoveries reminded Lily of her own findings in China, where she explored ancient tombs and artifacts. In China, she observed, the preservation of ancient historical sites was a matter of national pride and was taken very seriously. She also mentioned her visit to Japan, where she had discovered connections with China in their shared artifacts and folklore. The historical ties between Japan and China were profound and their histories intertwined in many ways. Adam, meanwhile, shared his experiences in South Africa, where he closely studied the rock paintings and cave art. He explained how South Africa had been a hub for early human civilizations, and was rich with history. Lily agreed with him, mentioning her visit to Egypt, which she described as a treasure through of archaeological wonders. The ancient pyramids and tombs of Egypt held within them the untold stories of a fascinating civilization. Recalling her research in Australia and New Zealand, she also highlighted the strong historical ties between these countries. Both Australia and New Zealand had shared indigenous cultures, whose traditions were reflected in their art and folklore. Their conversation ended with a sense of wonder, at how the far-flung corners of the world, from Mexico to New Zealand, were linked through a shared tapestry of history and culture.","[Mexico, Mexico, United States, United States, Canada, United States, Canada, China, China, Japan, China, Japan, China, South Africa, South Africa, Egypt, Egypt, Australia, New Zealand, Australia, New Zealand, Mexico, New Zealand]",16,2040
|
|
||||||
|
@ -1,535 +0,0 @@
|
|||||||
# Copyright (c) 2023 ETH Zurich.
|
|
||||||
# All rights reserved.
|
|
||||||
#
|
|
||||||
# Use of this source code is governed by a BSD-style license that can be
|
|
||||||
# found in the LICENSE file.
|
|
||||||
#
|
|
||||||
# main author: Ales Kubicek
|
|
||||||
|
|
||||||
import csv
|
|
||||||
from typing import List, Tuple
|
|
||||||
from graph_of_thoughts import controller
|
|
||||||
|
|
||||||
|
|
||||||
def find_country_indices(text: str, country: str) -> List[Tuple[int, str]]:
|
|
||||||
"""
|
|
||||||
Finds the indices of the occurences of a given country in the input text.
|
|
||||||
|
|
||||||
:param text: Input text.
|
|
||||||
:type text: str
|
|
||||||
:param country: Country to search for.
|
|
||||||
:type country: str
|
|
||||||
:return: List of tuples, where each tuple consists of index and country.
|
|
||||||
:rtype: List[Tuple[int, str]]
|
|
||||||
"""
|
|
||||||
|
|
||||||
indices = []
|
|
||||||
index = text.find(country)
|
|
||||||
while index != -1:
|
|
||||||
indices.append(index)
|
|
||||||
index = text.find(country, index + 1)
|
|
||||||
return [(index, country) for index in indices]
|
|
||||||
|
|
||||||
|
|
||||||
primary_countries = [
|
|
||||||
"Afghanistan",
|
|
||||||
"Argentina",
|
|
||||||
"Australia",
|
|
||||||
"Brazil",
|
|
||||||
"Canada",
|
|
||||||
"China",
|
|
||||||
"Colombia",
|
|
||||||
"Cuba",
|
|
||||||
"Egypt",
|
|
||||||
"France",
|
|
||||||
"Germany",
|
|
||||||
"Greece",
|
|
||||||
"India",
|
|
||||||
"Indonesia",
|
|
||||||
"Iran",
|
|
||||||
"Iraq",
|
|
||||||
"Ireland",
|
|
||||||
"Israel",
|
|
||||||
"Italy",
|
|
||||||
"Japan",
|
|
||||||
"Kenya",
|
|
||||||
"Mexico",
|
|
||||||
"Netherlands",
|
|
||||||
"New Zealand",
|
|
||||||
"Nigeria",
|
|
||||||
"North Korea",
|
|
||||||
"Pakistan",
|
|
||||||
"Peru",
|
|
||||||
"Philippines",
|
|
||||||
"Poland",
|
|
||||||
"Portugal",
|
|
||||||
"Russia",
|
|
||||||
"Saudi Arabia",
|
|
||||||
"South Africa",
|
|
||||||
"South Korea",
|
|
||||||
"Spain",
|
|
||||||
"Sweden",
|
|
||||||
"Switzerland",
|
|
||||||
"Thailand",
|
|
||||||
"Turkey",
|
|
||||||
"Ukraine",
|
|
||||||
"United Arab Emirates",
|
|
||||||
"United Kingdom",
|
|
||||||
"United States",
|
|
||||||
"Venezuela",
|
|
||||||
"Vietnam",
|
|
||||||
"Yemen",
|
|
||||||
"Zimbabwe",
|
|
||||||
"Belgium",
|
|
||||||
"Norway",
|
|
||||||
]
|
|
||||||
primary_adjectives = [
|
|
||||||
"Afghan",
|
|
||||||
"Argentine ",
|
|
||||||
"Argentinean",
|
|
||||||
"Australian",
|
|
||||||
"Brazilian",
|
|
||||||
"Canadian",
|
|
||||||
"Chinese",
|
|
||||||
"Colombian",
|
|
||||||
"Cuban",
|
|
||||||
"Egyptian",
|
|
||||||
"French",
|
|
||||||
"German",
|
|
||||||
"Greek",
|
|
||||||
"Indian",
|
|
||||||
"Indonesian",
|
|
||||||
"Iranian",
|
|
||||||
"Iraqi",
|
|
||||||
"Irish",
|
|
||||||
"Israeli",
|
|
||||||
"Italian",
|
|
||||||
"Japanese",
|
|
||||||
"Kenyan",
|
|
||||||
"Mexican",
|
|
||||||
"Dutch",
|
|
||||||
"New Zealander ",
|
|
||||||
"Kiwi",
|
|
||||||
"Nigerian",
|
|
||||||
"North Korean",
|
|
||||||
"Pakistani",
|
|
||||||
"Peruvian",
|
|
||||||
"Filipino",
|
|
||||||
"Philippine",
|
|
||||||
"Polish",
|
|
||||||
"Portuguese",
|
|
||||||
"Russian",
|
|
||||||
"Saudi ",
|
|
||||||
"Saudi Arabian",
|
|
||||||
"South African",
|
|
||||||
"South Korean",
|
|
||||||
"Spanish",
|
|
||||||
"Swedish",
|
|
||||||
"Swiss",
|
|
||||||
"Thai",
|
|
||||||
"Turkish",
|
|
||||||
"Ukrainian",
|
|
||||||
"United Arab Emirates",
|
|
||||||
"Emirati",
|
|
||||||
"British",
|
|
||||||
"American",
|
|
||||||
"Venezuelan",
|
|
||||||
"Vietnamese",
|
|
||||||
"Yemeni",
|
|
||||||
"Zimbabwean",
|
|
||||||
"Belgian",
|
|
||||||
"Norwegian",
|
|
||||||
]
|
|
||||||
rest_countries = [
|
|
||||||
"Albania",
|
|
||||||
"Algeria",
|
|
||||||
"Andorra",
|
|
||||||
"Angola",
|
|
||||||
"Antigua and Barbuda",
|
|
||||||
"Armenia",
|
|
||||||
"Austria",
|
|
||||||
"Azerbaijan",
|
|
||||||
"The Bahamas",
|
|
||||||
"Bahrain",
|
|
||||||
"Bangladesh",
|
|
||||||
"Barbados",
|
|
||||||
"Belarus",
|
|
||||||
"Belize",
|
|
||||||
"Benin",
|
|
||||||
"Bhutan",
|
|
||||||
"Bolivia",
|
|
||||||
"Bosnia and Herzegovina",
|
|
||||||
"Botswana",
|
|
||||||
"Brunei",
|
|
||||||
"Bulgaria",
|
|
||||||
"Burkina Faso",
|
|
||||||
"Burundi",
|
|
||||||
"Cabo Verde",
|
|
||||||
"Cambodia",
|
|
||||||
"Cameroon",
|
|
||||||
"Central African Republic",
|
|
||||||
"Chad",
|
|
||||||
"Chile",
|
|
||||||
"Comoros",
|
|
||||||
"Congo",
|
|
||||||
"Costa Rica",
|
|
||||||
"Côte d’Ivoire",
|
|
||||||
"Croatia",
|
|
||||||
"Cyprus",
|
|
||||||
"Czech Republic",
|
|
||||||
"Czechia",
|
|
||||||
"Denmark",
|
|
||||||
"Djibouti",
|
|
||||||
"Dominica",
|
|
||||||
"Dominican Republic",
|
|
||||||
"East Timor",
|
|
||||||
"Timor-Leste",
|
|
||||||
"Ecuador",
|
|
||||||
"El Salvador",
|
|
||||||
"Equatorial Guinea",
|
|
||||||
"Eritrea",
|
|
||||||
"Estonia",
|
|
||||||
"Eswatini",
|
|
||||||
"Ethiopia",
|
|
||||||
"Fiji",
|
|
||||||
"Finland",
|
|
||||||
"Gabon",
|
|
||||||
"The Gambia",
|
|
||||||
"Georgia",
|
|
||||||
"Ghana",
|
|
||||||
"Grenada",
|
|
||||||
"Guatemala",
|
|
||||||
"Guinea",
|
|
||||||
"Guinea-Bissau",
|
|
||||||
"Guyana",
|
|
||||||
"Haiti",
|
|
||||||
"Honduras",
|
|
||||||
"Hungary",
|
|
||||||
"Iceland",
|
|
||||||
"Jamaica",
|
|
||||||
"Jordan",
|
|
||||||
"Kazakhstan",
|
|
||||||
"Kiribati",
|
|
||||||
"Kosovo",
|
|
||||||
"Kuwait",
|
|
||||||
"Kyrgyzstan",
|
|
||||||
"Laos",
|
|
||||||
"Latvia",
|
|
||||||
"Lebanon",
|
|
||||||
"Lesotho",
|
|
||||||
"Liberia",
|
|
||||||
"Libya",
|
|
||||||
"Liechtenstein",
|
|
||||||
"Lithuania",
|
|
||||||
"Luxembourg",
|
|
||||||
"Madagascar",
|
|
||||||
"Malawi",
|
|
||||||
"Malaysia",
|
|
||||||
"Maldives",
|
|
||||||
"Mali",
|
|
||||||
"Malta",
|
|
||||||
"Marshall Islands",
|
|
||||||
"Mauritania",
|
|
||||||
"Mauritius",
|
|
||||||
"Micronesia",
|
|
||||||
"Moldova",
|
|
||||||
"Monaco",
|
|
||||||
"Mongolia",
|
|
||||||
"Montenegro",
|
|
||||||
"Morocco",
|
|
||||||
"Mozambique",
|
|
||||||
"Myanmar",
|
|
||||||
"Burma",
|
|
||||||
"Namibia",
|
|
||||||
"Nauru",
|
|
||||||
"Nepal",
|
|
||||||
"Nicaragua",
|
|
||||||
"Niger",
|
|
||||||
"North Macedonia",
|
|
||||||
"Oman",
|
|
||||||
"Palau",
|
|
||||||
"Panama",
|
|
||||||
"Papua New Guinea",
|
|
||||||
"Paraguay",
|
|
||||||
"Qatar",
|
|
||||||
"Romania",
|
|
||||||
"Rwanda",
|
|
||||||
"Saint Kitts and Nevis",
|
|
||||||
"Saint Lucia",
|
|
||||||
"Saint Vincent and the Grenadines",
|
|
||||||
"Samoa",
|
|
||||||
"San Marino",
|
|
||||||
"Sao Tome and Principe",
|
|
||||||
"Senegal",
|
|
||||||
"Serbia",
|
|
||||||
"Seychelles",
|
|
||||||
"Sierra Leone",
|
|
||||||
"Singapore",
|
|
||||||
"Slovakia",
|
|
||||||
"Slovenia",
|
|
||||||
"Solomon Islands",
|
|
||||||
"Somalia",
|
|
||||||
"Sri Lanka",
|
|
||||||
"Sudan",
|
|
||||||
"Suriname",
|
|
||||||
"Syria",
|
|
||||||
"Taiwan",
|
|
||||||
"Tajikistan",
|
|
||||||
"Tanzania",
|
|
||||||
"Togo",
|
|
||||||
"Tonga",
|
|
||||||
"Trinidad and Tobago",
|
|
||||||
"Tunisia",
|
|
||||||
"Turkmenistan",
|
|
||||||
"Tuvalu",
|
|
||||||
"Uganda",
|
|
||||||
"Uruguay",
|
|
||||||
"Uzbekistan",
|
|
||||||
"Vanuatu",
|
|
||||||
"Vatican City",
|
|
||||||
"Zambia",
|
|
||||||
]
|
|
||||||
rest_adjectives = [
|
|
||||||
"Albanian",
|
|
||||||
"Algerian",
|
|
||||||
"Andorran",
|
|
||||||
"Angolan",
|
|
||||||
"Antiguan and Barbudan",
|
|
||||||
"Armenian",
|
|
||||||
"Austrian",
|
|
||||||
"Azerbaijani",
|
|
||||||
"Bahamian",
|
|
||||||
"Bahraini",
|
|
||||||
"Bangladeshi",
|
|
||||||
"Barbadian",
|
|
||||||
"Belarusian",
|
|
||||||
"Belizean",
|
|
||||||
"Beninese",
|
|
||||||
"Bhutanese",
|
|
||||||
"Bolivian",
|
|
||||||
"Bosnian and Herzegovinian",
|
|
||||||
"Botswanan",
|
|
||||||
"Bruneian",
|
|
||||||
"Bulgarian",
|
|
||||||
"Burkinabè",
|
|
||||||
"Burundian",
|
|
||||||
"Cape Verdean",
|
|
||||||
"Cambodian",
|
|
||||||
"Cameroonian",
|
|
||||||
"Central African",
|
|
||||||
"Chadian",
|
|
||||||
"Chilean",
|
|
||||||
"Comorian",
|
|
||||||
"Congolese",
|
|
||||||
"Costa Rican",
|
|
||||||
"Ivorian",
|
|
||||||
"Croatian",
|
|
||||||
"Cypriot",
|
|
||||||
"Czech",
|
|
||||||
"Czech",
|
|
||||||
"Danish",
|
|
||||||
"Djiboutian",
|
|
||||||
"Dominican",
|
|
||||||
"Dominican",
|
|
||||||
"East Timorese",
|
|
||||||
"Timorese",
|
|
||||||
"Ecuadorian",
|
|
||||||
"Salvadoran",
|
|
||||||
"Equatorial Guinean",
|
|
||||||
"Eritrean",
|
|
||||||
"Estonian",
|
|
||||||
"Swazi",
|
|
||||||
"Ethiopian",
|
|
||||||
"Fijian",
|
|
||||||
"Finnish",
|
|
||||||
"Gabonese",
|
|
||||||
"Gambian",
|
|
||||||
"Georgian",
|
|
||||||
"Ghanaian",
|
|
||||||
"Grenadian",
|
|
||||||
"Guatemalan",
|
|
||||||
"Guinean",
|
|
||||||
"Bissau-Guinean",
|
|
||||||
"Guyanese",
|
|
||||||
"Haitian",
|
|
||||||
"Honduran",
|
|
||||||
"Hungarian",
|
|
||||||
"Icelandic",
|
|
||||||
"Jamaican",
|
|
||||||
"Jordanian",
|
|
||||||
"Kazakh",
|
|
||||||
"I-Kiribati",
|
|
||||||
"Kosovar",
|
|
||||||
"Kuwaiti",
|
|
||||||
"Kyrgyz",
|
|
||||||
"Laotian",
|
|
||||||
"Latvian",
|
|
||||||
"Lebanese",
|
|
||||||
"Basotho",
|
|
||||||
"Liberian",
|
|
||||||
"Libyan",
|
|
||||||
"Liechtensteiner",
|
|
||||||
"Lithuanian",
|
|
||||||
"Luxembourger",
|
|
||||||
"Malagasy",
|
|
||||||
"Malawian",
|
|
||||||
"Malaysian",
|
|
||||||
"Maldivian",
|
|
||||||
"Malian",
|
|
||||||
"Maltese",
|
|
||||||
"Marshallese",
|
|
||||||
"Mauritanian",
|
|
||||||
"Mauritian",
|
|
||||||
"Micronesian",
|
|
||||||
"Moldovan",
|
|
||||||
"Monégasque",
|
|
||||||
"Mongolian",
|
|
||||||
"Montenegrin",
|
|
||||||
"Moroccan",
|
|
||||||
"Mozambican",
|
|
||||||
"Myanmarese",
|
|
||||||
"Burmese",
|
|
||||||
"Namibian",
|
|
||||||
"Nauruan",
|
|
||||||
"Nepali",
|
|
||||||
"Nicaraguan",
|
|
||||||
"Nigerien",
|
|
||||||
"Macedonian",
|
|
||||||
"Omani",
|
|
||||||
"Palauan",
|
|
||||||
"Panamanian",
|
|
||||||
"Papua New Guinean",
|
|
||||||
"Paraguayan",
|
|
||||||
"Qatari",
|
|
||||||
"Romanian",
|
|
||||||
"Rwandan",
|
|
||||||
"Kittitian",
|
|
||||||
"Nevisian",
|
|
||||||
"Saint Lucian",
|
|
||||||
"Vincentian",
|
|
||||||
"Samoan",
|
|
||||||
"Sammarinese",
|
|
||||||
"Santomean",
|
|
||||||
"Senegalese",
|
|
||||||
"Serbian",
|
|
||||||
"Seychellois",
|
|
||||||
"Sierra Leonean",
|
|
||||||
"Singaporean",
|
|
||||||
"Slovak",
|
|
||||||
"Slovenian",
|
|
||||||
"Solomon Islander",
|
|
||||||
"Somali",
|
|
||||||
"Sri Lankan",
|
|
||||||
"Sudanese",
|
|
||||||
"Surinamese",
|
|
||||||
"Syrian",
|
|
||||||
"Taiwanese",
|
|
||||||
"Tajik",
|
|
||||||
"Tanzanian",
|
|
||||||
"Togolese",
|
|
||||||
"Tongan",
|
|
||||||
"Trinidadian ",
|
|
||||||
"Tobagonian",
|
|
||||||
"Tunisian",
|
|
||||||
"Turkmen",
|
|
||||||
"Tuvaluan",
|
|
||||||
"Ugandan",
|
|
||||||
"Uruguayan",
|
|
||||||
"Uzbek",
|
|
||||||
"Ni-Vanuatu",
|
|
||||||
"Vatican",
|
|
||||||
"Zambian",
|
|
||||||
]
|
|
||||||
|
|
||||||
lm = controller.ChatGPT(
|
|
||||||
"../../graph_of_thoughts/controller/config.json", model_name="chatgpt4"
|
|
||||||
)
|
|
||||||
|
|
||||||
prompt = """<Instruction> Generate a continuous passage (single paragraph) of 16 sentences following the provided restrictions precisely. </Instruction>
|
|
||||||
|
|
||||||
<Restrictions>
|
|
||||||
The following restrictions must apply to the generated text:
|
|
||||||
1. Single continuous passage of exactly 16 sentences without any paragraphs (line breaks).
|
|
||||||
2. Countries appearing in the passage must be only from the provided list. No other countries can be mentioned.
|
|
||||||
3. When a country is mentioned in the passage, it must be mentioned multiple times consecutively in the same or following sentences.
|
|
||||||
4. Passage should be creative and coherent.
|
|
||||||
5. Using adjectives of a country is NOT allowed (e.g., "Colombian coffee" should be "coffee from Colombia" instead)
|
|
||||||
</Restrictions>
|
|
||||||
|
|
||||||
<Example>
|
|
||||||
List of countries: [Afghanistan, Argentina, Australia, Brazil, Canada, China, Colombia, Cuba, Egypt, France, Germany, Greece, India, Indonesia, Iran, Iraq, Ireland, Israel, Italy, Japan, Kenya, Mexico, Netherlands, New Zealand, Nigeria, North Korea, Pakistan, Peru, Philippines, Poland, Portugal, Russia, Saudi Arabia, South Africa, South Korea, Spain, Sweden, Switzerland, Thailand, Turkey, Ukraine, United Arab Emirates, United Kingdom, United States, Venezuela, Vietnam, Yemen, Zimbabwe, Belgium, Norway]
|
|
||||||
Passage:
|
|
||||||
While exploring the ancient ruins in Greece, Sam discovered manuscripts that hinted at the hidden treasures of Egypt. It seemed these treasures were once stolen from Egypt by rogue merchants and secretly moved to Greece, only to be buried under layers of time. Intrigued, he shared the findings with his friend Maya from India, who was an expert in decoding ancient languages. She pointed out that there was a similar legend in India about treasures from China that had somehow ended up in the southern parts of India, possibly through trade or conquest. She also recounted tales from China that spoke of incredible artifacts from Indonesia, suggesting a rich tapestry of cultural exchanges throughout history. Their conversation took an interesting turn when Sam mentioned a book he'd read about the mysterious connections between Argentina and Brazil. The book detailed how both Argentina and Brazil, despite their differences, shared tales of lost civilizations and forgotten cities deep within their jungles. Maya excitedly mentioned that she'd been to the Philippines and had heard local legends about ancient ties with Indonesia and how traders from the Philippines would journey to Indonesia in search of spices and other goods. Thinking of spices, Sam fondly recalled his trip to Spain, where he had learned about the country's historical links with Portugal. Spain and Portugal, both maritime giants of their time, had extensively explored unknown lands and established trade routes. Maya, remembering her travels, said that she had been to Belgium once and was fascinated by its connections with the Netherlands. Both Belgium and the Netherlands, she explained, had rich histories of art, trade, and diplomacy that intertwined them for centuries. They both sat back, marveling at the interconnectedness of the world and how countries from Greece to the Netherlands shared tales of adventure, discovery, and mystery.
|
|
||||||
</Example>
|
|
||||||
|
|
||||||
List of countries: [Afghanistan, Argentina, Australia, Brazil, Canada, China, Colombia, Cuba, Egypt, France, Germany, Greece, India, Indonesia, Iran, Iraq, Ireland, Israel, Italy, Japan, Kenya, Mexico, Netherlands, New Zealand, Nigeria, North Korea, Pakistan, Peru, Philippines, Poland, Portugal, Russia, Saudi Arabia, South Africa, South Korea, Spain, Sweden, Switzerland, Thailand, Turkey, Ukraine, United Arab Emirates, United Kingdom, United States, Venezuela, Vietnam, Yemen, Zimbabwe, Belgium, Norway]
|
|
||||||
Passage:
|
|
||||||
"""
|
|
||||||
|
|
||||||
num_samples = 100
|
|
||||||
sample_id = 0
|
|
||||||
result = [["ID", "Text", "Countries", "Sentences", "Characters"]]
|
|
||||||
|
|
||||||
"""
|
|
||||||
Generate passages of text that contain country names to be used as input for the
|
|
||||||
keyword counting.
|
|
||||||
|
|
||||||
Input(x) : Number of samples
|
|
||||||
Output(y) : Passages written to a file in the CSV format.
|
|
||||||
File contains the sample ID, the passage, the countries the passage
|
|
||||||
contains, the sentences of the passages, number of characters of the
|
|
||||||
passage.
|
|
||||||
"""
|
|
||||||
|
|
||||||
# For x batches of y responses
|
|
||||||
for _ in range(num_samples):
|
|
||||||
response = lm.query(prompt, 1)
|
|
||||||
texts = lm.get_response_texts(response)
|
|
||||||
for text in texts:
|
|
||||||
# Clean paragraphs - single long passage
|
|
||||||
text = text.strip().replace("\n", "")
|
|
||||||
|
|
||||||
# Get all occurrences of all primary permissible countries
|
|
||||||
occurrences = []
|
|
||||||
for country in [country for country in primary_countries if country in text]:
|
|
||||||
occurrences.extend(find_country_indices(text, country))
|
|
||||||
# Order exactly how they appear in the text
|
|
||||||
ordered_occurrences = [country[1] for country in sorted(occurrences)]
|
|
||||||
|
|
||||||
# Check invalid countries and adjectives
|
|
||||||
invalid_primary_adjective = [
|
|
||||||
adjective for adjective in primary_adjectives if adjective in text
|
|
||||||
]
|
|
||||||
invalid_rest_country = [
|
|
||||||
country for country in rest_countries if country in text
|
|
||||||
]
|
|
||||||
invalid_rest_adjective = [
|
|
||||||
adjective for adjective in rest_adjectives if adjective in text
|
|
||||||
]
|
|
||||||
invalid_count = (
|
|
||||||
len(invalid_primary_adjective)
|
|
||||||
+ len(invalid_rest_country)
|
|
||||||
+ len(invalid_rest_adjective)
|
|
||||||
)
|
|
||||||
|
|
||||||
if invalid_count > 0:
|
|
||||||
print(
|
|
||||||
f"Invalid countries or adjectives present: {invalid_primary_adjective}, {invalid_rest_country}, {invalid_rest_adjective}"
|
|
||||||
)
|
|
||||||
continue
|
|
||||||
|
|
||||||
result.append(
|
|
||||||
[
|
|
||||||
sample_id,
|
|
||||||
text,
|
|
||||||
"[{0}]".format(", ".join(map(str, ordered_occurrences))),
|
|
||||||
len(text.split(".")) - 1,
|
|
||||||
len(text),
|
|
||||||
]
|
|
||||||
)
|
|
||||||
sample_id += 1
|
|
||||||
|
|
||||||
# Writing to csv file
|
|
||||||
with open("countries_script.csv", "w") as csvfile:
|
|
||||||
csvwriter = csv.writer(csvfile)
|
|
||||||
csvwriter.writerows(result)
|
|
||||||
@ -1,998 +0,0 @@
|
|||||||
# Keyword Counting 4-passage Split (GoT4) - Prompts and Examples
|
|
||||||
## Prompt Templates
|
|
||||||
### GENERATE: split_prompt
|
|
||||||
Replace `{input_text}` with an input text to be split.
|
|
||||||
```
|
|
||||||
<Instruction> Split the following input text into 4 paragraphs of approximately same length.
|
|
||||||
Only output the final 4 paragraphs in the following format without any additional text or thoughts:
|
|
||||||
{
|
|
||||||
"Paragraph 1": "Some paragraph text ...",
|
|
||||||
"Paragraph 2": "Some paragraph text ...",
|
|
||||||
"Paragraph 3": "Some paragraph text ...",
|
|
||||||
"Paragraph 4": "Some paragraph text ..."
|
|
||||||
}
|
|
||||||
</Instruction>
|
|
||||||
|
|
||||||
<Example>
|
|
||||||
Input:
|
|
||||||
Journeying westward, she admired the art in Italy and sipped coffee in France. The music of Spain and the history of Greece deepened her love for Europe. The Nordic beauty of Norway, Sweden, Finland, and Denmark took her breath away. She danced in Ireland, explored castles in Scotland, and marveled at the architecture in Germany and Russia. Italy, Norway, Sweden and Germany will always stay her favourite destinations to visit.
|
|
||||||
Output:
|
|
||||||
{
|
|
||||||
"Paragraph 1": "Journeying westward, she admired the art in Italy and sipped coffee in France.",
|
|
||||||
"Paragraph 2": "The music of Spain and the history of Greece deepened her love for Europe. The Nordic beauty of Norway, Sweden, Finland, and Denmark took her breath away.",
|
|
||||||
"Paragraph 3": "She danced in Ireland, explored castles in Scotland, and marveled at the architecture in Germany and Russia.",
|
|
||||||
"Paragraph 4": "Italy, Norway, Sweden and Germany will always stay her favourite destinations to visit."
|
|
||||||
}
|
|
||||||
</Example>
|
|
||||||
|
|
||||||
Input: {input_text}
|
|
||||||
```
|
|
||||||
|
|
||||||
### GENERATE: count_prompt
|
|
||||||
Replace `{input_text}` with a text where countries should be counted.
|
|
||||||
```
|
|
||||||
<Instruction> Count the frequency of how many times each country is explicitly named in the input text. You can generate any intermedate lists and states, but the final output should only contain the frequency of each country that appears at least once in the following json format, prefixed with "Output: " (make sure to keep the same spelling for each country in the output as in the input text):
|
|
||||||
{
|
|
||||||
"country1": frequency1,
|
|
||||||
"country2": frequency2,
|
|
||||||
...
|
|
||||||
}
|
|
||||||
</Instruction>
|
|
||||||
|
|
||||||
<Approach>
|
|
||||||
To count the frequency for each country follow these steps:
|
|
||||||
1. Split the input passage into four paragraphs of similar length.
|
|
||||||
2. Count the frequency of each country in each paragraph.
|
|
||||||
3. Combine the frequencies of each country from each paragraph by adding them together.
|
|
||||||
</Approach>
|
|
||||||
|
|
||||||
<Examples>
|
|
||||||
Input:
|
|
||||||
Alexandra boarded the first flight of her grand journey, starting from Canada. With a globe-trotting itinerary in hand, she was filled with excitement. Her first stop was Mexico, where she marveled at the Mayan ruins. From there, she explored the rainforests of Brazil and danced the tango in Argentina.
|
|
||||||
Paragraphs:
|
|
||||||
Alexandra boarded the first flight of her grand journey, starting from Canada. With a globe-trotting itinerary in hand, she was filled with excitement.
|
|
||||||
Her first stop was Mexico, where she marveled at the Mayan ruins. From there, she explored the rainforests of Brazil and danced the tango in Argentina.
|
|
||||||
Sublist frequencies:
|
|
||||||
{
|
|
||||||
"Canada": 1
|
|
||||||
}
|
|
||||||
{
|
|
||||||
"Mexico": 1,
|
|
||||||
"Brazil": 1,
|
|
||||||
"Argentina": 1
|
|
||||||
}
|
|
||||||
Output:
|
|
||||||
{
|
|
||||||
"Canada": 1,
|
|
||||||
"Mexico": 1,
|
|
||||||
"Brazil": 1,
|
|
||||||
"Argentina": 1
|
|
||||||
}
|
|
||||||
|
|
||||||
Input:
|
|
||||||
The adventure led him to the peaks of Peru where he trekked to see the mysteries of Machu Picchu. He then headed to Chile to gaze at the vastness of the Atacama Desert. A quick detour to Uruguay and Paraguay allowed him to experience the vibrancy of the local cultures before returning back to Canada through Peru, Brazil and Mexico.
|
|
||||||
Paragraphs:
|
|
||||||
The adventure led him to the peaks of Peru where he trekked to see the mysteries of Machu Picchu. He then headed to Chile to gaze at the vastness of the Atacama Desert.
|
|
||||||
A quick detour to Uruguay and Paraguay allowed him to experience the vibrancy of the local cultures before returning back to Canada through Peru, Brazil and Mexico.
|
|
||||||
Sublists:
|
|
||||||
{
|
|
||||||
"Peru": 1,
|
|
||||||
"Chile": 1
|
|
||||||
}
|
|
||||||
{
|
|
||||||
"Uruguay": 1,
|
|
||||||
"Paraguay": 1,
|
|
||||||
"Canada": 1,
|
|
||||||
"Peru": 1,
|
|
||||||
"Brazil": 1,
|
|
||||||
"Mexico": 1
|
|
||||||
}
|
|
||||||
Output:
|
|
||||||
{
|
|
||||||
"Peru": 2,
|
|
||||||
"Chile": 1,
|
|
||||||
"Uruguay": 1,
|
|
||||||
"Paraguay": 1,
|
|
||||||
"Canada": 1,
|
|
||||||
"Brazil": 1,
|
|
||||||
"Mexico": 1
|
|
||||||
}
|
|
||||||
|
|
||||||
Input:
|
|
||||||
Journeying westward, she admired the art in Italy and sipped coffee in France. The music of Spain and the history of Greece deepened her love for Europe. The Nordic beauty of Norway, Sweden, Finland, and Denmark took her breath away. She danced in Ireland, explored castles in Scotland, and marveled at the architecture in Germany and Russia. Italy, Norway, Sweden and Germany will always stay her favourite destinations to visit.
|
|
||||||
Paragraphs:
|
|
||||||
Journeying westward, she admired the art in Italy and sipped coffee in France.
|
|
||||||
The music of Spain and the history of Greece deepened her love for Europe. The Nordic beauty of Norway, Sweden, Finland, and Denmark took her breath away.
|
|
||||||
She danced in Ireland, explored castles in Scotland, and marveled at the architecture in Germany and Russia.
|
|
||||||
Italy, Norway, Sweden and Germany will always stay her favourite destinations to visit.
|
|
||||||
Sublists:
|
|
||||||
{
|
|
||||||
"Italy": 1,
|
|
||||||
"France": 1
|
|
||||||
}
|
|
||||||
{
|
|
||||||
"Spain": 1,
|
|
||||||
"Greece": 1,
|
|
||||||
"Norway": 1,
|
|
||||||
"Sweden": 1,
|
|
||||||
"Finland": 1,
|
|
||||||
"Denmark": 1
|
|
||||||
}
|
|
||||||
{
|
|
||||||
"Ireland": 1,
|
|
||||||
"Scotland": 1,
|
|
||||||
"Germany": 1,
|
|
||||||
"Russia": 1
|
|
||||||
}
|
|
||||||
{
|
|
||||||
"Italy": 1,
|
|
||||||
"Norway": 1,
|
|
||||||
"Sweden": 1,
|
|
||||||
"Germany": 1
|
|
||||||
}
|
|
||||||
Output:
|
|
||||||
{
|
|
||||||
"Italy": 2,
|
|
||||||
"France": 1,
|
|
||||||
"Spain": 1,
|
|
||||||
"Greece": 1,
|
|
||||||
"Norway": 2,
|
|
||||||
"Sweden": 2,
|
|
||||||
"Finland": 1,
|
|
||||||
"Denmark": 1,
|
|
||||||
"Ireland": 1,
|
|
||||||
"Scotland": 1,
|
|
||||||
"Germany": 2,
|
|
||||||
"Russia": 1
|
|
||||||
}
|
|
||||||
</Examples>
|
|
||||||
|
|
||||||
Input:
|
|
||||||
{input_text}
|
|
||||||
```
|
|
||||||
|
|
||||||
### AGGREGATE: merge_prompt
|
|
||||||
Replace `{dictionary_1}` and `{dictionary_2}` with dictionaries of countries to be merged.
|
|
||||||
```
|
|
||||||
<Instruction> Combine the following 2 dictionaries, each containing the frequency of countries in a text, into a single dictionary.
|
|
||||||
Simply add the frequencies together for each country and if a country is not present in one of the dictionaries, add it to the final dictionary with the frequency from the other dictionary.
|
|
||||||
Only output the final merged dictionary without any additional text or thoughts!
|
|
||||||
</Instruction>
|
|
||||||
|
|
||||||
<Approach>
|
|
||||||
To combine the 2 dictionaries into single one, follow these steps:
|
|
||||||
1. Create a new dictionary to store the combined frequencies.
|
|
||||||
2. Iterate through the keys of the first dictionary and add the frequency of each country to the new dictionary.
|
|
||||||
3. Iterate through the keys of the second dictionary and add the frequency of each country to the new dictionary and if it is already present, add the frequency to the existing value.
|
|
||||||
</Approach>
|
|
||||||
|
|
||||||
Combine the following 2 dictionaries into a single dictionary:
|
|
||||||
{dictionary_1}
|
|
||||||
{dictionary_2}
|
|
||||||
|
|
||||||
Combined Output:
|
|
||||||
```
|
|
||||||
|
|
||||||
### IMPROVE: improve_merge_prompt
|
|
||||||
Replace `{dictionary_1}` and `{dictionary_2}` with dictionaries of countries
|
|
||||||
to be merged and `{dictionary_incorrect}` with incorrectly merged dictionary
|
|
||||||
to be improved.
|
|
||||||
```
|
|
||||||
<Instruction> The following 2 dictionaries were combined into the third dictionary below. However, some mistakes occured and the third dictionary is incorrect. Please fix the third dictionary so that it contains the correct frequencies for each country. The correct frequencies are the sum of the frequencies from the first 2 dictionaries. If a country is not present in one of the dictionaries, add it to the final dictionary with the frequency from the other dictionary.
|
|
||||||
</Instruction>
|
|
||||||
|
|
||||||
<Example>
|
|
||||||
Dictionary 1:
|
|
||||||
{
|
|
||||||
"Peru": 2,
|
|
||||||
"Chile": 1,
|
|
||||||
"Uruguay": 1,
|
|
||||||
"Paraguay": 1
|
|
||||||
}
|
|
||||||
Dictionary 2:
|
|
||||||
{
|
|
||||||
"Peru": 1,
|
|
||||||
"Argentina": 1,
|
|
||||||
"Canada": 1,
|
|
||||||
"Chile": 3,
|
|
||||||
"Germany": 2
|
|
||||||
}
|
|
||||||
Incorrectly Combined Dictionary:
|
|
||||||
{
|
|
||||||
"Peru": 3,
|
|
||||||
"Chile": 2,
|
|
||||||
"Uruguay": 1,
|
|
||||||
"Paraguay": 1,
|
|
||||||
"Argentina": 1,
|
|
||||||
"Chile": 3,
|
|
||||||
"Germany": 2
|
|
||||||
}
|
|
||||||
Output:
|
|
||||||
{
|
|
||||||
"Peru": 3,
|
|
||||||
"Chile": 4,
|
|
||||||
"Uruguay": 1,
|
|
||||||
"Paraguay": 1,
|
|
||||||
"Argentina": 1,
|
|
||||||
"Canada": 1,
|
|
||||||
"Germany": 2
|
|
||||||
}
|
|
||||||
</Example>
|
|
||||||
|
|
||||||
Dictionary 1:
|
|
||||||
{dictionary_1}
|
|
||||||
Dictionary 2:
|
|
||||||
{dictionary_2}
|
|
||||||
Incorrectly Combined Dictionary:
|
|
||||||
{dictionary_incorrect}
|
|
||||||
Output:
|
|
||||||
```
|
|
||||||
|
|
||||||
## Complete Example Prompts
|
|
||||||
### The GoO Summarised
|
|
||||||
1. Split the input text into four paragraphs of roughly equal size (split prompt)
|
|
||||||
2. For each paragraph: Count the occurrences of individual countries (count prompt) 10 times; score each counting attempt; keep the best
|
|
||||||
3. Merge the country counts into one dictionary (merge prompt) 3 times;
|
|
||||||
validate and improve invalid merge attempts (improve merge prompt) up to 3 attempts each; score; keep the best
|
|
||||||
|
|
||||||
|
|
||||||
### Input
|
|
||||||
```
|
|
||||||
My friend, Alex from Peru, once recounted his journey to Argentina where he learned about the strong cultural ties between Argentina and Brazil due to their shared history.
|
|
||||||
He spoke fondly of his time in Argentina and Brazil, marveling at the similar music, dance, and culinary traditions that seamlessly bridged the borders of these countries.
|
|
||||||
It reminded me of a documentary I'd watched about the ancient Silk Road that spanned across Iran and China.
|
|
||||||
It touched upon the shared historical narratives of Iran and China, highlighting how goods, ideas, and culture flowed between these two countries for centuries.
|
|
||||||
Intriguingly, the documentary also brought up some parallels between this eastern route and the vibrant exchange between Italy and France during the Renaissance.
|
|
||||||
The cultural and intellectual bonds between Italy and France were indeed profound, just as the resilient trade relations that existed between the United States and Canada.
|
|
||||||
The United States and Canada, apart from their geographical proximity, shared an economic bond that strongly influenced their policies and international standing.
|
|
||||||
Similarly, the ties between Australia and New Zealand shed light on their gestalt identity in the Pacific region.
|
|
||||||
Despite their unique characteristics, Australia and New Zealand were often viewed as a single entity due to their remarkably similar backgrounds in terms of culture, language, and colonial history.
|
|
||||||
Inspired by these chronicles of interconnectedness, I decided to delve deeper into history and found a fascinating account of how Ukraine and Poland had influenced each other through years of coexistence.
|
|
||||||
Despite their tumultuous past, Ukraine and Poland shared a cultural tapestry that was deeply woven into their histories.
|
|
||||||
It was quite similar to the complex relationship between North Korea and South Korea, which, despite their political differences, shared common heritage and traditions.
|
|
||||||
Thus, from Argentina to South Korea, the world was an intricate web of countries intertwined with each other through shared histories, cultures, and sometimes, shared destinies.
|
|
||||||
```
|
|
||||||
|
|
||||||
### Step 1 - Prompt
|
|
||||||
```
|
|
||||||
<Instruction> Split the following input text into 4 paragraphs of approximately same length.
|
|
||||||
Only output the final 4 paragraphs in the following format without any additional text or thoughts:
|
|
||||||
{
|
|
||||||
"Paragraph 1": "Some paragraph text ...",
|
|
||||||
"Paragraph 2": "Some paragraph text ...",
|
|
||||||
"Paragraph 3": "Some paragraph text ...",
|
|
||||||
"Paragraph 4": "Some paragraph text ..."
|
|
||||||
}
|
|
||||||
</Instruction>
|
|
||||||
|
|
||||||
<Example>
|
|
||||||
Input:
|
|
||||||
Journeying westward, she admired the art in Italy and sipped coffee in France. The music of Spain and the history of Greece deepened her love for Europe. The Nordic beauty of Norway, Sweden, Finland, and Denmark took her breath away. She danced in Ireland, explored castles in Scotland, and marveled at the architecture in Germany and Russia. Italy, Norway, Sweden and Germany will always stay her favourite destinations to visit.
|
|
||||||
Output:
|
|
||||||
{
|
|
||||||
"Paragraph 1": "Journeying westward, she admired the art in Italy and sipped coffee in France.",
|
|
||||||
"Paragraph 2": "The music of Spain and the history of Greece deepened her love for Europe. The Nordic beauty of Norway, Sweden, Finland, and Denmark took her breath away.",
|
|
||||||
"Paragraph 3": "She danced in Ireland, explored castles in Scotland, and marveled at the architecture in Germany and Russia.",
|
|
||||||
"Paragraph 4": "Italy, Norway, Sweden and Germany will always stay her favourite destinations to visit."
|
|
||||||
}
|
|
||||||
</Example>
|
|
||||||
|
|
||||||
Input:
|
|
||||||
My friend, Alex from Peru, once recounted his journey to Argentina where he learned about the strong cultural ties between Argentina and Brazil due to their shared history.
|
|
||||||
He spoke fondly of his time in Argentina and Brazil, marveling at the similar music, dance, and culinary traditions that seamlessly bridged the borders of these countries.
|
|
||||||
It reminded me of a documentary I'd watched about the ancient Silk Road that spanned across Iran and China.
|
|
||||||
It touched upon the shared historical narratives of Iran and China, highlighting how goods, ideas, and culture flowed between these two countries for centuries.
|
|
||||||
Intriguingly, the documentary also brought up some parallels between this eastern route and the vibrant exchange between Italy and France during the Renaissance.
|
|
||||||
The cultural and intellectual bonds between Italy and France were indeed profound, just as the resilient trade relations that existed between the United States and Canada.
|
|
||||||
The United States and Canada, apart from their geographical proximity, shared an economic bond that strongly influenced their policies and international standing.
|
|
||||||
Similarly, the ties between Australia and New Zealand shed light on their gestalt identity in the Pacific region.
|
|
||||||
Despite their unique characteristics, Australia and New Zealand were often viewed as a single entity due to their remarkably similar backgrounds in terms of culture, language, and colonial history.
|
|
||||||
Inspired by these chronicles of interconnectedness, I decided to delve deeper into history and found a fascinating account of how Ukraine and Poland had influenced each other through years of coexistence.
|
|
||||||
Despite their tumultuous past, Ukraine and Poland shared a cultural tapestry that was deeply woven into their histories.
|
|
||||||
It was quite similar to the complex relationship between North Korea and South Korea, which, despite their political differences, shared common heritage and traditions.
|
|
||||||
Thus, from Argentina to South Korea, the world was an intricate web of countries intertwined with each other through shared histories, cultures, and sometimes, shared destinies.
|
|
||||||
```
|
|
||||||
|
|
||||||
### Step 1 - Response
|
|
||||||
```
|
|
||||||
{
|
|
||||||
"Paragraph 1": "My friend, Alex from Peru, once recounted his journey to Argentina where he learned about the strong cultural ties between Argentina and Brazil due to their shared history. He spoke fondly of his time in Argentina and Brazil, marveling at the similar music, dance, and culinary traditions that seamlessly bridged the borders of these countries.",
|
|
||||||
"Paragraph 2": "It reminded me of a documentary I'd watched about the ancient Silk Road that spanned across Iran and China. It touched upon the shared historical narratives of Iran and China, highlighting how goods, ideas, and culture flowed between these two countries for centuries. Intriguingly, the documentary also brought up some parallels between this eastern route and the vibrant exchange between Italy and France during the Renaissance. The cultural and intellectual bonds between Italy and France were indeed profound, just as the resilient trade relations that existed between the United States and Canada.",
|
|
||||||
"Paragraph 3": "The United States and Canada, apart from their geographical proximity, shared an economic bond that strongly influenced their policies and international standing. Similarly, the ties between Australia and New Zealand shed light on their gestalt identity in the Pacific region. Despite their unique characteristics, Australia and New Zealand were often viewed as a single entity due to their remarkably similar backgrounds in terms of culture, language, and colonial history. Inspired by these chronicles of interconnectedness, I decided to delve deeper into history and found a fascinating account of how Ukraine and Poland had influenced each other through years of coexistence.",
|
|
||||||
"Paragraph 4": "Despite their tumultuous past, Ukraine and Poland shared a cultural tapestry that was deeply woven into their histories. It was quite similar to the complex relationship between North Korea and South Korea, which, despite their political differences, shared common heritage and traditions. Thus, from Argentina to South Korea, the world was an intricate web of countries intertwined with each other through shared histories, cultures, and sometimes, shared destinies."
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### Step 2a - Prompt
|
|
||||||
```
|
|
||||||
<Instruction> Count the frequency of how many times each country is explicitly named in the input text. You can generate any intermedate lists and states, but the final output should only contain the frequency of each country that appears at least once in the following json format, prefixed with "Output: " (make sure to keep the same spelling for each country in the output as in the input text):
|
|
||||||
{
|
|
||||||
"country1": frequency1,
|
|
||||||
"country2": frequency2,
|
|
||||||
...
|
|
||||||
}
|
|
||||||
</Instruction>
|
|
||||||
|
|
||||||
<Approach>
|
|
||||||
To count the frequency for each country follow these steps:
|
|
||||||
1. Split the input passage into four paragraphs of similar length.
|
|
||||||
2. Count the frequency of each country in each paragraph.
|
|
||||||
3. Combine the frequencies of each country from each paragraph by adding them together.
|
|
||||||
</Approach>
|
|
||||||
|
|
||||||
<Examples>
|
|
||||||
Input:
|
|
||||||
Alexandra boarded the first flight of her grand journey, starting from Canada. With a globe-trotting itinerary in hand, she was filled with excitement. Her first stop was Mexico, where she marveled at the Mayan ruins. From there, she explored the rainforests of Brazil and danced the tango in Argentina.
|
|
||||||
Paragraphs:
|
|
||||||
Alexandra boarded the first flight of her grand journey, starting from Canada. With a globe-trotting itinerary in hand, she was filled with excitement.
|
|
||||||
Her first stop was Mexico, where she marveled at the Mayan ruins. From there, she explored the rainforests of Brazil and danced the tango in Argentina.
|
|
||||||
Sublist frequencies:
|
|
||||||
{
|
|
||||||
"Canada": 1
|
|
||||||
}
|
|
||||||
{
|
|
||||||
"Mexico": 1,
|
|
||||||
"Brazil": 1,
|
|
||||||
"Argentina": 1
|
|
||||||
}
|
|
||||||
Output:
|
|
||||||
{
|
|
||||||
"Canada": 1,
|
|
||||||
"Mexico": 1,
|
|
||||||
"Brazil": 1,
|
|
||||||
"Argentina": 1
|
|
||||||
}
|
|
||||||
|
|
||||||
Input:
|
|
||||||
The adventure led him to the peaks of Peru where he trekked to see the mysteries of Machu Picchu. He then headed to Chile to gaze at the vastness of the Atacama Desert. A quick detour to Uruguay and Paraguay allowed him to experience the vibrancy of the local cultures before returning back to Canada through Peru, Brazil and Mexico.
|
|
||||||
Paragraphs:
|
|
||||||
The adventure led him to the peaks of Peru where he trekked to see the mysteries of Machu Picchu. He then headed to Chile to gaze at the vastness of the Atacama Desert.
|
|
||||||
A quick detour to Uruguay and Paraguay allowed him to experience the vibrancy of the local cultures before returning back to Canada through Peru, Brazil and Mexico.
|
|
||||||
Sublists:
|
|
||||||
{
|
|
||||||
"Peru": 1,
|
|
||||||
"Chile": 1
|
|
||||||
}
|
|
||||||
{
|
|
||||||
"Uruguay": 1,
|
|
||||||
"Paraguay": 1,
|
|
||||||
"Canada": 1,
|
|
||||||
"Peru": 1,
|
|
||||||
"Brazil": 1,
|
|
||||||
"Mexico": 1
|
|
||||||
}
|
|
||||||
Output:
|
|
||||||
{
|
|
||||||
"Peru": 2,
|
|
||||||
"Chile": 1,
|
|
||||||
"Uruguay": 1,
|
|
||||||
"Paraguay": 1,
|
|
||||||
"Canada": 1,
|
|
||||||
"Brazil": 1,
|
|
||||||
"Mexico": 1
|
|
||||||
}
|
|
||||||
|
|
||||||
Input:
|
|
||||||
Journeying westward, she admired the art in Italy and sipped coffee in France. The music of Spain and the history of Greece deepened her love for Europe. The Nordic beauty of Norway, Sweden, Finland, and Denmark took her breath away. She danced in Ireland, explored castles in Scotland, and marveled at the architecture in Germany and Russia. Italy, Norway, Sweden and Germany will always stay her favourite destinations to visit.
|
|
||||||
Paragraphs:
|
|
||||||
Journeying westward, she admired the art in Italy and sipped coffee in France.
|
|
||||||
The music of Spain and the history of Greece deepened her love for Europe. The Nordic beauty of Norway, Sweden, Finland, and Denmark took her breath away.
|
|
||||||
She danced in Ireland, explored castles in Scotland, and marveled at the architecture in Germany and Russia.
|
|
||||||
Italy, Norway, Sweden and Germany will always stay her favourite destinations to visit.
|
|
||||||
Sublists:
|
|
||||||
{
|
|
||||||
"Italy": 1,
|
|
||||||
"France": 1
|
|
||||||
}
|
|
||||||
{
|
|
||||||
"Spain": 1,
|
|
||||||
"Greece": 1,
|
|
||||||
"Norway": 1,
|
|
||||||
"Sweden": 1,
|
|
||||||
"Finland": 1,
|
|
||||||
"Denmark": 1
|
|
||||||
}
|
|
||||||
{
|
|
||||||
"Ireland": 1,
|
|
||||||
"Scotland": 1,
|
|
||||||
"Germany": 1,
|
|
||||||
"Russia": 1
|
|
||||||
}
|
|
||||||
{
|
|
||||||
"Italy": 1,
|
|
||||||
"Norway": 1,
|
|
||||||
"Sweden": 1,
|
|
||||||
"Germany": 1
|
|
||||||
}
|
|
||||||
Output:
|
|
||||||
{
|
|
||||||
"Italy": 2,
|
|
||||||
"France": 1,
|
|
||||||
"Spain": 1,
|
|
||||||
"Greece": 1,
|
|
||||||
"Norway": 2,
|
|
||||||
"Sweden": 2,
|
|
||||||
"Finland": 1,
|
|
||||||
"Denmark": 1,
|
|
||||||
"Ireland": 1,
|
|
||||||
"Scotland": 1,
|
|
||||||
"Germany": 2,
|
|
||||||
"Russia": 1
|
|
||||||
}
|
|
||||||
</Examples>
|
|
||||||
|
|
||||||
Input:
|
|
||||||
My friend, Alex from Peru, once recounted his journey to Argentina where he learned about the strong cultural ties between Argentina and Brazil due to their shared history. He spoke fondly of his time in Argentina and Brazil, marveling at the similar music, dance, and culinary traditions that seamlessly bridged the borders of these countries.
|
|
||||||
```
|
|
||||||
|
|
||||||
### Step 2a - 10 Responses
|
|
||||||
```
|
|
||||||
1. { "Peru": 1, "Argentina": 1, "Brazil": 1 }
|
|
||||||
2. { "Peru": 1, "Argentina": 2, "Brazil": 2 }
|
|
||||||
3. { "Peru": 1, "Argentina": 2, "Brazil": 2 }
|
|
||||||
4. { "Peru": 1, "Argentina": 1, "Brazil": 1 }
|
|
||||||
5. { "Peru": 1, "Argentina": 2, "Brazil": 2 }
|
|
||||||
6. { "Peru": 1, "Argentina": 2, "Brazil": 2 }
|
|
||||||
7. { "Peru": 1, "Argentina": 1, "Brazil": 1 }
|
|
||||||
8. { "Peru": 1, "Argentina": 1, "Brazil": 1 }
|
|
||||||
9. { "Peru": 1, "Argentina": 1, "Brazil": 1 }
|
|
||||||
10. { "Peru": 1, "Argentina": 1, "Brazil": 1 }
|
|
||||||
```
|
|
||||||
|
|
||||||
### Step 2b - Prompt
|
|
||||||
```
|
|
||||||
<Instruction> Count the frequency of how many times each country is explicitly named in the input text. You can generate any intermedate lists and states, but the final output should only contain the frequency of each country that appears at least once in the following json format, prefixed with "Output: " (make sure to keep the same spelling for each country in the output as in the input text):
|
|
||||||
{
|
|
||||||
"country1": frequency1,
|
|
||||||
"country2": frequency2,
|
|
||||||
...
|
|
||||||
}
|
|
||||||
</Instruction>
|
|
||||||
|
|
||||||
<Approach>
|
|
||||||
To count the frequency for each country follow these steps:
|
|
||||||
1. Split the input passage into four paragraphs of similar length.
|
|
||||||
2. Count the frequency of each country in each paragraph.
|
|
||||||
3. Combine the frequencies of each country from each paragraph by adding them together.
|
|
||||||
</Approach>
|
|
||||||
|
|
||||||
<Examples>
|
|
||||||
Input:
|
|
||||||
Alexandra boarded the first flight of her grand journey, starting from Canada. With a globe-trotting itinerary in hand, she was filled with excitement. Her first stop was Mexico, where she marveled at the Mayan ruins. From there, she explored the rainforests of Brazil and danced the tango in Argentina.
|
|
||||||
Paragraphs:
|
|
||||||
Alexandra boarded the first flight of her grand journey, starting from Canada. With a globe-trotting itinerary in hand, she was filled with excitement.
|
|
||||||
Her first stop was Mexico, where she marveled at the Mayan ruins. From there, she explored the rainforests of Brazil and danced the tango in Argentina.
|
|
||||||
Sublist frequencies:
|
|
||||||
{
|
|
||||||
"Canada": 1
|
|
||||||
}
|
|
||||||
{
|
|
||||||
"Mexico": 1,
|
|
||||||
"Brazil": 1,
|
|
||||||
"Argentina": 1
|
|
||||||
}
|
|
||||||
Output:
|
|
||||||
{
|
|
||||||
"Canada": 1,
|
|
||||||
"Mexico": 1,
|
|
||||||
"Brazil": 1,
|
|
||||||
"Argentina": 1
|
|
||||||
}
|
|
||||||
|
|
||||||
Input:
|
|
||||||
The adventure led him to the peaks of Peru where he trekked to see the mysteries of Machu Picchu. He then headed to Chile to gaze at the vastness of the Atacama Desert. A quick detour to Uruguay and Paraguay allowed him to experience the vibrancy of the local cultures before returning back to Canada through Peru, Brazil and Mexico.
|
|
||||||
Paragraphs:
|
|
||||||
The adventure led him to the peaks of Peru where he trekked to see the mysteries of Machu Picchu. He then headed to Chile to gaze at the vastness of the Atacama Desert.
|
|
||||||
A quick detour to Uruguay and Paraguay allowed him to experience the vibrancy of the local cultures before returning back to Canada through Peru, Brazil and Mexico.
|
|
||||||
Sublists:
|
|
||||||
{
|
|
||||||
"Peru": 1,
|
|
||||||
"Chile": 1
|
|
||||||
}
|
|
||||||
{
|
|
||||||
"Uruguay": 1,
|
|
||||||
"Paraguay": 1,
|
|
||||||
"Canada": 1,
|
|
||||||
"Peru": 1,
|
|
||||||
"Brazil": 1,
|
|
||||||
"Mexico": 1
|
|
||||||
}
|
|
||||||
Output:
|
|
||||||
{
|
|
||||||
"Peru": 2,
|
|
||||||
"Chile": 1,
|
|
||||||
"Uruguay": 1,
|
|
||||||
"Paraguay": 1,
|
|
||||||
"Canada": 1,
|
|
||||||
"Brazil": 1,
|
|
||||||
"Mexico": 1
|
|
||||||
}
|
|
||||||
|
|
||||||
Input:
|
|
||||||
Journeying westward, she admired the art in Italy and sipped coffee in France. The music of Spain and the history of Greece deepened her love for Europe. The Nordic beauty of Norway, Sweden, Finland, and Denmark took her breath away. She danced in Ireland, explored castles in Scotland, and marveled at the architecture in Germany and Russia. Italy, Norway, Sweden and Germany will always stay her favourite destinations to visit.
|
|
||||||
Paragraphs:
|
|
||||||
Journeying westward, she admired the art in Italy and sipped coffee in France.
|
|
||||||
The music of Spain and the history of Greece deepened her love for Europe. The Nordic beauty of Norway, Sweden, Finland, and Denmark took her breath away.
|
|
||||||
She danced in Ireland, explored castles in Scotland, and marveled at the architecture in Germany and Russia.
|
|
||||||
Italy, Norway, Sweden and Germany will always stay her favourite destinations to visit.
|
|
||||||
Sublists:
|
|
||||||
{
|
|
||||||
"Italy": 1,
|
|
||||||
"France": 1
|
|
||||||
}
|
|
||||||
{
|
|
||||||
"Spain": 1,
|
|
||||||
"Greece": 1,
|
|
||||||
"Norway": 1,
|
|
||||||
"Sweden": 1,
|
|
||||||
"Finland": 1,
|
|
||||||
"Denmark": 1
|
|
||||||
}
|
|
||||||
{
|
|
||||||
"Ireland": 1,
|
|
||||||
"Scotland": 1,
|
|
||||||
"Germany": 1,
|
|
||||||
"Russia": 1
|
|
||||||
}
|
|
||||||
{
|
|
||||||
"Italy": 1,
|
|
||||||
"Norway": 1,
|
|
||||||
"Sweden": 1,
|
|
||||||
"Germany": 1
|
|
||||||
}
|
|
||||||
Output:
|
|
||||||
{
|
|
||||||
"Italy": 2,
|
|
||||||
"France": 1,
|
|
||||||
"Spain": 1,
|
|
||||||
"Greece": 1,
|
|
||||||
"Norway": 2,
|
|
||||||
"Sweden": 2,
|
|
||||||
"Finland": 1,
|
|
||||||
"Denmark": 1,
|
|
||||||
"Ireland": 1,
|
|
||||||
"Scotland": 1,
|
|
||||||
"Germany": 2,
|
|
||||||
"Russia": 1
|
|
||||||
}
|
|
||||||
</Examples>
|
|
||||||
|
|
||||||
Input:
|
|
||||||
It reminded me of a documentary I'd watched about the ancient Silk Road that spanned across Iran and China. It touched upon the shared historical narratives of Iran and China, highlighting how goods, ideas, and culture flowed between these two countries for centuries. Intriguingly, the documentary also brought up some parallels between this eastern route and the vibrant exchange between Italy and France during the Renaissance. The cultural and intellectual bonds between Italy and France were indeed profound, just as the resilient trade relations that existed between the United States and Canada.
|
|
||||||
```
|
|
||||||
|
|
||||||
### Step 2b - 10 Responses
|
|
||||||
```
|
|
||||||
1. { "Iran": 1, "China": 1, "Italy": 1, "France": 1, "United States": 1, "Canada": 1 }
|
|
||||||
2. { "Iran": 1, "China": 1, "Italy": 1, "France": 1, "United States": 1, "Canada": 1 }
|
|
||||||
3. { "Iran": 3, "China": 2, "Italy": 1, "France": 1, "United States": 1, "Canada": 1 }
|
|
||||||
4. { "Iran": 1, "China": 1, "Italy": 1, "France": 1, "United States": 1, "Canada": 1 }
|
|
||||||
5. { "Iran": 2, "China": 2, "Italy": 1, "France": 1, "United States": 1, "Canada": 1 }
|
|
||||||
6. { "Iran": 1, "China": 1, "Italy": 1, "France": 1, "United States": 1, "Canada": 1 }
|
|
||||||
7. { "Iran": 1, "China": 1, "Italy": 1, "France": 1, "United States": 1, "Canada": 1 }
|
|
||||||
8. { "Iran": 1, "China": 1, "Italy": 1, "France": 1, "United States": 1, "Canada": 1 }
|
|
||||||
9. { "Iran": 1, "China": 1, "Italy": 1, "France": 1, "United States": 1, "Canada": 1 }
|
|
||||||
10. { "Iran": 2, "China": 2, "Italy": 1, "France": 2, "United States": 1, "Canada": 1 }
|
|
||||||
```
|
|
||||||
|
|
||||||
### Step 2c - Prompt
|
|
||||||
```
|
|
||||||
<Instruction> Count the frequency of how many times each country is explicitly named in the input text. You can generate any intermedate lists and states, but the final output should only contain the frequency of each country that appears at least once in the following json format, prefixed with "Output: " (make sure to keep the same spelling for each country in the output as in the input text):
|
|
||||||
{
|
|
||||||
"country1": frequency1,
|
|
||||||
"country2": frequency2,
|
|
||||||
...
|
|
||||||
}
|
|
||||||
</Instruction>
|
|
||||||
|
|
||||||
<Approach>
|
|
||||||
To count the frequency for each country follow these steps:
|
|
||||||
1. Split the input passage into four paragraphs of similar length.
|
|
||||||
2. Count the frequency of each country in each paragraph.
|
|
||||||
3. Combine the frequencies of each country from each paragraph by adding them together.
|
|
||||||
</Approach>
|
|
||||||
|
|
||||||
<Examples>
|
|
||||||
Input:
|
|
||||||
Alexandra boarded the first flight of her grand journey, starting from Canada. With a globe-trotting itinerary in hand, she was filled with excitement. Her first stop was Mexico, where she marveled at the Mayan ruins. From there, she explored the rainforests of Brazil and danced the tango in Argentina.
|
|
||||||
Paragraphs:
|
|
||||||
Alexandra boarded the first flight of her grand journey, starting from Canada. With a globe-trotting itinerary in hand, she was filled with excitement.
|
|
||||||
Her first stop was Mexico, where she marveled at the Mayan ruins. From there, she explored the rainforests of Brazil and danced the tango in Argentina.
|
|
||||||
Sublist frequencies:
|
|
||||||
{
|
|
||||||
"Canada": 1
|
|
||||||
}
|
|
||||||
{
|
|
||||||
"Mexico": 1,
|
|
||||||
"Brazil": 1,
|
|
||||||
"Argentina": 1
|
|
||||||
}
|
|
||||||
Output:
|
|
||||||
{
|
|
||||||
"Canada": 1,
|
|
||||||
"Mexico": 1,
|
|
||||||
"Brazil": 1,
|
|
||||||
"Argentina": 1
|
|
||||||
}
|
|
||||||
|
|
||||||
Input:
|
|
||||||
The adventure led him to the peaks of Peru where he trekked to see the mysteries of Machu Picchu. He then headed to Chile to gaze at the vastness of the Atacama Desert. A quick detour to Uruguay and Paraguay allowed him to experience the vibrancy of the local cultures before returning back to Canada through Peru, Brazil and Mexico.
|
|
||||||
Paragraphs:
|
|
||||||
The adventure led him to the peaks of Peru where he trekked to see the mysteries of Machu Picchu. He then headed to Chile to gaze at the vastness of the Atacama Desert.
|
|
||||||
A quick detour to Uruguay and Paraguay allowed him to experience the vibrancy of the local cultures before returning back to Canada through Peru, Brazil and Mexico.
|
|
||||||
Sublists:
|
|
||||||
{
|
|
||||||
"Peru": 1,
|
|
||||||
"Chile": 1
|
|
||||||
}
|
|
||||||
{
|
|
||||||
"Uruguay": 1,
|
|
||||||
"Paraguay": 1,
|
|
||||||
"Canada": 1,
|
|
||||||
"Peru": 1,
|
|
||||||
"Brazil": 1,
|
|
||||||
"Mexico": 1
|
|
||||||
}
|
|
||||||
Output:
|
|
||||||
{
|
|
||||||
"Peru": 2,
|
|
||||||
"Chile": 1,
|
|
||||||
"Uruguay": 1,
|
|
||||||
"Paraguay": 1,
|
|
||||||
"Canada": 1,
|
|
||||||
"Brazil": 1,
|
|
||||||
"Mexico": 1
|
|
||||||
}
|
|
||||||
|
|
||||||
Input:
|
|
||||||
Journeying westward, she admired the art in Italy and sipped coffee in France. The music of Spain and the history of Greece deepened her love for Europe. The Nordic beauty of Norway, Sweden, Finland, and Denmark took her breath away. She danced in Ireland, explored castles in Scotland, and marveled at the architecture in Germany and Russia. Italy, Norway, Sweden and Germany will always stay her favourite destinations to visit.
|
|
||||||
Paragraphs:
|
|
||||||
Journeying westward, she admired the art in Italy and sipped coffee in France.
|
|
||||||
The music of Spain and the history of Greece deepened her love for Europe. The Nordic beauty of Norway, Sweden, Finland, and Denmark took her breath away.
|
|
||||||
She danced in Ireland, explored castles in Scotland, and marveled at the architecture in Germany and Russia.
|
|
||||||
Italy, Norway, Sweden and Germany will always stay her favourite destinations to visit.
|
|
||||||
Sublists:
|
|
||||||
{
|
|
||||||
"Italy": 1,
|
|
||||||
"France": 1
|
|
||||||
}
|
|
||||||
{
|
|
||||||
"Spain": 1,
|
|
||||||
"Greece": 1,
|
|
||||||
"Norway": 1,
|
|
||||||
"Sweden": 1,
|
|
||||||
"Finland": 1,
|
|
||||||
"Denmark": 1
|
|
||||||
}
|
|
||||||
{
|
|
||||||
"Ireland": 1,
|
|
||||||
"Scotland": 1,
|
|
||||||
"Germany": 1,
|
|
||||||
"Russia": 1
|
|
||||||
}
|
|
||||||
{
|
|
||||||
"Italy": 1,
|
|
||||||
"Norway": 1,
|
|
||||||
"Sweden": 1,
|
|
||||||
"Germany": 1
|
|
||||||
}
|
|
||||||
Output:
|
|
||||||
{
|
|
||||||
"Italy": 2,
|
|
||||||
"France": 1,
|
|
||||||
"Spain": 1,
|
|
||||||
"Greece": 1,
|
|
||||||
"Norway": 2,
|
|
||||||
"Sweden": 2,
|
|
||||||
"Finland": 1,
|
|
||||||
"Denmark": 1,
|
|
||||||
"Ireland": 1,
|
|
||||||
"Scotland": 1,
|
|
||||||
"Germany": 2,
|
|
||||||
"Russia": 1
|
|
||||||
}
|
|
||||||
</Examples>
|
|
||||||
|
|
||||||
Input:
|
|
||||||
The United States and Canada, apart from their geographical proximity, shared an economic bond that strongly influenced their policies and international standing. Similarly, the ties between Australia and New Zealand shed light on their gestalt identity in the Pacific region. Despite their unique characteristics, Australia and New Zealand were often viewed as a single entity due to their remarkably similar backgrounds in terms of culture, language, and colonial history. Inspired by these chronicles of interconnectedness, I decided to delve deeper into history and found a fascinating account of how Ukraine and Poland had influenced each other through years of coexistence.
|
|
||||||
```
|
|
||||||
|
|
||||||
### Step 2c - 10 Responses
|
|
||||||
```
|
|
||||||
1. { "United States": 1, "Canada": 1, "Australia": 2, "New Zealand": 2, "Ukraine": 1, "Poland": 1 }
|
|
||||||
2. { "United States": 1, "Canada": 1, "Australia": 2, "New Zealand": 2, "Ukraine": 1, "Poland": 1 }
|
|
||||||
3. { "United States": 1, "Canada": 1, "Australia": 1, "New Zealand": 1, "Australia and New Zealand": 1, "Ukraine": 1, "Poland": 1 }
|
|
||||||
4. { "United States": 1, "Canada": 1, "Australia": 2, "New Zealand": 2, "Ukraine": 1, "Poland": 1 }
|
|
||||||
5. { "United States": 1, "Canada": 1, "Australia": 1, "New Zealand": 1, "Australia and New Zealand": 1, "Ukraine": 1, "Poland": 1 }
|
|
||||||
6. { "United States": 1, "Canada": 1, "Australia": 1, "New Zealand": 1, "Ukraine": 1, "Poland": 1 }
|
|
||||||
7. { "United States": 1, "Canada": 1, "Australia": 2, "New Zealand": 2, "Ukraine": 1, "Poland": 1 }
|
|
||||||
8. { "United States": 1, "Canada": 1, "Australia": 1, "New Zealand": 1, "Ukraine": 1, "Poland": 1 }
|
|
||||||
9. { "United States": 1, "Canada": 1, "Australia": 2, "New Zealand": 2, "Ukraine": 1, "Poland": 1 }
|
|
||||||
10. { "United States": 1, "Canada": 1, "Australia": 2, "New Zealand": 2, "Ukraine": 1, "Poland": 1 }
|
|
||||||
```
|
|
||||||
|
|
||||||
### Step 2d - Prompt
|
|
||||||
```
|
|
||||||
<Instruction> Count the frequency of how many times each country is explicitly named in the input text. You can generate any intermedate lists and states, but the final output should only contain the frequency of each country that appears at least once in the following json format, prefixed with "Output: " (make sure to keep the same spelling for each country in the output as in the input text):
|
|
||||||
{
|
|
||||||
"country1": frequency1,
|
|
||||||
"country2": frequency2,
|
|
||||||
...
|
|
||||||
}
|
|
||||||
</Instruction>
|
|
||||||
|
|
||||||
<Approach>
|
|
||||||
To count the frequency for each country follow these steps:
|
|
||||||
1. Split the input passage into four paragraphs of similar length.
|
|
||||||
2. Count the frequency of each country in each paragraph.
|
|
||||||
3. Combine the frequencies of each country from each paragraph by adding them together.
|
|
||||||
</Approach>
|
|
||||||
|
|
||||||
<Examples>
|
|
||||||
Input:
|
|
||||||
Alexandra boarded the first flight of her grand journey, starting from Canada. With a globe-trotting itinerary in hand, she was filled with excitement. Her first stop was Mexico, where she marveled at the Mayan ruins. From there, she explored the rainforests of Brazil and danced the tango in Argentina.
|
|
||||||
Paragraphs:
|
|
||||||
Alexandra boarded the first flight of her grand journey, starting from Canada. With a globe-trotting itinerary in hand, she was filled with excitement.
|
|
||||||
Her first stop was Mexico, where she marveled at the Mayan ruins. From there, she explored the rainforests of Brazil and danced the tango in Argentina.
|
|
||||||
Sublist frequencies:
|
|
||||||
{
|
|
||||||
"Canada": 1
|
|
||||||
}
|
|
||||||
{
|
|
||||||
"Mexico": 1,
|
|
||||||
"Brazil": 1,
|
|
||||||
"Argentina": 1
|
|
||||||
}
|
|
||||||
Output:
|
|
||||||
{
|
|
||||||
"Canada": 1,
|
|
||||||
"Mexico": 1,
|
|
||||||
"Brazil": 1,
|
|
||||||
"Argentina": 1
|
|
||||||
}
|
|
||||||
|
|
||||||
Input:
|
|
||||||
The adventure led him to the peaks of Peru where he trekked to see the mysteries of Machu Picchu. He then headed to Chile to gaze at the vastness of the Atacama Desert. A quick detour to Uruguay and Paraguay allowed him to experience the vibrancy of the local cultures before returning back to Canada through Peru, Brazil and Mexico.
|
|
||||||
Paragraphs:
|
|
||||||
The adventure led him to the peaks of Peru where he trekked to see the mysteries of Machu Picchu. He then headed to Chile to gaze at the vastness of the Atacama Desert.
|
|
||||||
A quick detour to Uruguay and Paraguay allowed him to experience the vibrancy of the local cultures before returning back to Canada through Peru, Brazil and Mexico.
|
|
||||||
Sublists:
|
|
||||||
{
|
|
||||||
"Peru": 1,
|
|
||||||
"Chile": 1
|
|
||||||
}
|
|
||||||
{
|
|
||||||
"Uruguay": 1,
|
|
||||||
"Paraguay": 1,
|
|
||||||
"Canada": 1,
|
|
||||||
"Peru": 1,
|
|
||||||
"Brazil": 1,
|
|
||||||
"Mexico": 1
|
|
||||||
}
|
|
||||||
Output:
|
|
||||||
{
|
|
||||||
"Peru": 2,
|
|
||||||
"Chile": 1,
|
|
||||||
"Uruguay": 1,
|
|
||||||
"Paraguay": 1,
|
|
||||||
"Canada": 1,
|
|
||||||
"Brazil": 1,
|
|
||||||
"Mexico": 1
|
|
||||||
}
|
|
||||||
|
|
||||||
Input:
|
|
||||||
Journeying westward, she admired the art in Italy and sipped coffee in France. The music of Spain and the history of Greece deepened her love for Europe. The Nordic beauty of Norway, Sweden, Finland, and Denmark took her breath away. She danced in Ireland, explored castles in Scotland, and marveled at the architecture in Germany and Russia. Italy, Norway, Sweden and Germany will always stay her favourite destinations to visit.
|
|
||||||
Paragraphs:
|
|
||||||
Journeying westward, she admired the art in Italy and sipped coffee in France.
|
|
||||||
The music of Spain and the history of Greece deepened her love for Europe. The Nordic beauty of Norway, Sweden, Finland, and Denmark took her breath away.
|
|
||||||
She danced in Ireland, explored castles in Scotland, and marveled at the architecture in Germany and Russia.
|
|
||||||
Italy, Norway, Sweden and Germany will always stay her favourite destinations to visit.
|
|
||||||
Sublists:
|
|
||||||
{
|
|
||||||
"Italy": 1,
|
|
||||||
"France": 1
|
|
||||||
}
|
|
||||||
{
|
|
||||||
"Spain": 1,
|
|
||||||
"Greece": 1,
|
|
||||||
"Norway": 1,
|
|
||||||
"Sweden": 1,
|
|
||||||
"Finland": 1,
|
|
||||||
"Denmark": 1
|
|
||||||
}
|
|
||||||
{
|
|
||||||
"Ireland": 1,
|
|
||||||
"Scotland": 1,
|
|
||||||
"Germany": 1,
|
|
||||||
"Russia": 1
|
|
||||||
}
|
|
||||||
{
|
|
||||||
"Italy": 1,
|
|
||||||
"Norway": 1,
|
|
||||||
"Sweden": 1,
|
|
||||||
"Germany": 1
|
|
||||||
}
|
|
||||||
Output:
|
|
||||||
{
|
|
||||||
"Italy": 2,
|
|
||||||
"France": 1,
|
|
||||||
"Spain": 1,
|
|
||||||
"Greece": 1,
|
|
||||||
"Norway": 2,
|
|
||||||
"Sweden": 2,
|
|
||||||
"Finland": 1,
|
|
||||||
"Denmark": 1,
|
|
||||||
"Ireland": 1,
|
|
||||||
"Scotland": 1,
|
|
||||||
"Germany": 2,
|
|
||||||
"Russia": 1
|
|
||||||
}
|
|
||||||
</Examples>
|
|
||||||
|
|
||||||
Input:
|
|
||||||
Despite their tumultuous past, Ukraine and Poland shared a cultural tapestry that was deeply woven into their histories. It was quite similar to the complex relationship between North Korea and South Korea, which, despite their political differences, shared common heritage and traditions. Thus, from Argentina to South Korea, the world was an intricate web of countries intertwined with each other through shared histories, cultures, and sometimes, shared destinies.
|
|
||||||
```
|
|
||||||
|
|
||||||
### Step 2d - 10 Responses
|
|
||||||
```
|
|
||||||
1. { "Ukraine": 1, "Poland": 1, "North Korea": 1, "South Korea": 2, "Argentina": 1 }
|
|
||||||
2. { "Ukraine": 1, "Poland": 1, "North Korea": 1, "South Korea": 2, "Argentina": 1 }
|
|
||||||
3. { "Ukraine": 1, "Poland": 1, "North Korea": 1, "South Korea": 2, "Argentina": 1 }
|
|
||||||
4. { "Ukraine": 1, "Poland": 1, "North Korea": 1, "South Korea": 2, "Argentina": 1 }
|
|
||||||
5. { "Ukraine": 1, "Poland": 1, "North Korea": 1, "South Korea": 2, "Argentina": 1 }
|
|
||||||
6. { "Ukraine": 1, "Poland": 1, "North Korea": 1, "South Korea": 2, "Argentina": 1 }
|
|
||||||
7. { "Ukraine": 1, "Poland": 1, "North Korea": 1, "South Korea": 2, "Argentina": 1 }
|
|
||||||
8. { "Ukraine": 1, "Poland": 1, "North Korea": 1, "South Korea": 2, "Argentina": 1 }
|
|
||||||
9. { "Ukraine": 1, "Poland": 1, "North Korea": 1, "South Korea": 2, "Argentina": 1 }
|
|
||||||
10. { "Ukraine": 1, "Poland": 1, "North Korea": 1, "South Korea": 2, "Argentina": 1 }
|
|
||||||
```
|
|
||||||
|
|
||||||
### Step 3a - Prompt
|
|
||||||
```
|
|
||||||
<Instruction> Combine the following 2 dictionaries, each containing the frequency of countries in a text, into a single dictionary.
|
|
||||||
Simply add the frequencies together for each country and if a country is not present in one of the dictionaries, add it to the final dictionary with the frequency from the other dictionary.
|
|
||||||
Only output the final merged dictionary without any additional text or thoughts!
|
|
||||||
</Instruction>
|
|
||||||
|
|
||||||
<Approach>
|
|
||||||
To combine the 2 dictionaries into single one, follow these steps:
|
|
||||||
1. Create a new dictionary to store the combined frequencies.
|
|
||||||
2. Iterate through the keys of the first dictionary and add the frequency of each country to the new dictionary.
|
|
||||||
3. Iterate through the keys of the second dictionary and add the frequency of each country to the new dictionary and if it is already present, add the frequency to the existing value.
|
|
||||||
</Approach>
|
|
||||||
|
|
||||||
Combine the following 2 dictionaries into a single dictionary:
|
|
||||||
{
|
|
||||||
"Peru": 1,
|
|
||||||
"Argentina": 2,
|
|
||||||
"Brazil": 2
|
|
||||||
}
|
|
||||||
{
|
|
||||||
"Iran": 2,
|
|
||||||
"China": 2,
|
|
||||||
"Italy": 1,
|
|
||||||
"France": 2,
|
|
||||||
"United States": 1,
|
|
||||||
"Canada": 1
|
|
||||||
}
|
|
||||||
|
|
||||||
Combined Output:
|
|
||||||
```
|
|
||||||
|
|
||||||
### Step 3a - 3 Responses
|
|
||||||
```
|
|
||||||
1. { "Peru": 1, "Argentina": 2, "Brazil": 2, "Iran": 2, "China": 2, "Italy": 1, "France": 2, "United States": 1, "Canada": 1 }
|
|
||||||
2. { "Peru": 1, "Argentina": 2, "Brazil": 2, "Iran": 2, "China": 2, "Italy": 1, "France": 2, "United States": 1, "Canada": 1 }
|
|
||||||
3. { "Peru": 1, "Argentina": 2, "Brazil": 2, "Iran": 2, "China": 2, "Italy": 1, "France": 2, "United States": 1, "Canada": 1 }
|
|
||||||
```
|
|
||||||
|
|
||||||
### Step 3b - Prompt
|
|
||||||
```
|
|
||||||
<Instruction> Combine the following 2 dictionaries, each containing the frequency of countries in a text, into a single dictionary.
|
|
||||||
Simply add the frequencies together for each country and if a country is not present in one of the dictionaries, add it to the final dictionary with the frequency from the other dictionary.
|
|
||||||
Only output the final merged dictionary without any additional text or thoughts!
|
|
||||||
</Instruction>
|
|
||||||
|
|
||||||
<Approach>
|
|
||||||
To combine the 2 dictionaries into single one, follow these steps:
|
|
||||||
1. Create a new dictionary to store the combined frequencies.
|
|
||||||
2. Iterate through the keys of the first dictionary and add the frequency of each country to the new dictionary.
|
|
||||||
3. Iterate through the keys of the second dictionary and add the frequency of each country to the new dictionary and if it is already present, add the frequency to the existing value.
|
|
||||||
</Approach>
|
|
||||||
|
|
||||||
Combine the following 2 dictionaries into a single dictionary:
|
|
||||||
{
|
|
||||||
"United States": 1,
|
|
||||||
"Canada": 1,
|
|
||||||
"Australia": 2,
|
|
||||||
"New Zealand": 2,
|
|
||||||
"Ukraine": 1,
|
|
||||||
"Poland": 1
|
|
||||||
}
|
|
||||||
{
|
|
||||||
"Ukraine": 1,
|
|
||||||
"Poland": 1,
|
|
||||||
"North Korea": 1,
|
|
||||||
"South Korea": 2,
|
|
||||||
"Argentina": 1
|
|
||||||
}
|
|
||||||
|
|
||||||
Combined Output:
|
|
||||||
```
|
|
||||||
|
|
||||||
### Step 3b - 3 Responses
|
|
||||||
```
|
|
||||||
1. { "United States": 1, "Canada": 1, "Australia": 2, "New Zealand": 2, "Ukraine": 2, "Poland": 2, "North Korea": 1, "South Korea": 2, "Argentina": 1 }
|
|
||||||
2. { "United States": 1, "Canada": 1, "Australia": 2, "New Zealand": 2, "Ukraine": 2, "Poland": 2, "North Korea": 1, "South Korea": 2, "Argentina": 1 }
|
|
||||||
3. { "United States": 1, "Canada": 1, "Australia": 2, "New Zealand": 2, "Ukraine": 2, "Poland": 2, "North Korea": 1, "South Korea": 2, "Argentina": 1 }
|
|
||||||
```
|
|
||||||
|
|
||||||
### Step 3c - Prompt
|
|
||||||
```
|
|
||||||
<Instruction> Combine the following 2 dictionaries, each containing the frequency of countries in a text, into a single dictionary.
|
|
||||||
Simply add the frequencies together for each country and if a country is not present in one of the dictionaries, add it to the final dictionary with the frequency from the other dictionary.
|
|
||||||
Only output the final merged dictionary without any additional text or thoughts!
|
|
||||||
</Instruction>
|
|
||||||
|
|
||||||
<Approach>
|
|
||||||
To combine the 2 dictionaries into single one, follow these steps:
|
|
||||||
1. Create a new dictionary to store the combined frequencies.
|
|
||||||
2. Iterate through the keys of the first dictionary and add the frequency of each country to the new dictionary.
|
|
||||||
3. Iterate through the keys of the second dictionary and add the frequency of each country to the new dictionary and if it is already present, add the frequency to the existing value.
|
|
||||||
</Approach>
|
|
||||||
|
|
||||||
Combine the following 2 dictionaries into a single dictionary:
|
|
||||||
{
|
|
||||||
"Peru": 1,
|
|
||||||
"Argentina": 2,
|
|
||||||
"Brazil": 2,
|
|
||||||
"Iran": 2,
|
|
||||||
"China": 2,
|
|
||||||
"Italy": 1,
|
|
||||||
"France": 2,
|
|
||||||
"United States": 1,
|
|
||||||
"Canada": 1
|
|
||||||
}
|
|
||||||
{
|
|
||||||
"United States": 1,
|
|
||||||
"Canada": 1,
|
|
||||||
"Australia": 2,
|
|
||||||
"New Zealand": 2,
|
|
||||||
"Ukraine": 2,
|
|
||||||
"Poland": 2,
|
|
||||||
"North Korea": 1,
|
|
||||||
"South Korea": 2,
|
|
||||||
"Argentina": 1
|
|
||||||
}
|
|
||||||
|
|
||||||
Combined Output:
|
|
||||||
```
|
|
||||||
|
|
||||||
### Step 3c - 3 Responses
|
|
||||||
```
|
|
||||||
1. { "Peru": 1, "Argentina": 3, "Brazil": 2, "Iran": 2, "China": 2, "Italy": 1, "France": 2, "United States": 2, "Canada": 2, "Australia": 2, "New Zealand": 2, "Ukraine": 2, "Poland": 2, "North Korea": 1, "South Korea": 2 }
|
|
||||||
2. { "Peru": 1, "Argentina": 3, "Brazil": 2, "Iran": 2, "China": 2, "Italy": 1, "France": 2, "United States": 2, "Canada": 2, "Australia": 2, "New Zealand": 2, "Ukraine": 2, "Poland": 2, "North Korea": 1, "South Korea": 2 }
|
|
||||||
3. { "Peru": 1, "Argentina": 3, "Brazil": 2, "Iran": 2, "China": 2, "Italy": 1, "France": 2, "United States": 2, "Canada": 2, "Australia": 2, "New Zealand": 2, "Ukraine": 2, "Poland": 2, "North Korea": 1, "South Korea": 2 }
|
|
||||||
```
|
|
||||||
|
|
||||||
### Final Result
|
|
||||||
2 errors propagated from counting.
|
|
||||||
```
|
|
||||||
{
|
|
||||||
"Peru": 1,
|
|
||||||
"Argentina": 3,
|
|
||||||
"Brazil": 2,
|
|
||||||
"Iran": 2,
|
|
||||||
"China": 2,
|
|
||||||
"Italy": 1,
|
|
||||||
"France": 2,
|
|
||||||
"United States": 2,
|
|
||||||
"Canada": 2,
|
|
||||||
"Australia": 2,
|
|
||||||
"New Zealand": 2,
|
|
||||||
"Ukraine": 2,
|
|
||||||
"Poland": 2,
|
|
||||||
"North Korea": 1,
|
|
||||||
"South Korea": 2
|
|
||||||
}
|
|
||||||
```
|
|
||||||
File diff suppressed because it is too large
Load Diff
@ -1,167 +0,0 @@
|
|||||||
# Copyright (c) 2023 ETH Zurich.
|
|
||||||
# All rights reserved.
|
|
||||||
#
|
|
||||||
# Use of this source code is governed by a BSD-style license that can be
|
|
||||||
# found in the LICENSE file.
|
|
||||||
#
|
|
||||||
# main author: Nils Blach
|
|
||||||
# contributions: Ales Kubicek
|
|
||||||
|
|
||||||
import json
|
|
||||||
import os
|
|
||||||
import matplotlib.pyplot as plt
|
|
||||||
|
|
||||||
|
|
||||||
def get_complete_results(base_directory):
|
|
||||||
results_complete = {}
|
|
||||||
for folder_name in os.listdir(base_directory):
|
|
||||||
folder_path = os.path.join(base_directory, folder_name)
|
|
||||||
if os.path.isdir(folder_path):
|
|
||||||
results_complete[folder_name] = []
|
|
||||||
for file_name in os.listdir(folder_path):
|
|
||||||
if file_name.endswith(".json"):
|
|
||||||
file_path = os.path.join(folder_path, file_name)
|
|
||||||
with open(file_path, "r") as f:
|
|
||||||
data = json.load(f)
|
|
||||||
results_complete[folder_name].append(
|
|
||||||
{"key": int(file_name.split(".")[0]), "data": data}
|
|
||||||
)
|
|
||||||
for key in results_complete.keys():
|
|
||||||
results_complete[key] = sorted(
|
|
||||||
results_complete[key], key=lambda x: x["key"]
|
|
||||||
)
|
|
||||||
return results_complete
|
|
||||||
|
|
||||||
|
|
||||||
def get_final_scores(results_complete):
|
|
||||||
scores = {}
|
|
||||||
for method in results_complete.keys():
|
|
||||||
scores[method] = []
|
|
||||||
for result in results_complete[method]:
|
|
||||||
score = 100
|
|
||||||
solved = False
|
|
||||||
cost = 1
|
|
||||||
prompt_tokens = 0
|
|
||||||
completion_tokens = 0
|
|
||||||
for op in result["data"]:
|
|
||||||
if "operation" in op and op["operation"] == "ground_truth_evaluator":
|
|
||||||
try:
|
|
||||||
score = min(op["scores"])
|
|
||||||
solved = any(op["problem_solved"])
|
|
||||||
except:
|
|
||||||
continue
|
|
||||||
if "cost" in op:
|
|
||||||
cost = op["cost"]
|
|
||||||
prompt_tokens = op["prompt_tokens"]
|
|
||||||
completion_tokens = op["completion_tokens"]
|
|
||||||
scores[method].append(
|
|
||||||
[result["key"], score, solved, prompt_tokens, completion_tokens, cost]
|
|
||||||
)
|
|
||||||
scores[method] = sorted(scores[method], key=lambda x: x[0])
|
|
||||||
return scores
|
|
||||||
|
|
||||||
|
|
||||||
def get_plotting_data(base_directory):
|
|
||||||
results_complete = get_complete_results(base_directory)
|
|
||||||
scores = get_final_scores(results_complete)
|
|
||||||
results_plotting = {
|
|
||||||
method: {
|
|
||||||
"scores": [x[1] for x in scores[method]],
|
|
||||||
"solved": sum([1 for x in scores[method] if x[2]]),
|
|
||||||
"costs": [x[5] for x in scores[method]],
|
|
||||||
}
|
|
||||||
for method in scores.keys()
|
|
||||||
}
|
|
||||||
return results_plotting
|
|
||||||
|
|
||||||
|
|
||||||
def plot_results(
|
|
||||||
results,
|
|
||||||
methods_order=["io", "cot", "tot", "tot2", "got4", "got8", "gotx"],
|
|
||||||
model="GPT-3.5",
|
|
||||||
y_lower=0,
|
|
||||||
y_upper=40,
|
|
||||||
cost_upper=1.8,
|
|
||||||
display_solved=True,
|
|
||||||
annotation_offset=1,
|
|
||||||
display_left_ylabel=False,
|
|
||||||
display_right_ylabel=False,
|
|
||||||
):
|
|
||||||
methods_order = [method for method in methods_order if method in results]
|
|
||||||
# Extract scores based on the order
|
|
||||||
scores_ordered = [
|
|
||||||
[score for score in results[method]["scores"] if score != 100 and score != 300]
|
|
||||||
for method in methods_order
|
|
||||||
]
|
|
||||||
total_costs = [sum(results[method]["costs"]) for method in methods_order]
|
|
||||||
|
|
||||||
# Create figure and axis
|
|
||||||
fig, ax = plt.subplots(dpi=150, figsize=(3.75, 4))
|
|
||||||
|
|
||||||
# Create boxplots
|
|
||||||
positions = range(1, len(methods_order) + 1)
|
|
||||||
ax.boxplot(scores_ordered, positions=positions)
|
|
||||||
|
|
||||||
fig_fontsize = 12
|
|
||||||
|
|
||||||
# Set the ticks and labels
|
|
||||||
methods_labels = ["IO", "CoT", "ToT", "ToT2", "GoT4", "GoT8", "GoTx"]
|
|
||||||
ax.set_xticks(range(1, len(methods_order) + 1))
|
|
||||||
ax.set_xticks(range(1, len(methods_order) + 1))
|
|
||||||
ax.set_xticklabels(methods_labels, fontsize=10)
|
|
||||||
|
|
||||||
ax.set_ylim(y_lower, (y_upper + 2) if display_solved else y_upper + 1)
|
|
||||||
plt.yticks(fontsize=fig_fontsize)
|
|
||||||
if display_left_ylabel:
|
|
||||||
ax.set_ylabel(f"Number of errors; the lower the better", fontsize=fig_fontsize)
|
|
||||||
|
|
||||||
ax.set_title(f"Keyword Counting")
|
|
||||||
|
|
||||||
ax2 = ax.twinx()
|
|
||||||
ax2.bar(positions, total_costs, alpha=0.5, color="blue", label="Total Cost ($)")
|
|
||||||
ax2.yaxis.set_tick_params(colors="#1919ff", labelsize=fig_fontsize)
|
|
||||||
ax2.set_ylim(0, cost_upper)
|
|
||||||
number_of_ticks = len(ax.get_yticks())
|
|
||||||
tick_interval = cost_upper / (number_of_ticks)
|
|
||||||
ax2_ticks = [tick_interval * i for i in range(number_of_ticks)]
|
|
||||||
|
|
||||||
ax2.set_yticks(ax2_ticks)
|
|
||||||
|
|
||||||
if display_right_ylabel:
|
|
||||||
ax2.set_ylabel(
|
|
||||||
"Total Cost ($); the lower the better",
|
|
||||||
color="#1919ff",
|
|
||||||
fontsize=fig_fontsize,
|
|
||||||
)
|
|
||||||
|
|
||||||
if display_solved:
|
|
||||||
annotation_height = y_upper + annotation_offset
|
|
||||||
count = 1
|
|
||||||
for method in methods_order:
|
|
||||||
if method not in results:
|
|
||||||
continue
|
|
||||||
solved = results[method]["solved"]
|
|
||||||
ax.text(
|
|
||||||
count,
|
|
||||||
annotation_height,
|
|
||||||
f"{solved}",
|
|
||||||
ha="center",
|
|
||||||
va="bottom",
|
|
||||||
fontsize=fig_fontsize,
|
|
||||||
)
|
|
||||||
count += 1
|
|
||||||
|
|
||||||
model = model.replace(".", "").replace("-", "").lower()
|
|
||||||
fig.savefig(f"keyword_counting_{model}.pdf", bbox_inches="tight")
|
|
||||||
|
|
||||||
|
|
||||||
plot_results(
|
|
||||||
get_plotting_data("results/"),
|
|
||||||
display_solved=True,
|
|
||||||
annotation_offset=-0.3,
|
|
||||||
model="GPT-3.5",
|
|
||||||
y_upper=35,
|
|
||||||
display_left_ylabel=True,
|
|
||||||
display_right_ylabel=True,
|
|
||||||
cost_upper=9,
|
|
||||||
)
|
|
||||||
@ -1,52 +0,0 @@
|
|||||||
# Set Intersection
|
|
||||||
|
|
||||||
The use case in this directory computes the intersection of two input
|
|
||||||
sets. We provide implementations of five different approaches for 32, 64
|
|
||||||
and 128 elements:
|
|
||||||
- IO
|
|
||||||
- Chain-of-Thought (CoT)
|
|
||||||
- Tree of Thought (ToT):
|
|
||||||
- ToT: wider tree, meaning more branches per level
|
|
||||||
- ToT2: tree with more levels, but fewer branches per level
|
|
||||||
- Graph of Thoughts (GoT)
|
|
||||||
|
|
||||||
## Data
|
|
||||||
|
|
||||||
We provide input files with 100 precomputed samples for each set length:
|
|
||||||
`set_intersection_<number of elements>.csv`. It is also possible to use
|
|
||||||
the data generator `dataset_gen_intersection.py` to generate additional or
|
|
||||||
different samples. The parameters can be updated in lines 24 to 28 of
|
|
||||||
the main body:
|
|
||||||
- set_size = 32 # size of the generated sets
|
|
||||||
- int_value_ubound = 64 # (exclusive) upper limit of generated numbers
|
|
||||||
- seed = 42 # seed of the random number generator
|
|
||||||
- num_sample = 100 # number of samples
|
|
||||||
- filename = 'set_intersection_032.csv' # output filename
|
|
||||||
|
|
||||||
## Execution
|
|
||||||
|
|
||||||
The files to execute the use case are called
|
|
||||||
`set_intersection_<number of elements>.py`. In the main body, one can
|
|
||||||
select the specific samples to be run (variable sample) and the
|
|
||||||
approaches (variable approaches). It is also possible to set a budget in
|
|
||||||
dollars (variable budget).
|
|
||||||
The input filename for the samples is currently hardcoded to
|
|
||||||
`set_intersection_<number of elements>.csv`, but can be updated in the
|
|
||||||
function `run`.
|
|
||||||
|
|
||||||
The Python scripts will create the directory `result`, if it is not
|
|
||||||
already present. In the `result` directory, another directory is created
|
|
||||||
for each run: `{name of LLM}_{list of approaches}_{day}_{start time}`.
|
|
||||||
Inside each execution specific directory two files (`config.json`,
|
|
||||||
`log.log`) and a separate directory for each selected approach are
|
|
||||||
created. `config.json` contains the configuration of the run: input data,
|
|
||||||
selected approaches, name of the LLM, and the budget. `log.log` contains
|
|
||||||
the prompts and responses of the LLM as well as additional debug data.
|
|
||||||
The approach directories contain a separate json file for every sample
|
|
||||||
and the file contains the Graph Reasoning State (GRS) for that sample.
|
|
||||||
|
|
||||||
## Plot Data
|
|
||||||
|
|
||||||
Change the results directory in line 170 of `plot.py` and update the
|
|
||||||
length parameter in the subsequent line and run `python3 plot.py` to
|
|
||||||
plot your data.
|
|
||||||
@ -1,92 +0,0 @@
|
|||||||
# Copyright (c) 2023 ETH Zurich.
|
|
||||||
# All rights reserved.
|
|
||||||
#
|
|
||||||
# Use of this source code is governed by a BSD-style license that can be
|
|
||||||
# found in the LICENSE file.
|
|
||||||
#
|
|
||||||
# main author: Robert Gerstenberger
|
|
||||||
|
|
||||||
import csv
|
|
||||||
import numpy as np
|
|
||||||
|
|
||||||
|
|
||||||
def scramble(array: np.ndarray, rng: np.random.Generator) -> None:
|
|
||||||
"""
|
|
||||||
Helper function to change the order of the elements in an array randomly.
|
|
||||||
|
|
||||||
:param array: Array to be scrambled.
|
|
||||||
:type: numpy.ndarray
|
|
||||||
:param rng: Random number generator.
|
|
||||||
:type rng: numpy.random.Generator
|
|
||||||
"""
|
|
||||||
|
|
||||||
size = array.shape[0]
|
|
||||||
|
|
||||||
index_array = rng.integers(0, size, size)
|
|
||||||
|
|
||||||
for i in range(size):
|
|
||||||
temp = array[i]
|
|
||||||
array[i] = array[index_array[i]]
|
|
||||||
array[index_array[i]] = temp
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
"""
|
|
||||||
Input(u) : Set size.
|
|
||||||
Input(v) : Range of the integer numbers in the sets: 0..v (exclusive)
|
|
||||||
Input(w) : Seed for the random number generator.
|
|
||||||
Input(x) : Number of samples to be generated.
|
|
||||||
Input(y) : Filename for the output CSV file.
|
|
||||||
Output(z) : Input sets and intersected set written a file in the CSV format.
|
|
||||||
File contains the sample ID, input set 1, input set 2,
|
|
||||||
intersection set.
|
|
||||||
"""
|
|
||||||
|
|
||||||
set_size = 32 # size of the generated sets
|
|
||||||
int_value_ubound = 64 # (exclusive) upper limit of generated numbers
|
|
||||||
seed = 42 # seed of the random number generator
|
|
||||||
num_sample = 100 # number of samples
|
|
||||||
filename = "set_intersection_032.csv" # output filename
|
|
||||||
|
|
||||||
assert 2 * set_size <= int_value_ubound
|
|
||||||
|
|
||||||
rng = np.random.default_rng(seed)
|
|
||||||
|
|
||||||
intersection_sizes = rng.integers(set_size // 4, 3 * set_size // 4, num_sample)
|
|
||||||
|
|
||||||
np.set_printoptions(
|
|
||||||
linewidth=np.inf
|
|
||||||
) # no wrapping in the array fields in the output file
|
|
||||||
|
|
||||||
with open(filename, "w") as f:
|
|
||||||
fieldnames = ["ID", "SET1", "SET2", "INTERSECTION"]
|
|
||||||
writer = csv.DictWriter(f, delimiter=",", fieldnames=fieldnames)
|
|
||||||
writer.writeheader()
|
|
||||||
|
|
||||||
for i in range(num_sample):
|
|
||||||
intersection_size = intersection_sizes[i]
|
|
||||||
|
|
||||||
full_set = np.arange(0, int_value_ubound, dtype=np.int16)
|
|
||||||
|
|
||||||
scramble(full_set, rng)
|
|
||||||
|
|
||||||
intersection = full_set[:intersection_size].copy()
|
|
||||||
|
|
||||||
sorted_intersection = np.sort(intersection)
|
|
||||||
|
|
||||||
set1 = full_set[:set_size].copy()
|
|
||||||
set2 = np.concatenate(
|
|
||||||
[intersection, full_set[set_size : 2 * set_size - intersection_size]]
|
|
||||||
)
|
|
||||||
|
|
||||||
scramble(set1, rng)
|
|
||||||
scramble(set2, rng)
|
|
||||||
|
|
||||||
writer.writerow(
|
|
||||||
{
|
|
||||||
"ID": i,
|
|
||||||
"SET1": set1.tolist(),
|
|
||||||
"SET2": set2.tolist(),
|
|
||||||
"INTERSECTION": sorted_intersection.tolist(),
|
|
||||||
}
|
|
||||||
)
|
|
||||||
@ -1,188 +0,0 @@
|
|||||||
# Set Intersection for 32-Element Sets - Prompts and Examples
|
|
||||||
## Prompt Templates
|
|
||||||
### GENERATE: split_prompt
|
|
||||||
Replace `{input}` with the input list/set of numbers to be split.
|
|
||||||
```
|
|
||||||
<Instruction> Split the following list of 32 numbers into 2 lists of 16 numbers each, the first list should contain the first 16 numbers and the second list the second 16 numbers.
|
|
||||||
Only output the 2 lists in the following format without any additional text or thoughts!:
|
|
||||||
{
|
|
||||||
"List 1": [13, 16, 30, 6, 21, 7, 31, ...],
|
|
||||||
"List 2": [25, 24, 10, 4, 27, 0, 14, ...]
|
|
||||||
} </Instruction>
|
|
||||||
|
|
||||||
<Example>
|
|
||||||
Input: [26, 40, 42, 57, 15, 31, 5, 32, 11, 4, 24, 28, 51, 54, 12, 22, 33, 35, 7, 13, 2, 59, 8, 23, 43, 16, 29, 55, 25, 63, 21, 18]
|
|
||||||
Output:
|
|
||||||
{
|
|
||||||
"List 1": [26, 40, 42, 57, 15, 31, 5, 32, 11, 4, 24, 28, 51, 54, 12, 22],
|
|
||||||
"List 2": [33, 35, 7, 13, 2, 59, 8, 23, 43, 16, 29, 55, 25, 63, 21, 18]
|
|
||||||
}
|
|
||||||
</Example>
|
|
||||||
|
|
||||||
Input: {input}
|
|
||||||
```
|
|
||||||
|
|
||||||
### GENERATE: intersect_prompt
|
|
||||||
Replace `{set1}` and `{set2}` with the sets to be intersected.
|
|
||||||
```
|
|
||||||
<Instruction> Find the intersection of two sets of numbers. Output only the set of numbers that are present in both sets, no additional text. </Instruction>
|
|
||||||
|
|
||||||
<Examples>
|
|
||||||
Input Set 1: [13, 16, 30, 6, 21, 7, 31, 15, 11, 1, 24, 10, 9, 3, 20, 8]
|
|
||||||
Input Set 2: [25, 24, 10, 4, 27, 0, 14, 12, 8, 2, 29, 20, 17, 19, 26, 23]
|
|
||||||
Output: [24, 10, 20, 8]
|
|
||||||
|
|
||||||
Input Set 1: [26, 40, 42, 57, 15, 31, 5, 32, 11, 4, 24, 28, 51, 54, 12, 22, 33, 35, 7, 13, 2, 59, 8, 23, 43, 16, 29, 55, 25, 63, 21, 18]
|
|
||||||
Input Set 2: [16, 60, 36, 48, 0, 15, 5, 19, 46, 24, 1, 6, 61, 10, 38, 53, 58, 9, 44, 14, 35, 63, 52, 20, 27, 17, 39, 47, 34, 56, 40, 59]
|
|
||||||
Output: [40, 15, 5, 24, 35, 59, 16, 63]
|
|
||||||
|
|
||||||
Input Set 1: [115, 61, 35, 103, 90, 117, 86, 44, 63, 45, 40, 30, 74, 33, 31, 1, 118, 48, 38, 0, 119, 51, 64, 78, 15, 121, 89, 101, 79, 69, 120, 29, 58, 50, 116, 11, 60, 12, 39, 95, 23, 2, 109, 84, 7, 43, 99, 98, 52, 70, 75, 102, 57, 19, 94, 36, 114, 88, 71, 56, 83, 6, 96, 107]
|
|
||||||
Input Set 2: [13, 35, 20, 96, 34, 18, 47, 127, 126, 9, 21, 16, 77, 22, 111, 122, 85, 73, 42, 105, 123, 15, 33, 59, 67, 57, 104, 8, 30, 89, 76, 12, 65, 84, 32, 40, 7, 100, 108, 50, 14, 28, 24, 53, 90, 17, 91, 81, 124, 63, 5, 46, 125, 93, 49, 66, 117, 37, 115, 113, 2, 106, 41, 72]
|
|
||||||
Output: [115, 35, 90, 117, 63, 40, 30, 33, 15, 89, 50, 12, 2, 84, 7, 57, 96]
|
|
||||||
</Examples>
|
|
||||||
|
|
||||||
Input Set 1: {set1}
|
|
||||||
Input Set 2: {set2}
|
|
||||||
```
|
|
||||||
|
|
||||||
### AGGREGATE: merge_prompt
|
|
||||||
Replace `{input1}` and `{input2}` with the lists/sets to be merged.
|
|
||||||
```
|
|
||||||
<Instruction> Merge the following 2 lists of length 16 each, into one list of length 32 by appending the second list to the first list.
|
|
||||||
Only output the final list without any additional text or thoughts! </Instruction>
|
|
||||||
|
|
||||||
List 1: {input1}
|
|
||||||
List 2: {input2}
|
|
||||||
|
|
||||||
```
|
|
||||||
|
|
||||||
## Complete Example Prompts
|
|
||||||
### The GoO Summarised
|
|
||||||
|
|
||||||
1. Split the second input set into two sub-sets of equal size (split prompt)
|
|
||||||
2. For each sub-set: Intersect the sub-set with the first input set (intersect prompt) five times; score each sort attempt; keep the best
|
|
||||||
3. Merge the resulting intersections into one full intersection set (merge prompt) 10 times; score each merge attempt; keep the best
|
|
||||||
|
|
||||||
### Input
|
|
||||||
```
|
|
||||||
[31, 11, 4, 63, 38, 58, 59, 24, 61, 14, 32, 39, 27, 46, 48, 19, 52, 57, 50, 56, 3, 2, 53, 29, 5, 37, 62, 41, 36, 12, 49, 16]
|
|
||||||
```
|
|
||||||
|
|
||||||
### Step 1 - Prompt
|
|
||||||
```
|
|
||||||
<Instruction> Split the following list of 32 numbers into 2 lists of 16 numbers each, the first list should contain the first 16 numbers and the second list the second 16 numbers.
|
|
||||||
Only output the 2 lists in the following format without any additional text or thoughts!:
|
|
||||||
{
|
|
||||||
"List 1": [13, 16, 30, 6, 21, 7, 31, ...],
|
|
||||||
"List 2": [25, 24, 10, 4, 27, 0, 14, ...]
|
|
||||||
} </Instruction>
|
|
||||||
|
|
||||||
<Example>
|
|
||||||
Input: [26, 40, 42, 57, 15, 31, 5, 32, 11, 4, 24, 28, 51, 54, 12, 22, 33, 35, 7, 13, 2, 59, 8, 23, 43, 16, 29, 55, 25, 63, 21, 18]
|
|
||||||
Output:
|
|
||||||
{
|
|
||||||
"List 1": [26, 40, 42, 57, 15, 31, 5, 32, 11, 4, 24, 28, 51, 54, 12, 22],
|
|
||||||
"List 2": [33, 35, 7, 13, 2, 59, 8, 23, 43, 16, 29, 55, 25, 63, 21, 18]
|
|
||||||
}
|
|
||||||
</Example>
|
|
||||||
|
|
||||||
Input: [31, 11, 4, 63, 38, 58, 59, 24, 61, 14, 32, 39, 27, 46, 48, 19, 52, 57, 50, 56, 3, 2, 53, 29, 5, 37, 62, 41, 36, 12, 49, 16]
|
|
||||||
```
|
|
||||||
|
|
||||||
### Step 1 - Response
|
|
||||||
```
|
|
||||||
{
|
|
||||||
"List 1": [31, 11, 4, 63, 38, 58, 59, 24, 61, 14, 32, 39, 27, 46, 48, 19],
|
|
||||||
"List 2": [52, 57, 50, 56, 3, 2, 53, 29, 5, 37, 62, 41, 36, 12, 49, 16]
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### Step 2a - Prompt
|
|
||||||
```
|
|
||||||
<Instruction> Find the intersection of two sets of numbers. Output only the set of numbers that are present in both sets, no additional text. </Instruction>
|
|
||||||
|
|
||||||
<Examples>
|
|
||||||
Input Set 1: [13, 16, 30, 6, 21, 7, 31, 15, 11, 1, 24, 10, 9, 3, 20, 8]
|
|
||||||
Input Set 2: [25, 24, 10, 4, 27, 0, 14, 12, 8, 2, 29, 20, 17, 19, 26, 23]
|
|
||||||
Output: [24, 10, 20, 8]
|
|
||||||
|
|
||||||
Input Set 1: [26, 40, 42, 57, 15, 31, 5, 32, 11, 4, 24, 28, 51, 54, 12, 22, 33, 35, 7, 13, 2, 59, 8, 23, 43, 16, 29, 55, 25, 63, 21, 18]
|
|
||||||
Input Set 2: [16, 60, 36, 48, 0, 15, 5, 19, 46, 24, 1, 6, 61, 10, 38, 53, 58, 9, 44, 14, 35, 63, 52, 20, 27, 17, 39, 47, 34, 56, 40, 59]
|
|
||||||
Output: [40, 15, 5, 24, 35, 59, 16, 63]
|
|
||||||
|
|
||||||
Input Set 1: [115, 61, 35, 103, 90, 117, 86, 44, 63, 45, 40, 30, 74, 33, 31, 1, 118, 48, 38, 0, 119, 51, 64, 78, 15, 121, 89, 101, 79, 69, 120, 29, 58, 50, 116, 11, 60, 12, 39, 95, 23, 2, 109, 84, 7, 43, 99, 98, 52, 70, 75, 102, 57, 19, 94, 36, 114, 88, 71, 56, 83, 6, 96, 107]
|
|
||||||
Input Set 2: [13, 35, 20, 96, 34, 18, 47, 127, 126, 9, 21, 16, 77, 22, 111, 122, 85, 73, 42, 105, 123, 15, 33, 59, 67, 57, 104, 8, 30, 89, 76, 12, 65, 84, 32, 40, 7, 100, 108, 50, 14, 28, 24, 53, 90, 17, 91, 81, 124, 63, 5, 46, 125, 93, 49, 66, 117, 37, 115, 113, 2, 106, 41, 72]
|
|
||||||
Output: [115, 35, 90, 117, 63, 40, 30, 33, 15, 89, 50, 12, 2, 84, 7, 57, 96]
|
|
||||||
</Examples>
|
|
||||||
|
|
||||||
Input Set 1: [11, 60, 1, 49, 21, 33, 14, 56, 54, 15, 23, 40, 45, 22, 7, 28, 20, 46, 51, 6, 34, 37, 3, 50, 17, 8, 25, 0, 35, 47, 18, 19]
|
|
||||||
Input Set 2: [31, 11, 4, 63, 38, 58, 59, 24, 61, 14, 32, 39, 27, 46, 48, 19]
|
|
||||||
```
|
|
||||||
|
|
||||||
### Step 2a - 5 Responses
|
|
||||||
```
|
|
||||||
1. [11, 14, 46, 14, 19]
|
|
||||||
2. Output: [11, 14, 46, 19]
|
|
||||||
3. [11, 14, 46, 14, 19]
|
|
||||||
4. [11, 14, 46, 19]
|
|
||||||
5. [11, 14, 46, 14, 19]
|
|
||||||
```
|
|
||||||
|
|
||||||
### Step 2b - Prompt
|
|
||||||
```
|
|
||||||
<Instruction> Find the intersection of two sets of numbers. Output only the set of numbers that are present in both sets, no additional text. </Instruction>
|
|
||||||
|
|
||||||
<Examples>
|
|
||||||
Input Set 1: [13, 16, 30, 6, 21, 7, 31, 15, 11, 1, 24, 10, 9, 3, 20, 8]
|
|
||||||
Input Set 2: [25, 24, 10, 4, 27, 0, 14, 12, 8, 2, 29, 20, 17, 19, 26, 23]
|
|
||||||
Output: [24, 10, 20, 8]
|
|
||||||
|
|
||||||
Input Set 1: [26, 40, 42, 57, 15, 31, 5, 32, 11, 4, 24, 28, 51, 54, 12, 22, 33, 35, 7, 13, 2, 59, 8, 23, 43, 16, 29, 55, 25, 63, 21, 18]
|
|
||||||
Input Set 2: [16, 60, 36, 48, 0, 15, 5, 19, 46, 24, 1, 6, 61, 10, 38, 53, 58, 9, 44, 14, 35, 63, 52, 20, 27, 17, 39, 47, 34, 56, 40, 59]
|
|
||||||
Output: [40, 15, 5, 24, 35, 59, 16, 63]
|
|
||||||
|
|
||||||
Input Set 1: [115, 61, 35, 103, 90, 117, 86, 44, 63, 45, 40, 30, 74, 33, 31, 1, 118, 48, 38, 0, 119, 51, 64, 78, 15, 121, 89, 101, 79, 69, 120, 29, 58, 50, 116, 11, 60, 12, 39, 95, 23, 2, 109, 84, 7, 43, 99, 98, 52, 70, 75, 102, 57, 19, 94, 36, 114, 88, 71, 56, 83, 6, 96, 107]
|
|
||||||
Input Set 2: [13, 35, 20, 96, 34, 18, 47, 127, 126, 9, 21, 16, 77, 22, 111, 122, 85, 73, 42, 105, 123, 15, 33, 59, 67, 57, 104, 8, 30, 89, 76, 12, 65, 84, 32, 40, 7, 100, 108, 50, 14, 28, 24, 53, 90, 17, 91, 81, 124, 63, 5, 46, 125, 93, 49, 66, 117, 37, 115, 113, 2, 106, 41, 72]
|
|
||||||
Output: [115, 35, 90, 117, 63, 40, 30, 33, 15, 89, 50, 12, 2, 84, 7, 57, 96]
|
|
||||||
</Examples>
|
|
||||||
|
|
||||||
Input Set 1: [11, 60, 1, 49, 21, 33, 14, 56, 54, 15, 23, 40, 45, 22, 7, 28, 20, 46, 51, 6, 34, 37, 3, 50, 17, 8, 25, 0, 35, 47, 18, 19]
|
|
||||||
Input Set 2: [52, 57, 50, 56, 3, 2, 53, 29, 5, 37, 62, 41, 36, 12, 49, 16]
|
|
||||||
```
|
|
||||||
|
|
||||||
### Step 2b - 5 Responses
|
|
||||||
```
|
|
||||||
1. [50, 56, 49]
|
|
||||||
2. [50, 56, 37, 49]
|
|
||||||
3. [56, 49, 37, 3, 50]
|
|
||||||
4. [50, 56, 37, 49]
|
|
||||||
5. [50, 56, 37, 49]
|
|
||||||
```
|
|
||||||
|
|
||||||
### Step 3 - Prompt
|
|
||||||
```
|
|
||||||
<Instruction> Merge the following 2 lists of length 16 each, into one list of length 32 by appending the second list to the first list.
|
|
||||||
Only output the final list without any additional text or thoughts! </Instruction>
|
|
||||||
|
|
||||||
List 1: [11, 14, 46, 19]
|
|
||||||
List 2: [56, 49, 37, 3, 50]
|
|
||||||
```
|
|
||||||
|
|
||||||
### Step 3 - 10 Responses
|
|
||||||
```
|
|
||||||
1. [11, 14, 46, 19, 56, 49, 37, 3, 50]
|
|
||||||
2. [11, 14, 46, 19, 56, 49, 37, 3, 50]
|
|
||||||
3. [11, 14, 46, 19, 56, 49, 37, 3, 50]
|
|
||||||
4. [11, 14, 46, 19, 56, 49, 37, 3, 50]
|
|
||||||
5. [11, 14, 46, 19, 56, 49, 37, 3, 50]
|
|
||||||
6. [11, 14, 46, 19, 56, 49, 37, 3, 50]
|
|
||||||
7. [11, 14, 46, 19, 56, 49, 37, 3, 50]
|
|
||||||
8. [11, 14, 46, 19, 56, 49, 37, 3, 50]
|
|
||||||
9. [11, 14, 46, 19, 56, 49, 37, 3, 50]
|
|
||||||
10. [11, 14, 46, 19, 56, 49, 37, 3, 50]
|
|
||||||
```
|
|
||||||
|
|
||||||
### Final Result
|
|
||||||
```
|
|
||||||
[11, 14, 46, 19, 56, 49, 37, 3, 50]
|
|
||||||
```
|
|
||||||
@ -1,184 +0,0 @@
|
|||||||
# Copyright (c) 2023 ETH Zurich.
|
|
||||||
# All rights reserved.
|
|
||||||
#
|
|
||||||
# Use of this source code is governed by a BSD-style license that can be
|
|
||||||
# found in the LICENSE file.
|
|
||||||
#
|
|
||||||
# main author: Nils Blach
|
|
||||||
# contributions: Robert Gerstenberger
|
|
||||||
|
|
||||||
import json
|
|
||||||
import os
|
|
||||||
import matplotlib.pyplot as plt
|
|
||||||
|
|
||||||
|
|
||||||
def get_complete_results(base_directory):
|
|
||||||
results_complete = {}
|
|
||||||
for folder_name in os.listdir(base_directory):
|
|
||||||
folder_path = os.path.join(base_directory, folder_name)
|
|
||||||
if os.path.isdir(folder_path):
|
|
||||||
results_complete[folder_name] = []
|
|
||||||
for file_name in os.listdir(folder_path):
|
|
||||||
if file_name.endswith(".json"):
|
|
||||||
file_path = os.path.join(folder_path, file_name)
|
|
||||||
with open(file_path, "r") as f:
|
|
||||||
data = json.load(f)
|
|
||||||
results_complete[folder_name].append(
|
|
||||||
{"key": int(file_name.split(".")[0]), "data": data}
|
|
||||||
)
|
|
||||||
for key in results_complete.keys():
|
|
||||||
results_complete[key] = sorted(
|
|
||||||
results_complete[key], key=lambda x: x["key"]
|
|
||||||
)
|
|
||||||
return results_complete
|
|
||||||
|
|
||||||
|
|
||||||
def get_final_scores(results_complete):
|
|
||||||
scores = {}
|
|
||||||
for method in results_complete.keys():
|
|
||||||
scores[method] = []
|
|
||||||
for result in results_complete[method]:
|
|
||||||
score = 100
|
|
||||||
solved = False
|
|
||||||
cost = 1
|
|
||||||
prompt_tokens = 0
|
|
||||||
completion_tokens = 0
|
|
||||||
for op in result["data"]:
|
|
||||||
if "operation" in op and op["operation"] == "ground_truth_evaluator":
|
|
||||||
try:
|
|
||||||
score = min(op["scores"])
|
|
||||||
solved = any(op["problem_solved"])
|
|
||||||
except:
|
|
||||||
continue
|
|
||||||
if "cost" in op:
|
|
||||||
cost = op["cost"]
|
|
||||||
prompt_tokens = op["prompt_tokens"]
|
|
||||||
completion_tokens = op["completion_tokens"]
|
|
||||||
scores[method].append(
|
|
||||||
[result["key"], score, solved, prompt_tokens, completion_tokens, cost]
|
|
||||||
)
|
|
||||||
scores[method] = sorted(scores[method], key=lambda x: x[0])
|
|
||||||
return scores
|
|
||||||
|
|
||||||
|
|
||||||
def get_plotting_data(base_directory):
|
|
||||||
results_complete = get_complete_results(base_directory)
|
|
||||||
scores = get_final_scores(results_complete)
|
|
||||||
results_plotting = {
|
|
||||||
method: {
|
|
||||||
"scores": [x[1] for x in scores[method]],
|
|
||||||
"solved": sum([1 for x in scores[method] if x[2]]),
|
|
||||||
"costs": [x[5] for x in scores[method]],
|
|
||||||
}
|
|
||||||
for method in scores.keys()
|
|
||||||
}
|
|
||||||
return results_plotting
|
|
||||||
|
|
||||||
|
|
||||||
def plot_results(
|
|
||||||
results,
|
|
||||||
methods_order=["io", "cot", "tot", "tot2", "got"],
|
|
||||||
model="GPT-3.5",
|
|
||||||
length=32,
|
|
||||||
y_lower=0,
|
|
||||||
cost_upper=0.0,
|
|
||||||
display_solved=True,
|
|
||||||
annotation_offset=0,
|
|
||||||
display_left_ylabel=False,
|
|
||||||
display_right_ylabel=False,
|
|
||||||
):
|
|
||||||
methods_order = [method for method in methods_order if method in results]
|
|
||||||
# Extract scores based on the order
|
|
||||||
scores_ordered = [
|
|
||||||
[score for score in results[method]["scores"] if score != 1000]
|
|
||||||
for method in methods_order
|
|
||||||
]
|
|
||||||
total_costs = [sum(results[method]["costs"]) for method in methods_order]
|
|
||||||
|
|
||||||
# Create figure and axis
|
|
||||||
fig, ax = plt.subplots(dpi=150, figsize=(2.5, 5))
|
|
||||||
|
|
||||||
# Create boxplots
|
|
||||||
positions = range(1, len(methods_order) + 1)
|
|
||||||
ax.boxplot(scores_ordered, positions=positions)
|
|
||||||
|
|
||||||
fig_fontsize = 12
|
|
||||||
|
|
||||||
# Set the ticks and labels
|
|
||||||
methods_labels = ["IO", "CoT", "ToT", "ToT2", "GoT"]
|
|
||||||
plt.yticks(fontsize=fig_fontsize)
|
|
||||||
ax.set_xticks(range(1, len(methods_order) + 1))
|
|
||||||
ax.set_xticks(range(1, len(methods_order) + 1))
|
|
||||||
ax.set_xticklabels(methods_labels, fontsize=fig_fontsize)
|
|
||||||
|
|
||||||
y_upper = length
|
|
||||||
|
|
||||||
range_increase = 1
|
|
||||||
if display_solved:
|
|
||||||
if length < 48:
|
|
||||||
range_increase = 2
|
|
||||||
elif length < 96:
|
|
||||||
range_increase = 4
|
|
||||||
else:
|
|
||||||
range_increase = 8
|
|
||||||
|
|
||||||
ax.set_ylim(y_lower, y_upper + range_increase)
|
|
||||||
ax1_yticks = range(
|
|
||||||
y_lower, y_upper + 1, 2 if length < 48 else (4 if length < 96 else 8)
|
|
||||||
)
|
|
||||||
ax.set_yticks(ax1_yticks)
|
|
||||||
if display_left_ylabel:
|
|
||||||
ax.set_ylabel(
|
|
||||||
f"#incorrect elements; the lower the better", fontsize=fig_fontsize
|
|
||||||
)
|
|
||||||
|
|
||||||
ax.set_title(f"{length} elements")
|
|
||||||
|
|
||||||
ax2 = ax.twinx()
|
|
||||||
ax2.bar(positions, total_costs, alpha=0.5, color="blue", label="Total Cost ($)")
|
|
||||||
ax2.yaxis.set_tick_params(colors="#1919ff", labelsize=fig_fontsize)
|
|
||||||
if cost_upper > 0:
|
|
||||||
ax2.set_ylim(0, cost_upper)
|
|
||||||
number_of_ticks = len(ax.get_yticks())
|
|
||||||
tick_interval = cost_upper / (number_of_ticks)
|
|
||||||
ax2_ticks = [tick_interval * i for i in range(number_of_ticks)]
|
|
||||||
|
|
||||||
# Set custom tick positions for ax2
|
|
||||||
ax2.set_yticks(ax2_ticks)
|
|
||||||
|
|
||||||
if display_right_ylabel:
|
|
||||||
ax2.set_ylabel(
|
|
||||||
"Total Cost ($); the lower the better",
|
|
||||||
color="#1919ff",
|
|
||||||
fontsize=fig_fontsize,
|
|
||||||
)
|
|
||||||
|
|
||||||
if display_solved:
|
|
||||||
annotation_height = y_upper + annotation_offset
|
|
||||||
count = 1
|
|
||||||
for method in methods_order:
|
|
||||||
if method not in results:
|
|
||||||
continue
|
|
||||||
solved = results[method]["solved"]
|
|
||||||
ax.text(
|
|
||||||
count,
|
|
||||||
annotation_height,
|
|
||||||
f"{solved}",
|
|
||||||
ha="center",
|
|
||||||
va="bottom",
|
|
||||||
fontsize=fig_fontsize,
|
|
||||||
)
|
|
||||||
count += 1
|
|
||||||
|
|
||||||
model = model.replace(".", "").replace("-", "").lower()
|
|
||||||
fig.savefig(f"set_intersection_{model}_{length}.pdf", bbox_inches="tight")
|
|
||||||
|
|
||||||
|
|
||||||
plot_results(
|
|
||||||
get_plotting_data("results/"),
|
|
||||||
length=32,
|
|
||||||
display_solved=True,
|
|
||||||
model="GPT-3.5",
|
|
||||||
display_left_ylabel=True,
|
|
||||||
display_right_ylabel=True,
|
|
||||||
)
|
|
||||||
@ -1,101 +0,0 @@
|
|||||||
ID,SET1,SET2,INTERSECTION
|
|
||||||
0,"[11, 60, 1, 49, 21, 33, 14, 56, 54, 15, 23, 40, 45, 22, 7, 28, 20, 46, 51, 6, 34, 37, 3, 50, 17, 8, 25, 0, 35, 47, 18, 19]","[31, 11, 4, 63, 38, 58, 59, 24, 61, 14, 32, 39, 27, 46, 48, 19, 52, 57, 50, 56, 3, 2, 53, 29, 5, 37, 62, 41, 36, 12, 49, 16]","[3, 11, 14, 19, 37, 46, 49, 50, 56]"
|
|
||||||
1,"[28, 58, 36, 18, 37, 31, 44, 39, 34, 51, 12, 56, 21, 27, 7, 24, 46, 1, 25, 2, 41, 6, 45, 29, 49, 42, 35, 30, 54, 55, 10, 50]","[4, 16, 28, 46, 49, 21, 58, 30, 19, 1, 37, 15, 3, 59, 51, 24, 10, 12, 34, 20, 40, 44, 35, 23, 36, 0, 43, 54, 2, 31, 57, 41]","[1, 2, 10, 12, 21, 24, 28, 30, 31, 34, 35, 36, 37, 41, 44, 46, 49, 51, 54, 58]"
|
|
||||||
2,"[32, 24, 17, 27, 51, 19, 50, 59, 8, 18, 41, 36, 43, 39, 49, 4, 5, 31, 30, 60, 48, 61, 10, 16, 47, 58, 9, 54, 40, 37, 1, 11]","[44, 27, 33, 46, 51, 17, 28, 18, 4, 23, 32, 49, 58, 1, 20, 40, 39, 52, 30, 55, 26, 19, 63, 31, 57, 13, 47, 61, 10, 6, 38, 41]","[1, 4, 10, 17, 18, 19, 27, 30, 31, 32, 39, 40, 41, 47, 49, 51, 58, 61]"
|
|
||||||
3,"[22, 38, 13, 31, 63, 6, 59, 17, 27, 20, 21, 42, 24, 4, 25, 15, 28, 29, 19, 49, 40, 30, 34, 0, 53, 23, 12, 62, 36, 51, 39, 45]","[21, 29, 30, 28, 31, 55, 2, 22, 19, 4, 46, 32, 37, 43, 23, 15, 58, 56, 11, 50, 48, 54, 41, 45, 51, 7, 42, 38, 18, 9, 6, 1]","[4, 6, 15, 19, 21, 22, 23, 28, 29, 30, 31, 38, 42, 45, 51]"
|
|
||||||
4,"[4, 26, 48, 7, 42, 60, 31, 36, 25, 20, 14, 50, 21, 30, 12, 10, 23, 8, 39, 13, 46, 32, 11, 34, 6, 2, 57, 0, 17, 33, 18, 24]","[1, 37, 48, 28, 39, 58, 53, 10, 6, 50, 17, 9, 51, 56, 38, 52, 8, 30, 46, 62, 5, 24, 61, 22, 57, 14, 44, 11, 47, 35, 63, 36]","[6, 8, 10, 11, 14, 17, 24, 30, 36, 39, 46, 48, 50, 57]"
|
|
||||||
5,"[48, 50, 19, 59, 57, 63, 49, 61, 14, 11, 29, 39, 4, 17, 21, 12, 3, 35, 23, 2, 58, 60, 38, 43, 28, 47, 5, 44, 15, 55, 8, 26]","[50, 12, 30, 23, 11, 61, 17, 58, 19, 55, 62, 43, 33, 2, 37, 21, 56, 4, 15, 24, 38, 45, 1, 28, 59, 48, 35, 6, 47, 8, 27, 22]","[2, 4, 8, 11, 12, 15, 17, 19, 21, 23, 28, 35, 38, 43, 47, 48, 50, 55, 58, 59, 61]"
|
|
||||||
6,"[34, 2, 12, 54, 13, 41, 57, 42, 21, 59, 26, 10, 6, 35, 27, 16, 1, 44, 8, 11, 14, 7, 62, 20, 30, 19, 32, 9, 43, 39, 22, 24]","[18, 41, 49, 56, 1, 3, 45, 36, 17, 34, 50, 46, 29, 30, 61, 32, 14, 52, 24, 51, 5, 25, 31, 55, 37, 0, 23, 40, 28, 20, 22, 63]","[1, 14, 20, 22, 24, 30, 32, 34, 41]"
|
|
||||||
7,"[49, 61, 9, 23, 27, 21, 47, 37, 54, 32, 18, 14, 22, 13, 62, 46, 0, 26, 4, 11, 60, 34, 25, 50, 6, 51, 59, 8, 35, 41, 29, 17]","[33, 21, 49, 54, 6, 27, 3, 47, 4, 20, 43, 41, 8, 24, 0, 57, 28, 35, 15, 17, 46, 23, 13, 9, 45, 31, 22, 12, 5, 29, 50, 7]","[0, 4, 6, 8, 9, 13, 17, 21, 22, 23, 27, 29, 35, 41, 46, 47, 49, 50, 54]"
|
|
||||||
8,"[55, 59, 60, 13, 41, 15, 38, 52, 35, 5, 9, 61, 57, 34, 47, 22, 21, 1, 27, 11, 36, 2, 6, 58, 62, 51, 23, 50, 8, 39, 28, 54]","[37, 14, 4, 55, 50, 17, 8, 19, 10, 44, 0, 31, 27, 53, 39, 35, 30, 5, 62, 24, 26, 29, 54, 45, 40, 3, 43, 11, 12, 25, 41, 42]","[5, 8, 11, 27, 35, 39, 41, 50, 54, 55, 62]"
|
|
||||||
9,"[20, 12, 52, 56, 36, 51, 59, 40, 55, 33, 41, 38, 28, 45, 13, 2, 43, 27, 35, 7, 9, 0, 10, 37, 29, 15, 14, 49, 17, 63, 25, 32]","[11, 13, 19, 6, 21, 54, 36, 14, 58, 26, 2, 39, 53, 22, 50, 51, 24, 16, 5, 17, 1, 60, 57, 23, 35, 33, 42, 18, 61, 29, 47, 8]","[2, 13, 14, 17, 29, 33, 35, 36, 51]"
|
|
||||||
10,"[24, 21, 51, 37, 54, 22, 18, 38, 14, 30, 48, 49, 28, 47, 10, 44, 58, 29, 3, 45, 26, 15, 9, 6, 33, 2, 34, 17, 20, 25, 0, 1]","[3, 0, 20, 1, 24, 45, 25, 52, 13, 43, 38, 15, 22, 53, 27, 8, 42, 40, 59, 32, 21, 60, 30, 46, 51, 47, 14, 50, 5, 62, 57, 28]","[0, 1, 3, 14, 15, 20, 21, 22, 24, 25, 28, 30, 38, 45, 47, 51]"
|
|
||||||
11,"[58, 52, 51, 15, 42, 26, 19, 30, 28, 6, 40, 11, 25, 53, 18, 47, 21, 12, 41, 45, 43, 7, 35, 48, 27, 3, 32, 2, 4, 62, 17, 29]","[41, 4, 0, 35, 25, 29, 14, 62, 45, 2, 37, 11, 15, 8, 30, 39, 58, 19, 56, 42, 17, 51, 24, 28, 44, 27, 13, 18, 26, 47, 53, 43]","[2, 4, 11, 15, 17, 18, 19, 25, 26, 27, 28, 29, 30, 35, 41, 42, 43, 45, 47, 51, 53, 58, 62]"
|
|
||||||
12,"[8, 17, 23, 38, 24, 47, 61, 3, 46, 43, 58, 39, 27, 21, 26, 14, 62, 1, 15, 19, 37, 53, 55, 6, 51, 57, 52, 25, 16, 22, 33, 54]","[2, 18, 15, 17, 23, 45, 14, 36, 53, 7, 27, 32, 31, 21, 8, 38, 16, 61, 6, 58, 3, 39, 49, 19, 22, 42, 0, 12, 30, 60, 26, 46]","[3, 6, 8, 14, 15, 16, 17, 19, 21, 22, 23, 26, 27, 38, 39, 46, 53, 58, 61]"
|
|
||||||
13,"[58, 10, 28, 52, 51, 48, 49, 13, 53, 25, 55, 6, 4, 15, 30, 56, 54, 41, 38, 42, 21, 33, 44, 18, 37, 43, 31, 20, 3, 2, 50, 0]","[54, 3, 32, 44, 26, 49, 0, 9, 35, 51, 25, 33, 62, 55, 47, 2, 29, 11, 42, 18, 4, 23, 31, 41, 20, 7, 58, 52, 13, 36, 5, 53]","[0, 2, 3, 4, 13, 18, 20, 25, 31, 33, 41, 42, 44, 49, 51, 52, 53, 54, 55, 58]"
|
|
||||||
14,"[12, 18, 22, 38, 6, 23, 54, 37, 40, 44, 19, 46, 35, 47, 7, 10, 16, 24, 4, 42, 31, 30, 0, 63, 25, 57, 39, 51, 3, 60, 49, 28]","[40, 6, 12, 18, 19, 34, 17, 47, 4, 29, 53, 3, 15, 7, 37, 1, 59, 42, 33, 35, 52, 16, 61, 13, 9, 51, 32, 60, 24, 54, 38, 46]","[3, 4, 6, 7, 12, 16, 18, 19, 24, 35, 37, 38, 40, 42, 46, 47, 51, 54, 60]"
|
|
||||||
15,"[61, 62, 10, 53, 17, 20, 28, 54, 55, 59, 4, 1, 11, 34, 5, 35, 56, 38, 60, 29, 43, 42, 63, 50, 24, 40, 47, 52, 16, 39, 31, 19]","[60, 24, 50, 45, 23, 18, 20, 31, 29, 35, 43, 44, 59, 41, 51, 46, 16, 63, 38, 3, 42, 25, 4, 17, 5, 9, 61, 7, 27, 54, 19, 39]","[4, 5, 16, 17, 19, 20, 24, 29, 31, 35, 38, 39, 42, 43, 50, 54, 59, 60, 61, 63]"
|
|
||||||
16,"[12, 2, 21, 59, 49, 30, 31, 14, 23, 39, 5, 15, 4, 19, 27, 43, 6, 34, 0, 22, 16, 60, 17, 48, 40, 9, 56, 1, 63, 8, 45, 51]","[40, 4, 38, 7, 1, 47, 44, 16, 58, 11, 25, 35, 3, 51, 26, 10, 41, 60, 46, 6, 34, 19, 2, 61, 21, 31, 8, 22, 12, 53, 15, 50]","[1, 2, 4, 6, 8, 12, 15, 16, 19, 21, 22, 31, 34, 40, 51, 60]"
|
|
||||||
17,"[26, 8, 53, 37, 45, 60, 32, 44, 19, 39, 36, 16, 25, 9, 6, 50, 38, 47, 30, 63, 42, 52, 57, 10, 7, 27, 14, 1, 41, 58, 59, 62]","[28, 12, 29, 21, 63, 34, 17, 49, 56, 4, 15, 39, 1, 46, 22, 62, 9, 43, 0, 50, 23, 54, 44, 31, 5, 51, 11, 37, 8, 61, 13, 42]","[1, 8, 9, 37, 39, 42, 44, 50, 62, 63]"
|
|
||||||
18,"[9, 58, 28, 53, 3, 18, 45, 33, 11, 5, 2, 47, 4, 59, 38, 30, 61, 15, 21, 48, 41, 25, 26, 13, 60, 23, 6, 55, 17, 36, 62, 10]","[43, 15, 16, 10, 33, 5, 13, 62, 41, 53, 0, 32, 54, 23, 26, 19, 25, 9, 37, 49, 38, 21, 28, 27, 40, 45, 4, 61, 29, 3, 55, 18]","[3, 4, 5, 9, 10, 13, 15, 18, 21, 23, 25, 26, 28, 33, 38, 41, 45, 53, 55, 61, 62]"
|
|
||||||
19,"[26, 63, 42, 19, 6, 62, 39, 14, 24, 17, 44, 31, 8, 7, 10, 16, 1, 56, 45, 43, 50, 57, 54, 51, 58, 59, 5, 37, 3, 21, 13, 30]","[60, 3, 1, 34, 17, 55, 50, 21, 0, 46, 62, 59, 47, 36, 15, 22, 19, 5, 2, 38, 58, 12, 32, 27, 28, 18, 24, 31, 26, 63, 51, 53]","[1, 3, 5, 17, 19, 21, 24, 26, 31, 50, 51, 58, 59, 62, 63]"
|
|
||||||
20,"[30, 31, 28, 56, 54, 29, 61, 18, 1, 46, 34, 5, 15, 63, 26, 45, 4, 47, 40, 21, 3, 9, 2, 27, 7, 6, 24, 19, 42, 11, 35, 41]","[57, 5, 54, 42, 41, 31, 12, 13, 9, 34, 21, 23, 7, 48, 22, 29, 19, 14, 0, 43, 3, 27, 26, 4, 16, 25, 53, 51, 10, 11, 52, 20]","[3, 4, 5, 7, 9, 11, 19, 21, 26, 27, 29, 31, 34, 41, 42, 54]"
|
|
||||||
21,"[9, 12, 51, 48, 23, 6, 50, 14, 53, 15, 58, 44, 32, 59, 24, 42, 27, 41, 29, 10, 45, 34, 0, 46, 3, 36, 25, 16, 13, 63, 49, 52]","[16, 15, 51, 37, 20, 27, 47, 36, 45, 59, 0, 41, 40, 2, 28, 31, 17, 1, 21, 22, 5, 58, 18, 57, 34, 39, 54, 9, 53, 33, 60, 19]","[0, 9, 15, 16, 27, 34, 36, 41, 45, 51, 53, 58, 59]"
|
|
||||||
22,"[57, 44, 59, 3, 16, 5, 17, 6, 52, 0, 18, 40, 47, 11, 45, 14, 1, 49, 30, 55, 27, 12, 50, 10, 7, 39, 46, 22, 23, 38, 34, 41]","[44, 48, 9, 58, 28, 40, 4, 56, 49, 19, 53, 42, 47, 35, 25, 34, 29, 43, 33, 32, 6, 55, 37, 26, 20, 11, 2, 27, 31, 62, 54, 39]","[6, 11, 27, 34, 39, 40, 44, 47, 49, 55]"
|
|
||||||
23,"[32, 25, 2, 13, 11, 23, 54, 49, 9, 57, 59, 6, 61, 47, 37, 34, 18, 29, 46, 45, 56, 31, 14, 17, 48, 55, 7, 50, 27, 51, 63, 44]","[49, 34, 46, 32, 17, 50, 51, 54, 61, 14, 52, 21, 56, 16, 44, 33, 25, 57, 2, 28, 35, 19, 55, 11, 22, 9, 48, 20, 53, 29, 45, 18]","[2, 9, 11, 14, 17, 18, 25, 29, 32, 34, 44, 45, 46, 48, 49, 50, 51, 54, 55, 56, 57, 61]"
|
|
||||||
24,"[23, 16, 41, 14, 10, 42, 9, 27, 35, 51, 24, 4, 59, 47, 55, 12, 18, 58, 7, 63, 28, 32, 22, 49, 37, 31, 13, 1, 21, 39, 36, 30]","[39, 6, 63, 37, 27, 19, 40, 16, 18, 54, 22, 20, 2, 30, 23, 59, 46, 52, 43, 10, 21, 41, 31, 4, 12, 24, 32, 38, 3, 13, 47, 53]","[4, 10, 12, 13, 16, 18, 21, 22, 23, 24, 27, 30, 31, 32, 37, 39, 41, 47, 59, 63]"
|
|
||||||
25,"[20, 5, 41, 40, 24, 60, 2, 6, 59, 50, 34, 47, 10, 0, 14, 29, 35, 28, 39, 58, 13, 16, 27, 4, 18, 52, 56, 21, 45, 15, 32, 30]","[2, 23, 40, 31, 7, 52, 55, 18, 59, 56, 60, 44, 38, 54, 35, 4, 46, 1, 12, 47, 10, 3, 13, 29, 48, 43, 6, 11, 20, 24, 45, 34]","[2, 4, 6, 10, 13, 18, 20, 24, 29, 34, 35, 40, 45, 47, 52, 56, 59, 60]"
|
|
||||||
26,"[60, 39, 4, 62, 16, 57, 49, 41, 25, 59, 55, 35, 53, 33, 17, 5, 18, 13, 46, 30, 45, 38, 1, 20, 22, 44, 11, 15, 42, 7, 21, 28]","[57, 27, 24, 15, 44, 13, 60, 35, 62, 47, 38, 59, 42, 52, 49, 43, 10, 0, 53, 6, 37, 18, 19, 36, 51, 58, 3, 34, 23, 21, 63, 40]","[13, 15, 18, 21, 35, 38, 42, 44, 49, 53, 57, 59, 60, 62]"
|
|
||||||
27,"[9, 48, 28, 3, 61, 4, 1, 22, 62, 57, 18, 21, 34, 26, 25, 8, 12, 38, 27, 49, 46, 6, 40, 56, 10, 16, 36, 5, 45, 44, 51, 42]","[10, 36, 20, 5, 45, 34, 16, 18, 21, 51, 60, 17, 57, 50, 42, 54, 46, 63, 39, 38, 58, 3, 24, 53, 37, 26, 27, 48, 22, 44, 49, 8]","[3, 5, 8, 10, 16, 18, 21, 22, 26, 27, 34, 36, 38, 42, 44, 45, 46, 48, 49, 51, 57]"
|
|
||||||
28,"[17, 42, 40, 38, 58, 19, 63, 22, 50, 18, 8, 16, 12, 15, 61, 10, 52, 9, 44, 13, 60, 24, 41, 56, 35, 46, 21, 36, 28, 0, 48, 6]","[6, 33, 10, 5, 23, 41, 53, 30, 58, 22, 45, 42, 61, 7, 12, 19, 56, 60, 2, 28, 62, 36, 39, 43, 27, 20, 16, 55, 37, 40, 9, 32]","[6, 9, 10, 12, 16, 19, 22, 28, 36, 40, 41, 42, 56, 58, 60, 61]"
|
|
||||||
29,"[1, 4, 35, 57, 2, 11, 43, 56, 16, 9, 51, 8, 26, 0, 59, 31, 22, 25, 29, 50, 13, 44, 12, 32, 30, 27, 19, 36, 42, 61, 7, 38]","[59, 35, 54, 49, 7, 45, 33, 57, 32, 41, 18, 47, 55, 56, 58, 5, 4, 46, 21, 9, 63, 52, 34, 2, 62, 8, 48, 22, 51, 11, 1, 16]","[1, 2, 4, 7, 8, 9, 11, 16, 22, 32, 35, 51, 56, 57, 59]"
|
|
||||||
30,"[54, 2, 52, 36, 21, 37, 13, 28, 39, 17, 27, 0, 42, 56, 10, 4, 60, 48, 49, 24, 20, 16, 5, 62, 14, 47, 53, 1, 7, 32, 34, 12]","[38, 41, 13, 32, 31, 63, 11, 62, 1, 18, 52, 58, 56, 51, 7, 37, 36, 22, 9, 35, 43, 0, 46, 4, 55, 44, 25, 8, 24, 14, 34, 47]","[0, 1, 4, 7, 13, 14, 24, 32, 34, 36, 37, 47, 52, 56, 62]"
|
|
||||||
31,"[6, 32, 2, 58, 25, 19, 31, 0, 30, 28, 46, 49, 41, 48, 45, 44, 59, 51, 22, 3, 38, 36, 1, 15, 11, 26, 37, 24, 55, 9, 14, 61]","[10, 56, 39, 4, 34, 11, 29, 14, 57, 15, 60, 20, 9, 43, 53, 40, 8, 44, 50, 27, 47, 19, 36, 52, 6, 62, 45, 63, 12, 21, 51, 58]","[6, 9, 11, 14, 15, 19, 36, 44, 45, 51, 58]"
|
|
||||||
32,"[24, 51, 50, 19, 63, 42, 29, 59, 41, 9, 38, 28, 25, 4, 18, 40, 12, 1, 58, 34, 15, 30, 54, 6, 46, 27, 39, 55, 23, 61, 5, 36]","[56, 17, 24, 11, 35, 20, 15, 10, 3, 45, 26, 33, 21, 60, 8, 32, 22, 48, 25, 49, 41, 14, 13, 18, 19, 61, 2, 43, 52, 16, 5, 27]","[5, 15, 18, 19, 24, 25, 27, 41, 61]"
|
|
||||||
33,"[27, 37, 6, 7, 2, 32, 28, 11, 24, 43, 5, 15, 12, 18, 36, 26, 55, 35, 57, 56, 29, 54, 30, 9, 3, 20, 38, 48, 63, 25, 16, 10]","[21, 34, 3, 20, 2, 13, 56, 32, 25, 58, 61, 51, 59, 53, 6, 27, 8, 48, 50, 43, 46, 52, 63, 11, 42, 12, 19, 45, 9, 39, 55, 54]","[2, 3, 6, 9, 11, 12, 20, 25, 27, 32, 43, 48, 54, 55, 56, 63]"
|
|
||||||
34,"[22, 52, 23, 8, 46, 5, 19, 37, 31, 51, 6, 17, 21, 13, 29, 4, 58, 0, 32, 40, 9, 33, 55, 34, 11, 30, 24, 20, 3, 39, 2, 25]","[51, 20, 56, 37, 9, 19, 38, 53, 43, 35, 39, 23, 46, 8, 11, 3, 28, 49, 13, 22, 52, 2, 42, 26, 21, 30, 58, 24, 59, 40, 5, 6]","[2, 3, 5, 6, 8, 9, 11, 13, 19, 20, 21, 22, 23, 24, 30, 37, 39, 40, 46, 51, 52, 58]"
|
|
||||||
35,"[9, 37, 45, 60, 18, 38, 55, 14, 44, 49, 27, 19, 6, 2, 51, 26, 11, 24, 54, 32, 48, 57, 34, 3, 42, 5, 39, 31, 53, 62, 43, 41]","[61, 28, 30, 25, 33, 43, 46, 56, 58, 62, 21, 17, 16, 50, 29, 41, 13, 38, 54, 63, 40, 18, 35, 10, 36, 20, 7, 9, 8, 45, 19, 52]","[9, 18, 19, 38, 41, 43, 45, 54, 62]"
|
|
||||||
36,"[45, 26, 14, 2, 25, 41, 59, 58, 42, 39, 56, 46, 1, 48, 30, 31, 63, 36, 33, 5, 60, 19, 21, 57, 62, 35, 17, 37, 6, 3, 38, 13]","[43, 32, 60, 56, 36, 22, 59, 6, 39, 63, 46, 8, 26, 38, 57, 31, 29, 25, 45, 18, 55, 52, 13, 10, 62, 35, 30, 7, 47, 14, 37, 21]","[6, 13, 14, 21, 25, 26, 30, 31, 35, 36, 37, 38, 39, 45, 46, 56, 57, 59, 60, 62, 63]"
|
|
||||||
37,"[47, 52, 1, 22, 16, 56, 5, 30, 43, 54, 3, 48, 21, 17, 33, 9, 57, 11, 4, 46, 15, 26, 12, 7, 32, 44, 6, 41, 31, 14, 49, 59]","[42, 44, 49, 59, 58, 61, 57, 6, 46, 3, 7, 31, 43, 1, 34, 22, 12, 16, 4, 54, 55, 45, 27, 5, 26, 48, 13, 17, 19, 51, 39, 47]","[1, 3, 4, 5, 6, 7, 12, 16, 17, 22, 26, 31, 43, 44, 46, 47, 48, 49, 54, 57, 59]"
|
|
||||||
38,"[28, 27, 51, 11, 2, 31, 14, 6, 33, 25, 42, 49, 59, 60, 17, 10, 56, 12, 23, 36, 37, 53, 18, 52, 26, 21, 62, 63, 38, 22, 16, 44]","[52, 53, 6, 9, 40, 47, 14, 3, 34, 29, 18, 48, 7, 58, 20, 50, 51, 22, 5, 46, 28, 37, 60, 61, 55, 27, 1, 4, 49, 45, 39, 24]","[6, 14, 18, 22, 27, 28, 37, 49, 51, 52, 53, 60]"
|
|
||||||
39,"[41, 11, 6, 43, 60, 48, 37, 55, 31, 39, 7, 13, 58, 42, 62, 44, 21, 56, 35, 19, 49, 9, 36, 34, 14, 10, 8, 54, 30, 32, 52, 2]","[0, 36, 42, 58, 13, 38, 31, 45, 11, 61, 18, 28, 32, 60, 17, 57, 10, 2, 3, 59, 51, 35, 12, 9, 7, 50, 63, 41, 48, 34, 39, 49]","[2, 7, 9, 10, 11, 13, 31, 32, 34, 35, 36, 39, 41, 42, 48, 49, 58, 60]"
|
|
||||||
40,"[27, 26, 20, 51, 10, 56, 49, 22, 31, 61, 44, 24, 32, 17, 50, 29, 30, 19, 3, 4, 12, 0, 43, 41, 35, 58, 33, 15, 37, 34, 57, 25]","[54, 29, 52, 49, 45, 30, 48, 7, 11, 19, 47, 46, 10, 38, 16, 36, 40, 5, 41, 1, 23, 8, 42, 33, 56, 14, 59, 2, 58, 0, 63, 62]","[0, 10, 19, 29, 30, 33, 41, 49, 56, 58]"
|
|
||||||
41,"[44, 7, 43, 52, 38, 18, 47, 2, 49, 41, 36, 42, 57, 1, 23, 27, 39, 50, 31, 51, 16, 29, 30, 26, 40, 24, 10, 20, 6, 11, 62, 19]","[47, 21, 60, 51, 62, 55, 28, 48, 16, 2, 52, 36, 10, 18, 63, 3, 45, 32, 50, 43, 19, 44, 27, 56, 26, 1, 8, 29, 25, 41, 11, 38]","[1, 2, 10, 11, 16, 18, 19, 26, 27, 29, 36, 38, 41, 43, 44, 47, 50, 51, 52, 62]"
|
|
||||||
42,"[11, 29, 26, 5, 13, 32, 53, 10, 25, 60, 52, 39, 35, 55, 23, 17, 49, 47, 28, 20, 4, 50, 2, 31, 19, 48, 34, 6, 0, 54, 16, 37]","[59, 30, 29, 23, 49, 57, 25, 16, 37, 27, 1, 17, 20, 42, 55, 47, 41, 7, 33, 6, 52, 48, 34, 51, 13, 3, 56, 11, 26, 50, 54, 46]","[6, 11, 13, 16, 17, 20, 23, 25, 26, 29, 34, 37, 47, 48, 49, 50, 52, 54, 55]"
|
|
||||||
43,"[2, 21, 7, 46, 49, 10, 25, 20, 32, 23, 55, 26, 4, 34, 15, 16, 28, 50, 30, 42, 37, 45, 44, 1, 39, 5, 6, 13, 24, 11, 38, 14]","[29, 45, 42, 8, 21, 9, 1, 5, 15, 61, 62, 51, 54, 41, 38, 31, 7, 56, 43, 13, 36, 0, 60, 10, 59, 48, 25, 3, 34, 19, 23, 27]","[1, 5, 7, 10, 13, 15, 21, 23, 25, 34, 38, 42, 45]"
|
|
||||||
44,"[53, 2, 10, 19, 21, 8, 35, 56, 49, 5, 45, 42, 39, 16, 38, 32, 20, 9, 25, 41, 15, 29, 61, 7, 1, 11, 6, 47, 57, 36, 40, 28]","[23, 59, 3, 54, 22, 50, 47, 18, 37, 24, 27, 26, 48, 6, 34, 45, 16, 28, 55, 60, 33, 30, 52, 19, 7, 17, 13, 58, 35, 46, 62, 2]","[2, 6, 7, 16, 19, 28, 35, 45, 47]"
|
|
||||||
45,"[46, 13, 6, 58, 35, 57, 18, 2, 53, 30, 31, 28, 60, 5, 21, 56, 52, 48, 55, 39, 59, 12, 23, 45, 36, 22, 50, 0, 20, 4, 29, 32]","[45, 2, 20, 60, 13, 35, 22, 57, 24, 55, 5, 58, 51, 32, 56, 4, 0, 8, 7, 54, 41, 36, 48, 53, 23, 34, 38, 39, 28, 29, 61, 30]","[0, 2, 4, 5, 13, 20, 22, 23, 28, 29, 30, 32, 35, 36, 39, 45, 48, 53, 55, 56, 57, 58, 60]"
|
|
||||||
46,"[49, 54, 43, 3, 33, 63, 61, 57, 53, 42, 58, 48, 11, 51, 50, 55, 39, 1, 9, 28, 30, 6, 21, 20, 7, 47, 32, 19, 15, 5, 36, 40]","[44, 18, 40, 59, 58, 54, 0, 29, 38, 30, 48, 24, 55, 17, 52, 43, 11, 60, 31, 53, 20, 45, 41, 63, 3, 36, 23, 56, 35, 19, 14, 39]","[3, 11, 19, 20, 30, 36, 39, 40, 43, 48, 53, 54, 55, 58, 63]"
|
|
||||||
47,"[20, 13, 57, 11, 8, 58, 9, 4, 43, 10, 12, 61, 48, 40, 22, 32, 50, 2, 25, 41, 17, 21, 33, 5, 1, 37, 34, 19, 31, 47, 15, 3]","[58, 10, 51, 3, 1, 39, 41, 38, 46, 8, 42, 12, 20, 32, 43, 5, 2, 50, 25, 17, 18, 55, 27, 40, 16, 19, 30, 57, 31, 47, 48, 37]","[1, 2, 3, 5, 8, 10, 12, 17, 19, 20, 25, 31, 32, 37, 40, 41, 43, 47, 48, 50, 57, 58]"
|
|
||||||
48,"[39, 3, 45, 11, 22, 60, 1, 19, 25, 55, 18, 40, 35, 4, 58, 28, 8, 36, 41, 44, 30, 61, 50, 9, 2, 5, 15, 48, 12, 32, 31, 56]","[26, 19, 24, 8, 39, 44, 22, 30, 18, 31, 58, 11, 12, 63, 57, 17, 49, 45, 55, 23, 2, 42, 59, 60, 47, 0, 5, 21, 1, 29, 41, 27]","[1, 2, 5, 8, 11, 12, 18, 19, 22, 30, 31, 39, 41, 44, 45, 55, 58, 60]"
|
|
||||||
49,"[39, 40, 9, 0, 34, 57, 30, 43, 7, 35, 6, 24, 41, 63, 44, 36, 8, 13, 45, 15, 28, 55, 61, 42, 11, 48, 12, 38, 5, 27, 50, 59]","[9, 40, 0, 15, 52, 31, 45, 7, 57, 55, 28, 44, 26, 29, 11, 56, 38, 17, 3, 63, 42, 43, 5, 60, 30, 10, 58, 47, 41, 50, 54, 48]","[0, 5, 7, 9, 11, 15, 28, 30, 38, 40, 41, 42, 43, 44, 45, 48, 50, 55, 57, 63]"
|
|
||||||
50,"[27, 33, 14, 63, 0, 59, 54, 9, 62, 31, 21, 53, 39, 19, 10, 2, 22, 16, 29, 44, 17, 24, 60, 35, 12, 56, 4, 34, 36, 57, 30, 26]","[24, 36, 50, 3, 63, 58, 17, 22, 56, 0, 60, 7, 27, 55, 4, 11, 40, 31, 62, 14, 19, 35, 39, 33, 43, 21, 10, 6, 45, 59, 8, 18]","[0, 4, 10, 14, 17, 19, 21, 22, 24, 27, 31, 33, 35, 36, 39, 56, 59, 60, 62, 63]"
|
|
||||||
51,"[2, 27, 29, 42, 14, 34, 24, 58, 31, 4, 43, 62, 7, 1, 9, 5, 10, 53, 21, 49, 26, 23, 32, 3, 35, 51, 59, 55, 30, 39, 13, 16]","[57, 4, 11, 56, 37, 33, 63, 21, 30, 24, 55, 60, 48, 19, 46, 54, 44, 47, 7, 25, 31, 17, 18, 15, 45, 13, 10, 29, 0, 6, 61, 1]","[1, 4, 7, 10, 13, 21, 24, 29, 30, 31, 55]"
|
|
||||||
52,"[26, 63, 60, 20, 62, 17, 52, 35, 56, 12, 51, 10, 31, 42, 50, 18, 4, 39, 11, 44, 22, 25, 33, 5, 32, 13, 45, 0, 28, 23, 48, 59]","[36, 4, 35, 53, 22, 42, 55, 7, 18, 19, 13, 54, 52, 29, 27, 40, 25, 46, 15, 31, 63, 43, 57, 33, 38, 37, 1, 39, 9, 41, 45, 16]","[4, 13, 18, 22, 25, 31, 33, 35, 39, 42, 45, 52, 63]"
|
|
||||||
53,"[30, 10, 36, 20, 63, 2, 43, 62, 25, 33, 9, 4, 24, 60, 21, 57, 26, 40, 53, 18, 17, 45, 23, 50, 3, 48, 41, 12, 52, 47, 59, 0]","[15, 55, 6, 43, 41, 46, 32, 30, 48, 39, 9, 28, 58, 16, 8, 7, 33, 56, 20, 2, 21, 24, 34, 49, 1, 23, 17, 11, 45, 0, 44, 60]","[0, 2, 9, 17, 20, 21, 23, 24, 30, 33, 41, 43, 45, 48, 60]"
|
|
||||||
54,"[34, 24, 6, 4, 11, 52, 45, 54, 27, 49, 63, 16, 1, 28, 44, 33, 20, 22, 2, 57, 3, 40, 23, 30, 7, 41, 8, 62, 43, 58, 17, 46]","[61, 27, 43, 59, 55, 46, 10, 24, 12, 60, 1, 22, 23, 19, 52, 29, 40, 50, 39, 37, 26, 0, 63, 49, 56, 35, 6, 17, 41, 33, 9, 25]","[1, 6, 17, 22, 23, 24, 27, 33, 40, 41, 43, 46, 49, 52, 63]"
|
|
||||||
55,"[40, 62, 51, 34, 3, 26, 52, 29, 5, 44, 18, 23, 61, 10, 50, 24, 47, 43, 37, 7, 38, 11, 15, 59, 36, 49, 53, 42, 48, 45, 56, 58]","[4, 28, 22, 9, 35, 30, 5, 2, 54, 33, 26, 11, 13, 6, 25, 31, 20, 17, 34, 27, 36, 55, 48, 8, 32, 19, 56, 42, 46, 21, 57, 41]","[5, 11, 26, 34, 36, 42, 48, 56]"
|
|
||||||
56,"[49, 63, 30, 15, 54, 9, 6, 42, 8, 27, 3, 12, 48, 2, 36, 1, 26, 11, 43, 16, 61, 0, 62, 25, 53, 24, 51, 7, 19, 20, 14, 41]","[37, 55, 44, 8, 35, 43, 10, 36, 17, 32, 51, 7, 9, 59, 56, 16, 33, 19, 14, 58, 15, 6, 28, 53, 60, 50, 30, 41, 2, 23, 61, 34]","[2, 6, 7, 8, 9, 14, 15, 16, 19, 30, 36, 41, 43, 51, 53, 61]"
|
|
||||||
57,"[26, 10, 13, 42, 6, 50, 47, 34, 27, 36, 46, 3, 14, 61, 5, 44, 15, 55, 30, 41, 52, 28, 8, 2, 21, 38, 40, 11, 0, 62, 1, 16]","[19, 26, 14, 7, 13, 1, 15, 29, 4, 63, 22, 59, 47, 60, 51, 35, 23, 50, 58, 30, 57, 2, 49, 31, 62, 25, 24, 53, 20, 45, 54, 12]","[1, 2, 13, 14, 15, 26, 30, 47, 50, 62]"
|
|
||||||
58,"[43, 24, 41, 12, 33, 52, 34, 30, 39, 51, 1, 37, 49, 14, 50, 40, 23, 42, 17, 6, 31, 7, 3, 61, 21, 38, 63, 57, 20, 45, 9, 36]","[60, 34, 57, 3, 41, 49, 50, 23, 40, 14, 42, 58, 55, 19, 54, 22, 59, 21, 15, 20, 9, 28, 63, 17, 51, 27, 37, 13, 4, 2, 33, 1]","[1, 3, 9, 14, 17, 20, 21, 23, 33, 34, 37, 40, 41, 42, 49, 50, 51, 57, 63]"
|
|
||||||
59,"[5, 19, 20, 41, 52, 7, 49, 50, 28, 18, 1, 40, 33, 55, 53, 2, 22, 47, 27, 62, 29, 17, 58, 4, 43, 39, 14, 15, 45, 26, 10, 21]","[62, 61, 27, 26, 7, 25, 40, 4, 32, 14, 42, 50, 20, 39, 24, 3, 58, 2, 57, 5, 1, 17, 11, 9, 8, 38, 18, 46, 51, 29, 23, 33]","[1, 2, 4, 5, 7, 14, 17, 18, 20, 26, 27, 29, 33, 39, 40, 50, 58, 62]"
|
|
||||||
60,"[33, 32, 31, 53, 61, 4, 40, 55, 1, 43, 26, 25, 18, 17, 45, 46, 59, 21, 23, 35, 10, 27, 12, 50, 38, 42, 30, 2, 13, 29, 5, 14]","[13, 26, 30, 15, 50, 61, 2, 42, 52, 46, 32, 23, 17, 14, 21, 44, 40, 53, 8, 33, 59, 31, 28, 20, 4, 5, 55, 60, 56, 41, 24, 27]","[2, 4, 5, 13, 14, 17, 21, 23, 26, 27, 30, 31, 32, 33, 40, 42, 46, 50, 53, 55, 59, 61]"
|
|
||||||
61,"[44, 60, 25, 59, 56, 22, 42, 5, 12, 4, 31, 35, 50, 2, 43, 19, 62, 53, 24, 15, 47, 13, 37, 6, 29, 9, 34, 49, 1, 27, 55, 0]","[48, 18, 47, 60, 25, 34, 3, 13, 36, 15, 54, 35, 2, 57, 9, 40, 42, 4, 24, 43, 51, 61, 62, 26, 56, 0, 53, 11, 5, 28, 58, 19]","[0, 2, 4, 5, 9, 13, 15, 19, 24, 25, 34, 35, 42, 43, 47, 53, 56, 60, 62]"
|
|
||||||
62,"[16, 53, 6, 40, 31, 42, 45, 61, 37, 33, 27, 17, 57, 44, 2, 28, 15, 41, 8, 35, 5, 62, 1, 0, 22, 34, 21, 4, 18, 25, 36, 3]","[19, 28, 56, 32, 47, 62, 48, 43, 39, 3, 26, 38, 52, 31, 12, 33, 24, 46, 36, 51, 61, 27, 8, 57, 2, 63, 6, 9, 29, 54, 5, 13]","[2, 3, 5, 6, 8, 27, 28, 31, 33, 36, 57, 61, 62]"
|
|
||||||
63,"[46, 10, 18, 3, 15, 4, 23, 13, 31, 6, 30, 39, 16, 35, 56, 59, 28, 25, 27, 26, 50, 54, 34, 36, 12, 37, 33, 42, 40, 32, 0, 5]","[56, 41, 25, 26, 3, 4, 63, 30, 20, 59, 23, 35, 39, 62, 53, 47, 52, 27, 28, 6, 50, 18, 46, 55, 33, 13, 34, 32, 10, 29, 16, 0]","[0, 3, 4, 6, 10, 13, 16, 18, 23, 25, 26, 27, 28, 30, 32, 33, 34, 35, 39, 46, 50, 56, 59]"
|
|
||||||
64,"[47, 11, 4, 10, 52, 54, 43, 50, 18, 60, 34, 39, 36, 30, 37, 23, 41, 32, 5, 15, 2, 58, 40, 28, 27, 38, 12, 0, 20, 42, 26, 19]","[22, 18, 31, 62, 33, 15, 38, 4, 49, 17, 26, 50, 41, 27, 46, 35, 48, 36, 29, 9, 16, 24, 23, 56, 19, 5, 1, 12, 34, 61, 8, 55]","[4, 5, 12, 15, 18, 19, 23, 26, 27, 34, 36, 38, 41, 50]"
|
|
||||||
65,"[34, 4, 44, 39, 29, 25, 16, 35, 10, 51, 62, 40, 5, 20, 6, 32, 48, 42, 23, 22, 7, 33, 3, 19, 8, 27, 52, 21, 13, 53, 36, 12]","[38, 2, 46, 35, 7, 44, 1, 49, 45, 27, 57, 0, 19, 61, 59, 11, 16, 18, 51, 8, 60, 9, 12, 15, 23, 6, 40, 22, 14, 30, 50, 31]","[6, 7, 8, 12, 16, 19, 22, 23, 27, 35, 40, 44, 51]"
|
|
||||||
66,"[28, 39, 33, 2, 21, 53, 14, 12, 18, 7, 45, 43, 37, 60, 54, 42, 11, 38, 23, 9, 50, 26, 22, 62, 15, 5, 8, 0, 20, 13, 56, 30]","[51, 18, 47, 54, 4, 60, 48, 62, 61, 56, 50, 40, 14, 28, 39, 2, 25, 9, 0, 7, 13, 32, 33, 35, 30, 11, 8, 20, 23, 49, 12, 22]","[0, 2, 7, 8, 9, 11, 12, 13, 14, 18, 20, 22, 23, 28, 30, 33, 39, 50, 54, 56, 60, 62]"
|
|
||||||
67,"[56, 61, 32, 49, 34, 25, 16, 14, 0, 28, 10, 55, 46, 53, 27, 52, 39, 37, 63, 6, 43, 21, 19, 30, 29, 50, 41, 8, 31, 18, 33, 40]","[22, 51, 50, 8, 32, 45, 6, 38, 34, 3, 10, 43, 54, 37, 1, 13, 61, 55, 47, 5, 20, 23, 9, 11, 58, 59, 48, 27, 17, 42, 31, 25]","[6, 8, 10, 25, 27, 31, 32, 34, 37, 43, 50, 55, 61]"
|
|
||||||
68,"[32, 56, 63, 19, 46, 60, 23, 2, 62, 44, 53, 51, 49, 54, 33, 4, 31, 6, 47, 34, 61, 21, 12, 26, 55, 5, 27, 29, 15, 45, 52, 38]","[49, 8, 24, 36, 0, 3, 33, 51, 16, 11, 43, 40, 2, 29, 41, 38, 7, 39, 17, 55, 18, 62, 25, 20, 48, 14, 10, 13, 30, 22, 19, 50]","[2, 19, 29, 33, 38, 49, 51, 55, 62]"
|
|
||||||
69,"[10, 62, 7, 61, 52, 43, 6, 14, 42, 13, 23, 30, 11, 22, 3, 8, 21, 63, 1, 34, 35, 4, 19, 36, 27, 26, 33, 28, 29, 59, 60, 16]","[31, 63, 8, 6, 32, 52, 48, 30, 53, 49, 5, 26, 2, 14, 47, 43, 51, 18, 44, 33, 55, 4, 20, 17, 42, 7, 15, 10, 24, 3, 59, 39]","[3, 4, 6, 7, 8, 10, 14, 26, 30, 33, 42, 43, 52, 59, 63]"
|
|
||||||
70,"[9, 22, 57, 5, 0, 10, 56, 20, 8, 23, 4, 3, 58, 50, 38, 18, 1, 14, 17, 40, 21, 11, 48, 36, 27, 31, 2, 30, 34, 37, 49, 7]","[41, 36, 45, 4, 6, 42, 30, 10, 33, 54, 44, 47, 18, 40, 27, 9, 2, 21, 49, 60, 34, 1, 5, 23, 50, 46, 57, 32, 22, 26, 7, 11]","[1, 2, 4, 5, 7, 9, 10, 11, 18, 21, 22, 23, 27, 30, 34, 36, 40, 49, 50, 57]"
|
|
||||||
71,"[57, 29, 41, 52, 62, 54, 6, 11, 36, 0, 44, 9, 35, 59, 63, 32, 47, 49, 38, 34, 8, 53, 18, 13, 27, 60, 3, 10, 19, 61, 25, 28]","[10, 62, 39, 11, 16, 28, 14, 37, 23, 8, 17, 25, 58, 45, 7, 56, 36, 61, 48, 12, 13, 4, 33, 55, 27, 42, 20, 46, 32, 31, 43, 21]","[8, 10, 11, 13, 25, 27, 28, 32, 36, 61, 62]"
|
|
||||||
72,"[63, 8, 51, 13, 28, 52, 10, 41, 46, 34, 14, 9, 20, 39, 11, 25, 7, 21, 59, 5, 56, 23, 43, 47, 27, 35, 54, 30, 16, 37, 57, 2]","[19, 18, 24, 37, 58, 17, 9, 13, 2, 50, 16, 63, 3, 39, 53, 4, 35, 49, 22, 42, 45, 43, 20, 14, 47, 32, 12, 52, 26, 56, 11, 29]","[2, 9, 11, 13, 14, 16, 20, 35, 37, 39, 43, 47, 52, 56, 63]"
|
|
||||||
73,"[56, 35, 29, 20, 24, 27, 31, 12, 2, 4, 33, 11, 60, 16, 7, 8, 58, 63, 46, 49, 36, 52, 53, 41, 30, 32, 51, 47, 62, 21, 15, 0]","[51, 26, 31, 25, 47, 60, 49, 45, 9, 34, 61, 33, 38, 44, 53, 59, 3, 14, 48, 8, 22, 4, 19, 55, 6, 43, 37, 5, 50, 27, 40, 17]","[4, 8, 27, 31, 33, 47, 49, 51, 53, 60]"
|
|
||||||
74,"[17, 33, 36, 34, 27, 54, 37, 29, 38, 57, 21, 40, 43, 20, 48, 10, 41, 28, 22, 4, 32, 30, 31, 59, 47, 23, 14, 5, 1, 42, 50, 44]","[13, 23, 10, 14, 51, 12, 21, 40, 32, 3, 29, 56, 33, 50, 37, 22, 63, 47, 24, 38, 35, 43, 5, 26, 16, 27, 31, 1, 49, 46, 8, 52]","[1, 5, 10, 14, 21, 22, 23, 27, 29, 31, 32, 33, 37, 38, 40, 43, 47, 50]"
|
|
||||||
75,"[57, 37, 60, 46, 20, 41, 33, 42, 26, 55, 39, 4, 36, 10, 32, 31, 58, 9, 3, 59, 22, 2, 34, 17, 0, 61, 16, 53, 12, 25, 50, 23]","[54, 19, 15, 49, 33, 31, 8, 28, 17, 38, 27, 55, 14, 4, 45, 6, 9, 60, 10, 24, 61, 63, 13, 47, 46, 2, 20, 36, 48, 7, 53, 23]","[2, 4, 9, 10, 17, 20, 23, 31, 33, 36, 46, 53, 55, 60, 61]"
|
|
||||||
76,"[20, 56, 53, 3, 34, 4, 2, 6, 27, 15, 63, 23, 55, 39, 14, 36, 42, 25, 35, 38, 43, 1, 37, 51, 16, 29, 31, 47, 26, 28, 10, 45]","[33, 19, 20, 42, 10, 11, 48, 4, 47, 26, 9, 37, 55, 30, 58, 8, 41, 46, 59, 50, 34, 43, 18, 5, 62, 49, 23, 21, 61, 57, 51, 15]","[4, 10, 15, 20, 23, 26, 34, 37, 42, 43, 47, 51, 55]"
|
|
||||||
77,"[23, 14, 10, 28, 7, 25, 2, 0, 29, 1, 35, 8, 54, 58, 51, 24, 21, 41, 3, 38, 44, 45, 39, 5, 53, 55, 17, 48, 6, 20, 18, 34]","[2, 53, 46, 41, 62, 43, 12, 63, 10, 32, 22, 59, 51, 13, 25, 23, 14, 36, 16, 11, 58, 57, 9, 61, 42, 47, 37, 50, 33, 3, 6, 15]","[2, 3, 6, 10, 14, 23, 25, 41, 51, 53, 58]"
|
|
||||||
78,"[41, 50, 43, 38, 1, 44, 15, 61, 23, 26, 19, 9, 6, 31, 54, 18, 56, 10, 16, 29, 42, 39, 11, 12, 60, 52, 45, 63, 0, 30, 20, 53]","[58, 44, 4, 38, 11, 39, 60, 59, 22, 46, 50, 7, 15, 13, 8, 36, 5, 26, 28, 10, 53, 1, 23, 20, 43, 61, 51, 40, 57, 9, 54, 14]","[1, 9, 10, 11, 15, 20, 23, 26, 38, 39, 43, 44, 50, 53, 54, 60, 61]"
|
|
||||||
79,"[33, 36, 54, 26, 59, 28, 0, 10, 1, 63, 45, 11, 53, 56, 6, 4, 27, 40, 39, 61, 55, 18, 3, 48, 9, 22, 2, 34, 15, 16, 60, 42]","[54, 63, 16, 12, 53, 27, 34, 8, 1, 41, 36, 52, 58, 24, 59, 23, 30, 6, 44, 32, 37, 48, 3, 42, 9, 45, 51, 14, 22, 61, 50, 4]","[1, 3, 4, 6, 9, 16, 22, 27, 34, 36, 42, 45, 48, 53, 54, 59, 61, 63]"
|
|
||||||
80,"[50, 47, 26, 38, 60, 19, 14, 61, 55, 2, 31, 9, 29, 11, 62, 58, 5, 24, 25, 17, 13, 53, 37, 1, 34, 6, 43, 54, 33, 12, 41, 7]","[16, 62, 31, 60, 50, 29, 10, 20, 39, 2, 42, 14, 22, 1, 13, 59, 0, 26, 11, 34, 19, 6, 7, 54, 8, 38, 33, 61, 17, 25, 58, 47]","[1, 2, 6, 7, 11, 13, 14, 17, 19, 25, 26, 29, 31, 33, 34, 38, 47, 50, 54, 58, 60, 61, 62]"
|
|
||||||
81,"[61, 3, 10, 49, 21, 43, 46, 18, 16, 57, 4, 2, 19, 14, 34, 62, 26, 25, 1, 5, 6, 52, 63, 13, 36, 7, 45, 39, 48, 15, 54, 33]","[47, 48, 6, 16, 31, 13, 56, 17, 21, 55, 36, 40, 35, 2, 50, 3, 53, 38, 51, 23, 12, 11, 24, 34, 15, 46, 45, 8, 1, 20, 22, 39]","[1, 2, 3, 6, 13, 15, 16, 21, 34, 36, 39, 45, 46, 48]"
|
|
||||||
82,"[30, 47, 32, 19, 28, 45, 23, 13, 37, 0, 40, 16, 14, 26, 5, 43, 12, 41, 3, 6, 53, 51, 63, 24, 7, 61, 20, 22, 39, 11, 33, 34]","[9, 38, 46, 16, 6, 21, 39, 51, 1, 54, 13, 18, 57, 58, 8, 48, 29, 10, 31, 44, 17, 28, 27, 45, 34, 47, 60, 4, 5, 42, 15, 50]","[5, 6, 13, 16, 28, 34, 39, 45, 47, 51]"
|
|
||||||
83,"[18, 15, 28, 52, 23, 43, 45, 19, 34, 37, 21, 8, 27, 14, 2, 4, 47, 42, 16, 17, 60, 29, 39, 0, 58, 5, 46, 33, 12, 32, 55, 10]","[51, 27, 18, 40, 45, 32, 60, 0, 53, 4, 23, 28, 10, 15, 29, 17, 24, 2, 5, 34, 21, 22, 52, 11, 12, 50, 44, 9, 3, 42, 39, 48]","[0, 2, 4, 5, 10, 12, 15, 17, 18, 21, 23, 27, 28, 29, 32, 34, 39, 42, 45, 52, 60]"
|
|
||||||
84,"[16, 49, 22, 56, 0, 53, 12, 26, 9, 51, 20, 14, 38, 29, 36, 13, 19, 40, 6, 44, 48, 43, 32, 15, 42, 62, 39, 52, 23, 7, 60, 50]","[45, 50, 38, 19, 63, 9, 26, 20, 31, 4, 62, 54, 29, 61, 47, 16, 18, 10, 52, 39, 35, 17, 23, 56, 51, 14, 27, 49, 40, 30, 58, 15]","[9, 14, 15, 16, 19, 20, 23, 26, 29, 38, 39, 40, 49, 50, 51, 52, 56, 62]"
|
|
||||||
85,"[41, 56, 11, 28, 36, 57, 6, 23, 49, 29, 20, 5, 63, 39, 58, 51, 42, 37, 18, 0, 19, 31, 47, 10, 32, 48, 38, 7, 13, 44, 45, 8]","[29, 5, 60, 63, 11, 12, 43, 20, 42, 30, 21, 3, 6, 49, 58, 17, 36, 15, 1, 44, 24, 41, 59, 51, 53, 32, 38, 56, 37, 57, 23, 33]","[5, 6, 11, 20, 23, 29, 32, 36, 37, 38, 41, 42, 44, 49, 51, 56, 57, 58, 63]"
|
|
||||||
86,"[19, 38, 45, 33, 6, 24, 36, 10, 29, 30, 21, 9, 59, 25, 32, 11, 4, 2, 5, 57, 20, 40, 35, 55, 15, 1, 34, 44, 3, 8, 23, 18]","[37, 42, 17, 16, 6, 55, 43, 0, 56, 53, 31, 14, 21, 44, 26, 61, 49, 25, 12, 27, 28, 18, 32, 51, 13, 50, 52, 19, 54, 58, 30, 48]","[6, 18, 19, 21, 25, 30, 32, 44, 55]"
|
|
||||||
87,"[28, 38, 31, 55, 61, 37, 62, 60, 25, 6, 41, 15, 9, 4, 18, 7, 12, 49, 24, 0, 23, 46, 44, 3, 30, 59, 5, 8, 2, 34, 13, 11]","[22, 34, 53, 40, 60, 36, 27, 43, 1, 10, 17, 15, 25, 51, 48, 57, 13, 33, 54, 7, 30, 16, 38, 29, 26, 0, 44, 14, 12, 19, 42, 31]","[0, 7, 12, 13, 15, 25, 30, 31, 34, 38, 44, 60]"
|
|
||||||
88,"[60, 43, 34, 28, 48, 25, 29, 15, 16, 46, 26, 54, 4, 38, 11, 61, 39, 20, 17, 50, 27, 2, 49, 14, 36, 24, 5, 53, 41, 55, 10, 3]","[56, 48, 63, 6, 54, 38, 21, 42, 39, 47, 20, 11, 49, 14, 28, 46, 53, 61, 23, 9, 24, 34, 10, 43, 0, 52, 5, 29, 44, 59, 27, 16]","[5, 10, 11, 14, 16, 20, 24, 27, 28, 29, 34, 38, 39, 43, 46, 48, 49, 53, 54, 61]"
|
|
||||||
89,"[24, 23, 8, 45, 17, 37, 35, 10, 58, 57, 3, 61, 18, 29, 15, 4, 13, 43, 5, 19, 26, 49, 0, 1, 27, 39, 6, 20, 16, 50, 14, 34]","[38, 63, 5, 49, 47, 18, 39, 60, 29, 26, 44, 3, 0, 10, 52, 61, 14, 4, 15, 35, 59, 8, 57, 50, 54, 20, 28, 24, 16, 1, 55, 25]","[0, 1, 3, 4, 5, 8, 10, 14, 15, 16, 18, 20, 24, 26, 29, 35, 39, 49, 50, 57, 61]"
|
|
||||||
90,"[21, 35, 6, 49, 16, 39, 20, 27, 28, 57, 13, 29, 32, 61, 33, 46, 60, 38, 54, 45, 19, 59, 43, 36, 51, 31, 24, 4, 52, 50, 9, 63]","[13, 2, 42, 49, 59, 5, 27, 53, 33, 44, 16, 25, 15, 48, 52, 11, 47, 32, 7, 56, 17, 10, 14, 43, 24, 31, 0, 37, 51, 4, 30, 19]","[4, 13, 16, 19, 24, 27, 31, 32, 33, 43, 49, 51, 52, 59]"
|
|
||||||
91,"[38, 36, 34, 17, 18, 46, 20, 3, 15, 57, 23, 42, 4, 25, 2, 7, 55, 24, 19, 51, 52, 31, 21, 28, 16, 62, 9, 12, 27, 13, 48, 1]","[52, 28, 54, 22, 51, 18, 19, 8, 26, 43, 55, 4, 21, 35, 1, 37, 46, 9, 44, 39, 0, 23, 15, 42, 57, 25, 27, 49, 34, 17, 13, 11]","[1, 4, 9, 13, 15, 17, 18, 19, 21, 23, 25, 27, 28, 34, 42, 46, 51, 52, 55, 57]"
|
|
||||||
92,"[59, 17, 41, 57, 60, 46, 5, 52, 20, 38, 28, 45, 50, 58, 22, 54, 36, 56, 47, 32, 27, 3, 1, 10, 53, 24, 31, 23, 49, 6, 35, 0]","[59, 41, 20, 24, 53, 19, 46, 5, 35, 31, 38, 37, 36, 0, 52, 16, 2, 45, 17, 33, 21, 57, 50, 61, 56, 11, 8, 29, 18, 54, 10, 47]","[0, 5, 10, 17, 20, 24, 31, 35, 36, 38, 41, 45, 46, 47, 50, 52, 53, 54, 56, 57, 59]"
|
|
||||||
93,"[11, 52, 33, 43, 9, 23, 21, 61, 10, 49, 20, 15, 58, 63, 62, 16, 45, 2, 37, 28, 35, 34, 31, 24, 3, 44, 29, 30, 19, 8, 32, 38]","[49, 29, 32, 0, 20, 53, 39, 26, 44, 23, 6, 35, 10, 15, 57, 7, 61, 56, 60, 17, 4, 42, 18, 55, 46, 28, 21, 41, 24, 33, 13, 47]","[10, 15, 20, 21, 23, 24, 28, 29, 32, 33, 35, 44, 49, 61]"
|
|
||||||
94,"[8, 3, 10, 61, 33, 57, 38, 37, 20, 0, 35, 40, 58, 19, 25, 4, 24, 22, 13, 56, 45, 16, 27, 51, 46, 29, 59, 34, 15, 31, 44, 39]","[58, 55, 44, 60, 31, 8, 40, 18, 13, 26, 7, 61, 38, 19, 37, 51, 57, 53, 29, 4, 50, 46, 35, 33, 39, 5, 45, 24, 36, 34, 3, 63]","[3, 4, 8, 13, 19, 24, 29, 31, 33, 34, 35, 37, 38, 39, 40, 44, 45, 46, 51, 57, 58, 61]"
|
|
||||||
95,"[12, 23, 31, 47, 17, 22, 27, 29, 16, 37, 0, 7, 4, 50, 39, 45, 46, 19, 20, 3, 30, 41, 62, 60, 8, 18, 35, 56, 11, 32, 14, 24]","[16, 13, 5, 49, 42, 12, 40, 34, 23, 48, 36, 51, 18, 7, 62, 31, 19, 61, 25, 58, 53, 54, 60, 17, 46, 59, 41, 52, 10, 33, 63, 26]","[7, 12, 16, 17, 18, 19, 23, 31, 41, 46, 60, 62]"
|
|
||||||
96,"[22, 41, 48, 14, 11, 34, 2, 16, 3, 6, 21, 56, 61, 29, 19, 52, 18, 38, 49, 15, 12, 43, 62, 50, 63, 46, 55, 28, 0, 30, 51, 27]","[5, 2, 11, 26, 30, 40, 49, 47, 62, 54, 4, 17, 60, 44, 61, 28, 18, 35, 39, 24, 37, 57, 8, 7, 20, 45, 10, 12, 25, 3, 38, 36]","[2, 3, 11, 12, 18, 28, 30, 38, 49, 61, 62]"
|
|
||||||
97,"[2, 62, 8, 22, 15, 60, 59, 14, 0, 10, 33, 34, 50, 4, 19, 31, 58, 1, 16, 53, 17, 24, 27, 18, 32, 11, 51, 43, 54, 42, 28, 39]","[57, 54, 13, 47, 2, 15, 7, 58, 43, 16, 22, 0, 36, 35, 23, 38, 21, 55, 33, 19, 46, 8, 29, 42, 60, 27, 32, 53, 49, 31, 9, 28]","[0, 2, 8, 15, 16, 19, 22, 27, 28, 31, 32, 33, 42, 43, 53, 54, 58, 60]"
|
|
||||||
98,"[62, 54, 43, 14, 61, 0, 42, 40, 15, 12, 50, 30, 29, 47, 37, 27, 20, 33, 7, 38, 2, 5, 53, 60, 11, 16, 35, 25, 6, 21, 55, 17]","[38, 7, 54, 31, 47, 28, 34, 8, 12, 20, 3, 63, 0, 52, 36, 55, 60, 25, 17, 15, 13, 46, 33, 56, 26, 11, 14, 16, 40, 4, 37, 45]","[0, 7, 11, 12, 14, 15, 16, 17, 20, 25, 33, 37, 38, 40, 47, 54, 55, 60]"
|
|
||||||
99,"[2, 41, 3, 17, 26, 4, 9, 58, 7, 42, 49, 21, 55, 40, 53, 8, 50, 18, 29, 0, 13, 44, 63, 5, 39, 48, 24, 36, 47, 54, 19, 38]","[10, 45, 1, 18, 8, 0, 16, 15, 55, 39, 25, 6, 61, 14, 3, 34, 49, 51, 43, 46, 36, 33, 28, 57, 52, 12, 9, 35, 23, 42, 11, 31]","[0, 3, 8, 9, 18, 36, 39, 42, 49, 55]"
|
|
||||||
|
@ -1,725 +0,0 @@
|
|||||||
# Copyright (c) 2023 ETH Zurich.
|
|
||||||
# All rights reserved.
|
|
||||||
#
|
|
||||||
# Use of this source code is governed by a BSD-style license that can be
|
|
||||||
# found in the LICENSE file.
|
|
||||||
#
|
|
||||||
# The source code is adapted from the sorting source code written by
|
|
||||||
# Nils Blach.
|
|
||||||
#
|
|
||||||
# main author: Robert Gerstenberger
|
|
||||||
|
|
||||||
import os
|
|
||||||
import logging
|
|
||||||
import datetime
|
|
||||||
import json
|
|
||||||
import csv
|
|
||||||
from typing import Dict, List, Callable, Union
|
|
||||||
from graph_of_thoughts import controller, language_models, operations, prompter, parser
|
|
||||||
|
|
||||||
# This is a hack to also allow execution of this file from the examples directory
|
|
||||||
try:
|
|
||||||
from . import utils
|
|
||||||
except ImportError:
|
|
||||||
import utils
|
|
||||||
|
|
||||||
|
|
||||||
class SetIntersectionPrompter(prompter.Prompter):
|
|
||||||
"""
|
|
||||||
SetIntersectionPrompter provides the generation of prompts specific to the
|
|
||||||
set intersection example for the language models.
|
|
||||||
|
|
||||||
Inherits from the Prompter class and implements its abstract methods.
|
|
||||||
"""
|
|
||||||
|
|
||||||
intersection_prompt = """<Instruction> Find the intersection of two sets of numbers. Output only the set of numbers that are present in both sets, no additional text. </Instruction>
|
|
||||||
|
|
||||||
<Examples>
|
|
||||||
Input Set 1: [13, 16, 30, 6, 21, 7, 31, 15, 11, 1, 24, 10, 9, 3, 20, 8]
|
|
||||||
Input Set 2: [25, 24, 10, 4, 27, 0, 14, 12, 8, 2, 29, 20, 17, 19, 26, 23]
|
|
||||||
Output: [24, 10, 20, 8]
|
|
||||||
|
|
||||||
Input Set 1: [26, 40, 42, 57, 15, 31, 5, 32, 11, 4, 24, 28, 51, 54, 12, 22, 33, 35, 7, 13, 2, 59, 8, 23, 43, 16, 29, 55, 25, 63, 21, 18]
|
|
||||||
Input Set 2: [16, 60, 36, 48, 0, 15, 5, 19, 46, 24, 1, 6, 61, 10, 38, 53, 58, 9, 44, 14, 35, 63, 52, 20, 27, 17, 39, 47, 34, 56, 40, 59]
|
|
||||||
Output: [40, 15, 5, 24, 35, 59, 16, 63]
|
|
||||||
|
|
||||||
Input Set 1: [115, 61, 35, 103, 90, 117, 86, 44, 63, 45, 40, 30, 74, 33, 31, 1, 118, 48, 38, 0, 119, 51, 64, 78, 15, 121, 89, 101, 79, 69, 120, 29, 58, 50, 116, 11, 60, 12, 39, 95, 23, 2, 109, 84, 7, 43, 99, 98, 52, 70, 75, 102, 57, 19, 94, 36, 114, 88, 71, 56, 83, 6, 96, 107]
|
|
||||||
Input Set 2: [13, 35, 20, 96, 34, 18, 47, 127, 126, 9, 21, 16, 77, 22, 111, 122, 85, 73, 42, 105, 123, 15, 33, 59, 67, 57, 104, 8, 30, 89, 76, 12, 65, 84, 32, 40, 7, 100, 108, 50, 14, 28, 24, 53, 90, 17, 91, 81, 124, 63, 5, 46, 125, 93, 49, 66, 117, 37, 115, 113, 2, 106, 41, 72]
|
|
||||||
Output: [115, 35, 90, 117, 63, 40, 30, 33, 15, 89, 50, 12, 2, 84, 7, 57, 96]
|
|
||||||
</Examples>
|
|
||||||
|
|
||||||
Input Set 1: {set1}
|
|
||||||
Input Set 2: {set2}"""
|
|
||||||
|
|
||||||
intersection_prompt_cot = """<Instruction> Find the intersection of two sets of numbers. You can generate any intermediate solutions, but the final output should be the set of numbers that are present in both sets, prefixed with "Output: ". </Instruction>
|
|
||||||
|
|
||||||
<Approach>
|
|
||||||
To find the intersection of the two sets follow these steps:
|
|
||||||
1. Split the second input set of numbers into two to four subsets, each containing an equal number of elements from the original set (make sure they don't overlap).
|
|
||||||
2. For each subset find the set of numbers that are present in the subset and the first input set.
|
|
||||||
3. Merge the resulting sets into a single output set.
|
|
||||||
</Approach>
|
|
||||||
|
|
||||||
<Examples>
|
|
||||||
Input Set 1: [13, 16, 30, 6, 21, 7, 31, 15, 11, 1, 24, 10, 9, 3, 20, 8]
|
|
||||||
Input Set 2: [25, 24, 10, 4, 27, 0, 14, 12, 8, 2, 29, 20, 17, 19, 26, 23]
|
|
||||||
Subsets of Input Set 2:
|
|
||||||
[25, 24, 10, 4, 27, 0, 14, 12]
|
|
||||||
[8, 2, 29, 20, 17, 19, 26, 23]
|
|
||||||
Intersected Subsets with Input Set 1:
|
|
||||||
[24, 10]
|
|
||||||
[8, 20]
|
|
||||||
Output: [24, 10, 8, 20]
|
|
||||||
|
|
||||||
Input Set 1: [26, 40, 42, 57, 15, 31, 5, 32, 11, 4, 24, 28, 51, 54, 12, 22, 33, 35, 7, 13, 2, 59, 8, 23, 43, 16, 29, 55, 25, 63, 21, 18]
|
|
||||||
Input Set 2: [16, 60, 36, 48, 0, 15, 5, 19, 46, 24, 1, 6, 61, 10, 38, 53, 58, 9, 44, 14, 35, 63, 52, 20, 27, 17, 39, 47, 34, 56, 40, 59]
|
|
||||||
Subsets of Input Set 2:
|
|
||||||
[16, 60, 36, 48, 0, 15, 5, 19, 46, 24, 1, 6, 61, 10, 38, 53]
|
|
||||||
[58, 9, 44, 14, 35, 63, 52, 20, 27, 17, 39, 47, 34, 56, 40, 59]
|
|
||||||
Intersected Subsets with Input Set 1:
|
|
||||||
[16, 15, 5, 24]
|
|
||||||
[35, 63, 40, 59]
|
|
||||||
Output: [16, 15, 5, 24, 35, 63, 40, 59]
|
|
||||||
|
|
||||||
Input Set 1: [115, 61, 35, 103, 90, 117, 86, 44, 63, 45, 40, 30, 74, 33, 31, 1, 118, 48, 38, 0, 119, 51, 64, 78, 15, 121, 89, 101, 79, 69, 120, 29, 58, 50, 116, 11, 60, 12, 39, 95, 23, 2, 109, 84, 7, 43, 99, 98, 52, 70, 75, 102, 57, 19, 94, 36, 114, 88, 71, 56, 83, 6, 96, 107]
|
|
||||||
Input Set 2: [13, 35, 20, 96, 34, 18, 47, 127, 126, 9, 21, 16, 77, 22, 111, 122, 85, 73, 42, 105, 123, 15, 33, 59, 67, 57, 104, 8, 30, 89, 76, 12, 65, 84, 32, 40, 7, 100, 108, 50, 14, 28, 24, 53, 90, 17, 91, 81, 124, 63, 5, 46, 125, 93, 49, 66, 117, 37, 115, 113, 2, 106, 41, 72]
|
|
||||||
Subsets of Input Set 2:
|
|
||||||
[13, 35, 20, 96, 34, 18, 47, 127, 126, 9, 21, 16, 77, 22, 111, 122]
|
|
||||||
[85, 73, 42, 105, 123, 15, 33, 59, 67, 57, 104, 8, 30, 89, 76, 12]
|
|
||||||
[65, 84, 32, 40, 7, 100, 108, 50, 14, 28, 24, 53, 90, 17, 91, 81]
|
|
||||||
[124, 63, 5, 46, 125, 93, 49, 66, 117, 37, 115, 113, 2, 106, 41, 72]
|
|
||||||
Intersected Subsets with Input Set 1:
|
|
||||||
[35, 96]
|
|
||||||
[15, 33, 57, 30, 89, 12]
|
|
||||||
[84, 40, 7, 50, 90]
|
|
||||||
[63, 117, 115, 2]
|
|
||||||
Output: [35, 96, 15, 33, 57, 30, 89, 12, 84, 40, 7, 50, 90, 63, 117, 115, 2]
|
|
||||||
</Examples>
|
|
||||||
|
|
||||||
Input Set 1: {set1}
|
|
||||||
Input Set 2: {set2}"""
|
|
||||||
|
|
||||||
tot_improve_prompt = """<Instruction> The following three sets represent two sets and an intersection set of those two sets. The intersection set is not correct. Fix the intersection set so that it is correct.
|
|
||||||
Make sure that the numbers in the intersection set can be found in both input sets. </Instruction>
|
|
||||||
|
|
||||||
<Approach>
|
|
||||||
To fix the incorrectly intersection set follow these steps:
|
|
||||||
1. Check for each number in the incorrect intersection set, whether it can be found in both input sets. If not, remove that number from the intersection set.
|
|
||||||
2. Iterate through the second input set and check whether each number is already in the incorrect intersection set and if not, check whether that number can also be found in the first input set. If so, add that number to the intersection set.
|
|
||||||
</Approach>
|
|
||||||
|
|
||||||
<Examples>
|
|
||||||
Input Set 1: [13, 16, 30, 6, 21, 7, 31, 15, 11, 1, 24, 10, 9, 3, 20, 8]
|
|
||||||
Input Set 2: [25, 24, 10, 4, 27, 0, 14, 12, 8, 2, 29, 20, 17, 19, 26, 23]
|
|
||||||
Incorrect Intersection Set: [24, 20, 25]
|
|
||||||
Reason: The incorrect intersection set contains the number 25, which is not present in the first input set and is missing the numbers 10 and 8.
|
|
||||||
Output: [24, 10, 20, 8]
|
|
||||||
|
|
||||||
Input Set 1: [26, 40, 42, 57, 15, 31, 5, 32, 11, 4, 24, 28, 51, 54, 12, 22, 33, 35, 7, 13, 2, 59, 8, 23, 43, 16, 29, 55, 25, 63, 21, 18]
|
|
||||||
Input Set 2: [16, 60, 36, 48, 0, 15, 5, 19, 46, 24, 1, 6, 61, 10, 38, 53, 58, 9, 44, 14, 35, 63, 52, 20, 27, 17, 39, 47, 34, 56, 40, 59]
|
|
||||||
Incorrect Intersection Set: [57, 16, 15, 24, 35, 10, 40]
|
|
||||||
Reason: The incorrect intersection set contains the numbers 57, which is not present in the second input set, and 10, which is not present in the first input set, and is missing the numbers 5, 63 and 59.
|
|
||||||
Output: [16, 15, 5, 24, 35, 63, 40, 59]
|
|
||||||
|
|
||||||
Input Set 1: [115, 61, 35, 103, 90, 117, 86, 44, 63, 45, 40, 30, 74, 33, 31, 1, 118, 48, 38, 0, 119, 51, 64, 78, 15, 121, 89, 101, 79, 69, 120, 29, 58, 50, 116, 11, 60, 12, 39, 95, 23, 2, 109, 84, 7, 43, 99, 98, 52, 70, 75, 102, 57, 19, 94, 36, 114, 88, 71, 56, 83, 6, 96, 107]
|
|
||||||
Input Set 2: [13, 35, 20, 96, 34, 18, 47, 127, 126, 9, 21, 16, 77, 22, 111, 122, 85, 73, 42, 105, 123, 15, 33, 59, 67, 57, 104, 8, 30, 89, 76, 12, 65, 84, 32, 40, 7, 100, 108, 50, 14, 28, 24, 53, 90, 17, 91, 81, 124, 63, 5, 46, 125, 93, 49, 66, 117, 37, 115, 113, 2, 106, 41, 72]
|
|
||||||
Incorrect Intersection Set: [35, 96, 44, 15, 33, 57, 30, 50, 90, 119, 123, 63, 117, 115, 2]
|
|
||||||
Reason: The incorrect intersection set contains the numbers 44 and 119, which are not present in the second input set, and 123, which is not present in the first input set, and is missing the numbers 89, 12, 84, 40 and 7.
|
|
||||||
Output: [35, 96, 15, 33, 57, 30, 89, 12, 84, 40, 7, 50, 90, 63, 117, 115, 2]
|
|
||||||
</Examples>
|
|
||||||
|
|
||||||
Input Set 1: {set1}
|
|
||||||
Input Set 2: {set2}
|
|
||||||
Incorrect Intersection Set: {incorrect_intersection}
|
|
||||||
"""
|
|
||||||
|
|
||||||
got_split_prompt = """<Instruction> Split the following list of 32 numbers into 2 lists of 16 numbers each, the first list should contain the first 16 numbers and the second list the second 16 numbers.
|
|
||||||
Only output the 2 lists in the following format without any additional text or thoughts!:
|
|
||||||
{{
|
|
||||||
"List 1": [13, 16, 30, 6, 21, 7, 31, ...],
|
|
||||||
"List 2": [25, 24, 10, 4, 27, 0, 14, ...]
|
|
||||||
}} </Instruction>
|
|
||||||
|
|
||||||
<Example>
|
|
||||||
Input: [26, 40, 42, 57, 15, 31, 5, 32, 11, 4, 24, 28, 51, 54, 12, 22, 33, 35, 7, 13, 2, 59, 8, 23, 43, 16, 29, 55, 25, 63, 21, 18]
|
|
||||||
Output:
|
|
||||||
{{
|
|
||||||
"List 1": [26, 40, 42, 57, 15, 31, 5, 32, 11, 4, 24, 28, 51, 54, 12, 22],
|
|
||||||
"List 2": [33, 35, 7, 13, 2, 59, 8, 23, 43, 16, 29, 55, 25, 63, 21, 18]
|
|
||||||
}}
|
|
||||||
</Example>
|
|
||||||
|
|
||||||
Input: {input}"""
|
|
||||||
|
|
||||||
got_merge_prompt = """<Instruction> Merge the following 2 lists into one list by appending the second list to the first list.
|
|
||||||
Only output the final list without any additional text or thoughts! </Instruction>
|
|
||||||
|
|
||||||
List 1: {input1}
|
|
||||||
List 2: {input2}
|
|
||||||
"""
|
|
||||||
|
|
||||||
def aggregation_prompt(self, state_dicts: List[Dict], **kwargs) -> str:
|
|
||||||
"""
|
|
||||||
Generate an aggregation prompt for the language model.
|
|
||||||
|
|
||||||
:param state_dicts: The thought states that should be aggregated.
|
|
||||||
:type state_dicts: List[Dict]
|
|
||||||
:param kwargs: Additional keyword arguments.
|
|
||||||
:return: The aggregation prompt.
|
|
||||||
:rtype: str
|
|
||||||
:raise AssertionError: If not exactly two thought states are provided.
|
|
||||||
"""
|
|
||||||
assert len(state_dicts) == 2, "Expected two states for aggregation prompt."
|
|
||||||
|
|
||||||
return self.got_merge_prompt.format(
|
|
||||||
input1=state_dicts[0]["current"],
|
|
||||||
input2=state_dicts[1]["current"],
|
|
||||||
)
|
|
||||||
|
|
||||||
def generate_prompt(
|
|
||||||
self,
|
|
||||||
num_branches: int,
|
|
||||||
set1: str,
|
|
||||||
set2: str,
|
|
||||||
current: str,
|
|
||||||
method: str,
|
|
||||||
**kwargs,
|
|
||||||
) -> str:
|
|
||||||
"""
|
|
||||||
Generate a generate prompt for the language model.
|
|
||||||
|
|
||||||
:param num_branches: The number of responses the prompt should ask the LM to generate.
|
|
||||||
:type num_branches: int
|
|
||||||
:param set1: First input set.
|
|
||||||
:type set1: str
|
|
||||||
:param set2: Second input set.
|
|
||||||
:type set2: str
|
|
||||||
:param current: Intermediate solution.
|
|
||||||
:type current: str
|
|
||||||
:param method: Method for which the generate prompt is generated.
|
|
||||||
:type method: str
|
|
||||||
:param kwargs: Additional keyword arguments.
|
|
||||||
:return: The generate prompt.
|
|
||||||
:rtype: str
|
|
||||||
:raise AssertionError: If the requested number of branches is not one.
|
|
||||||
"""
|
|
||||||
|
|
||||||
assert num_branches == 1, "Branching should be done via multiple requests."
|
|
||||||
if method.startswith("io"):
|
|
||||||
return self.intersection_prompt.format(set1=set1, set2=set2)
|
|
||||||
elif method.startswith("cot"):
|
|
||||||
return self.intersection_prompt_cot.format(set1=set1, set2=set2)
|
|
||||||
elif method.startswith("tot"):
|
|
||||||
if current is None or current == "":
|
|
||||||
return self.intersection_prompt.format(set1=set1, set2=set2)
|
|
||||||
return self.tot_improve_prompt.format(
|
|
||||||
set1=set1, set2=set2, incorrect_intersection=current
|
|
||||||
)
|
|
||||||
elif method.startswith("got"):
|
|
||||||
if kwargs["phase"] == 0:
|
|
||||||
return self.got_split_prompt.format(input=set2)
|
|
||||||
|
|
||||||
input_set = set2
|
|
||||||
if "subset" in kwargs and kwargs["subset"] != "":
|
|
||||||
input_set = kwargs["subset"]
|
|
||||||
|
|
||||||
return self.intersection_prompt.format(set1=set1, set2=input_set)
|
|
||||||
|
|
||||||
def improve_prompt(self, **kwargs) -> str:
|
|
||||||
"""
|
|
||||||
Generate an improve prompt for the language model.
|
|
||||||
|
|
||||||
:param kwargs: Additional keyword arguments.
|
|
||||||
:return: The improve prompt.
|
|
||||||
:rtype: str
|
|
||||||
"""
|
|
||||||
pass
|
|
||||||
|
|
||||||
def validation_prompt(self, **kwargs) -> str:
|
|
||||||
"""
|
|
||||||
Generate a validation prompt for the language model.
|
|
||||||
|
|
||||||
:param kwargs: Additional keyword arguments.
|
|
||||||
:return: The validation prompt.
|
|
||||||
:rtype: str
|
|
||||||
"""
|
|
||||||
pass
|
|
||||||
|
|
||||||
def score_prompt(self, state_dicts: List[Dict], **kwargs) -> str:
|
|
||||||
"""
|
|
||||||
Generate a score prompt for the language model.
|
|
||||||
|
|
||||||
:param state_dicts: The thought states that should be scored,
|
|
||||||
if more than one, they should be scored together.
|
|
||||||
:type state_dicts: List[Dict]
|
|
||||||
:param kwargs: Additional keyword arguments.
|
|
||||||
:return: The score prompt.
|
|
||||||
:rtype: str
|
|
||||||
"""
|
|
||||||
pass
|
|
||||||
|
|
||||||
|
|
||||||
class SetIntersectionParser(parser.Parser):
|
|
||||||
"""
|
|
||||||
SetIntersectionParser provides the parsing of language model reponses
|
|
||||||
specific to the set intersection example.
|
|
||||||
|
|
||||||
Inherits from the Parser class and implements its abstract methods.
|
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(self) -> None:
|
|
||||||
"""
|
|
||||||
Inits the response cache.
|
|
||||||
"""
|
|
||||||
self.cache = {}
|
|
||||||
|
|
||||||
def parse_aggregation_answer(
|
|
||||||
self, states: List[Dict], texts: List[str]
|
|
||||||
) -> Union[Dict, List[Dict]]:
|
|
||||||
"""
|
|
||||||
Parse the response from the language model for an aggregation prompt.
|
|
||||||
|
|
||||||
:param states: The thought states used to generate the prompt.
|
|
||||||
:type states: List[Dict]
|
|
||||||
:param texts: The responses to the prompt from the language model.
|
|
||||||
:type texts: List[str]
|
|
||||||
:return: The new thought states after parsing the respones from the language model.
|
|
||||||
:rtype: Union[Dict, List[Dict]]
|
|
||||||
:raise AssertionError: If not exactly two thought states are provided.
|
|
||||||
"""
|
|
||||||
|
|
||||||
assert len(states) == 2, "Expected two states for aggregation answer."
|
|
||||||
new_states = []
|
|
||||||
for text in texts:
|
|
||||||
answers = text.strip().split("\n")
|
|
||||||
if any(["Output" in answer for answer in answers]):
|
|
||||||
# cut elements until last output is found
|
|
||||||
for answer in reversed(answers):
|
|
||||||
if "Output" in answer:
|
|
||||||
answers = answers[answers.index(answer) :]
|
|
||||||
break
|
|
||||||
|
|
||||||
answers_stripped = [
|
|
||||||
answer for answer in answers if "[" in answer and "]" in answer
|
|
||||||
]
|
|
||||||
if len(answers_stripped) == 0:
|
|
||||||
for answer in answers:
|
|
||||||
answer = "[" + answer + "]"
|
|
||||||
try:
|
|
||||||
answer_converted = utils.string_to_list(answer)
|
|
||||||
if len(answer_converted) > 0:
|
|
||||||
answers_stripped.append(answer)
|
|
||||||
except:
|
|
||||||
pass
|
|
||||||
if len(answers_stripped) == 0:
|
|
||||||
logging.warning(
|
|
||||||
f"Could not parse aggregation answer: {text}. Returning empty list."
|
|
||||||
)
|
|
||||||
answer = "[]"
|
|
||||||
else:
|
|
||||||
answer = [
|
|
||||||
answer[answer.index("[") : answer.index("]") + 1]
|
|
||||||
for answer in answers_stripped
|
|
||||||
][0]
|
|
||||||
states = sorted(states, key=lambda x: x["part"])
|
|
||||||
merged_subsets = states[0]["subset"][:-1] + ", " + states[1]["subset"][1:]
|
|
||||||
new_state = states[0].copy()
|
|
||||||
new_state["current"] = answer
|
|
||||||
new_state["subset"] = merged_subsets
|
|
||||||
new_states.append(new_state)
|
|
||||||
return new_states
|
|
||||||
|
|
||||||
def parse_improve_answer(self, state: Dict, texts: List[str]) -> Dict:
|
|
||||||
"""
|
|
||||||
Parse the response from the language model for an improve prompt.
|
|
||||||
|
|
||||||
:param state: The thought state used to generate the prompt.
|
|
||||||
:type state: Dict
|
|
||||||
:param texts: The responses to the prompt from the language model.
|
|
||||||
:type texts: List[str]
|
|
||||||
:return: The new thought state after parsing the responses from the language model.
|
|
||||||
:rtype: Dict
|
|
||||||
"""
|
|
||||||
pass
|
|
||||||
|
|
||||||
def parse_generate_answer(self, state: Dict, texts: List[str]) -> List[Dict]:
|
|
||||||
"""
|
|
||||||
Parse the response from the language model for a generate prompt.
|
|
||||||
|
|
||||||
:param state: The thought state used to generate the prompt.
|
|
||||||
:type state: Dict
|
|
||||||
:param texts: The responses to the prompt from the language model.
|
|
||||||
:type texts: List[str]
|
|
||||||
:return: The new thought states after parsing the respones from the language model.
|
|
||||||
:rtype: List[Dict]
|
|
||||||
"""
|
|
||||||
|
|
||||||
new_states = []
|
|
||||||
for text in texts:
|
|
||||||
if state["method"].startswith("got") and state["phase"] == 0:
|
|
||||||
# We expect a json which contains the two lists named "List 1" and "List 2"
|
|
||||||
# cut everything until the opening bracket and everything after the closing bracket
|
|
||||||
|
|
||||||
try:
|
|
||||||
text = text[text.index("{") : text.index("}") + 1]
|
|
||||||
json_dict = json.loads(text)
|
|
||||||
if len(json_dict.keys()) != 2:
|
|
||||||
logging.warning(
|
|
||||||
f"Expected 2 lists in json, but found {len(json_dict.keys())}."
|
|
||||||
)
|
|
||||||
for key, value in json_dict.items():
|
|
||||||
if "List" not in key:
|
|
||||||
logging.warning(
|
|
||||||
f"Expected key to contain 'List', but found {key}."
|
|
||||||
)
|
|
||||||
continue
|
|
||||||
if not isinstance(value, list):
|
|
||||||
value = utils.string_to_list(value)
|
|
||||||
new_state = state.copy()
|
|
||||||
new_state["current"] = ""
|
|
||||||
new_state["subset"] = str(value)
|
|
||||||
new_state["phase"] = 1
|
|
||||||
new_state["part"] = key
|
|
||||||
new_states.append(new_state)
|
|
||||||
except Exception as e:
|
|
||||||
logging.error(
|
|
||||||
f"Could not parse step answer: {text}. Encountered exception: {e}"
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
answers = text.strip().split("\n")
|
|
||||||
answers = [
|
|
||||||
answer for answer in answers if "[" in answer and "]" in answer
|
|
||||||
]
|
|
||||||
if any(["Output" in answer for answer in answers]):
|
|
||||||
# cut elements until last output is found
|
|
||||||
for answer in reversed(answers):
|
|
||||||
if "Output" in answer:
|
|
||||||
answers = answers[answers.index(answer) :]
|
|
||||||
break
|
|
||||||
|
|
||||||
answers = [
|
|
||||||
answer[answer.index("[") : answer.index("]") + 1]
|
|
||||||
for answer in answers
|
|
||||||
]
|
|
||||||
if len(answers) == 0:
|
|
||||||
logging.warning(
|
|
||||||
f"Could not parse step answer: {text}. Returning empty list."
|
|
||||||
)
|
|
||||||
answer = "[]"
|
|
||||||
else:
|
|
||||||
if len(answers) > 1:
|
|
||||||
logging.warning(
|
|
||||||
f"Multiple answers found for step answer: {text}. Using the first one."
|
|
||||||
)
|
|
||||||
answer = answers[0]
|
|
||||||
|
|
||||||
new_state = state.copy()
|
|
||||||
new_state["current"] = answer
|
|
||||||
new_state["phase"] = 2
|
|
||||||
new_states.append(new_state)
|
|
||||||
return new_states
|
|
||||||
|
|
||||||
def parse_validation_answer(self, state: Dict, texts: List[str]) -> bool:
|
|
||||||
"""
|
|
||||||
Parse the response from the language model for a validation prompt.
|
|
||||||
|
|
||||||
:param state: The thought state used to generate the prompt.
|
|
||||||
:type state: Dict
|
|
||||||
:param texts: The responses to the prompt from the language model.
|
|
||||||
:type texts: List[str]
|
|
||||||
:return: Whether the thought state is valid or not.
|
|
||||||
:rtype: bool
|
|
||||||
"""
|
|
||||||
pass
|
|
||||||
|
|
||||||
def parse_score_answer(self, states: List[Dict], texts: List[str]) -> List[float]:
|
|
||||||
"""
|
|
||||||
Parse the response from the language model for a score prompt.
|
|
||||||
|
|
||||||
:param states: The thought states used to generate the prompt.
|
|
||||||
:type states: List[Dict]
|
|
||||||
:param texts: The responses to the prompt from the language model.
|
|
||||||
:type texts: List[str]
|
|
||||||
:return: The scores for the thought states.
|
|
||||||
:rtype: List[float]
|
|
||||||
"""
|
|
||||||
pass
|
|
||||||
|
|
||||||
|
|
||||||
def io() -> operations.GraphOfOperations:
|
|
||||||
"""
|
|
||||||
Generates the Graph of Operations for the IO method.
|
|
||||||
|
|
||||||
:return: Graph of Operations
|
|
||||||
:rtype: GraphOfOperations
|
|
||||||
"""
|
|
||||||
operations_graph = operations.GraphOfOperations()
|
|
||||||
|
|
||||||
operations_graph.append_operation(operations.Generate(1, 1))
|
|
||||||
operations_graph.append_operation(operations.Score(1, False, utils.num_errors))
|
|
||||||
operations_graph.append_operation(
|
|
||||||
operations.GroundTruth(utils.test_set_intersection)
|
|
||||||
)
|
|
||||||
|
|
||||||
return operations_graph
|
|
||||||
|
|
||||||
|
|
||||||
def cot() -> operations.GraphOfOperations:
|
|
||||||
"""
|
|
||||||
Generates the Graph of Operations for the CoT method.
|
|
||||||
|
|
||||||
:return: Graph of Operations
|
|
||||||
:rtype: GraphOfOperations
|
|
||||||
"""
|
|
||||||
operations_graph = operations.GraphOfOperations()
|
|
||||||
|
|
||||||
operations_graph.append_operation(operations.Generate(1, 1))
|
|
||||||
operations_graph.append_operation(operations.Score(1, False, utils.num_errors))
|
|
||||||
operations_graph.append_operation(
|
|
||||||
operations.GroundTruth(utils.test_set_intersection)
|
|
||||||
)
|
|
||||||
|
|
||||||
return operations_graph
|
|
||||||
|
|
||||||
|
|
||||||
def tot() -> operations.GraphOfOperations:
|
|
||||||
"""
|
|
||||||
Generates the Graph of Operations for the ToT method.
|
|
||||||
ToT uses a wider tree, where on each level there are more branches.
|
|
||||||
|
|
||||||
:return: Graph of Operations
|
|
||||||
:rtype: GraphOfOperations
|
|
||||||
"""
|
|
||||||
operations_graph = operations.GraphOfOperations()
|
|
||||||
|
|
||||||
operations_graph.append_operation(operations.Generate(1, 20))
|
|
||||||
operations_graph.append_operation(operations.Score(1, False, utils.num_errors))
|
|
||||||
op_1 = operations.KeepBestN(1, False)
|
|
||||||
operations_graph.append_operation(op_1)
|
|
||||||
|
|
||||||
for _ in range(1):
|
|
||||||
operations_graph.append_operation(operations.Generate(1, 20))
|
|
||||||
operations_graph.append_operation(operations.Score(1, False, utils.num_errors))
|
|
||||||
op_2 = operations.KeepBestN(1, False)
|
|
||||||
op_2.add_predecessor(op_1)
|
|
||||||
operations_graph.append_operation(op_2)
|
|
||||||
op_1 = op_2
|
|
||||||
|
|
||||||
operations_graph.append_operation(
|
|
||||||
operations.GroundTruth(utils.test_set_intersection)
|
|
||||||
)
|
|
||||||
|
|
||||||
return operations_graph
|
|
||||||
|
|
||||||
|
|
||||||
def tot2() -> operations.GraphOfOperations:
|
|
||||||
"""
|
|
||||||
Generates the Graph of Operations for the ToT2 method.
|
|
||||||
ToT2 uses a tree with more levels, but with fewer branches per level.
|
|
||||||
|
|
||||||
:return: Graph of Operations
|
|
||||||
:rtype: GraphOfOperations
|
|
||||||
"""
|
|
||||||
operations_graph = operations.GraphOfOperations()
|
|
||||||
|
|
||||||
operations_graph.append_operation(operations.Generate(1, 10))
|
|
||||||
operations_graph.append_operation(operations.Score(1, False, utils.num_errors))
|
|
||||||
op_1 = operations.KeepBestN(1, False)
|
|
||||||
operations_graph.append_operation(op_1)
|
|
||||||
|
|
||||||
for _ in range(2):
|
|
||||||
operations_graph.append_operation(operations.Generate(1, 10))
|
|
||||||
operations_graph.append_operation(operations.Score(1, False, utils.num_errors))
|
|
||||||
op_2 = operations.KeepBestN(1, False)
|
|
||||||
op_2.add_predecessor(op_1)
|
|
||||||
operations_graph.append_operation(op_2)
|
|
||||||
op_1 = op_2
|
|
||||||
|
|
||||||
operations_graph.append_operation(
|
|
||||||
operations.GroundTruth(utils.test_set_intersection)
|
|
||||||
)
|
|
||||||
|
|
||||||
return operations_graph
|
|
||||||
|
|
||||||
|
|
||||||
def got() -> operations.GraphOfOperations:
|
|
||||||
"""
|
|
||||||
Generates the Graph of Operations for the GoT method.
|
|
||||||
|
|
||||||
:return: Graph of Operations
|
|
||||||
:rtype: GraphOfOperations
|
|
||||||
"""
|
|
||||||
operations_graph = operations.GraphOfOperations()
|
|
||||||
|
|
||||||
plans = operations.Generate(1, 1)
|
|
||||||
operations_graph.append_operation(plans) # generate the sublists
|
|
||||||
for i in range(1, 3):
|
|
||||||
list_id = f"List {i}"
|
|
||||||
sub_list = operations.Selector(
|
|
||||||
lambda thoughts, list_id=list_id: [
|
|
||||||
thought for thought in thoughts if thought.state["part"] == list_id
|
|
||||||
]
|
|
||||||
)
|
|
||||||
sub_list.add_predecessor(plans)
|
|
||||||
operations_graph.add_operation(sub_list)
|
|
||||||
intersected_subset = operations.Generate(1, 5)
|
|
||||||
intersected_subset.add_predecessor(sub_list)
|
|
||||||
operations_graph.add_operation(intersected_subset)
|
|
||||||
score_sub_list = operations.Score(1, False, utils.num_errors)
|
|
||||||
score_sub_list.add_predecessor(intersected_subset)
|
|
||||||
operations_graph.add_operation(score_sub_list)
|
|
||||||
keep_best_sub_list = operations.KeepBestN(1, False)
|
|
||||||
keep_best_sub_list.add_predecessor(score_sub_list)
|
|
||||||
operations_graph.add_operation(keep_best_sub_list)
|
|
||||||
|
|
||||||
final_aggregate = operations.Aggregate(10)
|
|
||||||
operations_graph.append_operation(final_aggregate)
|
|
||||||
operations_graph.append_operation(operations.Score(1, False, utils.num_errors))
|
|
||||||
keep_best_aggregate_final = operations.KeepBestN(1, False)
|
|
||||||
operations_graph.append_operation(keep_best_aggregate_final)
|
|
||||||
|
|
||||||
operations_graph.append_operation(
|
|
||||||
operations.GroundTruth(utils.test_set_intersection)
|
|
||||||
)
|
|
||||||
|
|
||||||
return operations_graph
|
|
||||||
|
|
||||||
|
|
||||||
def run(
|
|
||||||
data_ids: List[int],
|
|
||||||
methods: List[Callable[[], operations.GraphOfOperations]],
|
|
||||||
budget: float,
|
|
||||||
lm_name: str,
|
|
||||||
) -> float:
|
|
||||||
"""
|
|
||||||
Controller function that executes each specified method for each specified
|
|
||||||
sample while the budget is not exhausted.
|
|
||||||
|
|
||||||
:param data_ids: Indices of the sample to be run.
|
|
||||||
:type data_ids: List[int]
|
|
||||||
:param methods: List of functions to generate Graphs of Operations.
|
|
||||||
:type methods: Each function generates a Graph of Operation.
|
|
||||||
:param budget: Language model budget for the execution in dollars.
|
|
||||||
:type budget: float
|
|
||||||
:param lm_name: Name of the language model to be used.
|
|
||||||
:type lm_name: str
|
|
||||||
:return: Spent budget in dollars.
|
|
||||||
:rtype: float
|
|
||||||
"""
|
|
||||||
|
|
||||||
orig_budget = budget
|
|
||||||
data_path = os.path.join(os.path.dirname(__file__), "set_intersection_032.csv")
|
|
||||||
data = []
|
|
||||||
with open(data_path, "r") as f:
|
|
||||||
reader = csv.reader(f)
|
|
||||||
next(reader)
|
|
||||||
for row in reader:
|
|
||||||
data.append([int(row[0]), row[1], row[2], row[3]])
|
|
||||||
|
|
||||||
if data_ids is None or len(data_ids) == 0:
|
|
||||||
data_ids = list(range(len(data)))
|
|
||||||
selected_data = [data[i] for i in data_ids]
|
|
||||||
|
|
||||||
results_dir = os.path.join(os.path.dirname(__file__), "results")
|
|
||||||
|
|
||||||
if not os.path.exists(results_dir):
|
|
||||||
os.makedirs(results_dir)
|
|
||||||
timestamp = datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
|
|
||||||
extra_info = f"{lm_name}_{'-'.join([method.__name__ for method in methods])}"
|
|
||||||
folder_name = f"{extra_info}_{timestamp}"
|
|
||||||
results_folder = os.path.join(results_dir, folder_name)
|
|
||||||
os.makedirs(results_folder)
|
|
||||||
|
|
||||||
config = {
|
|
||||||
"data": selected_data,
|
|
||||||
"methods": [method.__name__ for method in methods],
|
|
||||||
"lm": lm_name,
|
|
||||||
"budget": budget,
|
|
||||||
}
|
|
||||||
with open(os.path.join(results_folder, "config.json"), "w") as f:
|
|
||||||
json.dump(config, f)
|
|
||||||
|
|
||||||
logging.basicConfig(
|
|
||||||
filename=os.path.join(results_folder, "log.log"),
|
|
||||||
filemode="w",
|
|
||||||
format="%(name)s - %(levelname)s - %(message)s",
|
|
||||||
level=logging.DEBUG,
|
|
||||||
)
|
|
||||||
|
|
||||||
for method in methods:
|
|
||||||
# create a results directory for the method
|
|
||||||
os.makedirs(os.path.join(results_folder, method.__name__))
|
|
||||||
|
|
||||||
for data in selected_data:
|
|
||||||
logging.info(f"Running data {data[0]}: {data[1]} {data[2]}")
|
|
||||||
if budget <= 0.0:
|
|
||||||
logging.error(
|
|
||||||
f"Budget has been depleted, stopping. Data {data[0]} has not been run."
|
|
||||||
)
|
|
||||||
break
|
|
||||||
for method in methods:
|
|
||||||
logging.info(f"Running method {method.__name__}")
|
|
||||||
logging.info(f"Budget left: {budget}")
|
|
||||||
if budget <= 0.0:
|
|
||||||
logging.error(
|
|
||||||
f"Budget has been depleted, stopping. Method {method.__name__} has not been run."
|
|
||||||
)
|
|
||||||
break
|
|
||||||
lm = language_models.ChatGPT(
|
|
||||||
os.path.join(
|
|
||||||
os.path.dirname(__file__),
|
|
||||||
"../../graph_of_thoughts/language_models/config.json",
|
|
||||||
),
|
|
||||||
model_name=lm_name,
|
|
||||||
cache=True,
|
|
||||||
)
|
|
||||||
operations_graph = method()
|
|
||||||
executor = controller.Controller(
|
|
||||||
lm,
|
|
||||||
operations_graph,
|
|
||||||
SetIntersectionPrompter(),
|
|
||||||
SetIntersectionParser(),
|
|
||||||
{
|
|
||||||
"set1": data[1],
|
|
||||||
"set2": data[2],
|
|
||||||
"result": data[3],
|
|
||||||
"current": "",
|
|
||||||
"phase": 0,
|
|
||||||
"method": method.__name__,
|
|
||||||
},
|
|
||||||
)
|
|
||||||
try:
|
|
||||||
executor.run()
|
|
||||||
except Exception as e:
|
|
||||||
logging.error(f"Exception: {e}")
|
|
||||||
path = os.path.join(
|
|
||||||
results_folder,
|
|
||||||
method.__name__,
|
|
||||||
f"{data[0]}.json",
|
|
||||||
)
|
|
||||||
executor.output_graph(path)
|
|
||||||
budget -= lm.cost
|
|
||||||
|
|
||||||
return orig_budget - budget
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
"""
|
|
||||||
Input(x) : a list of 32 numbers between 0 and 63 (inclusive)
|
|
||||||
Input(y) : a list of 32 numbers between 0 and 63 (inclusive)
|
|
||||||
Output(z) : a list of the intersection between x and y
|
|
||||||
Correct : z = intersection(x, y)
|
|
||||||
Input Example:
|
|
||||||
[13, 16, 30, 6, 21, 7, 31, 15, 11, 1, 24, 10, 9, 3, 20, 8]
|
|
||||||
[25, 24, 10, 4, 27, 0, 14, 12, 8, 2, 29, 20, 17, 19, 26, 23]
|
|
||||||
Output Example:
|
|
||||||
[24, 10, 20, 8]
|
|
||||||
"""
|
|
||||||
|
|
||||||
budget = 15
|
|
||||||
samples = [item for item in range(0, 100)]
|
|
||||||
approaches = [io, cot, tot, tot2, got]
|
|
||||||
|
|
||||||
spent = run(samples, approaches, budget, "chatgpt")
|
|
||||||
|
|
||||||
logging.info(f"Spent {spent} out of {budget} budget.")
|
|
||||||
@ -1,101 +0,0 @@
|
|||||||
ID,SET1,SET2,INTERSECTION
|
|
||||||
0,"[42, 73, 86, 39, 85, 77, 69, 59, 43, 127, 121, 88, 109, 53, 70, 66, 25, 51, 34, 78, 45, 11, 40, 99, 68, 47, 49, 41, 101, 31, 24, 84, 36, 29, 118, 75, 3, 27, 30, 80, 125, 8, 37, 46, 90, 21, 60, 83, 19, 6, 95, 117, 87, 18, 100, 13, 22, 10, 110, 102, 35, 81, 17, 63]","[34, 49, 116, 106, 112, 23, 5, 80, 18, 62, 90, 54, 32, 103, 37, 43, 9, 25, 92, 16, 111, 79, 64, 91, 107, 58, 72, 94, 7, 60, 33, 14, 19, 104, 28, 74, 96, 76, 38, 52, 114, 50, 17, 0, 3, 100, 69, 98, 2, 1, 99, 12, 95, 97, 123, 4, 126, 124, 82, 27, 67, 57, 115, 46]","[3, 17, 18, 19, 25, 27, 34, 37, 43, 46, 49, 60, 69, 80, 90, 95, 99, 100]"
|
|
||||||
1,"[50, 23, 70, 14, 54, 85, 106, 108, 6, 113, 79, 48, 99, 112, 28, 69, 0, 19, 64, 37, 90, 4, 15, 30, 94, 109, 96, 87, 71, 43, 21, 117, 8, 63, 35, 66, 83, 40, 119, 26, 46, 5, 53, 62, 12, 126, 116, 100, 111, 29, 97, 27, 25, 36, 82, 42, 127, 11, 56, 121, 59, 52, 124, 2]","[112, 74, 109, 56, 57, 5, 43, 126, 61, 95, 60, 79, 21, 20, 100, 85, 66, 84, 19, 37, 67, 15, 75, 30, 106, 70, 11, 121, 55, 28, 34, 76, 62, 6, 53, 97, 69, 80, 114, 39, 82, 27, 26, 14, 23, 77, 65, 40, 108, 2, 50, 118, 59, 41, 54, 45, 0, 42, 110, 94, 92, 38, 91, 4]","[0, 2, 4, 5, 6, 11, 14, 15, 19, 21, 23, 26, 27, 28, 30, 37, 40, 42, 43, 50, 53, 54, 56, 59, 62, 66, 69, 70, 79, 82, 85, 94, 97, 100, 106, 108, 109, 112, 121, 126]"
|
|
||||||
2,"[91, 23, 41, 37, 31, 22, 50, 35, 111, 115, 77, 55, 2, 97, 68, 85, 122, 62, 110, 87, 27, 113, 96, 81, 117, 18, 83, 71, 34, 64, 32, 92, 116, 3, 1, 10, 106, 42, 30, 86, 121, 52, 108, 84, 102, 8, 100, 125, 16, 39, 58, 29, 24, 38, 15, 17, 6, 40, 9, 94, 51, 61, 54, 65]","[41, 50, 8, 115, 51, 91, 29, 104, 6, 85, 21, 5, 42, 58, 36, 93, 89, 113, 1, 46, 119, 90, 55, 123, 98, 84, 34, 71, 117, 94, 53, 65, 26, 86, 28, 44, 33, 37, 95, 22, 87, 63, 100, 17, 81, 68, 72, 39, 127, 2, 74, 30, 76, 111, 66, 106, 107, 4, 73, 121, 0, 67, 9, 10]","[1, 2, 6, 8, 9, 10, 17, 22, 29, 30, 34, 37, 39, 41, 42, 50, 51, 55, 58, 65, 68, 71, 81, 84, 85, 86, 87, 91, 94, 100, 106, 111, 113, 115, 117, 121]"
|
|
||||||
3,"[12, 43, 39, 18, 95, 36, 51, 26, 30, 100, 85, 29, 75, 32, 79, 57, 101, 34, 108, 10, 23, 106, 16, 9, 7, 72, 82, 68, 105, 0, 37, 65, 64, 67, 62, 25, 13, 66, 77, 33, 121, 15, 122, 17, 3, 8, 45, 1, 70, 119, 76, 69, 92, 5, 2, 28, 110, 20, 102, 6, 93, 44, 98, 103]","[83, 118, 32, 94, 115, 39, 91, 74, 34, 70, 79, 76, 61, 126, 106, 31, 25, 11, 123, 5, 40, 58, 78, 24, 101, 82, 4, 109, 71, 86, 17, 29, 37, 92, 73, 66, 50, 111, 95, 30, 124, 116, 62, 0, 84, 108, 63, 104, 107, 38, 27, 18, 33, 9, 49, 64, 26, 13, 42, 35, 20, 43, 77, 127]","[0, 5, 9, 13, 17, 18, 20, 25, 26, 29, 30, 32, 33, 34, 37, 39, 43, 62, 64, 66, 70, 76, 77, 79, 82, 92, 95, 101, 106, 108]"
|
|
||||||
4,"[20, 116, 36, 54, 27, 67, 105, 51, 43, 57, 55, 44, 113, 38, 114, 64, 63, 37, 92, 76, 90, 35, 42, 117, 69, 2, 14, 81, 6, 78, 68, 11, 30, 52, 58, 98, 104, 21, 4, 85, 13, 47, 87, 62, 48, 109, 101, 0, 34, 53, 56, 59, 19, 88, 112, 28, 71, 18, 80, 50, 121, 75, 111, 49]","[5, 34, 55, 53, 113, 60, 22, 98, 16, 92, 126, 27, 70, 30, 61, 37, 64, 9, 36, 123, 79, 101, 15, 90, 62, 52, 72, 3, 29, 4, 110, 102, 17, 125, 26, 99, 86, 1, 20, 112, 24, 28, 31, 47, 85, 11, 42, 78, 25, 91, 7, 104, 12, 74, 71, 89, 48, 95, 43, 100, 103, 97, 107, 2]","[2, 4, 11, 20, 27, 28, 30, 34, 36, 37, 42, 43, 47, 48, 52, 53, 55, 62, 64, 71, 78, 85, 90, 92, 98, 101, 104, 112, 113]"
|
|
||||||
5,"[123, 114, 111, 64, 115, 14, 75, 67, 4, 125, 61, 11, 38, 104, 105, 7, 45, 113, 93, 41, 86, 77, 44, 0, 89, 112, 90, 52, 57, 74, 26, 29, 69, 108, 32, 121, 107, 102, 19, 42, 60, 51, 116, 39, 24, 37, 118, 68, 92, 87, 95, 79, 106, 13, 40, 53, 81, 85, 99, 36, 23, 97, 48, 47]","[61, 53, 116, 0, 75, 43, 89, 38, 101, 60, 51, 83, 29, 112, 36, 81, 55, 34, 94, 125, 68, 64, 84, 26, 40, 107, 82, 16, 24, 42, 108, 19, 111, 54, 6, 41, 120, 79, 9, 48, 44, 4, 106, 93, 30, 13, 39, 20, 124, 66, 33, 52, 57, 11, 105, 114, 72, 123, 67, 102, 71, 113, 3, 69]","[0, 4, 11, 13, 19, 24, 26, 29, 36, 38, 39, 40, 41, 42, 44, 48, 51, 52, 53, 57, 60, 61, 64, 67, 68, 69, 75, 79, 81, 89, 93, 102, 105, 106, 107, 108, 111, 112, 113, 114, 116, 123, 125]"
|
|
||||||
6,"[118, 20, 53, 85, 46, 51, 22, 52, 98, 28, 49, 64, 35, 124, 69, 58, 125, 34, 104, 106, 2, 74, 67, 111, 96, 0, 3, 33, 8, 91, 23, 43, 92, 72, 76, 26, 50, 6, 18, 68, 56, 101, 97, 37, 47, 24, 7, 57, 54, 75, 73, 1, 61, 12, 114, 87, 19, 48, 40, 62, 10, 36, 16, 123]","[94, 59, 117, 86, 6, 22, 110, 82, 34, 13, 89, 19, 122, 30, 7, 27, 44, 85, 112, 42, 17, 24, 31, 56, 43, 39, 115, 23, 15, 46, 83, 57, 127, 116, 47, 63, 102, 78, 121, 21, 103, 113, 88, 38, 16, 76, 100, 84, 9, 14, 49, 8, 60, 109, 41, 79, 99, 37, 105, 55, 108, 107, 77, 4]","[6, 7, 8, 16, 19, 22, 23, 24, 34, 37, 43, 46, 47, 49, 56, 57, 76, 85]"
|
|
||||||
7,"[2, 67, 120, 31, 16, 77, 61, 75, 4, 8, 76, 111, 23, 85, 97, 7, 19, 36, 54, 55, 124, 20, 74, 92, 90, 25, 114, 32, 123, 21, 38, 50, 6, 109, 70, 26, 64, 47, 45, 51, 60, 63, 62, 41, 53, 12, 11, 28, 122, 103, 105, 49, 68, 91, 100, 46, 17, 52, 121, 73, 1, 42, 37, 48]","[68, 89, 71, 107, 76, 98, 34, 54, 64, 42, 92, 84, 55, 93, 57, 53, 90, 91, 62, 120, 82, 51, 38, 121, 48, 109, 36, 126, 8, 50, 94, 43, 101, 37, 79, 72, 12, 27, 60, 81, 6, 88, 113, 103, 1, 17, 11, 119, 32, 18, 19, 105, 73, 111, 40, 100, 95, 2, 49, 58, 16, 65, 45, 29]","[1, 2, 6, 8, 11, 12, 16, 17, 19, 32, 36, 37, 38, 42, 45, 48, 49, 50, 51, 53, 54, 55, 60, 62, 64, 68, 73, 76, 90, 91, 92, 100, 103, 105, 109, 111, 120, 121]"
|
|
||||||
8,"[89, 83, 32, 27, 46, 99, 81, 73, 109, 80, 100, 84, 72, 29, 124, 122, 90, 25, 31, 38, 60, 62, 97, 104, 10, 108, 70, 16, 118, 6, 94, 23, 49, 7, 117, 92, 76, 55, 54, 15, 111, 85, 91, 67, 61, 95, 47, 114, 13, 42, 43, 11, 8, 14, 82, 52, 9, 78, 37, 71, 119, 22, 40, 123]","[83, 110, 121, 80, 20, 109, 119, 11, 114, 2, 12, 104, 64, 23, 66, 51, 93, 100, 41, 127, 4, 54, 44, 96, 14, 26, 42, 62, 21, 45, 112, 68, 77, 122, 69, 35, 115, 117, 79, 98, 15, 113, 50, 86, 40, 107, 34, 78, 105, 106, 49, 17, 90, 85, 125, 22, 58, 48, 102, 88, 24, 33, 0, 57]","[11, 14, 15, 22, 23, 40, 42, 49, 54, 62, 78, 80, 83, 85, 90, 100, 104, 109, 114, 117, 119, 122]"
|
|
||||||
9,"[122, 95, 15, 106, 92, 43, 38, 117, 75, 64, 16, 96, 71, 105, 1, 49, 57, 80, 101, 116, 77, 63, 39, 21, 66, 35, 120, 100, 85, 26, 99, 2, 46, 19, 74, 3, 113, 37, 108, 65, 32, 67, 11, 47, 27, 81, 73, 17, 22, 4, 121, 60, 40, 9, 87, 50, 119, 104, 28, 79, 124, 78, 94, 34]","[100, 37, 17, 8, 7, 75, 51, 107, 10, 93, 94, 124, 38, 6, 109, 24, 26, 47, 59, 86, 123, 0, 5, 54, 70, 18, 27, 19, 4, 56, 101, 68, 48, 61, 41, 112, 117, 125, 77, 69, 13, 102, 35, 97, 111, 110, 89, 72, 115, 83, 52, 76, 2, 44, 120, 42, 30, 53, 36, 88, 31, 55, 64, 103]","[2, 4, 17, 19, 26, 27, 35, 37, 38, 47, 64, 75, 77, 94, 100, 101, 117, 120, 124]"
|
|
||||||
10,"[55, 57, 69, 7, 29, 14, 73, 18, 10, 94, 8, 38, 25, 24, 78, 30, 84, 51, 60, 65, 28, 44, 116, 3, 81, 62, 26, 102, 120, 118, 125, 91, 4, 99, 106, 100, 52, 54, 74, 80, 17, 49, 19, 0, 47, 53, 108, 46, 1, 5, 86, 2, 107, 93, 109, 43, 50, 59, 33, 87, 119, 115, 40, 12]","[43, 4, 55, 46, 68, 32, 19, 15, 66, 16, 29, 112, 9, 108, 121, 125, 49, 6, 74, 126, 110, 100, 124, 34, 92, 7, 73, 47, 22, 63, 91, 87, 103, 101, 37, 105, 109, 97, 93, 42, 64, 62, 72, 84, 23, 82, 79, 114, 26, 48, 40, 60, 96, 30, 99, 88, 28, 41, 86, 50, 51, 0, 38, 24]","[0, 4, 7, 19, 24, 26, 28, 29, 30, 38, 40, 43, 46, 47, 49, 50, 51, 55, 60, 62, 73, 74, 84, 86, 87, 91, 93, 99, 100, 108, 109, 125]"
|
|
||||||
11,"[51, 76, 124, 93, 123, 53, 67, 4, 125, 103, 46, 64, 83, 72, 104, 106, 55, 7, 44, 114, 23, 58, 37, 31, 19, 63, 101, 42, 60, 92, 57, 2, 81, 21, 115, 32, 12, 66, 1, 112, 38, 5, 50, 35, 59, 34, 47, 78, 79, 127, 13, 9, 61, 96, 97, 8, 126, 86, 118, 6, 56, 48, 0, 121]","[108, 101, 72, 34, 124, 123, 0, 82, 121, 126, 70, 116, 104, 77, 61, 118, 112, 96, 95, 45, 71, 39, 115, 51, 48, 79, 66, 78, 58, 37, 44, 5, 42, 53, 57, 32, 56, 6, 16, 103, 80, 92, 35, 50, 127, 12, 69, 1, 102, 47, 97, 4, 109, 55, 86, 67, 9, 33, 100, 10, 38, 23, 19, 63]","[0, 1, 4, 5, 6, 9, 12, 19, 23, 32, 34, 35, 37, 38, 42, 44, 47, 48, 50, 51, 53, 55, 56, 57, 58, 61, 63, 66, 67, 72, 78, 79, 86, 92, 96, 97, 101, 103, 104, 112, 115, 118, 121, 123, 124, 126, 127]"
|
|
||||||
12,"[43, 67, 127, 22, 2, 85, 112, 0, 81, 6, 27, 116, 119, 32, 39, 95, 55, 121, 61, 123, 70, 49, 52, 37, 91, 117, 54, 97, 120, 107, 104, 51, 74, 111, 18, 108, 105, 44, 64, 4, 16, 122, 71, 88, 7, 47, 30, 48, 28, 38, 69, 35, 63, 40, 73, 29, 17, 13, 115, 53, 62, 96, 12, 3]","[2, 60, 126, 55, 39, 16, 97, 51, 12, 49, 122, 35, 116, 13, 25, 54, 69, 21, 87, 100, 31, 120, 92, 44, 23, 34, 85, 40, 96, 111, 67, 117, 108, 70, 95, 113, 99, 28, 121, 103, 37, 36, 68, 61, 42, 115, 91, 27, 10, 79, 72, 86, 77, 45, 29, 32, 7, 46, 105, 93, 53, 127, 114, 71]","[2, 7, 12, 13, 16, 27, 28, 29, 32, 35, 37, 39, 40, 44, 49, 51, 53, 54, 55, 61, 67, 69, 70, 71, 85, 91, 95, 96, 97, 105, 108, 111, 115, 116, 117, 120, 121, 122, 127]"
|
|
||||||
13,"[113, 60, 114, 85, 127, 46, 104, 111, 1, 68, 97, 109, 2, 108, 0, 36, 73, 65, 93, 43, 37, 67, 59, 116, 38, 121, 90, 98, 42, 35, 75, 78, 7, 8, 52, 3, 15, 81, 25, 57, 101, 69, 33, 13, 58, 72, 11, 99, 86, 4, 27, 107, 64, 30, 39, 18, 17, 100, 122, 32, 55, 54, 14, 76]","[78, 58, 41, 42, 100, 85, 44, 4, 6, 20, 55, 114, 65, 61, 40, 107, 11, 32, 29, 33, 28, 97, 76, 16, 38, 86, 39, 103, 102, 104, 30, 72, 62, 75, 15, 110, 108, 31, 53, 45, 54, 37, 57, 109, 46, 117, 60, 111, 36, 12, 116, 21, 7, 82, 43, 18, 3, 23, 123, 124, 99, 121, 35, 5]","[3, 4, 7, 11, 15, 18, 30, 32, 33, 35, 36, 37, 38, 39, 42, 43, 46, 54, 55, 57, 58, 60, 65, 72, 75, 76, 78, 85, 86, 97, 99, 100, 104, 107, 108, 109, 111, 114, 116, 121]"
|
|
||||||
14,"[41, 89, 93, 70, 47, 46, 17, 83, 113, 107, 111, 82, 58, 65, 12, 126, 124, 32, 68, 100, 108, 34, 67, 60, 4, 39, 9, 48, 81, 36, 127, 57, 20, 11, 2, 30, 110, 64, 61, 23, 79, 69, 35, 18, 71, 31, 21, 90, 119, 43, 66, 120, 105, 29, 97, 92, 74, 56, 27, 51, 91, 55, 22, 96]","[89, 64, 120, 26, 56, 16, 81, 100, 39, 125, 55, 28, 21, 123, 22, 52, 127, 66, 77, 107, 69, 97, 47, 4, 35, 85, 78, 101, 70, 5, 112, 102, 1, 124, 51, 74, 10, 36, 3, 60, 65, 6, 68, 113, 86, 88, 111, 72, 7, 41, 29, 48, 82, 93, 96, 104, 108, 75, 58, 87, 109, 45, 91, 31]","[4, 21, 22, 29, 31, 35, 36, 39, 41, 47, 48, 51, 55, 56, 58, 60, 64, 65, 66, 68, 69, 70, 74, 81, 82, 89, 91, 93, 96, 97, 100, 107, 108, 111, 113, 120, 124, 127]"
|
|
||||||
15,"[70, 41, 44, 50, 67, 77, 7, 81, 38, 125, 32, 115, 109, 61, 60, 56, 13, 85, 27, 4, 21, 19, 114, 68, 105, 14, 107, 31, 126, 0, 84, 54, 34, 108, 106, 98, 127, 92, 121, 95, 57, 3, 99, 48, 79, 110, 104, 45, 30, 73, 87, 111, 49, 93, 33, 103, 15, 42, 78, 94, 47, 58, 40, 80]","[90, 30, 17, 119, 56, 80, 16, 105, 34, 97, 48, 58, 19, 93, 84, 1, 98, 23, 110, 117, 118, 75, 126, 35, 25, 123, 114, 121, 33, 86, 67, 54, 49, 91, 41, 78, 15, 77, 14, 7, 61, 113, 79, 102, 109, 47, 92, 101, 31, 0, 72, 32, 52, 103, 68, 42, 100, 122, 13, 95, 3, 81, 18, 38]","[0, 3, 7, 13, 14, 15, 19, 30, 31, 32, 33, 34, 38, 41, 42, 47, 48, 49, 54, 56, 58, 61, 67, 68, 77, 78, 79, 80, 81, 84, 92, 93, 95, 98, 103, 105, 109, 110, 114, 121, 126]"
|
|
||||||
16,"[44, 65, 118, 22, 68, 125, 1, 34, 18, 58, 121, 63, 7, 23, 117, 6, 64, 105, 67, 25, 51, 88, 116, 40, 123, 53, 87, 13, 27, 8, 21, 16, 84, 48, 32, 26, 94, 19, 10, 4, 103, 2, 3, 41, 69, 127, 82, 50, 86, 78, 107, 20, 62, 79, 24, 52, 59, 106, 33, 36, 55, 70, 60, 29]","[107, 97, 1, 124, 116, 76, 105, 68, 87, 53, 67, 127, 99, 44, 65, 9, 66, 98, 34, 121, 63, 117, 83, 126, 94, 113, 11, 64, 62, 96, 35, 73, 47, 71, 37, 39, 70, 16, 23, 24, 42, 85, 17, 43, 106, 29, 45, 90, 88, 81, 41, 75, 46, 93, 110, 21, 100, 95, 102, 51, 123, 78, 4, 33]","[1, 4, 16, 21, 23, 24, 29, 33, 34, 41, 44, 51, 53, 62, 63, 64, 65, 67, 68, 70, 78, 87, 88, 94, 105, 106, 107, 116, 117, 121, 123, 127]"
|
|
||||||
17,"[43, 22, 51, 64, 25, 16, 67, 104, 58, 35, 33, 109, 50, 97, 18, 117, 36, 44, 98, 14, 103, 121, 27, 53, 82, 105, 119, 56, 10, 46, 20, 102, 127, 41, 71, 37, 116, 13, 89, 73, 79, 62, 78, 125, 106, 83, 21, 124, 59, 24, 65, 85, 76, 95, 57, 40, 100, 108, 66, 99, 90, 32, 47, 61]","[1, 81, 122, 77, 13, 107, 102, 124, 93, 99, 21, 100, 31, 6, 126, 110, 12, 96, 26, 75, 45, 86, 70, 32, 105, 63, 61, 9, 34, 5, 49, 119, 113, 4, 118, 123, 11, 2, 14, 48, 43, 115, 112, 57, 90, 7, 17, 30, 65, 120, 28, 52, 104, 79, 68, 23, 3, 18, 92, 91, 29, 94, 82, 51]","[13, 14, 18, 21, 32, 43, 51, 57, 61, 65, 79, 82, 90, 99, 100, 102, 104, 105, 119, 124]"
|
|
||||||
18,"[74, 113, 18, 102, 49, 28, 37, 34, 79, 67, 77, 17, 115, 43, 6, 53, 7, 12, 38, 122, 69, 123, 19, 71, 59, 60, 85, 29, 44, 125, 21, 2, 48, 50, 88, 15, 56, 35, 31, 109, 52, 47, 80, 87, 90, 99, 36, 22, 121, 107, 73, 98, 110, 120, 20, 23, 89, 94, 118, 51, 5, 46, 4, 41]","[125, 113, 1, 4, 34, 118, 37, 27, 75, 79, 92, 94, 87, 107, 46, 120, 106, 49, 52, 59, 124, 28, 41, 21, 72, 102, 29, 5, 67, 26, 7, 6, 89, 112, 111, 35, 65, 98, 93, 23, 69, 31, 96, 85, 80, 110, 9, 8, 2, 127, 50, 76, 71, 36, 19, 10, 33, 60, 25, 51, 17, 32, 16, 73]","[2, 4, 5, 6, 7, 17, 19, 21, 23, 28, 29, 31, 34, 35, 36, 37, 41, 46, 49, 50, 51, 52, 59, 60, 67, 69, 71, 73, 79, 80, 85, 87, 89, 94, 98, 102, 107, 110, 113, 118, 120, 125]"
|
|
||||||
19,"[38, 0, 14, 10, 87, 56, 113, 94, 45, 86, 39, 88, 111, 121, 74, 98, 102, 52, 107, 48, 50, 95, 90, 112, 118, 82, 19, 109, 49, 58, 99, 92, 110, 124, 59, 20, 43, 80, 73, 68, 18, 63, 78, 42, 77, 126, 15, 115, 57, 75, 53, 25, 83, 1, 5, 36, 104, 46, 61, 71, 119, 33, 81, 93]","[14, 104, 96, 108, 98, 65, 33, 44, 46, 72, 58, 42, 15, 90, 83, 126, 85, 109, 125, 32, 38, 61, 117, 118, 101, 103, 97, 55, 1, 47, 115, 7, 30, 40, 102, 41, 91, 10, 36, 75, 43, 62, 26, 71, 16, 4, 25, 51, 116, 31, 24, 23, 12, 107, 110, 119, 124, 67, 3, 79, 28, 54, 18, 52]","[1, 10, 14, 15, 18, 25, 33, 36, 38, 42, 43, 46, 52, 58, 61, 71, 75, 83, 90, 98, 102, 104, 107, 109, 110, 115, 118, 119, 124, 126]"
|
|
||||||
20,"[30, 109, 92, 124, 41, 116, 33, 94, 106, 71, 72, 119, 17, 125, 113, 120, 84, 80, 5, 117, 75, 52, 39, 20, 123, 61, 79, 44, 36, 115, 101, 100, 99, 81, 28, 58, 91, 54, 53, 64, 104, 74, 51, 1, 83, 60, 89, 48, 68, 78, 27, 82, 110, 35, 63, 69, 46, 43, 34, 0, 24, 67, 76, 105]","[40, 71, 122, 34, 63, 41, 15, 2, 102, 77, 38, 116, 104, 5, 0, 59, 9, 11, 86, 101, 51, 105, 99, 8, 53, 45, 92, 96, 67, 52, 26, 28, 10, 79, 118, 42, 39, 22, 84, 108, 3, 18, 73, 35, 98, 36, 13, 113, 21, 20, 126, 55, 117, 119, 30, 76, 23, 95, 82, 44, 37, 110, 48, 121]","[0, 5, 20, 28, 30, 34, 35, 36, 39, 41, 44, 48, 51, 52, 53, 63, 67, 71, 76, 79, 82, 84, 92, 99, 101, 104, 105, 110, 113, 116, 117, 119]"
|
|
||||||
21,"[66, 28, 126, 114, 37, 14, 51, 100, 0, 63, 2, 87, 101, 6, 85, 75, 72, 12, 111, 120, 96, 91, 19, 71, 11, 9, 3, 121, 88, 89, 103, 17, 52, 50, 84, 41, 78, 107, 124, 68, 27, 32, 57, 29, 104, 61, 93, 70, 86, 67, 56, 64, 125, 22, 102, 7, 10, 80, 117, 81, 74, 4, 99, 73]","[87, 47, 88, 38, 14, 52, 122, 77, 23, 126, 45, 4, 2, 85, 28, 70, 48, 98, 32, 13, 109, 94, 61, 36, 68, 121, 116, 108, 100, 95, 26, 92, 114, 58, 76, 123, 115, 112, 11, 99, 79, 6, 35, 20, 24, 9, 17, 65, 80, 25, 60, 39, 113, 19, 30, 71, 63, 18, 34, 44, 105, 37, 83, 74]","[2, 4, 6, 9, 11, 14, 17, 19, 28, 32, 37, 52, 61, 63, 68, 70, 71, 74, 80, 85, 87, 88, 99, 100, 114, 121, 126]"
|
|
||||||
22,"[37, 119, 60, 45, 97, 34, 12, 69, 89, 61, 112, 42, 77, 44, 51, 126, 100, 13, 17, 43, 66, 106, 25, 28, 15, 122, 101, 86, 49, 23, 83, 14, 48, 74, 39, 62, 80, 76, 71, 117, 96, 125, 114, 59, 67, 52, 79, 47, 113, 98, 50, 33, 118, 107, 57, 99, 70, 31, 1, 56, 55, 68, 4, 41]","[58, 28, 0, 32, 16, 47, 87, 116, 109, 44, 123, 45, 121, 93, 24, 103, 53, 41, 26, 6, 119, 111, 79, 52, 92, 22, 81, 36, 59, 99, 115, 73, 2, 82, 127, 55, 114, 100, 88, 63, 13, 40, 124, 113, 5, 117, 38, 49, 85, 21, 105, 108, 89, 83, 11, 27, 31, 18, 7, 9, 96, 72, 110, 91]","[13, 28, 31, 41, 44, 45, 47, 49, 52, 55, 59, 79, 83, 89, 96, 99, 100, 113, 114, 117, 119]"
|
|
||||||
23,"[83, 70, 69, 31, 72, 13, 6, 15, 56, 115, 29, 127, 109, 23, 111, 93, 27, 90, 49, 53, 22, 30, 105, 125, 76, 106, 37, 67, 104, 47, 65, 96, 52, 19, 81, 82, 112, 10, 63, 89, 88, 94, 62, 28, 26, 58, 121, 1, 36, 16, 44, 92, 11, 54, 98, 84, 103, 39, 123, 41, 42, 87, 100, 97]","[81, 88, 29, 94, 47, 106, 84, 95, 56, 74, 87, 69, 39, 5, 104, 16, 112, 45, 13, 41, 9, 22, 109, 82, 54, 96, 113, 0, 110, 49, 77, 90, 30, 26, 28, 125, 12, 120, 40, 58, 14, 115, 108, 72, 105, 65, 71, 111, 83, 62, 76, 20, 11, 127, 70, 31, 42, 89, 24, 6, 63, 50, 107, 37]","[6, 11, 13, 16, 22, 26, 28, 29, 30, 31, 37, 39, 41, 42, 47, 49, 54, 56, 58, 62, 63, 65, 69, 70, 72, 76, 81, 82, 83, 84, 87, 88, 89, 90, 94, 96, 104, 105, 106, 109, 111, 112, 115, 125, 127]"
|
|
||||||
24,"[11, 16, 14, 24, 106, 59, 73, 61, 77, 102, 96, 26, 78, 18, 67, 7, 113, 127, 34, 72, 48, 88, 104, 101, 87, 89, 75, 74, 44, 90, 83, 97, 29, 64, 81, 54, 79, 124, 38, 41, 6, 76, 5, 35, 91, 47, 121, 12, 114, 17, 109, 108, 42, 111, 99, 93, 36, 92, 28, 65, 9, 94, 115, 3]","[87, 54, 90, 3, 85, 42, 5, 36, 113, 49, 41, 10, 115, 114, 117, 59, 33, 105, 100, 108, 18, 19, 55, 65, 47, 73, 77, 37, 101, 99, 45, 44, 28, 83, 13, 63, 53, 98, 64, 25, 96, 12, 16, 6, 23, 14, 46, 88, 15, 52, 93, 71, 48, 97, 116, 92, 17, 124, 95, 89, 9, 75, 78, 72]","[3, 5, 6, 9, 12, 14, 16, 17, 18, 28, 36, 41, 42, 44, 47, 48, 54, 59, 64, 65, 72, 73, 75, 77, 78, 83, 87, 88, 89, 90, 92, 93, 96, 97, 99, 101, 108, 113, 114, 115, 124]"
|
|
||||||
25,"[66, 46, 43, 29, 31, 35, 69, 79, 113, 49, 108, 84, 18, 61, 65, 72, 104, 115, 47, 98, 2, 75, 76, 19, 73, 114, 119, 89, 101, 105, 125, 23, 16, 17, 6, 50, 88, 56, 52, 20, 24, 27, 34, 95, 38, 116, 15, 39, 77, 83, 51, 100, 106, 44, 107, 4, 112, 71, 36, 30, 41, 67, 14, 55]","[87, 101, 45, 97, 93, 12, 35, 77, 80, 30, 102, 89, 98, 119, 3, 106, 0, 105, 34, 4, 108, 61, 100, 65, 55, 5, 52, 56, 54, 32, 114, 81, 19, 49, 107, 59, 29, 22, 70, 86, 31, 113, 26, 39, 60, 69, 33, 92, 82, 67, 115, 66, 125, 68, 117, 38, 53, 43, 109, 28, 2, 20, 1, 46]","[2, 4, 19, 20, 29, 30, 31, 34, 35, 38, 39, 43, 46, 49, 52, 55, 56, 61, 65, 66, 67, 69, 77, 89, 98, 100, 101, 105, 106, 107, 108, 113, 114, 115, 119, 125]"
|
|
||||||
26,"[119, 125, 121, 34, 33, 46, 113, 56, 38, 15, 57, 21, 83, 12, 72, 48, 65, 85, 4, 24, 29, 31, 50, 26, 1, 86, 70, 63, 122, 92, 22, 10, 106, 59, 58, 76, 19, 87, 123, 20, 43, 81, 62, 30, 0, 97, 7, 11, 14, 114, 75, 41, 36, 80, 117, 64, 73, 118, 47, 98, 102, 89, 90, 25]","[70, 4, 91, 89, 82, 123, 31, 61, 74, 99, 22, 103, 47, 68, 90, 110, 77, 108, 6, 50, 58, 78, 85, 16, 5, 23, 46, 34, 96, 121, 124, 95, 55, 79, 60, 57, 69, 39, 93, 81, 44, 113, 51, 36, 56, 66, 14, 27, 38, 42, 43, 84, 26, 25, 111, 45, 100, 12, 127, 97, 114, 87, 28, 116]","[4, 12, 14, 22, 25, 26, 31, 34, 36, 38, 43, 46, 47, 50, 56, 57, 58, 70, 81, 85, 87, 89, 90, 97, 113, 114, 121, 123]"
|
|
||||||
27,"[90, 47, 3, 53, 98, 92, 80, 76, 32, 52, 28, 66, 94, 103, 70, 29, 91, 62, 46, 56, 68, 59, 86, 125, 77, 35, 116, 124, 101, 88, 73, 54, 9, 23, 38, 74, 41, 39, 31, 84, 69, 40, 24, 21, 97, 37, 19, 106, 123, 112, 78, 107, 34, 4, 65, 49, 85, 79, 57, 2, 6, 30, 63, 15]","[52, 107, 33, 85, 38, 8, 105, 2, 63, 117, 40, 77, 44, 20, 90, 94, 23, 116, 76, 84, 65, 86, 21, 29, 60, 97, 9, 22, 34, 83, 35, 124, 3, 16, 69, 72, 67, 39, 98, 37, 99, 45, 58, 25, 28, 112, 15, 127, 11, 17, 30, 24, 66, 78, 14, 41, 89, 113, 49, 19, 31, 57, 73, 92]","[2, 3, 9, 15, 19, 21, 23, 24, 28, 29, 30, 31, 34, 35, 37, 38, 39, 40, 41, 49, 52, 57, 63, 65, 66, 69, 73, 76, 77, 78, 84, 85, 86, 90, 92, 94, 97, 98, 107, 112, 116, 124]"
|
|
||||||
28,"[28, 75, 38, 70, 106, 16, 99, 7, 37, 0, 110, 33, 120, 65, 127, 19, 66, 95, 50, 67, 34, 52, 105, 71, 93, 36, 125, 41, 55, 43, 54, 15, 4, 123, 46, 63, 11, 22, 109, 27, 13, 48, 81, 58, 49, 103, 45, 85, 31, 112, 92, 44, 89, 56, 62, 42, 2, 104, 25, 84, 77, 24, 61, 102]","[111, 13, 70, 30, 52, 45, 87, 103, 69, 21, 33, 98, 35, 65, 28, 15, 53, 93, 73, 92, 38, 67, 37, 84, 83, 32, 100, 62, 58, 118, 54, 115, 108, 51, 80, 90, 26, 119, 4, 81, 127, 85, 29, 95, 105, 47, 74, 122, 44, 91, 16, 31, 124, 64, 104, 101, 22, 75, 102, 7, 43, 60, 3, 57]","[4, 7, 13, 15, 16, 22, 28, 31, 33, 37, 38, 43, 44, 45, 52, 54, 58, 62, 65, 67, 70, 75, 81, 84, 85, 92, 93, 95, 102, 103, 104, 105, 127]"
|
|
||||||
29,"[29, 9, 37, 58, 97, 77, 19, 8, 32, 70, 62, 85, 44, 6, 7, 90, 21, 102, 108, 95, 67, 63, 12, 52, 18, 66, 110, 98, 68, 87, 38, 59, 5, 48, 83, 27, 104, 40, 46, 94, 53, 88, 106, 35, 43, 15, 36, 17, 28, 119, 42, 3, 56, 86, 109, 64, 123, 11, 51, 22, 60, 76, 79, 111]","[80, 121, 61, 7, 45, 116, 55, 71, 48, 24, 95, 87, 37, 76, 1, 58, 65, 124, 68, 9, 74, 26, 92, 5, 100, 90, 6, 103, 66, 86, 54, 104, 49, 96, 2, 113, 85, 78, 51, 17, 25, 32, 35, 64, 20, 13, 82, 125, 102, 77, 106, 10, 110, 43, 41, 62, 29, 16, 72, 4, 93, 127, 15, 12]","[5, 6, 7, 9, 12, 15, 17, 29, 32, 35, 37, 43, 48, 51, 58, 62, 64, 66, 68, 76, 77, 85, 86, 87, 90, 95, 102, 104, 106, 110]"
|
|
||||||
30,"[65, 87, 107, 50, 53, 7, 82, 62, 49, 112, 105, 90, 93, 45, 28, 122, 51, 24, 66, 41, 1, 18, 109, 94, 63, 72, 91, 47, 59, 119, 78, 56, 48, 42, 46, 115, 21, 92, 10, 39, 113, 96, 67, 37, 98, 121, 57, 111, 15, 86, 120, 100, 19, 23, 110, 69, 88, 95, 40, 6, 5, 33, 77, 16]","[24, 120, 43, 47, 12, 28, 69, 66, 113, 8, 23, 51, 25, 89, 14, 78, 114, 91, 101, 54, 32, 90, 31, 42, 116, 29, 76, 107, 64, 105, 40, 39, 70, 109, 71, 18, 127, 68, 111, 73, 87, 58, 13, 126, 121, 81, 30, 115, 77, 63, 5, 82, 124, 60, 21, 86, 65, 85, 74, 9, 17, 3, 2, 80]","[5, 18, 21, 23, 24, 28, 39, 40, 42, 47, 51, 63, 65, 66, 69, 77, 78, 82, 86, 87, 90, 91, 105, 107, 109, 111, 113, 115, 120, 121]"
|
|
||||||
31,"[89, 62, 125, 43, 101, 24, 123, 16, 113, 55, 8, 14, 3, 98, 65, 78, 57, 64, 114, 59, 110, 103, 30, 12, 72, 126, 42, 97, 74, 20, 69, 22, 91, 51, 109, 48, 81, 23, 6, 27, 2, 67, 49, 11, 111, 83, 73, 35, 66, 53, 25, 18, 120, 56, 115, 58, 4, 28, 32, 105, 41, 38, 124, 40]","[99, 88, 100, 17, 31, 40, 111, 72, 112, 70, 68, 63, 0, 116, 10, 1, 123, 19, 26, 15, 82, 44, 117, 41, 104, 62, 12, 76, 32, 85, 53, 52, 101, 120, 39, 119, 35, 115, 36, 86, 80, 125, 33, 71, 114, 11, 108, 13, 77, 107, 113, 45, 55, 2, 124, 90, 54, 25, 75, 5, 94, 89, 42, 122]","[2, 11, 12, 25, 32, 35, 40, 41, 42, 53, 55, 62, 72, 89, 101, 111, 113, 114, 115, 120, 123, 124, 125]"
|
|
||||||
32,"[45, 17, 7, 105, 16, 102, 10, 49, 97, 29, 90, 28, 18, 36, 84, 38, 35, 76, 73, 21, 120, 13, 79, 113, 127, 54, 110, 52, 44, 40, 8, 55, 11, 20, 125, 57, 75, 60, 39, 9, 43, 71, 3, 101, 33, 19, 95, 123, 37, 41, 1, 26, 87, 58, 85, 109, 51, 34, 66, 80, 91, 5, 12, 30]","[28, 47, 78, 96, 89, 69, 112, 10, 114, 99, 83, 49, 79, 72, 116, 50, 80, 3, 122, 64, 117, 59, 19, 38, 95, 33, 62, 70, 24, 94, 22, 25, 126, 46, 100, 101, 74, 92, 0, 42, 103, 115, 21, 55, 102, 60, 63, 81, 14, 104, 15, 77, 119, 93, 61, 111, 98, 27, 56, 121, 41, 18, 65, 73]","[3, 10, 18, 19, 21, 28, 33, 38, 41, 49, 55, 60, 73, 79, 80, 95, 101, 102]"
|
|
||||||
33,"[55, 120, 37, 77, 1, 33, 54, 91, 3, 27, 12, 101, 75, 119, 8, 58, 19, 68, 110, 84, 10, 124, 74, 60, 16, 125, 63, 104, 24, 25, 81, 100, 88, 50, 90, 40, 116, 23, 9, 47, 52, 4, 78, 39, 56, 57, 108, 15, 127, 66, 89, 32, 44, 85, 65, 53, 71, 14, 36, 45, 67, 115, 123, 48]","[14, 16, 49, 2, 101, 120, 74, 46, 64, 6, 78, 110, 5, 44, 65, 9, 92, 112, 57, 8, 15, 81, 40, 102, 100, 4, 34, 24, 22, 82, 90, 7, 70, 26, 80, 13, 42, 41, 73, 59, 123, 18, 33, 12, 125, 36, 1, 107, 122, 105, 98, 23, 63, 47, 77, 53, 97, 89, 21, 114, 38, 54, 96, 45]","[1, 4, 8, 9, 12, 14, 15, 16, 23, 24, 33, 36, 40, 44, 45, 47, 53, 54, 57, 63, 65, 74, 77, 78, 81, 89, 90, 100, 101, 110, 120, 123, 125]"
|
|
||||||
34,"[17, 28, 1, 97, 85, 70, 65, 68, 98, 19, 6, 55, 60, 82, 90, 41, 31, 125, 95, 112, 48, 86, 14, 49, 127, 92, 99, 15, 122, 43, 106, 58, 34, 80, 16, 91, 119, 46, 81, 2, 64, 8, 51, 52, 89, 20, 74, 12, 113, 72, 66, 36, 121, 37, 30, 115, 40, 44, 62, 117, 32, 123, 87, 79]","[62, 93, 113, 66, 122, 73, 68, 99, 112, 81, 25, 1, 127, 55, 0, 19, 60, 28, 18, 117, 49, 10, 43, 80, 95, 40, 4, 46, 31, 102, 3, 125, 20, 2, 21, 97, 121, 65, 52, 79, 41, 91, 17, 14, 33, 111, 100, 70, 13, 51, 35, 98, 105, 23, 64, 85, 61, 39, 92, 32, 48, 86, 118, 44]","[1, 2, 14, 17, 19, 20, 28, 31, 32, 40, 41, 43, 44, 46, 48, 49, 51, 52, 55, 60, 62, 64, 65, 66, 68, 70, 79, 80, 81, 85, 86, 91, 92, 95, 97, 98, 99, 112, 113, 117, 121, 122, 125, 127]"
|
|
||||||
35,"[51, 85, 64, 97, 39, 74, 25, 19, 3, 49, 62, 29, 45, 1, 115, 32, 8, 123, 15, 65, 50, 48, 58, 28, 82, 114, 22, 87, 31, 21, 76, 57, 125, 7, 55, 67, 88, 53, 43, 18, 27, 61, 36, 118, 26, 0, 68, 60, 46, 86, 13, 109, 92, 83, 56, 91, 23, 110, 12, 95, 81, 30, 34, 9]","[72, 20, 21, 90, 17, 68, 83, 33, 41, 51, 77, 84, 6, 108, 79, 115, 52, 54, 70, 61, 9, 57, 94, 107, 29, 63, 14, 53, 111, 47, 96, 18, 42, 36, 98, 106, 102, 100, 2, 120, 64, 121, 105, 73, 4, 104, 85, 69, 37, 89, 110, 127, 59, 24, 112, 16, 113, 101, 103, 7, 38, 10, 12, 19]","[7, 9, 12, 18, 19, 21, 29, 36, 51, 53, 57, 61, 64, 68, 83, 85, 110, 115]"
|
|
||||||
36,"[44, 96, 39, 51, 114, 97, 4, 124, 108, 77, 22, 80, 38, 26, 45, 3, 27, 25, 36, 120, 70, 46, 29, 57, 8, 99, 90, 64, 58, 9, 93, 56, 89, 106, 50, 122, 74, 48, 69, 102, 40, 37, 2, 86, 88, 47, 12, 33, 18, 13, 95, 105, 123, 110, 31, 73, 83, 82, 126, 0, 14, 54, 34, 11]","[39, 95, 123, 86, 88, 70, 89, 79, 50, 80, 56, 73, 25, 36, 71, 57, 3, 101, 21, 31, 30, 0, 121, 22, 12, 105, 2, 51, 82, 81, 9, 28, 14, 58, 4, 64, 108, 1, 109, 100, 23, 119, 61, 102, 63, 26, 29, 47, 53, 114, 126, 74, 54, 8, 16, 37, 69, 11, 60, 94, 62, 113, 27, 46]","[0, 2, 3, 4, 8, 9, 11, 12, 14, 22, 25, 26, 27, 29, 31, 36, 37, 39, 46, 47, 50, 51, 54, 56, 57, 58, 64, 69, 70, 73, 74, 80, 82, 86, 88, 89, 95, 102, 105, 108, 114, 123, 126]"
|
|
||||||
37,"[13, 43, 88, 51, 44, 101, 30, 71, 34, 127, 120, 28, 17, 60, 16, 4, 93, 107, 117, 75, 118, 18, 64, 50, 67, 46, 11, 52, 78, 27, 95, 12, 14, 97, 56, 5, 82, 74, 53, 15, 58, 54, 86, 36, 68, 47, 119, 87, 35, 8, 83, 104, 108, 76, 55, 29, 24, 21, 9, 92, 19, 102, 41, 91]","[66, 14, 109, 4, 115, 94, 34, 7, 45, 35, 101, 119, 50, 9, 25, 57, 30, 88, 71, 52, 19, 118, 17, 56, 38, 91, 5, 64, 24, 48, 29, 78, 21, 10, 70, 72, 2, 18, 104, 15, 13, 44, 108, 75, 58, 60, 97, 46, 36, 86, 93, 59, 100, 0, 43, 107, 125, 85, 80, 87, 32, 69, 55, 12]","[4, 5, 9, 12, 13, 14, 15, 17, 18, 19, 21, 24, 29, 30, 34, 35, 36, 43, 44, 46, 50, 52, 55, 56, 58, 60, 64, 71, 75, 78, 86, 87, 88, 91, 93, 97, 101, 104, 107, 108, 118, 119]"
|
|
||||||
38,"[27, 11, 109, 10, 64, 7, 43, 84, 17, 106, 71, 3, 5, 51, 73, 116, 54, 4, 80, 124, 38, 99, 33, 111, 81, 35, 95, 125, 25, 30, 67, 110, 82, 89, 49, 20, 39, 52, 68, 40, 32, 65, 105, 72, 96, 115, 93, 126, 24, 76, 102, 100, 97, 123, 101, 9, 50, 47, 44, 36, 23, 127, 66, 28]","[18, 106, 53, 121, 15, 49, 69, 86, 116, 114, 47, 73, 16, 112, 119, 52, 3, 45, 64, 33, 94, 91, 43, 1, 7, 75, 83, 55, 35, 23, 63, 5, 84, 56, 78, 12, 124, 46, 113, 2, 90, 126, 115, 120, 71, 0, 95, 93, 87, 77, 88, 19, 14, 70, 20, 29, 24, 31, 62, 37, 118, 79, 74, 111]","[3, 5, 7, 20, 23, 24, 33, 35, 43, 47, 49, 52, 64, 71, 73, 84, 93, 95, 106, 111, 115, 116, 124, 126]"
|
|
||||||
39,"[90, 50, 42, 2, 46, 10, 29, 55, 33, 108, 35, 53, 56, 83, 51, 114, 65, 75, 86, 30, 62, 22, 121, 32, 5, 43, 44, 77, 116, 106, 80, 23, 31, 69, 15, 93, 68, 96, 54, 97, 92, 73, 76, 123, 125, 79, 101, 24, 39, 9, 36, 72, 28, 37, 41, 105, 52, 6, 34, 16, 3, 104, 27, 1]","[76, 96, 83, 69, 104, 35, 15, 100, 33, 44, 91, 58, 10, 9, 29, 86, 70, 125, 112, 13, 108, 53, 65, 72, 66, 26, 117, 127, 126, 1, 120, 49, 109, 88, 43, 111, 39, 3, 123, 64, 54, 23, 77, 118, 30, 14, 12, 106, 37, 6, 19, 27, 48, 36, 63, 121, 74, 119, 22, 84, 50, 8, 107, 28]","[1, 3, 6, 9, 10, 15, 22, 23, 27, 28, 29, 30, 33, 35, 36, 37, 39, 43, 44, 50, 53, 54, 65, 69, 72, 76, 77, 83, 86, 96, 104, 106, 108, 121, 123, 125]"
|
|
||||||
40,"[34, 28, 122, 54, 84, 37, 59, 107, 5, 99, 80, 20, 73, 91, 88, 21, 50, 76, 1, 15, 100, 108, 78, 109, 7, 36, 6, 112, 114, 2, 16, 85, 118, 77, 49, 62, 66, 63, 95, 60, 19, 56, 105, 104, 57, 117, 17, 113, 92, 68, 102, 86, 23, 127, 11, 35, 69, 43, 74, 61, 3, 126, 47, 98]","[18, 57, 70, 79, 95, 24, 78, 111, 4, 9, 123, 110, 115, 90, 10, 125, 65, 32, 118, 8, 37, 83, 121, 3, 53, 116, 106, 40, 22, 98, 7, 81, 0, 21, 5, 97, 33, 46, 113, 12, 124, 119, 87, 29, 50, 2, 31, 103, 122, 41, 94, 76, 27, 99, 48, 45, 107, 114, 67, 96, 38, 91, 105, 80]","[2, 3, 5, 7, 21, 37, 50, 57, 76, 78, 80, 91, 95, 98, 99, 105, 107, 113, 114, 118, 122]"
|
|
||||||
41,"[76, 23, 20, 62, 38, 0, 32, 104, 80, 34, 58, 106, 36, 87, 15, 73, 44, 117, 40, 124, 35, 46, 114, 100, 53, 10, 66, 79, 88, 37, 31, 121, 48, 28, 122, 16, 63, 26, 72, 1, 126, 45, 4, 22, 9, 71, 5, 110, 118, 61, 13, 54, 78, 93, 30, 39, 103, 33, 41, 82, 113, 55, 68, 83]","[120, 125, 91, 103, 37, 0, 38, 15, 30, 23, 13, 117, 46, 7, 44, 66, 106, 20, 123, 29, 71, 74, 99, 77, 27, 54, 87, 126, 19, 124, 34, 85, 2, 48, 40, 42, 33, 93, 83, 80, 26, 57, 69, 100, 114, 45, 36, 14, 105, 9, 82, 122, 89, 58, 51, 59, 22, 81, 32, 62, 78, 24, 60, 10]","[0, 9, 10, 13, 15, 20, 22, 23, 26, 30, 32, 33, 34, 36, 37, 38, 40, 44, 45, 46, 48, 54, 58, 62, 66, 71, 78, 80, 82, 83, 87, 93, 100, 103, 106, 114, 117, 122, 124, 126]"
|
|
||||||
42,"[64, 26, 78, 51, 2, 113, 105, 6, 47, 79, 60, 18, 36, 7, 110, 4, 72, 32, 73, 27, 88, 45, 104, 14, 118, 123, 91, 121, 42, 68, 98, 111, 66, 106, 9, 10, 43, 52, 48, 102, 55, 95, 74, 5, 54, 56, 115, 114, 58, 120, 38, 37, 112, 99, 39, 62, 22, 20, 125, 81, 28, 82, 94, 71]","[85, 56, 80, 10, 91, 36, 126, 21, 18, 27, 86, 34, 121, 23, 72, 92, 50, 106, 70, 62, 12, 93, 7, 66, 52, 22, 81, 14, 47, 101, 64, 61, 90, 28, 41, 40, 99, 111, 74, 98, 94, 63, 120, 33, 45, 122, 123, 48, 82, 32, 0, 42, 57, 96, 110, 31, 71, 39, 44, 87, 5, 26, 54, 102]","[5, 7, 10, 14, 18, 22, 26, 27, 28, 32, 36, 39, 42, 45, 47, 48, 52, 54, 56, 62, 64, 66, 71, 72, 74, 81, 82, 91, 94, 98, 99, 102, 106, 110, 111, 120, 121, 123]"
|
|
||||||
43,"[5, 8, 7, 105, 3, 17, 46, 1, 104, 82, 118, 106, 91, 54, 102, 0, 39, 38, 19, 88, 94, 25, 85, 55, 113, 14, 18, 52, 35, 76, 57, 112, 9, 124, 123, 29, 80, 97, 33, 89, 78, 43, 22, 109, 126, 10, 34, 11, 108, 12, 96, 79, 121, 125, 86, 50, 92, 119, 42, 13, 58, 110, 60, 56]","[35, 73, 112, 30, 21, 77, 17, 88, 90, 14, 107, 113, 72, 44, 117, 31, 1, 105, 79, 47, 124, 10, 53, 28, 5, 96, 33, 119, 18, 66, 7, 83, 114, 23, 101, 13, 63, 74, 100, 85, 25, 115, 84, 103, 45, 99, 16, 2, 46, 62, 69, 81, 38, 41, 64, 126, 0, 65, 12, 19, 43, 49, 111, 67]","[0, 1, 5, 7, 10, 12, 13, 14, 17, 18, 19, 25, 33, 35, 38, 43, 46, 79, 85, 88, 96, 105, 112, 113, 119, 124, 126]"
|
|
||||||
44,"[26, 109, 77, 7, 110, 114, 125, 38, 98, 90, 57, 68, 72, 18, 13, 29, 82, 118, 30, 42, 31, 75, 21, 4, 37, 65, 56, 35, 8, 100, 105, 113, 5, 78, 70, 87, 92, 67, 59, 25, 9, 16, 79, 46, 6, 47, 55, 23, 44, 66, 49, 1, 20, 64, 40, 28, 48, 34, 115, 97, 85, 91, 22, 95]","[27, 117, 112, 38, 121, 98, 123, 43, 26, 88, 94, 35, 41, 29, 122, 116, 106, 73, 93, 36, 104, 126, 47, 12, 80, 58, 60, 84, 72, 18, 46, 32, 83, 119, 95, 52, 7, 53, 65, 127, 89, 86, 50, 109, 33, 39, 90, 107, 19, 111, 71, 61, 14, 96, 124, 2, 100, 118, 45, 15, 97, 17, 51, 34]","[7, 18, 26, 29, 34, 35, 38, 46, 47, 65, 72, 90, 95, 97, 98, 100, 109, 118]"
|
|
||||||
45,"[82, 6, 108, 31, 55, 89, 75, 40, 96, 45, 118, 44, 18, 116, 125, 81, 59, 65, 28, 14, 0, 117, 98, 91, 12, 35, 103, 120, 24, 99, 84, 43, 101, 66, 11, 5, 27, 53, 88, 104, 64, 62, 105, 70, 54, 2, 113, 50, 74, 49, 67, 114, 22, 1, 21, 85, 29, 77, 87, 63, 26, 111, 46, 79]","[35, 73, 18, 11, 5, 40, 113, 95, 2, 122, 103, 4, 104, 87, 58, 28, 125, 47, 118, 97, 85, 52, 124, 1, 72, 108, 44, 26, 55, 50, 81, 64, 117, 78, 92, 37, 76, 43, 66, 0, 98, 63, 91, 12, 75, 27, 53, 62, 99, 33, 45, 14, 20, 77, 105, 89, 101, 65, 84, 82, 49, 59, 21, 102]","[0, 1, 2, 5, 11, 12, 14, 18, 21, 26, 27, 28, 35, 40, 43, 44, 45, 49, 50, 53, 55, 59, 62, 63, 64, 65, 66, 75, 77, 81, 82, 84, 85, 87, 89, 91, 98, 99, 101, 103, 104, 105, 108, 113, 117, 118, 125]"
|
|
||||||
46,"[47, 75, 82, 0, 40, 46, 1, 6, 61, 66, 27, 55, 116, 71, 126, 12, 123, 69, 107, 49, 112, 28, 77, 120, 30, 88, 16, 91, 15, 21, 36, 124, 13, 81, 34, 70, 17, 68, 85, 22, 95, 53, 51, 39, 80, 5, 7, 106, 89, 19, 60, 44, 72, 25, 9, 64, 97, 42, 58, 35, 114, 57, 115, 59]","[71, 54, 96, 34, 27, 81, 0, 68, 63, 18, 20, 58, 44, 13, 113, 21, 117, 72, 127, 35, 75, 78, 89, 86, 17, 97, 110, 47, 24, 41, 12, 65, 92, 73, 74, 60, 120, 3, 7, 93, 108, 111, 26, 1, 48, 50, 116, 123, 14, 126, 80, 107, 38, 4, 98, 16, 46, 100, 90, 105, 11, 52, 31, 114]","[0, 1, 7, 12, 13, 16, 17, 21, 27, 34, 35, 44, 46, 47, 58, 60, 68, 71, 72, 75, 80, 81, 89, 97, 107, 114, 116, 120, 123, 126]"
|
|
||||||
47,"[17, 23, 27, 1, 67, 111, 69, 112, 19, 100, 45, 37, 21, 29, 14, 9, 116, 30, 85, 54, 117, 75, 108, 79, 11, 4, 33, 109, 114, 102, 90, 127, 55, 2, 34, 68, 74, 92, 97, 120, 47, 32, 78, 16, 8, 91, 6, 26, 89, 86, 76, 94, 15, 77, 13, 80, 18, 52, 25, 115, 64, 98, 124, 87]","[127, 65, 21, 48, 19, 20, 84, 1, 123, 87, 103, 33, 125, 79, 16, 105, 27, 64, 94, 109, 117, 23, 69, 25, 9, 45, 88, 90, 47, 5, 54, 74, 116, 102, 98, 12, 80, 37, 4, 106, 8, 96, 115, 18, 67, 95, 70, 60, 14, 89, 52, 26, 39, 34, 86, 44, 43, 112, 114, 29, 120, 75, 13, 72]","[1, 4, 8, 9, 13, 14, 16, 18, 19, 21, 23, 25, 26, 27, 29, 33, 34, 37, 45, 47, 52, 54, 64, 67, 69, 74, 75, 79, 80, 86, 87, 89, 90, 94, 98, 102, 109, 112, 114, 115, 116, 117, 120, 127]"
|
|
||||||
48,"[70, 41, 29, 23, 78, 32, 79, 125, 95, 63, 112, 39, 14, 100, 51, 20, 94, 98, 97, 52, 5, 73, 1, 75, 103, 62, 71, 88, 67, 69, 49, 127, 25, 99, 64, 17, 19, 45, 44, 80, 85, 89, 114, 34, 3, 50, 55, 12, 111, 9, 7, 38, 90, 53, 28, 82, 46, 36, 24, 65, 108, 119, 110, 40]","[103, 116, 24, 105, 82, 114, 56, 88, 39, 74, 63, 104, 36, 99, 32, 22, 61, 71, 51, 26, 72, 79, 57, 41, 43, 29, 83, 54, 62, 5, 106, 94, 64, 37, 110, 49, 98, 127, 112, 91, 3, 9, 84, 76, 10, 77, 8, 60, 34, 40, 78, 93, 44, 2, 102, 12, 45, 87, 1, 118, 19, 53, 125, 97]","[1, 3, 5, 9, 12, 19, 24, 29, 32, 34, 36, 39, 40, 41, 44, 45, 49, 51, 53, 62, 63, 64, 71, 78, 79, 82, 88, 94, 97, 98, 99, 103, 110, 112, 114, 125, 127]"
|
|
||||||
49,"[94, 92, 8, 3, 95, 14, 49, 69, 44, 105, 77, 74, 23, 81, 36, 35, 126, 127, 96, 123, 76, 62, 72, 104, 118, 65, 28, 70, 102, 4, 54, 0, 10, 12, 98, 122, 60, 108, 33, 84, 40, 87, 110, 43, 15, 97, 29, 16, 93, 83, 18, 11, 99, 64, 34, 75, 48, 53, 86, 125, 85, 116, 90, 66]","[121, 40, 44, 52, 94, 72, 76, 105, 60, 100, 104, 106, 83, 65, 91, 110, 95, 122, 20, 10, 99, 54, 57, 1, 15, 18, 66, 12, 14, 126, 28, 42, 3, 127, 56, 125, 88, 77, 6, 48, 5, 101, 17, 37, 113, 108, 120, 43, 67, 68, 51, 7, 11, 33, 29, 32, 8, 13, 90, 35, 23, 118, 34, 85]","[3, 8, 10, 11, 12, 14, 15, 18, 23, 28, 29, 33, 34, 35, 40, 43, 44, 48, 54, 60, 65, 66, 72, 76, 77, 83, 85, 90, 94, 95, 99, 104, 105, 108, 110, 118, 122, 125, 126, 127]"
|
|
||||||
50,"[49, 34, 40, 27, 1, 57, 46, 122, 73, 93, 77, 58, 42, 0, 36, 109, 96, 115, 98, 79, 8, 126, 68, 97, 102, 92, 60, 78, 24, 59, 26, 80, 100, 101, 85, 6, 39, 44, 94, 52, 56, 43, 107, 28, 114, 76, 11, 21, 65, 75, 63, 10, 104, 95, 23, 62, 103, 14, 16, 3, 118, 125, 5, 123]","[108, 43, 105, 23, 36, 26, 114, 104, 62, 6, 24, 92, 1, 7, 83, 5, 89, 28, 107, 42, 34, 110, 39, 64, 122, 87, 53, 55, 68, 40, 52, 106, 2, 78, 103, 56, 111, 98, 85, 79, 124, 60, 116, 15, 119, 27, 3, 54, 47, 126, 118, 65, 113, 100, 125, 49, 57, 9, 46, 32, 38, 123, 67, 95]","[1, 3, 5, 6, 23, 24, 26, 27, 28, 34, 36, 39, 40, 42, 43, 46, 49, 52, 56, 57, 60, 62, 65, 68, 78, 79, 85, 92, 95, 98, 100, 103, 104, 107, 114, 118, 122, 123, 125, 126]"
|
|
||||||
51,"[24, 41, 100, 35, 125, 3, 107, 51, 80, 82, 123, 71, 93, 47, 13, 14, 55, 30, 86, 64, 70, 122, 67, 38, 88, 8, 99, 117, 76, 89, 26, 29, 27, 39, 45, 102, 0, 53, 124, 7, 31, 79, 56, 23, 52, 75, 20, 42, 101, 108, 43, 33, 95, 69, 61, 98, 78, 115, 54, 40, 105, 87, 19, 44]","[1, 13, 122, 37, 35, 91, 54, 6, 89, 71, 50, 58, 16, 104, 116, 5, 25, 0, 95, 73, 2, 126, 110, 97, 103, 12, 90, 117, 72, 17, 53, 3, 43, 60, 68, 11, 109, 107, 105, 4, 62, 65, 34, 46, 81, 82, 40, 84, 106, 75, 74, 78, 9, 45, 15, 123, 111, 48, 41, 119, 83, 125, 28, 96]","[0, 3, 13, 35, 40, 41, 43, 45, 53, 54, 71, 75, 78, 82, 89, 95, 105, 107, 117, 122, 123, 125]"
|
|
||||||
52,"[89, 51, 33, 37, 49, 0, 84, 4, 65, 87, 125, 67, 90, 7, 19, 68, 11, 17, 46, 106, 39, 82, 62, 126, 120, 72, 45, 86, 93, 16, 50, 54, 105, 104, 55, 36, 58, 73, 43, 57, 103, 94, 76, 75, 70, 64, 127, 85, 53, 10, 35, 107, 31, 113, 47, 24, 124, 32, 108, 26, 60, 42, 74, 80]","[96, 34, 10, 67, 6, 86, 87, 82, 78, 15, 54, 59, 75, 23, 89, 88, 28, 99, 107, 27, 69, 94, 70, 120, 29, 46, 63, 71, 47, 83, 52, 122, 115, 7, 36, 13, 119, 116, 18, 112, 93, 80, 55, 8, 2, 51, 30, 9, 33, 60, 111, 38, 20, 21, 17, 35, 14, 81, 5, 101, 26, 19, 25, 39]","[7, 10, 17, 19, 26, 33, 35, 36, 39, 46, 47, 51, 54, 55, 60, 67, 70, 75, 80, 82, 86, 87, 89, 93, 94, 107, 120]"
|
|
||||||
53,"[115, 62, 77, 59, 8, 48, 85, 117, 63, 79, 34, 32, 3, 49, 105, 33, 23, 52, 57, 96, 121, 94, 90, 61, 27, 92, 30, 4, 64, 81, 20, 29, 106, 10, 98, 88, 60, 2, 45, 1, 46, 56, 26, 107, 19, 25, 82, 97, 36, 103, 100, 16, 110, 28, 70, 114, 80, 41, 84, 122, 53, 65, 76, 11]","[126, 16, 120, 111, 30, 86, 41, 42, 71, 7, 3, 74, 88, 89, 54, 67, 14, 121, 100, 73, 26, 77, 2, 103, 81, 83, 55, 6, 33, 13, 40, 69, 17, 85, 123, 80, 118, 29, 92, 1, 50, 39, 72, 84, 45, 46, 87, 4, 75, 34, 57, 95, 32, 117, 15, 76, 109, 66, 99, 61, 27, 112, 98, 104]","[1, 2, 3, 4, 16, 26, 27, 29, 30, 32, 33, 34, 41, 45, 46, 57, 61, 76, 77, 80, 81, 84, 85, 88, 92, 98, 100, 103, 117, 121]"
|
|
||||||
54,"[60, 111, 21, 57, 87, 100, 80, 20, 40, 63, 56, 90, 8, 76, 108, 94, 82, 73, 121, 45, 97, 65, 29, 34, 43, 114, 127, 72, 35, 4, 12, 125, 1, 123, 109, 105, 31, 5, 91, 124, 0, 99, 103, 78, 26, 6, 101, 92, 52, 10, 59, 41, 18, 36, 55, 96, 13, 42, 49, 28, 23, 66, 38, 2]","[90, 79, 63, 78, 29, 76, 106, 16, 103, 38, 48, 64, 82, 89, 73, 61, 60, 62, 21, 26, 18, 43, 2, 22, 52, 95, 70, 56, 102, 32, 33, 114, 53, 4, 111, 101, 15, 20, 113, 30, 3, 84, 44, 110, 119, 35, 68, 7, 91, 118, 57, 31, 55, 112, 14, 75, 0, 77, 121, 115, 46, 105, 104, 108]","[0, 2, 4, 18, 20, 21, 26, 29, 31, 35, 38, 43, 52, 55, 56, 57, 60, 63, 73, 76, 78, 82, 90, 91, 101, 103, 105, 108, 111, 114, 121]"
|
|
||||||
55,"[54, 42, 53, 36, 27, 101, 62, 3, 32, 20, 93, 122, 12, 69, 97, 46, 66, 5, 47, 19, 48, 10, 22, 6, 116, 91, 35, 63, 75, 45, 96, 14, 82, 59, 72, 102, 33, 76, 15, 23, 99, 4, 65, 52, 81, 114, 113, 94, 1, 43, 80, 24, 104, 31, 95, 49, 44, 120, 51, 118, 103, 106, 86, 8]","[86, 59, 83, 27, 13, 21, 85, 100, 98, 70, 18, 46, 0, 78, 25, 88, 19, 115, 10, 87, 2, 64, 117, 84, 121, 80, 33, 20, 65, 60, 110, 31, 125, 92, 63, 109, 57, 68, 74, 123, 67, 34, 50, 73, 41, 40, 42, 38, 56, 124, 37, 1, 29, 111, 119, 126, 91, 77, 61, 36, 6, 107, 17, 58]","[1, 6, 10, 19, 20, 27, 31, 33, 36, 42, 46, 59, 63, 65, 80, 86, 91]"
|
|
||||||
56,"[41, 9, 111, 20, 69, 47, 17, 35, 26, 25, 89, 93, 2, 37, 22, 118, 84, 5, 21, 126, 100, 60, 107, 104, 33, 1, 63, 86, 46, 12, 39, 29, 80, 3, 0, 77, 6, 108, 59, 76, 55, 30, 27, 14, 48, 4, 28, 53, 127, 11, 52, 58, 67, 92, 44, 122, 23, 51, 98, 94, 49, 90, 32, 31]","[92, 116, 84, 72, 88, 100, 94, 64, 52, 98, 122, 58, 106, 78, 112, 8, 111, 67, 27, 45, 2, 90, 125, 5, 50, 6, 42, 66, 39, 53, 16, 113, 69, 17, 123, 102, 93, 7, 13, 127, 68, 105, 91, 119, 37, 49, 3, 28, 20, 55, 35, 71, 99, 124, 121, 54, 70, 47, 120, 26, 126, 75, 25, 32]","[2, 3, 5, 6, 17, 20, 25, 26, 27, 28, 32, 35, 37, 39, 47, 49, 52, 53, 55, 58, 67, 69, 84, 90, 92, 93, 94, 98, 100, 111, 122, 126, 127]"
|
|
||||||
57,"[8, 71, 59, 118, 45, 96, 108, 7, 30, 73, 11, 92, 50, 28, 70, 19, 110, 0, 26, 53, 106, 49, 25, 88, 10, 121, 67, 1, 105, 61, 65, 4, 74, 86, 46, 37, 54, 16, 34, 38, 52, 100, 29, 39, 85, 27, 6, 94, 77, 107, 98, 33, 66, 40, 55, 14, 58, 44, 103, 2, 20, 112, 75, 114]","[95, 18, 101, 50, 108, 54, 126, 104, 62, 16, 76, 15, 99, 83, 115, 30, 3, 80, 7, 72, 125, 84, 117, 85, 28, 127, 79, 27, 56, 89, 78, 13, 2, 9, 68, 61, 22, 17, 5, 43, 40, 35, 102, 75, 121, 12, 63, 51, 23, 42, 109, 39, 21, 59, 49, 29, 81, 77, 48, 47, 73, 123, 113, 97]","[2, 7, 16, 27, 28, 29, 30, 39, 40, 49, 50, 54, 59, 61, 73, 75, 77, 85, 108, 121]"
|
|
||||||
58,"[32, 108, 121, 15, 20, 54, 115, 109, 19, 48, 9, 81, 57, 127, 27, 98, 60, 21, 47, 100, 76, 23, 126, 1, 36, 52, 2, 25, 66, 70, 31, 50, 46, 78, 14, 116, 45, 96, 92, 120, 93, 72, 4, 10, 17, 49, 103, 29, 82, 69, 102, 88, 125, 13, 110, 118, 18, 80, 62, 55, 39, 67, 94, 59]","[52, 74, 88, 111, 19, 116, 110, 78, 55, 121, 14, 73, 36, 104, 49, 86, 82, 102, 22, 124, 67, 51, 43, 10, 27, 123, 60, 79, 100, 127, 87, 2, 23, 98, 96, 44, 30, 24, 63, 89, 45, 90, 9, 108, 46, 68, 120, 64, 99, 26, 17, 91, 1, 32, 29, 21, 70, 56, 48, 62, 83, 103, 39, 57]","[1, 2, 9, 10, 14, 17, 19, 21, 23, 27, 29, 32, 36, 39, 45, 46, 48, 49, 52, 55, 57, 60, 62, 67, 70, 78, 82, 88, 96, 98, 100, 102, 103, 108, 110, 116, 120, 121, 127]"
|
|
||||||
59,"[115, 40, 65, 102, 105, 47, 27, 31, 121, 69, 54, 76, 67, 35, 15, 110, 118, 50, 32, 84, 42, 87, 108, 62, 9, 63, 94, 20, 106, 99, 86, 81, 122, 12, 89, 79, 68, 38, 7, 36, 75, 6, 55, 83, 48, 97, 95, 23, 1, 16, 93, 74, 70, 28, 77, 13, 72, 24, 92, 91, 5, 37, 57, 64]","[35, 50, 99, 76, 96, 42, 5, 71, 46, 12, 17, 28, 14, 6, 100, 87, 101, 27, 3, 37, 77, 48, 32, 52, 38, 80, 102, 49, 25, 118, 62, 31, 105, 59, 98, 20, 119, 18, 55, 45, 121, 24, 57, 56, 95, 54, 47, 73, 1, 70, 9, 117, 67, 41, 2, 33, 39, 92, 106, 90, 65, 88, 8, 108]","[1, 5, 6, 9, 12, 20, 24, 27, 28, 31, 32, 35, 37, 38, 42, 47, 48, 50, 54, 55, 57, 62, 65, 67, 70, 76, 77, 87, 92, 95, 99, 102, 105, 106, 108, 118, 121]"
|
|
||||||
60,"[49, 35, 9, 18, 108, 52, 69, 33, 29, 44, 80, 50, 94, 70, 64, 27, 32, 48, 59, 119, 92, 116, 78, 41, 76, 54, 19, 90, 61, 83, 13, 4, 42, 67, 101, 115, 22, 37, 102, 62, 11, 25, 85, 47, 56, 127, 55, 71, 2, 15, 122, 107, 100, 39, 30, 84, 38, 58, 34, 6, 43, 120, 72, 68]","[49, 114, 22, 29, 76, 94, 39, 4, 67, 16, 122, 36, 55, 47, 84, 74, 20, 6, 79, 100, 88, 44, 15, 14, 69, 119, 32, 64, 63, 52, 53, 72, 34, 102, 118, 57, 30, 11, 125, 35, 85, 21, 71, 83, 43, 18, 59, 40, 48, 50, 19, 60, 2, 37, 101, 86, 70, 98, 9, 93, 116, 58, 33, 61]","[2, 4, 6, 9, 11, 15, 18, 19, 22, 29, 30, 32, 33, 34, 35, 37, 39, 43, 44, 47, 48, 49, 50, 52, 55, 58, 59, 61, 64, 67, 69, 70, 71, 72, 76, 83, 84, 85, 94, 100, 101, 102, 116, 119, 122]"
|
|
||||||
61,"[87, 24, 107, 84, 36, 60, 94, 63, 57, 52, 82, 15, 13, 49, 114, 39, 0, 89, 74, 78, 44, 5, 102, 9, 28, 80, 90, 59, 27, 53, 73, 123, 124, 33, 119, 7, 45, 2, 50, 108, 75, 37, 69, 62, 66, 64, 79, 4, 14, 76, 29, 25, 101, 40, 20, 111, 51, 35, 121, 103, 96, 47, 58, 126]","[92, 88, 64, 83, 14, 33, 79, 44, 126, 119, 75, 11, 56, 60, 20, 69, 13, 118, 110, 63, 114, 102, 29, 74, 7, 113, 41, 97, 34, 2, 58, 89, 23, 82, 78, 87, 100, 19, 52, 57, 53, 77, 40, 5, 15, 107, 47, 105, 98, 67, 108, 27, 35, 54, 59, 55, 127, 91, 94, 16, 9, 18, 25, 8]","[2, 5, 7, 9, 13, 14, 15, 20, 25, 27, 29, 33, 35, 40, 44, 47, 52, 53, 57, 58, 59, 60, 63, 64, 69, 74, 75, 78, 79, 82, 87, 89, 94, 102, 107, 108, 114, 119, 126]"
|
|
||||||
62,"[86, 117, 27, 89, 65, 115, 24, 12, 40, 62, 77, 88, 2, 97, 21, 110, 64, 107, 34, 76, 37, 69, 108, 91, 124, 51, 72, 7, 75, 30, 120, 106, 70, 18, 10, 95, 116, 50, 122, 71, 94, 61, 41, 56, 67, 6, 59, 42, 111, 126, 19, 16, 92, 105, 104, 43, 81, 17, 29, 23, 8, 102, 48, 103]","[117, 88, 63, 32, 105, 23, 19, 87, 36, 67, 118, 112, 17, 73, 123, 35, 4, 7, 94, 15, 18, 57, 25, 84, 3, 100, 43, 66, 83, 114, 98, 44, 46, 41, 42, 121, 20, 37, 62, 40, 8, 52, 29, 93, 31, 61, 6, 30, 49, 21, 71, 80, 54, 16, 53, 28, 47, 60, 70, 120, 38, 39, 34, 26]","[6, 7, 8, 16, 17, 18, 19, 21, 23, 29, 30, 34, 37, 40, 41, 42, 43, 61, 62, 67, 70, 71, 88, 94, 105, 117, 120]"
|
|
||||||
63,"[22, 70, 26, 109, 94, 0, 29, 116, 19, 57, 3, 113, 59, 6, 101, 31, 54, 64, 118, 28, 91, 110, 25, 41, 16, 17, 12, 44, 51, 126, 82, 5, 72, 115, 84, 43, 7, 68, 98, 120, 102, 71, 14, 21, 85, 10, 47, 74, 121, 81, 56, 20, 87, 48, 15, 37, 32, 36, 96, 49, 67, 42, 30, 46]","[64, 121, 120, 62, 71, 46, 68, 84, 56, 82, 115, 78, 34, 59, 49, 57, 54, 116, 90, 47, 48, 21, 110, 102, 119, 75, 126, 42, 2, 67, 92, 3, 25, 127, 12, 32, 89, 74, 73, 17, 5, 30, 99, 26, 51, 1, 20, 58, 28, 100, 70, 0, 18, 118, 109, 91, 6, 36, 16, 113, 72, 106, 10, 66]","[0, 3, 5, 6, 10, 12, 16, 17, 20, 21, 25, 26, 28, 30, 32, 36, 42, 46, 47, 48, 49, 51, 54, 56, 57, 59, 64, 67, 68, 70, 71, 72, 74, 82, 84, 91, 102, 109, 110, 113, 115, 116, 118, 120, 121, 126]"
|
|
||||||
64,"[4, 0, 113, 53, 97, 107, 15, 18, 84, 54, 82, 110, 74, 11, 13, 39, 92, 115, 87, 116, 28, 37, 106, 99, 67, 111, 90, 88, 21, 96, 51, 64, 47, 26, 36, 30, 5, 3, 78, 9, 32, 75, 98, 73, 14, 89, 71, 2, 27, 19, 77, 69, 12, 49, 20, 63, 59, 101, 85, 65, 120, 124, 80, 91]","[19, 11, 96, 106, 127, 39, 2, 121, 87, 18, 51, 59, 76, 82, 41, 123, 13, 79, 46, 57, 77, 40, 15, 60, 93, 48, 9, 63, 4, 70, 73, 10, 108, 116, 102, 122, 95, 24, 113, 21, 36, 20, 12, 94, 99, 29, 83, 104, 28, 114, 117, 86, 118, 50, 14, 88, 8, 16, 105, 109, 52, 27, 55, 61]","[2, 4, 9, 11, 12, 13, 14, 15, 18, 19, 20, 21, 27, 28, 36, 39, 51, 59, 63, 73, 77, 82, 87, 88, 96, 99, 106, 113, 116]"
|
|
||||||
65,"[22, 72, 41, 42, 8, 113, 92, 108, 11, 39, 99, 27, 53, 80, 28, 16, 13, 88, 76, 81, 46, 43, 3, 2, 26, 40, 12, 0, 115, 10, 50, 117, 52, 44, 25, 111, 107, 96, 85, 78, 63, 23, 89, 59, 18, 124, 32, 126, 90, 84, 101, 122, 20, 34, 60, 93, 98, 69, 51, 6, 95, 21, 64, 97]","[62, 93, 117, 8, 61, 114, 98, 57, 16, 1, 47, 119, 99, 121, 32, 11, 66, 92, 110, 38, 53, 113, 77, 44, 85, 112, 28, 25, 17, 107, 71, 73, 68, 67, 116, 4, 56, 19, 120, 87, 29, 31, 7, 127, 50, 79, 72, 75, 27, 83, 21, 24, 109, 108, 80, 118, 26, 34, 58, 9, 86, 0, 22, 94]","[0, 8, 11, 16, 21, 22, 25, 26, 27, 28, 32, 34, 44, 50, 53, 72, 80, 85, 92, 93, 98, 99, 107, 108, 113, 117]"
|
|
||||||
66,"[42, 117, 88, 35, 1, 25, 45, 85, 123, 105, 27, 54, 74, 46, 47, 24, 40, 36, 52, 29, 30, 53, 93, 122, 99, 39, 98, 63, 121, 50, 94, 72, 89, 84, 70, 2, 126, 112, 75, 69, 34, 95, 37, 71, 125, 66, 118, 67, 12, 33, 62, 101, 115, 65, 13, 55, 17, 14, 51, 76, 10, 120, 92, 59]","[74, 27, 85, 12, 125, 5, 96, 118, 104, 63, 40, 81, 57, 71, 69, 112, 92, 30, 65, 107, 75, 121, 99, 76, 1, 116, 39, 32, 115, 38, 51, 16, 17, 67, 25, 109, 59, 10, 28, 89, 33, 58, 42, 61, 13, 82, 119, 11, 54, 14, 45, 77, 34, 44, 24, 37, 46, 36, 70, 84, 80, 101, 29, 35]","[1, 10, 12, 13, 14, 17, 24, 25, 27, 29, 30, 33, 34, 35, 36, 37, 39, 40, 42, 45, 46, 51, 54, 59, 63, 65, 67, 69, 70, 71, 74, 75, 76, 84, 85, 89, 92, 99, 101, 112, 115, 118, 121, 125]"
|
|
||||||
67,"[87, 120, 29, 81, 48, 4, 89, 60, 106, 44, 76, 119, 125, 68, 55, 95, 69, 1, 78, 50, 56, 59, 114, 54, 5, 13, 108, 2, 115, 98, 28, 94, 20, 118, 116, 21, 45, 80, 117, 25, 65, 27, 74, 110, 58, 96, 77, 23, 16, 52, 93, 26, 71, 17, 22, 83, 49, 79, 47, 46, 9, 123, 102, 91]","[30, 79, 52, 6, 16, 38, 72, 119, 17, 90, 63, 95, 82, 51, 76, 22, 48, 32, 88, 124, 126, 15, 41, 7, 122, 93, 21, 10, 68, 19, 106, 108, 65, 99, 89, 111, 84, 64, 46, 112, 85, 44, 8, 14, 103, 115, 75, 31, 57, 13, 1, 118, 83, 23, 55, 60, 113, 20, 40, 67, 39, 11, 18, 107]","[1, 13, 16, 17, 20, 21, 22, 23, 44, 46, 48, 52, 55, 60, 65, 68, 76, 79, 83, 89, 93, 95, 106, 108, 115, 118, 119]"
|
|
||||||
68,"[33, 92, 85, 97, 21, 116, 15, 29, 24, 9, 42, 58, 72, 73, 5, 70, 25, 90, 108, 18, 59, 91, 66, 67, 19, 96, 88, 3, 99, 62, 71, 44, 68, 114, 89, 74, 117, 121, 107, 60, 53, 51, 27, 76, 34, 17, 122, 82, 1, 50, 106, 46, 4, 119, 95, 54, 75, 112, 52, 37, 13, 100, 28, 45]","[17, 23, 62, 9, 120, 22, 55, 61, 10, 105, 77, 49, 78, 84, 89, 36, 70, 124, 48, 79, 106, 101, 32, 34, 38, 109, 8, 66, 81, 97, 63, 56, 47, 42, 126, 93, 69, 7, 113, 43, 39, 87, 21, 115, 6, 0, 20, 123, 25, 86, 104, 102, 11, 33, 50, 98, 16, 24, 112, 12, 65, 28, 76, 80]","[9, 17, 21, 24, 25, 28, 33, 34, 42, 50, 62, 66, 70, 76, 89, 97, 106, 112]"
|
|
||||||
69,"[28, 3, 77, 42, 16, 67, 63, 24, 78, 23, 109, 45, 118, 7, 88, 30, 43, 76, 59, 21, 99, 37, 52, 25, 5, 19, 89, 115, 35, 95, 93, 13, 2, 121, 57, 80, 8, 17, 101, 98, 1, 106, 9, 96, 82, 111, 10, 22, 107, 113, 20, 38, 56, 79, 36, 91, 74, 31, 126, 29, 64, 71, 55, 60]","[120, 48, 73, 49, 45, 40, 82, 21, 7, 34, 31, 109, 41, 72, 83, 22, 47, 118, 99, 74, 100, 101, 54, 106, 116, 127, 123, 124, 75, 50, 44, 112, 96, 108, 76, 0, 10, 29, 94, 3, 11, 58, 39, 8, 92, 59, 30, 51, 37, 16, 9, 97, 28, 86, 2, 24, 55, 19, 122, 105, 67, 1, 63, 102]","[1, 2, 3, 7, 8, 9, 10, 16, 19, 21, 22, 24, 28, 29, 30, 31, 37, 45, 55, 59, 63, 67, 74, 76, 82, 96, 99, 101, 106, 109, 118]"
|
|
||||||
70,"[22, 105, 56, 122, 33, 11, 121, 42, 58, 110, 66, 7, 46, 104, 123, 99, 108, 52, 76, 92, 115, 124, 28, 75, 38, 119, 80, 67, 81, 47, 17, 49, 39, 71, 101, 82, 94, 68, 117, 78, 50, 69, 8, 21, 83, 62, 126, 61, 16, 51, 30, 10, 116, 98, 91, 114, 113, 34, 20, 73, 118, 63, 0, 35]","[39, 72, 40, 105, 126, 25, 93, 120, 29, 81, 7, 99, 102, 19, 97, 50, 90, 121, 57, 18, 41, 22, 86, 51, 95, 11, 8, 104, 80, 34, 92, 17, 61, 84, 124, 52, 33, 58, 116, 122, 38, 68, 0, 83, 44, 30, 49, 66, 73, 13, 27, 56, 28, 70, 101, 78, 62, 94, 53, 69, 63, 125, 67, 24]","[0, 7, 8, 11, 17, 22, 28, 30, 33, 34, 38, 39, 49, 50, 51, 52, 56, 58, 61, 62, 63, 66, 67, 68, 69, 73, 78, 80, 81, 83, 92, 94, 99, 101, 104, 105, 116, 121, 122, 124, 126]"
|
|
||||||
71,"[46, 93, 57, 66, 9, 83, 126, 120, 95, 16, 118, 42, 90, 37, 39, 96, 85, 31, 3, 17, 10, 45, 114, 100, 58, 87, 116, 106, 78, 113, 89, 51, 19, 86, 111, 105, 65, 53, 59, 55, 88, 84, 44, 82, 68, 27, 121, 11, 26, 108, 63, 30, 48, 12, 110, 15, 77, 29, 73, 20, 56, 119, 8, 32]","[23, 30, 37, 57, 65, 47, 106, 50, 25, 15, 49, 28, 77, 22, 43, 127, 12, 39, 112, 81, 52, 90, 2, 36, 35, 62, 115, 101, 109, 21, 103, 97, 92, 80, 104, 19, 67, 69, 29, 96, 86, 122, 117, 73, 125, 120, 99, 83, 20, 60, 3, 33, 61, 91, 38, 89, 26, 87, 123, 0, 6, 41, 14, 72]","[3, 12, 15, 19, 20, 26, 29, 30, 37, 39, 57, 65, 73, 77, 83, 86, 87, 89, 90, 96, 106, 120]"
|
|
||||||
72,"[37, 123, 98, 88, 11, 6, 14, 25, 23, 79, 59, 8, 18, 74, 110, 119, 109, 118, 7, 117, 78, 27, 56, 81, 101, 1, 47, 32, 48, 77, 46, 70, 52, 96, 116, 40, 71, 122, 85, 26, 2, 112, 125, 49, 42, 10, 36, 13, 22, 102, 68, 38, 19, 73, 89, 45, 62, 43, 55, 35, 121, 58, 114, 24]","[10, 90, 88, 104, 107, 116, 8, 81, 42, 109, 125, 56, 57, 89, 26, 92, 21, 28, 99, 87, 41, 82, 79, 86, 4, 18, 72, 123, 80, 91, 39, 50, 100, 32, 40, 46, 33, 3, 69, 30, 84, 115, 62, 74, 117, 127, 108, 35, 55, 101, 63, 96, 0, 53, 60, 25, 75, 49, 95, 27, 23, 94, 6, 98]","[6, 8, 10, 18, 23, 25, 26, 27, 32, 35, 40, 42, 46, 49, 55, 56, 62, 74, 79, 81, 88, 89, 96, 98, 101, 109, 116, 117, 123, 125]"
|
|
||||||
73,"[17, 89, 0, 40, 25, 32, 54, 114, 59, 39, 49, 3, 100, 85, 78, 84, 44, 81, 69, 123, 7, 13, 1, 43, 103, 30, 27, 127, 122, 53, 4, 28, 99, 126, 110, 46, 64, 11, 94, 41, 58, 5, 125, 113, 111, 77, 36, 104, 42, 62, 10, 63, 72, 8, 90, 73, 61, 22, 109, 79, 82, 51, 6, 118]","[96, 9, 18, 83, 81, 22, 84, 98, 38, 122, 35, 27, 29, 36, 106, 10, 37, 66, 21, 76, 56, 57, 101, 16, 53, 55, 59, 24, 60, 74, 12, 8, 95, 92, 67, 121, 97, 80, 126, 11, 87, 34, 25, 104, 33, 113, 105, 2, 116, 124, 119, 70, 71, 72, 39, 115, 93, 100, 88, 31, 107, 117, 85, 3]","[3, 8, 10, 11, 22, 25, 27, 36, 39, 53, 59, 72, 81, 84, 85, 100, 104, 113, 122, 126]"
|
|
||||||
74,"[74, 110, 68, 46, 84, 86, 98, 6, 10, 23, 116, 48, 117, 59, 31, 109, 36, 51, 55, 27, 29, 14, 20, 40, 124, 106, 47, 9, 2, 42, 71, 26, 92, 11, 44, 115, 111, 24, 101, 50, 70, 28, 17, 65, 76, 78, 81, 89, 94, 1, 122, 121, 39, 72, 45, 100, 112, 102, 62, 113, 126, 18, 38, 88]","[29, 98, 46, 11, 15, 24, 7, 42, 117, 106, 99, 60, 78, 108, 116, 72, 18, 110, 123, 20, 45, 31, 121, 5, 37, 23, 68, 69, 113, 94, 71, 38, 12, 61, 51, 120, 82, 127, 81, 102, 104, 16, 6, 115, 95, 28, 84, 59, 10, 73, 70, 80, 79, 32, 96, 22, 50, 25, 47, 49, 107, 26, 13, 27]","[6, 10, 11, 18, 20, 23, 24, 26, 27, 28, 29, 31, 38, 42, 45, 46, 47, 50, 51, 59, 68, 70, 71, 72, 78, 81, 84, 94, 98, 102, 106, 110, 113, 115, 116, 117, 121]"
|
|
||||||
75,"[82, 37, 121, 60, 8, 56, 98, 39, 14, 112, 87, 119, 85, 71, 36, 33, 11, 22, 81, 123, 83, 91, 99, 43, 126, 42, 9, 49, 64, 4, 68, 72, 84, 26, 78, 10, 5, 27, 76, 51, 12, 93, 48, 38, 75, 20, 25, 63, 124, 62, 21, 73, 47, 100, 46, 107, 103, 23, 28, 34, 61, 104, 53, 52]","[10, 23, 9, 119, 33, 31, 69, 78, 27, 13, 29, 63, 56, 26, 24, 60, 120, 6, 77, 99, 8, 86, 0, 80, 40, 50, 57, 114, 55, 42, 65, 94, 87, 82, 54, 30, 97, 122, 21, 38, 49, 102, 2, 104, 44, 52, 20, 4, 47, 39, 118, 7, 83, 109, 15, 1, 116, 115, 75, 18, 46, 124, 11, 36]","[4, 8, 9, 10, 11, 20, 21, 23, 26, 27, 33, 36, 38, 39, 42, 46, 47, 49, 52, 56, 60, 63, 75, 78, 82, 83, 87, 99, 104, 119, 124]"
|
|
||||||
76,"[43, 28, 91, 48, 52, 30, 45, 77, 44, 40, 2, 26, 5, 82, 101, 11, 47, 67, 112, 41, 49, 114, 69, 32, 92, 39, 85, 35, 68, 86, 78, 65, 23, 60, 117, 56, 75, 38, 118, 71, 58, 120, 64, 124, 14, 62, 16, 51, 127, 96, 12, 66, 24, 76, 7, 74, 10, 21, 13, 61, 108, 18, 27, 95]","[107, 57, 42, 11, 2, 45, 93, 7, 59, 83, 27, 112, 10, 21, 95, 26, 98, 104, 110, 60, 48, 18, 115, 71, 1, 3, 4, 116, 76, 75, 82, 25, 38, 100, 96, 28, 113, 47, 94, 5, 15, 24, 87, 55, 111, 105, 56, 6, 97, 92, 99, 80, 20, 117, 125, 37, 90, 119, 54, 123, 31, 50, 19, 0]","[2, 5, 7, 10, 11, 18, 21, 24, 26, 27, 28, 38, 45, 47, 48, 56, 60, 71, 75, 76, 82, 92, 95, 96, 112, 117]"
|
|
||||||
77,"[52, 40, 0, 107, 60, 91, 103, 81, 121, 85, 31, 72, 1, 80, 125, 63, 68, 109, 61, 76, 55, 22, 7, 111, 108, 93, 14, 39, 33, 15, 74, 12, 53, 106, 48, 71, 120, 46, 87, 100, 27, 102, 99, 126, 35, 62, 9, 38, 114, 64, 56, 86, 59, 84, 5, 36, 42, 25, 13, 66, 6, 49, 101, 50]","[84, 106, 72, 57, 82, 125, 34, 42, 19, 96, 89, 118, 103, 24, 23, 50, 124, 97, 17, 110, 22, 100, 107, 27, 9, 45, 43, 98, 47, 104, 109, 35, 16, 26, 60, 12, 90, 31, 69, 38, 11, 77, 95, 115, 5, 41, 44, 92, 21, 120, 25, 2, 105, 20, 78, 67, 123, 81, 83, 113, 51, 10, 122, 74]","[5, 9, 12, 22, 25, 27, 31, 35, 38, 42, 50, 60, 72, 74, 81, 84, 100, 103, 106, 107, 109, 120, 125]"
|
|
||||||
78,"[38, 19, 81, 65, 13, 100, 120, 33, 64, 66, 113, 63, 101, 107, 123, 46, 79, 109, 114, 7, 3, 91, 124, 35, 111, 60, 47, 68, 57, 4, 27, 110, 10, 5, 77, 62, 115, 117, 54, 84, 75, 44, 59, 72, 98, 83, 73, 92, 28, 82, 6, 99, 76, 58, 49, 88, 103, 9, 94, 22, 50, 40, 121, 21]","[108, 32, 116, 59, 4, 79, 104, 29, 97, 46, 100, 115, 98, 114, 83, 122, 20, 60, 7, 102, 15, 119, 35, 37, 63, 33, 21, 109, 62, 84, 85, 39, 31, 110, 44, 23, 72, 25, 61, 75, 28, 52, 55, 27, 89, 117, 17, 105, 86, 91, 53, 64, 40, 45, 6, 2, 80, 81, 47, 16, 54, 42, 120, 3]","[3, 4, 6, 7, 21, 27, 28, 33, 35, 40, 44, 46, 47, 54, 59, 60, 62, 63, 64, 72, 75, 79, 81, 83, 84, 91, 98, 100, 109, 110, 114, 115, 117, 120]"
|
|
||||||
79,"[53, 29, 12, 87, 104, 41, 63, 74, 73, 31, 92, 33, 69, 42, 46, 25, 19, 54, 101, 97, 52, 20, 39, 66, 7, 3, 55, 88, 32, 64, 28, 76, 40, 30, 44, 16, 102, 61, 113, 10, 95, 108, 37, 118, 116, 14, 77, 58, 91, 106, 62, 109, 18, 86, 59, 111, 78, 47, 21, 107, 49, 56, 115, 51]","[55, 59, 126, 89, 115, 105, 51, 107, 22, 108, 30, 100, 69, 40, 80, 91, 21, 66, 127, 12, 31, 24, 53, 7, 36, 8, 41, 117, 102, 85, 17, 120, 3, 68, 83, 38, 42, 43, 98, 58, 28, 99, 97, 9, 35, 103, 62, 44, 113, 118, 37, 49, 20, 106, 56, 10, 57, 114, 112, 104, 64, 82, 39, 73]","[3, 7, 10, 12, 20, 21, 28, 30, 31, 37, 39, 40, 41, 42, 44, 49, 51, 53, 55, 56, 58, 59, 62, 64, 66, 69, 73, 91, 97, 102, 104, 106, 107, 108, 113, 115, 118]"
|
|
||||||
80,"[16, 82, 98, 73, 97, 48, 8, 90, 104, 114, 120, 0, 34, 68, 38, 95, 46, 4, 74, 18, 6, 77, 111, 96, 50, 14, 3, 59, 28, 94, 62, 1, 9, 126, 30, 70, 78, 10, 39, 33, 81, 117, 84, 12, 122, 42, 118, 32, 103, 87, 85, 5, 112, 58, 123, 110, 109, 26, 54, 83, 105, 102, 31, 44]","[110, 4, 60, 31, 0, 28, 74, 59, 125, 48, 61, 87, 95, 54, 118, 94, 83, 45, 90, 38, 91, 73, 109, 78, 82, 51, 46, 2, 16, 85, 18, 10, 92, 56, 43, 111, 9, 70, 79, 8, 93, 3, 98, 50, 63, 53, 75, 96, 34, 104, 27, 126, 105, 84, 62, 39, 114, 42, 6, 113, 58, 29, 1, 44]","[0, 1, 3, 4, 6, 8, 9, 10, 16, 18, 28, 31, 34, 38, 39, 42, 44, 46, 48, 50, 54, 58, 59, 62, 70, 73, 74, 78, 82, 83, 84, 85, 87, 90, 94, 95, 96, 98, 104, 105, 109, 110, 111, 114, 118, 126]"
|
|
||||||
81,"[52, 46, 35, 22, 33, 103, 21, 19, 54, 40, 28, 27, 77, 127, 9, 44, 16, 39, 53, 29, 30, 65, 23, 70, 55, 118, 72, 6, 24, 0, 62, 32, 38, 69, 63, 47, 79, 67, 13, 20, 41, 66, 105, 64, 14, 112, 85, 25, 60, 100, 1, 59, 106, 113, 50, 17, 88, 82, 76, 80, 26, 92, 89, 49]","[5, 66, 94, 21, 15, 16, 85, 92, 47, 107, 105, 98, 77, 38, 99, 90, 14, 96, 45, 53, 35, 95, 42, 80, 32, 3, 9, 44, 57, 29, 91, 18, 59, 74, 33, 116, 119, 37, 4, 83, 97, 40, 52, 86, 121, 30, 76, 87, 36, 72, 54, 114, 126, 109, 17, 55, 62, 117, 102, 100, 78, 43, 101, 68]","[9, 14, 16, 17, 21, 29, 30, 32, 33, 35, 38, 40, 44, 47, 52, 53, 54, 55, 59, 62, 66, 72, 76, 77, 80, 85, 92, 100, 105]"
|
|
||||||
82,"[0, 9, 96, 11, 39, 70, 64, 105, 29, 80, 45, 82, 27, 117, 90, 46, 121, 99, 97, 42, 107, 66, 119, 108, 37, 26, 51, 8, 18, 3, 50, 47, 76, 106, 91, 24, 69, 58, 15, 12, 75, 49, 85, 7, 112, 118, 34, 55, 25, 30, 120, 38, 72, 5, 84, 104, 53, 102, 67, 101, 74, 62, 110, 71]","[27, 126, 16, 28, 50, 124, 117, 0, 7, 94, 88, 59, 18, 13, 81, 22, 25, 111, 23, 63, 47, 87, 41, 17, 20, 67, 127, 14, 61, 73, 15, 33, 85, 103, 105, 71, 56, 52, 57, 35, 83, 6, 82, 115, 86, 70, 1, 60, 21, 68, 4, 75, 26, 65, 116, 11, 9, 64, 36, 37, 100, 122, 93, 79]","[0, 7, 9, 11, 15, 18, 25, 26, 27, 37, 47, 50, 64, 67, 70, 71, 75, 82, 85, 105, 117]"
|
|
||||||
83,"[5, 3, 19, 26, 63, 92, 123, 86, 83, 121, 29, 60, 100, 0, 47, 61, 41, 126, 68, 94, 115, 111, 73, 93, 50, 10, 1, 64, 84, 37, 95, 116, 96, 51, 108, 15, 11, 62, 13, 55, 30, 57, 72, 8, 45, 104, 118, 23, 79, 103, 39, 90, 32, 78, 66, 122, 80, 88, 74, 124, 40, 54, 35, 76]","[63, 96, 39, 71, 15, 111, 87, 83, 95, 86, 54, 26, 122, 0, 22, 115, 66, 10, 107, 73, 58, 61, 91, 32, 90, 14, 34, 37, 44, 88, 93, 57, 70, 64, 40, 33, 74, 78, 46, 41, 84, 2, 82, 110, 77, 104, 5, 102, 3, 113, 125, 1, 116, 29, 9, 30, 59, 51, 72, 126, 108, 45, 120, 62]","[0, 1, 3, 5, 10, 15, 26, 29, 30, 32, 37, 39, 40, 41, 45, 51, 54, 57, 61, 62, 63, 64, 66, 72, 73, 74, 78, 83, 84, 86, 88, 90, 93, 95, 96, 104, 108, 111, 115, 116, 122, 126]"
|
|
||||||
84,"[110, 117, 67, 83, 25, 2, 27, 123, 127, 59, 57, 95, 68, 118, 73, 124, 50, 63, 71, 12, 8, 46, 106, 3, 7, 1, 16, 40, 49, 77, 55, 15, 94, 89, 125, 0, 108, 4, 80, 38, 10, 85, 79, 122, 9, 61, 11, 22, 105, 58, 18, 75, 92, 45, 69, 23, 62, 41, 26, 51, 24, 35, 52, 33]","[105, 12, 99, 67, 110, 4, 95, 8, 119, 46, 41, 103, 88, 73, 80, 111, 62, 7, 9, 24, 78, 10, 18, 21, 30, 123, 32, 19, 81, 116, 40, 36, 113, 27, 3, 37, 109, 55, 85, 1, 92, 26, 127, 106, 72, 101, 98, 38, 91, 44, 39, 22, 25, 104, 66, 75, 20, 45, 33, 120, 93, 34, 59, 108]","[1, 3, 4, 7, 8, 9, 10, 12, 18, 22, 24, 25, 26, 27, 33, 38, 40, 41, 45, 46, 55, 59, 62, 67, 73, 75, 80, 85, 92, 95, 105, 106, 108, 110, 123, 127]"
|
|
||||||
85,"[6, 67, 36, 125, 62, 56, 64, 7, 49, 103, 115, 52, 66, 13, 21, 93, 27, 110, 82, 58, 20, 16, 108, 17, 61, 2, 98, 121, 23, 92, 102, 107, 45, 78, 19, 120, 100, 112, 8, 40, 46, 55, 14, 122, 50, 124, 63, 3, 41, 90, 91, 15, 35, 9, 59, 42, 60, 76, 88, 39, 33, 0, 53, 80]","[19, 33, 82, 51, 103, 22, 64, 116, 52, 63, 127, 55, 72, 7, 100, 0, 20, 89, 3, 99, 113, 92, 115, 59, 40, 44, 28, 6, 101, 14, 80, 102, 11, 123, 56, 10, 76, 73, 50, 57, 43, 38, 18, 36, 15, 74, 16, 13, 37, 94, 24, 12, 84, 107, 61, 91, 39, 60, 62, 21, 49, 77, 122, 2]","[0, 2, 3, 6, 7, 13, 14, 15, 16, 19, 20, 21, 33, 36, 39, 40, 49, 50, 52, 55, 56, 59, 60, 61, 62, 63, 64, 76, 80, 82, 91, 92, 100, 102, 103, 107, 115, 122]"
|
|
||||||
86,"[10, 119, 45, 90, 12, 86, 66, 39, 99, 94, 97, 82, 52, 72, 120, 15, 65, 81, 30, 114, 76, 42, 61, 79, 26, 105, 27, 75, 123, 83, 98, 64, 11, 122, 6, 43, 8, 40, 41, 55, 23, 74, 58, 3, 31, 95, 13, 49, 17, 84, 77, 54, 51, 108, 89, 1, 0, 29, 63, 111, 22, 33, 24, 35]","[19, 100, 87, 102, 92, 73, 58, 112, 53, 35, 70, 52, 107, 119, 106, 48, 26, 85, 2, 47, 34, 14, 6, 117, 80, 4, 104, 12, 91, 13, 20, 56, 113, 96, 84, 18, 126, 38, 121, 60, 88, 114, 69, 46, 116, 97, 16, 71, 9, 124, 93, 120, 22, 109, 42, 29, 10, 44, 118, 63, 67, 15, 21, 11]","[6, 10, 11, 12, 13, 15, 22, 26, 29, 35, 42, 52, 58, 63, 84, 97, 114, 119, 120]"
|
|
||||||
87,"[47, 94, 0, 113, 89, 21, 105, 77, 80, 114, 12, 53, 64, 70, 10, 35, 123, 118, 30, 119, 85, 23, 111, 101, 8, 54, 125, 59, 60, 13, 14, 49, 124, 42, 65, 120, 74, 62, 108, 18, 103, 90, 9, 46, 121, 11, 81, 22, 16, 25, 93, 63, 33, 107, 102, 1, 40, 76, 45, 88, 73, 37, 7, 5]","[107, 92, 77, 102, 91, 87, 79, 76, 22, 116, 44, 111, 2, 29, 0, 51, 31, 47, 52, 10, 104, 20, 26, 112, 68, 36, 106, 14, 120, 86, 64, 95, 19, 78, 83, 118, 127, 59, 62, 71, 84, 73, 56, 80, 25, 9, 89, 35, 41, 40, 17, 61, 66, 98, 32, 37, 8, 69, 115, 109, 34, 75, 27, 21]","[0, 8, 9, 10, 14, 21, 22, 25, 35, 37, 40, 47, 59, 62, 64, 73, 76, 77, 80, 89, 102, 107, 111, 118, 120]"
|
|
||||||
88,"[71, 50, 28, 79, 54, 57, 45, 22, 65, 80, 18, 44, 67, 113, 127, 126, 91, 7, 14, 99, 58, 47, 51, 69, 4, 27, 77, 73, 110, 119, 60, 88, 97, 68, 115, 46, 81, 39, 106, 52, 29, 6, 42, 5, 86, 55, 66, 95, 104, 124, 123, 103, 90, 26, 108, 63, 24, 10, 61, 25, 17, 116, 83, 8]","[119, 88, 13, 76, 52, 102, 14, 97, 47, 50, 46, 85, 94, 112, 23, 96, 19, 83, 99, 7, 21, 26, 116, 71, 103, 6, 24, 73, 78, 8, 65, 45, 126, 56, 51, 124, 91, 106, 55, 109, 122, 68, 75, 22, 15, 113, 107, 74, 9, 29, 38, 63, 84, 53, 79, 58, 48, 81, 104, 54, 77, 34, 28, 80]","[6, 7, 8, 14, 22, 24, 26, 28, 29, 45, 46, 47, 50, 51, 52, 54, 55, 58, 63, 65, 68, 71, 73, 77, 79, 80, 81, 83, 88, 91, 97, 99, 103, 104, 106, 113, 116, 119, 124, 126]"
|
|
||||||
89,"[101, 65, 94, 68, 0, 96, 37, 81, 79, 107, 60, 109, 3, 11, 78, 36, 38, 52, 14, 41, 71, 23, 64, 93, 45, 21, 48, 54, 120, 6, 83, 43, 34, 121, 56, 63, 51, 61, 2, 115, 104, 40, 95, 17, 44, 55, 114, 4, 80, 39, 12, 99, 16, 8, 35, 32, 86, 91, 72, 5, 18, 15, 97, 59]","[83, 13, 45, 6, 66, 108, 5, 124, 99, 35, 119, 10, 43, 107, 94, 82, 15, 86, 65, 81, 95, 16, 101, 80, 4, 97, 56, 72, 39, 0, 77, 102, 69, 37, 122, 91, 114, 74, 41, 89, 11, 113, 64, 32, 21, 61, 53, 90, 63, 24, 48, 71, 60, 93, 120, 59, 20, 54, 23, 123, 7, 76, 36, 46]","[0, 4, 5, 6, 11, 15, 16, 21, 23, 32, 35, 36, 37, 39, 41, 43, 45, 48, 54, 56, 59, 60, 61, 63, 64, 65, 71, 72, 80, 81, 83, 86, 91, 93, 94, 95, 97, 99, 101, 107, 114, 120]"
|
|
||||||
90,"[25, 2, 118, 120, 0, 53, 21, 7, 55, 117, 37, 97, 72, 94, 122, 88, 29, 13, 24, 91, 19, 115, 27, 34, 4, 75, 112, 65, 116, 119, 31, 80, 42, 82, 79, 78, 111, 33, 93, 85, 41, 48, 49, 123, 95, 64, 108, 92, 32, 106, 12, 105, 38, 46, 124, 103, 77, 26, 30, 125, 8, 23, 16, 74]","[13, 28, 15, 70, 55, 76, 87, 24, 22, 119, 123, 20, 122, 92, 31, 35, 126, 98, 30, 14, 29, 68, 88, 115, 42, 41, 1, 57, 83, 8, 97, 54, 85, 21, 81, 113, 19, 109, 63, 37, 40, 51, 27, 104, 34, 3, 32, 79, 121, 18, 58, 59, 25, 0, 103, 16, 11, 6, 67, 84, 45, 73, 106, 5]","[0, 8, 13, 16, 19, 21, 24, 25, 27, 29, 30, 31, 32, 34, 37, 41, 42, 55, 79, 85, 88, 92, 97, 103, 106, 115, 119, 122, 123]"
|
|
||||||
91,"[15, 24, 121, 26, 100, 32, 107, 19, 48, 2, 49, 4, 0, 104, 94, 101, 5, 82, 86, 14, 90, 40, 22, 55, 20, 71, 97, 125, 98, 16, 119, 85, 6, 41, 117, 61, 63, 58, 37, 73, 12, 120, 17, 50, 127, 38, 66, 45, 109, 80, 115, 84, 88, 29, 54, 76, 99, 89, 11, 79, 59, 75, 23, 42]","[59, 34, 95, 14, 104, 20, 121, 127, 36, 19, 50, 76, 49, 63, 65, 97, 116, 66, 94, 17, 93, 84, 120, 55, 3, 16, 100, 107, 82, 69, 33, 83, 109, 7, 105, 12, 41, 64, 71, 67, 37, 23, 91, 10, 26, 38, 62, 115, 24, 72, 112, 98, 40, 117, 35, 99, 22, 45, 101, 48, 15, 47, 8, 1]","[12, 14, 15, 16, 17, 19, 20, 22, 23, 24, 26, 37, 38, 40, 41, 45, 48, 49, 50, 55, 59, 63, 66, 71, 76, 82, 84, 94, 97, 98, 99, 100, 101, 104, 107, 109, 115, 117, 120, 121, 127]"
|
|
||||||
92,"[8, 52, 23, 16, 2, 66, 123, 94, 4, 31, 24, 44, 28, 75, 113, 82, 17, 53, 98, 124, 117, 45, 99, 87, 6, 57, 102, 15, 108, 37, 0, 114, 120, 9, 126, 46, 71, 116, 25, 49, 65, 110, 12, 92, 50, 21, 88, 18, 26, 90, 11, 68, 77, 14, 51, 33, 104, 72, 109, 69, 40, 81, 61, 34]","[100, 1, 76, 125, 54, 82, 67, 45, 2, 127, 116, 123, 6, 52, 0, 41, 124, 27, 77, 58, 57, 72, 25, 104, 8, 75, 5, 34, 30, 9, 91, 51, 95, 15, 113, 71, 35, 79, 87, 73, 24, 83, 126, 110, 66, 90, 49, 117, 16, 81, 31, 50, 23, 37, 120, 43, 68, 106, 40, 46, 55, 26, 85, 109]","[0, 2, 6, 8, 9, 15, 16, 23, 24, 25, 26, 31, 34, 37, 40, 45, 46, 49, 50, 51, 52, 57, 66, 68, 71, 72, 75, 77, 81, 82, 87, 90, 104, 109, 110, 113, 116, 117, 120, 123, 124, 126]"
|
|
||||||
93,"[18, 26, 30, 96, 37, 115, 48, 29, 31, 42, 24, 64, 47, 81, 8, 41, 59, 14, 25, 105, 110, 52, 70, 17, 10, 127, 54, 76, 124, 88, 82, 63, 67, 9, 100, 90, 92, 84, 0, 66, 51, 79, 68, 20, 97, 72, 62, 28, 60, 7, 33, 65, 95, 35, 5, 23, 119, 2, 106, 108, 102, 22, 123, 12]","[67, 110, 62, 63, 65, 7, 2, 119, 120, 98, 125, 86, 123, 93, 5, 91, 77, 37, 89, 108, 39, 69, 117, 3, 54, 97, 112, 101, 68, 25, 48, 87, 53, 116, 94, 51, 72, 103, 84, 109, 22, 99, 49, 27, 26, 4, 104, 118, 80, 96, 111, 126, 0, 81, 33, 113, 71, 31, 47, 32, 73, 107, 50, 15]","[0, 2, 5, 7, 22, 25, 26, 31, 33, 37, 47, 48, 51, 54, 62, 63, 65, 67, 68, 72, 81, 84, 96, 97, 108, 110, 119, 123]"
|
|
||||||
94,"[17, 84, 49, 51, 65, 67, 123, 90, 20, 35, 44, 11, 79, 94, 62, 110, 25, 3, 80, 113, 5, 64, 7, 32, 16, 59, 19, 100, 4, 78, 0, 29, 2, 86, 24, 27, 102, 58, 47, 82, 45, 68, 42, 37, 112, 55, 34, 14, 96, 85, 39, 116, 66, 77, 56, 125, 74, 88, 63, 48, 87, 26, 106, 69]","[85, 25, 15, 44, 68, 0, 17, 3, 49, 96, 9, 113, 55, 67, 121, 110, 107, 91, 59, 5, 7, 57, 102, 92, 119, 48, 109, 56, 62, 123, 101, 53, 90, 118, 87, 122, 19, 14, 74, 72, 76, 26, 112, 43, 79, 60, 42, 29, 77, 51, 47, 31, 106, 86, 18, 39, 20, 64, 32, 78, 1, 84, 16, 65]","[0, 3, 5, 7, 14, 16, 17, 19, 20, 25, 26, 29, 32, 39, 42, 44, 47, 48, 49, 51, 55, 56, 59, 62, 64, 65, 67, 68, 74, 77, 78, 79, 84, 85, 86, 87, 90, 96, 102, 106, 110, 112, 113, 123]"
|
|
||||||
95,"[123, 63, 33, 14, 0, 47, 6, 29, 70, 1, 85, 26, 19, 23, 112, 10, 51, 17, 104, 36, 82, 55, 27, 60, 81, 125, 64, 94, 5, 2, 22, 101, 15, 28, 41, 119, 62, 77, 67, 42, 120, 116, 127, 24, 48, 11, 97, 69, 39, 37, 89, 110, 103, 115, 3, 43, 49, 58, 32, 106, 122, 95, 53, 46]","[2, 80, 104, 109, 21, 7, 91, 123, 37, 78, 41, 35, 22, 33, 38, 4, 47, 34, 114, 108, 56, 81, 8, 39, 43, 124, 51, 0, 71, 113, 83, 3, 74, 87, 26, 59, 73, 10, 95, 52, 20, 17, 119, 99, 53, 58, 121, 107, 65, 111, 93, 69, 96, 49, 127, 29, 25, 18, 105, 50, 13, 100, 30, 40]","[0, 2, 3, 10, 17, 22, 26, 29, 33, 37, 39, 41, 43, 47, 49, 51, 53, 58, 69, 81, 95, 104, 119, 123, 127]"
|
|
||||||
96,"[19, 119, 55, 69, 33, 44, 6, 88, 20, 62, 8, 41, 3, 71, 101, 26, 65, 22, 53, 77, 84, 100, 57, 106, 5, 47, 80, 16, 114, 116, 112, 32, 12, 79, 60, 102, 35, 70, 87, 117, 11, 83, 66, 10, 113, 120, 61, 0, 118, 90, 108, 98, 68, 39, 36, 54, 111, 31, 7, 28, 86, 92, 126, 23]","[50, 84, 88, 29, 43, 107, 40, 68, 72, 123, 4, 93, 57, 85, 30, 124, 78, 18, 69, 81, 13, 115, 103, 109, 38, 110, 19, 79, 113, 5, 3, 89, 2, 53, 24, 102, 25, 125, 15, 116, 86, 1, 64, 112, 94, 61, 96, 49, 91, 67, 32, 73, 90, 59, 83, 63, 34, 7, 121, 28, 42, 122, 71, 16]","[3, 5, 7, 16, 19, 28, 32, 53, 57, 61, 68, 69, 71, 79, 83, 84, 86, 88, 90, 102, 112, 113, 116]"
|
|
||||||
97,"[32, 52, 10, 96, 43, 70, 103, 100, 55, 119, 28, 98, 41, 67, 54, 71, 66, 3, 15, 81, 9, 104, 97, 108, 22, 34, 42, 106, 1, 86, 61, 78, 21, 76, 4, 127, 40, 107, 5, 7, 112, 75, 74, 8, 57, 14, 6, 17, 90, 117, 44, 27, 20, 115, 83, 85, 63, 16, 51, 69, 60, 38, 53, 13]","[81, 79, 75, 58, 97, 77, 83, 90, 7, 54, 29, 37, 6, 118, 43, 107, 22, 96, 88, 15, 127, 3, 51, 10, 69, 80, 71, 17, 27, 101, 111, 95, 31, 0, 94, 1, 100, 82, 116, 52, 72, 41, 59, 113, 16, 34, 28, 18, 67, 122, 114, 86, 33, 9, 103, 73, 64, 119, 13, 87, 14, 76, 12, 106]","[1, 3, 6, 7, 9, 10, 13, 14, 15, 16, 17, 22, 27, 28, 34, 41, 43, 51, 52, 54, 67, 69, 71, 75, 76, 81, 83, 86, 90, 96, 97, 100, 103, 106, 107, 119, 127]"
|
|
||||||
98,"[108, 111, 30, 21, 113, 120, 70, 0, 72, 78, 41, 82, 54, 6, 89, 20, 22, 96, 115, 87, 61, 52, 116, 64, 66, 34, 65, 74, 118, 7, 50, 60, 59, 24, 126, 5, 16, 18, 51, 71, 104, 124, 28, 69, 1, 33, 79, 48, 95, 42, 122, 25, 81, 105, 43, 67, 77, 100, 36, 31, 107, 56, 2, 37]","[34, 12, 76, 98, 82, 96, 102, 70, 75, 113, 116, 125, 106, 63, 122, 3, 10, 57, 49, 84, 6, 72, 11, 83, 118, 37, 52, 31, 20, 4, 123, 46, 62, 108, 38, 69, 54, 25, 95, 119, 126, 100, 45, 124, 103, 71, 51, 41, 115, 67, 50, 86, 53, 78, 88, 42, 21, 2, 87, 13, 28, 39, 1, 81]","[1, 2, 6, 20, 21, 25, 28, 31, 34, 37, 41, 42, 50, 51, 52, 54, 67, 69, 70, 71, 72, 78, 81, 82, 87, 95, 96, 100, 108, 113, 115, 116, 118, 122, 124, 126]"
|
|
||||||
99,"[58, 108, 54, 110, 77, 65, 56, 98, 83, 99, 61, 12, 15, 116, 14, 67, 103, 63, 17, 39, 104, 51, 52, 5, 100, 49, 76, 48, 41, 25, 35, 111, 43, 81, 23, 47, 62, 127, 1, 88, 26, 69, 75, 85, 7, 24, 18, 112, 66, 71, 42, 73, 40, 107, 64, 87, 11, 9, 109, 45, 0, 31, 28, 29]","[46, 63, 9, 89, 111, 26, 101, 98, 84, 122, 53, 83, 42, 37, 4, 5, 79, 24, 92, 113, 57, 117, 126, 16, 121, 82, 114, 103, 70, 34, 115, 48, 6, 93, 15, 87, 107, 124, 66, 68, 97, 74, 60, 0, 32, 102, 10, 95, 105, 21, 120, 88, 2, 52, 33, 47, 86, 123, 55, 119, 96, 50, 14, 59]","[0, 5, 9, 14, 15, 24, 26, 42, 47, 48, 52, 63, 66, 83, 87, 88, 98, 103, 107, 111]"
|
|
||||||
|
@ -1,755 +0,0 @@
|
|||||||
# Copyright (c) 2023 ETH Zurich.
|
|
||||||
# All rights reserved.
|
|
||||||
#
|
|
||||||
# Use of this source code is governed by a BSD-style license that can be
|
|
||||||
# found in the LICENSE file.
|
|
||||||
#
|
|
||||||
# The source code is adapted from the sorting source code written by
|
|
||||||
# Nils Blach.
|
|
||||||
#
|
|
||||||
# main author: Robert Gerstenberger
|
|
||||||
|
|
||||||
import os
|
|
||||||
import logging
|
|
||||||
import datetime
|
|
||||||
import json
|
|
||||||
import csv
|
|
||||||
from typing import Dict, List, Callable, Union
|
|
||||||
from graph_of_thoughts import controller, language_models, operations, prompter, parser
|
|
||||||
|
|
||||||
# This is a hack to also allow execution of this file from the examples directory
|
|
||||||
try:
|
|
||||||
from . import utils
|
|
||||||
except ImportError:
|
|
||||||
import utils
|
|
||||||
|
|
||||||
|
|
||||||
class SetIntersectionPrompter(prompter.Prompter):
|
|
||||||
"""
|
|
||||||
SetIntersectionPrompter provides the generation of prompts specific to the
|
|
||||||
set intersection example for the language models.
|
|
||||||
|
|
||||||
Inherits from the Prompter class and implements its abstract methods.
|
|
||||||
"""
|
|
||||||
|
|
||||||
intersection_prompt = """<Instruction> Find the intersection of two sets of numbers. Output only the set of numbers that are present in both sets, no additional text. </Instruction>
|
|
||||||
|
|
||||||
<Examples>
|
|
||||||
Input Set 1: [13, 16, 30, 6, 21, 7, 31, 15, 11, 1, 24, 10, 9, 3, 20, 8]
|
|
||||||
Input Set 2: [25, 24, 10, 4, 27, 0, 14, 12, 8, 2, 29, 20, 17, 19, 26, 23]
|
|
||||||
Output: [24, 10, 20, 8]
|
|
||||||
|
|
||||||
Input Set 1: [26, 40, 42, 57, 15, 31, 5, 32, 11, 4, 24, 28, 51, 54, 12, 22, 33, 35, 7, 13, 2, 59, 8, 23, 43, 16, 29, 55, 25, 63, 21, 18]
|
|
||||||
Input Set 2: [16, 60, 36, 48, 0, 15, 5, 19, 46, 24, 1, 6, 61, 10, 38, 53, 58, 9, 44, 14, 35, 63, 52, 20, 27, 17, 39, 47, 34, 56, 40, 59]
|
|
||||||
Output: [40, 15, 5, 24, 35, 59, 16, 63]
|
|
||||||
|
|
||||||
Input Set 1: [115, 61, 35, 103, 90, 117, 86, 44, 63, 45, 40, 30, 74, 33, 31, 1, 118, 48, 38, 0, 119, 51, 64, 78, 15, 121, 89, 101, 79, 69, 120, 29, 58, 50, 116, 11, 60, 12, 39, 95, 23, 2, 109, 84, 7, 43, 99, 98, 52, 70, 75, 102, 57, 19, 94, 36, 114, 88, 71, 56, 83, 6, 96, 107]
|
|
||||||
Input Set 2: [13, 35, 20, 96, 34, 18, 47, 127, 126, 9, 21, 16, 77, 22, 111, 122, 85, 73, 42, 105, 123, 15, 33, 59, 67, 57, 104, 8, 30, 89, 76, 12, 65, 84, 32, 40, 7, 100, 108, 50, 14, 28, 24, 53, 90, 17, 91, 81, 124, 63, 5, 46, 125, 93, 49, 66, 117, 37, 115, 113, 2, 106, 41, 72]
|
|
||||||
Output: [115, 35, 90, 117, 63, 40, 30, 33, 15, 89, 50, 12, 2, 84, 7, 57, 96]
|
|
||||||
</Examples>
|
|
||||||
|
|
||||||
Input Set 1: {set1}
|
|
||||||
Input Set 2: {set2}"""
|
|
||||||
|
|
||||||
intersection_prompt_cot = """<Instruction> Find the intersection of two sets of numbers. You can generate any intermediate solutions, but the final output should be the set of numbers that are present in both sets, prefixed with "Output: ". </Instruction>
|
|
||||||
|
|
||||||
<Approach>
|
|
||||||
To find the intersection of the two sets follow these steps:
|
|
||||||
1. Split the second input set of numbers into two to four subsets, each containing an equal number of elements from the original set (make sure they don't overlap).
|
|
||||||
2. For each subset find the set of numbers that are present in the subset and the first input set.
|
|
||||||
3. Merge the resulting sets into a single output set.
|
|
||||||
</Approach>
|
|
||||||
|
|
||||||
<Examples>
|
|
||||||
Input Set 1: [13, 16, 30, 6, 21, 7, 31, 15, 11, 1, 24, 10, 9, 3, 20, 8]
|
|
||||||
Input Set 2: [25, 24, 10, 4, 27, 0, 14, 12, 8, 2, 29, 20, 17, 19, 26, 23]
|
|
||||||
Subsets of Input Set 2:
|
|
||||||
[25, 24, 10, 4, 27, 0, 14, 12]
|
|
||||||
[8, 2, 29, 20, 17, 19, 26, 23]
|
|
||||||
Intersected Subsets with Input Set 1:
|
|
||||||
[24, 10]
|
|
||||||
[8, 20]
|
|
||||||
Output: [24, 10, 8, 20]
|
|
||||||
|
|
||||||
Input Set 1: [26, 40, 42, 57, 15, 31, 5, 32, 11, 4, 24, 28, 51, 54, 12, 22, 33, 35, 7, 13, 2, 59, 8, 23, 43, 16, 29, 55, 25, 63, 21, 18]
|
|
||||||
Input Set 2: [16, 60, 36, 48, 0, 15, 5, 19, 46, 24, 1, 6, 61, 10, 38, 53, 58, 9, 44, 14, 35, 63, 52, 20, 27, 17, 39, 47, 34, 56, 40, 59]
|
|
||||||
Subsets of Input Set 2:
|
|
||||||
[16, 60, 36, 48, 0, 15, 5, 19, 46, 24, 1, 6, 61, 10, 38, 53]
|
|
||||||
[58, 9, 44, 14, 35, 63, 52, 20, 27, 17, 39, 47, 34, 56, 40, 59]
|
|
||||||
Intersected Subsets with Input Set 1:
|
|
||||||
[16, 15, 5, 24]
|
|
||||||
[35, 63, 40, 59]
|
|
||||||
Output: [16, 15, 5, 24, 35, 63, 40, 59]
|
|
||||||
|
|
||||||
Input Set 1: [115, 61, 35, 103, 90, 117, 86, 44, 63, 45, 40, 30, 74, 33, 31, 1, 118, 48, 38, 0, 119, 51, 64, 78, 15, 121, 89, 101, 79, 69, 120, 29, 58, 50, 116, 11, 60, 12, 39, 95, 23, 2, 109, 84, 7, 43, 99, 98, 52, 70, 75, 102, 57, 19, 94, 36, 114, 88, 71, 56, 83, 6, 96, 107]
|
|
||||||
Input Set 2: [13, 35, 20, 96, 34, 18, 47, 127, 126, 9, 21, 16, 77, 22, 111, 122, 85, 73, 42, 105, 123, 15, 33, 59, 67, 57, 104, 8, 30, 89, 76, 12, 65, 84, 32, 40, 7, 100, 108, 50, 14, 28, 24, 53, 90, 17, 91, 81, 124, 63, 5, 46, 125, 93, 49, 66, 117, 37, 115, 113, 2, 106, 41, 72]
|
|
||||||
Subsets of Input Set 2:
|
|
||||||
[13, 35, 20, 96, 34, 18, 47, 127, 126, 9, 21, 16, 77, 22, 111, 122]
|
|
||||||
[85, 73, 42, 105, 123, 15, 33, 59, 67, 57, 104, 8, 30, 89, 76, 12]
|
|
||||||
[65, 84, 32, 40, 7, 100, 108, 50, 14, 28, 24, 53, 90, 17, 91, 81]
|
|
||||||
[124, 63, 5, 46, 125, 93, 49, 66, 117, 37, 115, 113, 2, 106, 41, 72]
|
|
||||||
Intersected Subsets with Input Set 1:
|
|
||||||
[35, 96]
|
|
||||||
[15, 33, 57, 30, 89, 12]
|
|
||||||
[84, 40, 7, 50, 90]
|
|
||||||
[63, 117, 115, 2]
|
|
||||||
Output: [35, 96, 15, 33, 57, 30, 89, 12, 84, 40, 7, 50, 90, 63, 117, 115, 2]
|
|
||||||
</Examples>
|
|
||||||
|
|
||||||
Input Set 1: {set1}
|
|
||||||
Input Set 2: {set2}"""
|
|
||||||
|
|
||||||
tot_improve_prompt = """<Instruction> The following three sets represent two sets and an intersection set of those two sets. The intersection set is not correct. Fix the intersection set so that it is correct.
|
|
||||||
Make sure that the numbers in the intersection set can be found in both input sets. </Instruction>
|
|
||||||
|
|
||||||
<Approach>
|
|
||||||
To fix the incorrectly intersection set follow these steps:
|
|
||||||
1. Check for each number in the incorrect intersection set, whether it can be found in both input sets. If not, remove that number from the intersection set.
|
|
||||||
2. Iterate through the second input set and check whether each number is already in the incorrect intersection set and if not, check whether that number can also be found in the first input set. If so, add that number to the intersection set.
|
|
||||||
</Approach>
|
|
||||||
|
|
||||||
<Examples>
|
|
||||||
Input Set 1: [13, 16, 30, 6, 21, 7, 31, 15, 11, 1, 24, 10, 9, 3, 20, 8]
|
|
||||||
Input Set 2: [25, 24, 10, 4, 27, 0, 14, 12, 8, 2, 29, 20, 17, 19, 26, 23]
|
|
||||||
Incorrect Intersection Set: [24, 20, 25]
|
|
||||||
Reason: The incorrect intersection set contains the number 25, which is not present in the first input set and is missing the numbers 10 and 8.
|
|
||||||
Output: [24, 10, 20, 8]
|
|
||||||
|
|
||||||
Input Set 1: [26, 40, 42, 57, 15, 31, 5, 32, 11, 4, 24, 28, 51, 54, 12, 22, 33, 35, 7, 13, 2, 59, 8, 23, 43, 16, 29, 55, 25, 63, 21, 18]
|
|
||||||
Input Set 2: [16, 60, 36, 48, 0, 15, 5, 19, 46, 24, 1, 6, 61, 10, 38, 53, 58, 9, 44, 14, 35, 63, 52, 20, 27, 17, 39, 47, 34, 56, 40, 59]
|
|
||||||
Incorrect Intersection Set: [57, 16, 15, 24, 35, 10, 40]
|
|
||||||
Reason: The incorrect intersection set contains the numbers 57, which is not present in the second input set, and 10, which is not present in the first input set, and is missing the numbers 5, 63 and 59.
|
|
||||||
Output: [16, 15, 5, 24, 35, 63, 40, 59]
|
|
||||||
|
|
||||||
Input Set 1: [115, 61, 35, 103, 90, 117, 86, 44, 63, 45, 40, 30, 74, 33, 31, 1, 118, 48, 38, 0, 119, 51, 64, 78, 15, 121, 89, 101, 79, 69, 120, 29, 58, 50, 116, 11, 60, 12, 39, 95, 23, 2, 109, 84, 7, 43, 99, 98, 52, 70, 75, 102, 57, 19, 94, 36, 114, 88, 71, 56, 83, 6, 96, 107]
|
|
||||||
Input Set 2: [13, 35, 20, 96, 34, 18, 47, 127, 126, 9, 21, 16, 77, 22, 111, 122, 85, 73, 42, 105, 123, 15, 33, 59, 67, 57, 104, 8, 30, 89, 76, 12, 65, 84, 32, 40, 7, 100, 108, 50, 14, 28, 24, 53, 90, 17, 91, 81, 124, 63, 5, 46, 125, 93, 49, 66, 117, 37, 115, 113, 2, 106, 41, 72]
|
|
||||||
Incorrect Intersection Set: [35, 96, 44, 15, 33, 57, 30, 50, 90, 119, 123, 63, 117, 115, 2]
|
|
||||||
Reason: The incorrect intersection set contains the numbers 44 and 119, which are not present in the second input set, and 123, which is not present in the first input set, and is missing the numbers 89, 12, 84, 40 and 7.
|
|
||||||
Output: [35, 96, 15, 33, 57, 30, 89, 12, 84, 40, 7, 50, 90, 63, 117, 115, 2]
|
|
||||||
</Examples>
|
|
||||||
|
|
||||||
Input Set 1: {set1}
|
|
||||||
Input Set 2: {set2}
|
|
||||||
Incorrect Intersection Set: {incorrect_intersection}
|
|
||||||
"""
|
|
||||||
|
|
||||||
got_split_prompt = """<Instruction> Split the following list of 64 numbers into 4 lists of 16 numbers each, the first list should contain the first 16 numbers, the second list the second 16 numbers, the third list the third 16 numbers and the fourth list the fourth 16 numbers.
|
|
||||||
Only output the 4 lists in the following format without any additional text or thoughts!:
|
|
||||||
{{
|
|
||||||
"List 1": [13, 35, 20, 96, 34, 18, 47, ...],
|
|
||||||
"List 2": [127, 126, 9, 21, 16, 77, 22, ...],
|
|
||||||
"List 3": [111, 122, 85, 73, 42, 105, 123, ...],
|
|
||||||
"List 4": [15, 33, 59, 67, 57, 104, 8, ...]
|
|
||||||
}} </Instruction>
|
|
||||||
|
|
||||||
<Example>
|
|
||||||
Input: [115, 61, 35, 103, 90, 117, 86, 44, 63, 45, 40, 30, 74, 33, 31, 1, 118, 48, 38, 0, 119, 51, 64, 78, 15, 121, 89, 101, 79, 69, 120, 29, 58, 50, 116, 11, 60, 12, 39, 95, 23, 2, 109, 84, 7, 43, 99, 98, 52, 70, 75, 102, 57, 19, 94, 36, 114, 88, 71, 56, 83, 6, 96, 107]
|
|
||||||
Output:
|
|
||||||
{{
|
|
||||||
"List 1": [115, 61, 35, 103, 90, 117, 86, 44, 63, 45, 40, 30, 74, 33, 31, 1],
|
|
||||||
"List 2": [118, 48, 38, 0, 119, 51, 64, 78, 15, 121, 89, 101, 79, 69, 120, 29],
|
|
||||||
"List 3": [58, 50, 116, 11, 60, 12, 39, 95, 23, 2, 109, 84, 7, 43, 99, 98],
|
|
||||||
"List 4": [52, 70, 75, 102, 57, 19, 94, 36, 114, 88, 71, 56, 83, 6, 96, 107]
|
|
||||||
}}
|
|
||||||
</Example>
|
|
||||||
|
|
||||||
Input: {input}"""
|
|
||||||
|
|
||||||
got_merge_prompt = """<Instruction> Merge the following 2 lists into one list by appending the second list to the first list.
|
|
||||||
Only output the final list without any additional text or thoughts! </Instruction>
|
|
||||||
|
|
||||||
List 1: {input1}
|
|
||||||
List 2: {input2}
|
|
||||||
"""
|
|
||||||
|
|
||||||
def aggregation_prompt(self, state_dicts: List[Dict], **kwargs) -> str:
|
|
||||||
"""
|
|
||||||
Generate an aggregation prompt for the language model.
|
|
||||||
|
|
||||||
:param state_dicts: The thought states that should be aggregated.
|
|
||||||
:type state_dicts: List[Dict]
|
|
||||||
:param kwargs: Additional keyword arguments.
|
|
||||||
:return: The aggregation prompt.
|
|
||||||
:rtype: str
|
|
||||||
:raise AssertionError: If not exactly two thought states are provided.
|
|
||||||
"""
|
|
||||||
|
|
||||||
assert len(state_dicts) == 2, "Expected two states for aggregation prompt."
|
|
||||||
|
|
||||||
return self.got_merge_prompt.format(
|
|
||||||
input1=state_dicts[0]["current"],
|
|
||||||
input2=state_dicts[1]["current"],
|
|
||||||
)
|
|
||||||
|
|
||||||
def generate_prompt(
|
|
||||||
self,
|
|
||||||
num_branches: int,
|
|
||||||
set1: str,
|
|
||||||
set2: str,
|
|
||||||
current: str,
|
|
||||||
method: str,
|
|
||||||
**kwargs,
|
|
||||||
) -> str:
|
|
||||||
"""
|
|
||||||
Generate a generate prompt for the language model.
|
|
||||||
|
|
||||||
:param num_branches: The number of responses the prompt should ask the LM to generate.
|
|
||||||
:type num_branches: int
|
|
||||||
:param set1: First input set.
|
|
||||||
:type set1: str
|
|
||||||
:param set2: Second input set.
|
|
||||||
:type set2: str
|
|
||||||
:param current: Intermediate solution.
|
|
||||||
:type current: str
|
|
||||||
:param method: Method for which the generate prompt is generated.
|
|
||||||
:type method: str
|
|
||||||
:param kwargs: Additional keyword arguments.
|
|
||||||
:return: The generate prompt.
|
|
||||||
:rtype: str
|
|
||||||
:raise AssertionError: If the requested number of branches is not one.
|
|
||||||
"""
|
|
||||||
|
|
||||||
assert num_branches == 1, "Branching should be done via multiple requests."
|
|
||||||
if method.startswith("io"):
|
|
||||||
return self.intersection_prompt.format(set1=set1, set2=set2)
|
|
||||||
elif method.startswith("cot"):
|
|
||||||
return self.intersection_prompt_cot.format(set1=set1, set2=set2)
|
|
||||||
elif method.startswith("tot"):
|
|
||||||
if current is None or current == "":
|
|
||||||
return self.intersection_prompt.format(set1=set1, set2=set2)
|
|
||||||
return self.tot_improve_prompt.format(
|
|
||||||
set1=set1, set2=set2, incorrect_intersection=current
|
|
||||||
)
|
|
||||||
elif method.startswith("got"):
|
|
||||||
if kwargs["phase"] == 0:
|
|
||||||
return self.got_split_prompt.format(input=set2)
|
|
||||||
|
|
||||||
input_set = set2
|
|
||||||
if "subset" in kwargs and kwargs["subset"] != "":
|
|
||||||
input_set = kwargs["subset"]
|
|
||||||
|
|
||||||
return self.intersection_prompt.format(set1=set1, set2=input_set)
|
|
||||||
|
|
||||||
def improve_prompt(self, **kwargs) -> str:
|
|
||||||
"""
|
|
||||||
Generate an improve prompt for the language model.
|
|
||||||
|
|
||||||
:param kwargs: Additional keyword arguments.
|
|
||||||
:return: The improve prompt.
|
|
||||||
:rtype: str
|
|
||||||
"""
|
|
||||||
pass
|
|
||||||
|
|
||||||
def validation_prompt(self, **kwargs) -> str:
|
|
||||||
"""
|
|
||||||
Generate a validation prompt for the language model.
|
|
||||||
|
|
||||||
:param kwargs: Additional keyword arguments.
|
|
||||||
:return: The validation prompt.
|
|
||||||
:rtype: str
|
|
||||||
"""
|
|
||||||
pass
|
|
||||||
|
|
||||||
def score_prompt(self, state_dicts: List[Dict], **kwargs) -> str:
|
|
||||||
"""
|
|
||||||
Generate a score prompt for the language model.
|
|
||||||
|
|
||||||
:param state_dicts: The thought states that should be scored,
|
|
||||||
if more than one, they should be scored together.
|
|
||||||
:type state_dicts: List[Dict]
|
|
||||||
:param kwargs: Additional keyword arguments.
|
|
||||||
:return: The score prompt.
|
|
||||||
:rtype: str
|
|
||||||
"""
|
|
||||||
pass
|
|
||||||
|
|
||||||
|
|
||||||
class SetIntersectionParser(parser.Parser):
|
|
||||||
"""
|
|
||||||
SetIntersectionParser provides the parsing of language model reponses
|
|
||||||
specific to the set intersection example.
|
|
||||||
|
|
||||||
Inherits from the Parser class and implements its abstract methods.
|
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(self) -> None:
|
|
||||||
"""
|
|
||||||
Inits the response cache.
|
|
||||||
"""
|
|
||||||
self.cache = {}
|
|
||||||
|
|
||||||
def parse_aggregation_answer(
|
|
||||||
self, states: List[Dict], texts: List[str]
|
|
||||||
) -> Union[Dict, List[Dict]]:
|
|
||||||
"""
|
|
||||||
Parse the response from the language model for an aggregation prompt.
|
|
||||||
|
|
||||||
:param states: The thought states used to generate the prompt.
|
|
||||||
:type states: List[Dict]
|
|
||||||
:param texts: The responses to the prompt from the language model.
|
|
||||||
:type texts: List[str]
|
|
||||||
:return: The new thought states after parsing the respones from the language model.
|
|
||||||
:rtype: Union[Dict, List[Dict]]
|
|
||||||
:raise AssertionError: If not exactly two thought states are provided.
|
|
||||||
"""
|
|
||||||
|
|
||||||
assert len(states) == 2, "Expected two states for aggregation answer."
|
|
||||||
new_states = []
|
|
||||||
for text in texts:
|
|
||||||
answers = text.strip().split("\n")
|
|
||||||
if any(["Output" in answer for answer in answers]):
|
|
||||||
# cut elements until last output is found
|
|
||||||
for answer in reversed(answers):
|
|
||||||
if "Output" in answer:
|
|
||||||
answers = answers[answers.index(answer) :]
|
|
||||||
break
|
|
||||||
|
|
||||||
answers_stripped = [
|
|
||||||
answer for answer in answers if "[" in answer and "]" in answer
|
|
||||||
]
|
|
||||||
if len(answers_stripped) == 0:
|
|
||||||
for answer in answers:
|
|
||||||
answer = "[" + answer + "]"
|
|
||||||
try:
|
|
||||||
answer_converted = utils.string_to_list(answer)
|
|
||||||
if len(answer_converted) > 0:
|
|
||||||
answers_stripped.append(answer)
|
|
||||||
except:
|
|
||||||
pass
|
|
||||||
if len(answers_stripped) == 0:
|
|
||||||
logging.warning(
|
|
||||||
f"Could not parse aggregation answer: {text}. Returning empty list."
|
|
||||||
)
|
|
||||||
answer = "[]"
|
|
||||||
else:
|
|
||||||
answer = [
|
|
||||||
answer[answer.index("[") : answer.index("]") + 1]
|
|
||||||
for answer in answers_stripped
|
|
||||||
][0]
|
|
||||||
states = sorted(states, key=lambda x: x["part"])
|
|
||||||
merged_subsets = states[0]["subset"][:-1] + ", " + states[1]["subset"][1:]
|
|
||||||
new_state = states[0].copy()
|
|
||||||
new_state["current"] = answer
|
|
||||||
new_state["subset"] = merged_subsets
|
|
||||||
new_states.append(new_state)
|
|
||||||
return new_states
|
|
||||||
|
|
||||||
def parse_improve_answer(self, state: Dict, texts: List[str]) -> Dict:
|
|
||||||
"""
|
|
||||||
Parse the response from the language model for an improve prompt.
|
|
||||||
|
|
||||||
:param state: The thought state used to generate the prompt.
|
|
||||||
:type state: Dict
|
|
||||||
:param texts: The responses to the prompt from the language model.
|
|
||||||
:type texts: List[str]
|
|
||||||
:return: The new thought state after parsing the responses from the language model.
|
|
||||||
:rtype: Dict
|
|
||||||
"""
|
|
||||||
pass
|
|
||||||
|
|
||||||
def parse_generate_answer(self, state: Dict, texts: List[str]) -> List[Dict]:
|
|
||||||
"""
|
|
||||||
Parse the response from the language model for a generate prompt.
|
|
||||||
|
|
||||||
:param state: The thought state used to generate the prompt.
|
|
||||||
:type state: Dict
|
|
||||||
:param texts: The responses to the prompt from the language model.
|
|
||||||
:type texts: List[str]
|
|
||||||
:return: The new thought states after parsing the respones from the language model.
|
|
||||||
:rtype: List[Dict]
|
|
||||||
"""
|
|
||||||
|
|
||||||
new_states = []
|
|
||||||
for text in texts:
|
|
||||||
if state["method"].startswith("got") and state["phase"] == 0:
|
|
||||||
# We expect a json which contains the two lists named "List 1" and "List 2"
|
|
||||||
# cut everything until the opening bracket and everything after the closing bracket
|
|
||||||
|
|
||||||
try:
|
|
||||||
text = text[text.index("{") : text.index("}") + 1]
|
|
||||||
json_dict = json.loads(text)
|
|
||||||
if len(json_dict.keys()) != 4:
|
|
||||||
logging.warning(
|
|
||||||
f"Expected 4 lists in json, but found {len(json_dict.keys())}."
|
|
||||||
)
|
|
||||||
for key, value in json_dict.items():
|
|
||||||
if "List" not in key:
|
|
||||||
logging.warning(
|
|
||||||
f"Expected key to contain 'List', but found {key}."
|
|
||||||
)
|
|
||||||
continue
|
|
||||||
if not isinstance(value, list):
|
|
||||||
value = utils.string_to_list(value)
|
|
||||||
new_state = state.copy()
|
|
||||||
new_state["current"] = ""
|
|
||||||
new_state["subset"] = str(value)
|
|
||||||
new_state["phase"] = 1
|
|
||||||
new_state["part"] = key
|
|
||||||
new_states.append(new_state)
|
|
||||||
except Exception as e:
|
|
||||||
logging.error(
|
|
||||||
f"Could not parse step answer: {text}. Encountered exception: {e}"
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
answers = text.strip().split("\n")
|
|
||||||
answers = [
|
|
||||||
answer for answer in answers if "[" in answer and "]" in answer
|
|
||||||
]
|
|
||||||
if any(["Output" in answer for answer in answers]):
|
|
||||||
# cut elements until last output is found
|
|
||||||
for answer in reversed(answers):
|
|
||||||
if "Output" in answer:
|
|
||||||
answers = answers[answers.index(answer) :]
|
|
||||||
break
|
|
||||||
|
|
||||||
answers = [
|
|
||||||
answer[answer.index("[") : answer.index("]") + 1]
|
|
||||||
for answer in answers
|
|
||||||
]
|
|
||||||
if len(answers) == 0:
|
|
||||||
logging.warning(
|
|
||||||
f"Could not parse step answer: {text}. Returning empty list."
|
|
||||||
)
|
|
||||||
answer = "[]"
|
|
||||||
else:
|
|
||||||
if len(answers) > 1:
|
|
||||||
logging.warning(
|
|
||||||
f"Multiple answers found for step answer: {text}. Using the first one."
|
|
||||||
)
|
|
||||||
answer = answers[0]
|
|
||||||
|
|
||||||
new_state = state.copy()
|
|
||||||
new_state["current"] = answer
|
|
||||||
new_state["phase"] = 2
|
|
||||||
new_states.append(new_state)
|
|
||||||
return new_states
|
|
||||||
|
|
||||||
def parse_validation_answer(self, state: Dict, texts: List[str]) -> bool:
|
|
||||||
"""
|
|
||||||
Parse the response from the language model for a validation prompt.
|
|
||||||
|
|
||||||
:param state: The thought state used to generate the prompt.
|
|
||||||
:type state: Dict
|
|
||||||
:param texts: The responses to the prompt from the language model.
|
|
||||||
:type texts: List[str]
|
|
||||||
:return: Whether the thought state is valid or not.
|
|
||||||
:rtype: bool
|
|
||||||
"""
|
|
||||||
pass
|
|
||||||
|
|
||||||
def parse_score_answer(self, states: List[Dict], texts: List[str]) -> List[float]:
|
|
||||||
"""
|
|
||||||
Parse the response from the language model for a score prompt.
|
|
||||||
|
|
||||||
:param states: The thought states used to generate the prompt.
|
|
||||||
:type states: List[Dict]
|
|
||||||
:param texts: The responses to the prompt from the language model.
|
|
||||||
:type texts: List[str]
|
|
||||||
:return: The scores for the thought states.
|
|
||||||
:rtype: List[float]
|
|
||||||
"""
|
|
||||||
pass
|
|
||||||
|
|
||||||
|
|
||||||
def io() -> operations.GraphOfOperations:
|
|
||||||
"""
|
|
||||||
Generates the Graph of Operations for the IO method.
|
|
||||||
|
|
||||||
:return: Graph of Operations
|
|
||||||
:rtype: GraphOfOperations
|
|
||||||
"""
|
|
||||||
operations_graph = operations.GraphOfOperations()
|
|
||||||
|
|
||||||
operations_graph.append_operation(operations.Generate(1, 1))
|
|
||||||
operations_graph.append_operation(operations.Score(1, False, utils.num_errors))
|
|
||||||
operations_graph.append_operation(
|
|
||||||
operations.GroundTruth(utils.test_set_intersection)
|
|
||||||
)
|
|
||||||
|
|
||||||
return operations_graph
|
|
||||||
|
|
||||||
|
|
||||||
def cot() -> operations.GraphOfOperations:
|
|
||||||
"""
|
|
||||||
Generates the Graph of Operations for the CoT method.
|
|
||||||
|
|
||||||
:return: Graph of Operations
|
|
||||||
:rtype: GraphOfOperations
|
|
||||||
"""
|
|
||||||
operations_graph = operations.GraphOfOperations()
|
|
||||||
|
|
||||||
operations_graph.append_operation(operations.Generate(1, 1))
|
|
||||||
operations_graph.append_operation(operations.Score(1, False, utils.num_errors))
|
|
||||||
operations_graph.append_operation(
|
|
||||||
operations.GroundTruth(utils.test_set_intersection)
|
|
||||||
)
|
|
||||||
|
|
||||||
return operations_graph
|
|
||||||
|
|
||||||
|
|
||||||
def tot() -> operations.GraphOfOperations:
|
|
||||||
"""
|
|
||||||
Generates the Graph of Operations for the ToT method.
|
|
||||||
ToT uses a wider tree, where on each level there are more branches.
|
|
||||||
|
|
||||||
:return: Graph of Operations
|
|
||||||
:rtype: GraphOfOperations
|
|
||||||
"""
|
|
||||||
operations_graph = operations.GraphOfOperations()
|
|
||||||
|
|
||||||
operations_graph.append_operation(operations.Generate(1, 20))
|
|
||||||
operations_graph.append_operation(operations.Score(1, False, utils.num_errors))
|
|
||||||
op_1 = operations.KeepBestN(1, False)
|
|
||||||
operations_graph.append_operation(op_1)
|
|
||||||
|
|
||||||
for _ in range(3):
|
|
||||||
operations_graph.append_operation(operations.Generate(1, 20))
|
|
||||||
operations_graph.append_operation(operations.Score(1, False, utils.num_errors))
|
|
||||||
op_2 = operations.KeepBestN(1, False)
|
|
||||||
op_2.add_predecessor(op_1)
|
|
||||||
operations_graph.append_operation(op_2)
|
|
||||||
op_1 = op_2
|
|
||||||
|
|
||||||
operations_graph.append_operation(
|
|
||||||
operations.GroundTruth(utils.test_set_intersection)
|
|
||||||
)
|
|
||||||
|
|
||||||
return operations_graph
|
|
||||||
|
|
||||||
|
|
||||||
def tot2() -> operations.GraphOfOperations:
|
|
||||||
"""
|
|
||||||
Generates the Graph of Operations for the ToT2 method.
|
|
||||||
ToT2 uses a tree with more levels, but with fewer branches per level.
|
|
||||||
|
|
||||||
:return: Graph of Operations
|
|
||||||
:rtype: GraphOfOperations
|
|
||||||
"""
|
|
||||||
operations_graph = operations.GraphOfOperations()
|
|
||||||
|
|
||||||
operations_graph.append_operation(operations.Generate(1, 10))
|
|
||||||
operations_graph.append_operation(operations.Score(1, False, utils.num_errors))
|
|
||||||
op_1 = operations.KeepBestN(1, False)
|
|
||||||
operations_graph.append_operation(op_1)
|
|
||||||
|
|
||||||
for _ in range(6):
|
|
||||||
operations_graph.append_operation(operations.Generate(1, 10))
|
|
||||||
operations_graph.append_operation(operations.Score(1, False, utils.num_errors))
|
|
||||||
op_2 = operations.KeepBestN(1, False)
|
|
||||||
op_2.add_predecessor(op_1)
|
|
||||||
operations_graph.append_operation(op_2)
|
|
||||||
op_1 = op_2
|
|
||||||
|
|
||||||
operations_graph.append_operation(
|
|
||||||
operations.GroundTruth(utils.test_set_intersection)
|
|
||||||
)
|
|
||||||
|
|
||||||
return operations_graph
|
|
||||||
|
|
||||||
|
|
||||||
def got() -> operations.GraphOfOperations:
|
|
||||||
"""
|
|
||||||
Generates the Graph of Operations for the GoT method.
|
|
||||||
|
|
||||||
:return: Graph of Operations
|
|
||||||
:rtype: GraphOfOperations
|
|
||||||
"""
|
|
||||||
operations_graph = operations.GraphOfOperations()
|
|
||||||
|
|
||||||
plans = operations.Generate(1, 1)
|
|
||||||
operations_graph.append_operation(plans) # generate the sublists
|
|
||||||
solved_subsets = []
|
|
||||||
for i in range(1, 5):
|
|
||||||
list_id = f"List {i}"
|
|
||||||
sub_list = operations.Selector(
|
|
||||||
lambda thoughts, list_id=list_id: [
|
|
||||||
thought for thought in thoughts if thought.state["part"] == list_id
|
|
||||||
]
|
|
||||||
)
|
|
||||||
sub_list.add_predecessor(plans)
|
|
||||||
operations_graph.add_operation(sub_list)
|
|
||||||
intersected_subset = operations.Generate(1, 5)
|
|
||||||
intersected_subset.add_predecessor(sub_list)
|
|
||||||
operations_graph.add_operation(intersected_subset)
|
|
||||||
score_sub_list = operations.Score(1, False, utils.num_errors)
|
|
||||||
score_sub_list.add_predecessor(intersected_subset)
|
|
||||||
operations_graph.add_operation(score_sub_list)
|
|
||||||
keep_best_sub_list = operations.KeepBestN(1, False)
|
|
||||||
keep_best_sub_list.add_predecessor(score_sub_list)
|
|
||||||
operations_graph.add_operation(keep_best_sub_list)
|
|
||||||
|
|
||||||
solved_subsets.append(keep_best_sub_list)
|
|
||||||
|
|
||||||
aggregate_1 = operations.Aggregate(10)
|
|
||||||
aggregate_1.add_predecessor(solved_subsets[0])
|
|
||||||
aggregate_1.add_predecessor(solved_subsets[1])
|
|
||||||
operations_graph.add_operation(aggregate_1)
|
|
||||||
score_aggregate_1 = operations.Score(1, False, utils.num_errors)
|
|
||||||
score_aggregate_1.add_predecessor(aggregate_1)
|
|
||||||
operations_graph.add_operation(score_aggregate_1)
|
|
||||||
keep_best_aggregate_1 = operations.KeepBestN(1, False)
|
|
||||||
keep_best_aggregate_1.add_predecessor(score_aggregate_1)
|
|
||||||
operations_graph.add_operation(keep_best_aggregate_1)
|
|
||||||
|
|
||||||
aggregate_2 = operations.Aggregate(10)
|
|
||||||
aggregate_2.add_predecessor(solved_subsets[2])
|
|
||||||
aggregate_2.add_predecessor(solved_subsets[3])
|
|
||||||
operations_graph.add_operation(aggregate_2)
|
|
||||||
score_aggregate_2 = operations.Score(1, False, utils.num_errors)
|
|
||||||
score_aggregate_2.add_predecessor(aggregate_2)
|
|
||||||
operations_graph.add_operation(score_aggregate_2)
|
|
||||||
keep_best_aggregate_2 = operations.KeepBestN(1, False)
|
|
||||||
keep_best_aggregate_2.add_predecessor(score_aggregate_2)
|
|
||||||
operations_graph.add_operation(keep_best_aggregate_2)
|
|
||||||
|
|
||||||
final_aggregate = operations.Aggregate(10)
|
|
||||||
operations_graph.append_operation(final_aggregate)
|
|
||||||
operations_graph.append_operation(operations.Score(1, False, utils.num_errors))
|
|
||||||
keep_best_aggregate_final = operations.KeepBestN(1, False)
|
|
||||||
operations_graph.append_operation(keep_best_aggregate_final)
|
|
||||||
|
|
||||||
operations_graph.append_operation(
|
|
||||||
operations.GroundTruth(utils.test_set_intersection)
|
|
||||||
)
|
|
||||||
|
|
||||||
return operations_graph
|
|
||||||
|
|
||||||
|
|
||||||
def run(
|
|
||||||
data_ids: List[int],
|
|
||||||
methods: List[Callable[[], operations.GraphOfOperations]],
|
|
||||||
budget: float,
|
|
||||||
lm_name: str,
|
|
||||||
) -> float:
|
|
||||||
"""
|
|
||||||
Controller function that executes each specified method for each specified
|
|
||||||
sample while the budget is not exhausted.
|
|
||||||
|
|
||||||
:param data_ids: Indices of the sample to be run.
|
|
||||||
:type data_ids: List[int]
|
|
||||||
:param methods: List of functions to generate Graphs of Operations.
|
|
||||||
:type methods: Each function generates a Graph of Operation.
|
|
||||||
:param budget: Language model budget for the execution in dollars.
|
|
||||||
:type budget: float
|
|
||||||
:param lm_name: Name of the language model to be used.
|
|
||||||
:type lm_name: str
|
|
||||||
:return: Spent budget in dollars.
|
|
||||||
:rtype: float
|
|
||||||
"""
|
|
||||||
|
|
||||||
orig_budget = budget
|
|
||||||
data_path = os.path.join(os.path.dirname(__file__), "set_intersection_064.csv")
|
|
||||||
data = []
|
|
||||||
with open(data_path, "r") as f:
|
|
||||||
reader = csv.reader(f)
|
|
||||||
next(reader)
|
|
||||||
for row in reader:
|
|
||||||
data.append([int(row[0]), row[1], row[2], row[3]])
|
|
||||||
|
|
||||||
if data_ids is None or len(data_ids) == 0:
|
|
||||||
data_ids = list(range(len(data)))
|
|
||||||
selected_data = [data[i] for i in data_ids]
|
|
||||||
|
|
||||||
results_dir = os.path.join(os.path.dirname(__file__), "results")
|
|
||||||
|
|
||||||
if not os.path.exists(results_dir):
|
|
||||||
os.makedirs(results_dir)
|
|
||||||
timestamp = datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
|
|
||||||
extra_info = f"{lm_name}_{'-'.join([method.__name__ for method in methods])}"
|
|
||||||
folder_name = f"{extra_info}_{timestamp}"
|
|
||||||
results_folder = os.path.join(results_dir, folder_name)
|
|
||||||
os.makedirs(results_folder)
|
|
||||||
|
|
||||||
config = {
|
|
||||||
"data": selected_data,
|
|
||||||
"methods": [method.__name__ for method in methods],
|
|
||||||
"lm": lm_name,
|
|
||||||
"budget": budget,
|
|
||||||
}
|
|
||||||
with open(os.path.join(results_folder, "config.json"), "w") as f:
|
|
||||||
json.dump(config, f)
|
|
||||||
|
|
||||||
logging.basicConfig(
|
|
||||||
filename=os.path.join(results_folder, "log.log"),
|
|
||||||
filemode="w",
|
|
||||||
format="%(name)s - %(levelname)s - %(message)s",
|
|
||||||
level=logging.DEBUG,
|
|
||||||
)
|
|
||||||
|
|
||||||
for method in methods:
|
|
||||||
# create a results directory for the method
|
|
||||||
os.makedirs(os.path.join(results_folder, method.__name__))
|
|
||||||
|
|
||||||
for data in selected_data:
|
|
||||||
logging.info(f"Running data {data[0]}: {data[1]} {data[2]}")
|
|
||||||
if budget <= 0.0:
|
|
||||||
logging.error(
|
|
||||||
f"Budget has been depleted, stopping. Data {data[0]} has not been run."
|
|
||||||
)
|
|
||||||
break
|
|
||||||
for method in methods:
|
|
||||||
logging.info(f"Running method {method.__name__}")
|
|
||||||
logging.info(f"Budget left: {budget}")
|
|
||||||
if budget <= 0.0:
|
|
||||||
logging.error(
|
|
||||||
f"Budget has been depleted, stopping. Method {method.__name__} has not been run."
|
|
||||||
)
|
|
||||||
break
|
|
||||||
lm = language_models.ChatGPT(
|
|
||||||
os.path.join(
|
|
||||||
os.path.dirname(__file__),
|
|
||||||
"../../graph_of_thoughts/language_models/config.json",
|
|
||||||
),
|
|
||||||
model_name=lm_name,
|
|
||||||
cache=True,
|
|
||||||
)
|
|
||||||
operations_graph = method()
|
|
||||||
executor = controller.Controller(
|
|
||||||
lm,
|
|
||||||
operations_graph,
|
|
||||||
SetIntersectionPrompter(),
|
|
||||||
SetIntersectionParser(),
|
|
||||||
{
|
|
||||||
"set1": data[1],
|
|
||||||
"set2": data[2],
|
|
||||||
"result": data[3],
|
|
||||||
"current": "",
|
|
||||||
"phase": 0,
|
|
||||||
"method": method.__name__,
|
|
||||||
},
|
|
||||||
)
|
|
||||||
try:
|
|
||||||
executor.run()
|
|
||||||
except Exception as e:
|
|
||||||
logging.error(f"Exception: {e}")
|
|
||||||
path = os.path.join(
|
|
||||||
results_folder,
|
|
||||||
method.__name__,
|
|
||||||
f"{data[0]}.json",
|
|
||||||
)
|
|
||||||
executor.output_graph(path)
|
|
||||||
budget -= lm.cost
|
|
||||||
|
|
||||||
return orig_budget - budget
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
"""
|
|
||||||
Input(x) : a list of 64 numbers between 0 and 127 (inclusive)
|
|
||||||
Input(y) : a list of 64 numbers between 0 and 127 (inclusive)
|
|
||||||
Output(z) : a list of the intersection between x and y
|
|
||||||
Correct : z = intersection(x, y)
|
|
||||||
Input Example:
|
|
||||||
[26, 40, 42, 57, 15, 31, 5, 32, 11, 4, 24, 28, 51, 54, 12, 22, 33, 35, 7, 13, 2, 59, 8, 23, 43, 16, 29, 55, 25, 63, 21, 18]
|
|
||||||
[16, 60, 36, 48, 0, 15, 5, 19, 46, 24, 1, 6, 61, 10, 38, 53, 58, 9, 44, 14, 35, 63, 52, 20, 27, 17, 39, 47, 34, 56, 40, 59]
|
|
||||||
Output Example:
|
|
||||||
[40, 15, 5, 24, 35, 59, 16, 63]
|
|
||||||
"""
|
|
||||||
|
|
||||||
budget = 15
|
|
||||||
samples = [item for item in range(0, 100)]
|
|
||||||
approaches = [io, cot, tot, tot2, got]
|
|
||||||
|
|
||||||
spent = run(samples, approaches, budget, "chatgpt")
|
|
||||||
|
|
||||||
logging.info(f"Spent {spent} out of {budget} budget.")
|
|
||||||
@ -1,101 +0,0 @@
|
|||||||
ID,SET1,SET2,INTERSECTION
|
|
||||||
0,"[132, 75, 157, 25, 199, 202, 147, 109, 221, 110, 220, 251, 213, 11, 224, 101, 200, 170, 155, 71, 119, 122, 39, 1, 29, 113, 189, 212, 10, 219, 49, 28, 151, 40, 103, 8, 145, 214, 114, 91, 175, 107, 152, 163, 148, 246, 176, 181, 18, 106, 74, 115, 144, 0, 205, 121, 46, 234, 142, 223, 228, 162, 96, 97, 130, 156, 172, 241, 33, 186, 137, 150, 65, 161, 226, 116, 111, 12, 146, 38, 167, 4, 108, 169, 61, 93, 190, 252, 22, 31, 3, 9, 13, 35, 23, 141, 129, 198, 85, 84, 62, 158, 201, 67, 117, 59, 41, 191, 56, 90, 51, 227, 143, 83, 184, 174, 125, 98, 232, 238, 57, 225, 54, 179, 177, 237, 37, 95]","[27, 162, 187, 254, 128, 227, 2, 165, 143, 109, 140, 46, 160, 26, 139, 171, 42, 199, 207, 30, 205, 117, 213, 48, 40, 212, 185, 196, 197, 94, 136, 35, 229, 193, 36, 7, 15, 43, 4, 203, 142, 144, 49, 31, 240, 124, 116, 69, 37, 250, 95, 105, 103, 168, 126, 64, 73, 206, 24, 157, 135, 118, 34, 134, 45, 62, 153, 5, 47, 239, 216, 222, 80, 231, 102, 21, 57, 215, 149, 141, 236, 32, 188, 204, 194, 23, 233, 83, 154, 210, 159, 70, 202, 253, 20, 71, 166, 242, 221, 228, 78, 230, 29, 145, 147, 81, 104, 235, 66, 100, 131, 132, 244, 195, 68, 72, 53, 182, 79, 248, 3, 82, 211, 173, 180, 17, 77, 51]","[3, 4, 23, 29, 31, 35, 37, 40, 46, 49, 51, 57, 62, 71, 83, 95, 103, 109, 116, 117, 132, 141, 142, 143, 144, 145, 147, 157, 162, 199, 202, 205, 212, 213, 221, 227, 228]"
|
|
||||||
1,"[95, 68, 8, 76, 195, 92, 25, 19, 109, 146, 36, 2, 191, 164, 200, 189, 113, 203, 45, 52, 244, 210, 167, 91, 133, 139, 231, 202, 93, 22, 158, 239, 39, 94, 97, 40, 6, 213, 56, 16, 153, 14, 145, 172, 182, 226, 161, 130, 174, 44, 84, 112, 99, 204, 105, 64, 227, 198, 196, 42, 38, 129, 4, 208, 107, 70, 47, 49, 243, 178, 28, 148, 125, 80, 55, 104, 132, 90, 100, 175, 229, 185, 46, 154, 187, 63, 18, 184, 241, 12, 31, 218, 29, 72, 21, 255, 103, 150, 83, 73, 89, 71, 157, 11, 9, 188, 220, 65, 140, 27, 162, 66, 119, 116, 152, 205, 143, 147, 61, 88, 235, 5, 141, 33, 75, 13, 102, 128]","[239, 240, 54, 97, 1, 39, 70, 25, 125, 243, 155, 19, 115, 133, 213, 227, 31, 29, 130, 40, 203, 209, 174, 64, 36, 128, 161, 83, 57, 139, 107, 212, 112, 90, 147, 149, 14, 118, 129, 42, 110, 188, 235, 16, 51, 108, 92, 73, 196, 158, 126, 113, 79, 208, 71, 63, 103, 167, 109, 255, 91, 10, 181, 6, 68, 238, 191, 189, 201, 156, 20, 43, 220, 205, 144, 215, 4, 162, 74, 18, 183, 185, 67, 194, 231, 81, 84, 101, 69, 24, 94, 152, 104, 22, 241, 11, 48, 76, 95, 237, 199, 135, 140, 9, 173, 58, 190, 8, 65, 210, 12, 169, 37, 200, 246, 178, 98, 93, 202, 226, 157, 166, 47, 142, 150, 197, 244, 121]","[4, 6, 8, 9, 11, 12, 14, 16, 18, 19, 22, 25, 29, 31, 36, 39, 40, 42, 47, 63, 64, 65, 68, 70, 71, 73, 76, 83, 84, 90, 91, 92, 93, 94, 95, 97, 103, 104, 107, 109, 112, 113, 125, 128, 129, 130, 133, 139, 140, 147, 150, 152, 157, 158, 161, 162, 167, 174, 178, 185, 188, 189, 191, 196, 200, 202, 203, 205, 208, 210, 213, 220, 226, 227, 231, 235, 239, 241, 243, 244, 255]"
|
|
||||||
2,"[249, 62, 255, 65, 140, 234, 237, 148, 166, 170, 6, 36, 215, 179, 208, 213, 194, 145, 58, 241, 83, 81, 176, 239, 216, 33, 235, 117, 250, 14, 94, 39, 178, 132, 201, 37, 231, 177, 93, 183, 23, 125, 152, 159, 78, 1, 246, 108, 205, 247, 160, 175, 238, 49, 127, 77, 103, 27, 242, 34, 167, 196, 133, 143, 124, 182, 225, 106, 146, 104, 97, 10, 162, 233, 55, 163, 184, 66, 31, 224, 217, 68, 20, 131, 84, 147, 24, 38, 218, 232, 203, 137, 57, 154, 47, 3, 21, 115, 169, 198, 2, 110, 197, 161, 67, 136, 191, 149, 61, 53, 227, 80, 128, 188, 42, 153, 230, 109, 220, 5, 116, 252, 85, 11, 228, 105, 113, 151]","[76, 196, 243, 230, 147, 154, 57, 64, 140, 49, 149, 223, 215, 207, 255, 174, 79, 188, 54, 106, 77, 33, 245, 240, 2, 175, 165, 254, 97, 62, 89, 66, 234, 219, 136, 42, 176, 38, 195, 212, 241, 135, 20, 41, 35, 55, 128, 166, 95, 171, 0, 68, 127, 114, 186, 137, 162, 161, 69, 225, 181, 192, 153, 14, 239, 8, 110, 115, 104, 156, 78, 7, 81, 129, 133, 96, 50, 36, 141, 113, 47, 23, 172, 39, 112, 151, 244, 204, 145, 99, 93, 6, 169, 236, 4, 116, 216, 70, 132, 179, 217, 10, 19, 53, 84, 88, 74, 24, 155, 18, 193, 203, 189, 94, 237, 63, 163, 218, 9, 178, 48, 180, 201, 202, 233, 185, 183, 197]","[2, 6, 10, 14, 20, 23, 24, 33, 36, 38, 39, 42, 47, 49, 53, 55, 57, 62, 66, 68, 77, 78, 81, 84, 93, 94, 97, 104, 106, 110, 113, 115, 116, 127, 128, 132, 133, 136, 137, 140, 145, 147, 149, 151, 153, 154, 161, 162, 163, 166, 169, 175, 176, 178, 179, 183, 188, 196, 197, 201, 203, 215, 216, 217, 218, 225, 230, 233, 234, 237, 239, 241, 255]"
|
|
||||||
3,"[229, 37, 83, 1, 214, 179, 159, 5, 235, 120, 88, 61, 213, 234, 194, 108, 59, 200, 118, 170, 73, 91, 250, 215, 68, 255, 35, 122, 150, 33, 193, 227, 133, 197, 56, 225, 137, 17, 173, 18, 171, 82, 31, 140, 254, 39, 217, 10, 36, 75, 64, 246, 8, 22, 232, 238, 223, 162, 85, 15, 121, 201, 220, 221, 218, 101, 13, 100, 12, 177, 92, 105, 157, 199, 29, 188, 69, 4, 58, 9, 224, 47, 182, 115, 25, 119, 131, 117, 80, 183, 132, 135, 219, 62, 129, 2, 42, 106, 205, 191, 207, 87, 45, 202, 172, 154, 139, 142, 102, 14, 41, 16, 241, 94, 156, 89, 86, 78, 99, 252, 253, 77, 6, 116, 204, 206, 110, 251]","[61, 155, 205, 169, 2, 83, 114, 74, 191, 131, 185, 130, 172, 0, 95, 84, 118, 76, 78, 213, 214, 195, 8, 234, 173, 64, 59, 113, 48, 170, 109, 244, 41, 183, 23, 248, 73, 68, 150, 149, 223, 46, 91, 133, 175, 212, 33, 165, 29, 108, 94, 79, 227, 249, 112, 138, 70, 89, 232, 164, 245, 250, 87, 30, 202, 57, 174, 101, 1, 242, 67, 239, 148, 188, 126, 143, 21, 226, 122, 102, 19, 97, 129, 7, 134, 56, 182, 60, 81, 45, 32, 28, 34, 186, 22, 176, 208, 159, 136, 24, 43, 139, 194, 85, 26, 75, 218, 254, 111, 217, 50, 233, 144, 36, 82, 147, 16, 128, 222, 90, 241, 203, 51, 189, 15, 58, 251, 211]","[1, 2, 8, 15, 16, 22, 29, 33, 36, 41, 45, 56, 58, 59, 61, 64, 68, 73, 75, 78, 82, 83, 85, 87, 89, 91, 94, 101, 102, 108, 118, 122, 129, 131, 133, 139, 150, 159, 170, 172, 173, 182, 183, 188, 191, 194, 202, 205, 213, 214, 217, 218, 223, 227, 232, 234, 241, 250, 251, 254]"
|
|
||||||
4,"[96, 78, 17, 90, 12, 255, 235, 160, 229, 219, 83, 102, 163, 158, 142, 215, 28, 106, 45, 47, 95, 36, 65, 30, 181, 146, 59, 197, 170, 118, 42, 13, 29, 178, 192, 186, 168, 79, 222, 39, 161, 26, 56, 88, 180, 33, 179, 69, 206, 144, 230, 38, 32, 82, 250, 99, 72, 159, 204, 9, 140, 58, 14, 50, 22, 19, 116, 169, 173, 35, 80, 200, 183, 149, 124, 213, 171, 216, 24, 141, 148, 175, 242, 86, 37, 221, 245, 1, 48, 157, 184, 89, 34, 54, 214, 198, 93, 6, 109, 44, 125, 153, 208, 132, 52, 211, 165, 167, 251, 147, 239, 103, 98, 240, 61, 203, 143, 67, 226, 176, 66, 131, 166, 207, 43, 232, 133, 15]","[57, 250, 63, 207, 20, 27, 80, 205, 96, 166, 78, 37, 164, 100, 217, 134, 163, 135, 115, 155, 200, 125, 82, 174, 83, 50, 0, 231, 75, 232, 84, 93, 239, 86, 2, 196, 222, 215, 58, 172, 220, 61, 122, 247, 66, 42, 161, 234, 153, 6, 236, 162, 230, 194, 8, 35, 168, 74, 70, 112, 87, 195, 81, 145, 121, 40, 137, 227, 38, 60, 139, 17, 201, 76, 118, 245, 229, 41, 95, 173, 146, 218, 176, 19, 225, 3, 209, 211, 5, 109, 228, 51, 108, 131, 170, 182, 179, 183, 159, 254, 152, 212, 190, 28, 13, 97, 219, 91, 54, 157, 104, 79, 253, 68, 103, 177, 148, 114, 116, 185, 25, 208, 191, 49, 22, 221, 31, 45]","[6, 13, 17, 19, 22, 28, 35, 37, 38, 42, 45, 50, 54, 58, 61, 66, 78, 79, 80, 82, 83, 86, 93, 95, 96, 103, 109, 116, 118, 125, 131, 146, 148, 153, 157, 159, 161, 163, 166, 168, 170, 173, 176, 179, 183, 200, 207, 208, 211, 215, 219, 221, 222, 229, 230, 232, 239, 245, 250]"
|
|
||||||
5,"[76, 34, 18, 92, 100, 226, 228, 56, 80, 29, 126, 55, 109, 182, 27, 176, 179, 68, 246, 128, 137, 218, 187, 189, 0, 183, 162, 74, 255, 12, 15, 96, 202, 135, 210, 177, 51, 142, 111, 152, 205, 14, 75, 171, 107, 133, 233, 38, 32, 253, 180, 242, 25, 129, 10, 247, 156, 39, 48, 221, 203, 197, 19, 155, 16, 4, 166, 40, 188, 131, 49, 191, 185, 208, 122, 72, 108, 245, 227, 206, 115, 231, 3, 9, 174, 103, 232, 195, 17, 73, 144, 151, 158, 141, 62, 67, 21, 211, 123, 114, 209, 30, 252, 31, 229, 91, 125, 219, 79, 58, 204, 127, 136, 214, 148, 59, 238, 169, 87, 23, 37, 192, 119, 254, 94, 134, 90, 11]","[31, 18, 105, 11, 25, 165, 47, 58, 94, 72, 221, 171, 157, 228, 229, 242, 65, 40, 92, 111, 237, 16, 209, 62, 155, 186, 3, 119, 117, 27, 158, 235, 148, 24, 215, 177, 0, 120, 144, 114, 135, 180, 91, 59, 208, 198, 55, 96, 44, 134, 243, 89, 121, 214, 56, 73, 210, 64, 52, 8, 126, 38, 12, 227, 9, 238, 4, 127, 204, 212, 103, 15, 46, 131, 48, 90, 76, 241, 156, 150, 108, 128, 218, 80, 222, 6, 203, 81, 202, 35, 29, 151, 175, 142, 140, 2, 84, 192, 61, 206, 87, 146, 247, 51, 170, 244, 234, 125, 70, 232, 123, 10, 182, 153, 50, 60, 100, 115, 137, 133, 129, 19, 188, 68, 179, 97, 205, 75]","[0, 3, 4, 9, 10, 11, 12, 15, 16, 18, 19, 25, 27, 29, 31, 38, 40, 48, 51, 55, 56, 58, 59, 62, 68, 72, 73, 75, 76, 80, 87, 90, 91, 92, 94, 96, 100, 103, 108, 111, 114, 115, 119, 123, 125, 126, 127, 128, 129, 131, 133, 134, 135, 137, 142, 144, 148, 151, 155, 156, 158, 171, 177, 179, 180, 182, 188, 192, 202, 203, 204, 205, 206, 208, 209, 210, 214, 218, 221, 227, 228, 229, 232, 238, 242, 247]"
|
|
||||||
6,"[105, 245, 208, 74, 185, 27, 32, 157, 60, 169, 173, 94, 237, 140, 151, 124, 193, 142, 198, 247, 43, 63, 120, 125, 141, 189, 110, 231, 98, 96, 72, 175, 80, 100, 207, 53, 252, 16, 107, 114, 49, 180, 164, 184, 11, 119, 129, 218, 66, 61, 155, 18, 213, 170, 73, 243, 148, 205, 147, 106, 68, 211, 143, 95, 152, 75, 203, 192, 99, 145, 235, 233, 44, 54, 171, 116, 81, 0, 136, 3, 103, 165, 59, 52, 45, 230, 177, 33, 31, 50, 82, 232, 13, 255, 183, 111, 176, 181, 39, 122, 101, 221, 92, 88, 70, 135, 115, 130, 79, 7, 89, 137, 191, 179, 236, 65, 250, 174, 48, 134, 104, 90, 108, 113, 42, 226, 85, 132]","[43, 187, 54, 201, 121, 66, 163, 154, 109, 62, 171, 19, 166, 81, 135, 227, 240, 238, 246, 172, 78, 10, 223, 241, 108, 58, 250, 107, 239, 153, 191, 204, 149, 15, 210, 150, 213, 180, 103, 28, 131, 17, 184, 1, 57, 86, 29, 138, 162, 206, 159, 243, 127, 199, 79, 136, 37, 75, 235, 5, 216, 14, 133, 20, 93, 32, 77, 67, 118, 234, 190, 25, 76, 217, 44, 7, 147, 249, 160, 175, 219, 174, 35, 12, 102, 47, 41, 94, 61, 55, 46, 170, 69, 16, 214, 53, 242, 194, 140, 168, 165, 228, 244, 202, 9, 161, 56, 2, 157, 142, 117, 114, 40, 233, 123, 6, 182, 36, 18, 26, 87, 146, 71, 224, 139, 222, 212, 23]","[7, 16, 18, 32, 43, 44, 53, 54, 61, 66, 75, 79, 81, 94, 103, 107, 108, 114, 135, 136, 140, 142, 147, 157, 165, 170, 171, 174, 175, 180, 184, 191, 213, 233, 235, 243, 250]"
|
|
||||||
7,"[22, 201, 171, 46, 62, 25, 104, 220, 219, 160, 91, 200, 174, 42, 145, 26, 80, 255, 234, 214, 11, 228, 44, 147, 69, 167, 73, 185, 115, 162, 252, 4, 105, 64, 72, 126, 204, 157, 249, 92, 88, 202, 27, 203, 155, 111, 5, 15, 124, 61, 208, 224, 0, 1, 18, 85, 28, 239, 76, 55, 79, 48, 140, 19, 36, 8, 21, 66, 217, 169, 142, 56, 35, 113, 193, 103, 131, 39, 130, 45, 161, 84, 143, 141, 232, 97, 94, 209, 57, 173, 150, 41, 159, 53, 86, 23, 49, 17, 250, 40, 213, 241, 195, 50, 78, 121, 178, 137, 199, 149, 106, 74, 235, 226, 245, 135, 166, 144, 134, 120, 186, 102, 148, 225, 132, 125, 70, 89]","[255, 23, 91, 158, 203, 120, 209, 212, 38, 105, 73, 226, 187, 145, 163, 219, 70, 216, 57, 76, 241, 225, 249, 30, 131, 181, 224, 178, 69, 31, 153, 134, 168, 210, 4, 127, 242, 125, 33, 87, 86, 102, 100, 175, 121, 133, 77, 9, 35, 54, 149, 55, 2, 247, 166, 250, 234, 143, 232, 215, 5, 64, 78, 227, 179, 117, 171, 90, 204, 60, 177, 27, 148, 24, 49, 113, 88, 160, 34, 180, 140, 235, 20, 220, 122, 106, 185, 186, 108, 150, 46, 63, 200, 191, 95, 146, 82, 6, 26, 176, 50, 174, 61, 245, 217, 202, 45, 254, 65, 137, 193, 115, 107, 79, 164, 135, 39, 197, 213, 111, 126, 118, 195, 252, 97, 190, 40, 182]","[4, 5, 23, 26, 27, 35, 39, 40, 45, 46, 49, 50, 55, 57, 61, 64, 69, 70, 73, 76, 78, 79, 86, 88, 91, 97, 102, 105, 106, 111, 113, 115, 120, 121, 125, 126, 131, 134, 135, 137, 140, 143, 145, 148, 149, 150, 160, 166, 171, 174, 178, 185, 186, 193, 195, 200, 202, 203, 204, 209, 213, 217, 219, 220, 224, 225, 226, 232, 234, 235, 241, 245, 249, 250, 252, 255]"
|
|
||||||
8,"[134, 245, 157, 213, 19, 34, 38, 230, 144, 18, 168, 146, 73, 64, 91, 89, 198, 87, 65, 32, 36, 148, 131, 221, 58, 84, 132, 41, 83, 67, 249, 42, 237, 136, 250, 35, 60, 69, 176, 125, 116, 117, 85, 106, 61, 133, 244, 46, 96, 23, 166, 188, 120, 139, 15, 123, 238, 162, 129, 212, 175, 227, 138, 97, 72, 6, 9, 47, 119, 103, 192, 88, 63, 8, 233, 77, 251, 70, 200, 30, 16, 126, 202, 183, 173, 81, 169, 223, 102, 164, 105, 218, 231, 180, 33, 78, 118, 150, 165, 191, 152, 178, 53, 153, 205, 189, 51, 243, 141, 40, 204, 151, 20, 5, 239, 254, 48, 14, 135, 98, 197, 2, 54, 111, 122, 232, 59, 107]","[26, 3, 87, 214, 121, 5, 23, 123, 254, 27, 66, 248, 152, 209, 191, 39, 171, 154, 149, 193, 92, 45, 228, 199, 223, 173, 21, 104, 226, 101, 44, 136, 224, 140, 155, 220, 117, 29, 109, 194, 49, 89, 216, 55, 4, 114, 202, 61, 184, 225, 13, 76, 183, 174, 75, 57, 48, 71, 38, 190, 7, 210, 243, 138, 246, 67, 163, 18, 247, 34, 6, 103, 237, 54, 102, 82, 97, 169, 221, 147, 74, 72, 144, 222, 128, 37, 217, 162, 12, 68, 33, 80, 17, 203, 164, 195, 125, 240, 127, 252, 22, 115, 151, 130, 143, 200, 106, 253, 201, 11, 10, 113, 16, 28, 234, 63, 150, 141, 24, 124, 79, 232, 215, 167, 31, 108, 181, 206]","[5, 6, 16, 18, 23, 33, 34, 38, 48, 54, 61, 63, 67, 72, 87, 89, 97, 102, 103, 106, 117, 123, 125, 136, 138, 141, 144, 150, 151, 152, 162, 164, 169, 173, 183, 191, 200, 202, 221, 223, 232, 237, 243, 254]"
|
|
||||||
9,"[71, 207, 57, 0, 118, 134, 100, 10, 216, 119, 218, 42, 102, 184, 82, 146, 249, 90, 136, 12, 164, 248, 220, 176, 158, 47, 9, 224, 250, 25, 159, 110, 14, 125, 231, 39, 174, 116, 143, 147, 81, 124, 40, 156, 11, 103, 217, 152, 87, 196, 80, 28, 52, 43, 121, 4, 138, 72, 140, 85, 169, 91, 153, 198, 53, 206, 128, 93, 246, 188, 243, 68, 59, 212, 240, 31, 142, 139, 60, 65, 18, 67, 221, 187, 226, 127, 89, 48, 44, 252, 154, 74, 84, 222, 75, 208, 45, 66, 199, 170, 200, 235, 227, 233, 78, 145, 168, 113, 186, 107, 209, 27, 6, 79, 55, 228, 144, 17, 22, 49, 236, 117, 223, 69, 133, 32, 15, 173]","[128, 234, 195, 96, 104, 83, 34, 78, 120, 244, 97, 189, 239, 38, 139, 188, 56, 6, 185, 109, 46, 26, 89, 118, 157, 35, 243, 170, 115, 108, 226, 49, 213, 123, 149, 225, 71, 184, 74, 64, 165, 2, 182, 15, 3, 162, 31, 20, 137, 155, 135, 177, 4, 114, 196, 151, 207, 29, 132, 88, 105, 214, 197, 167, 211, 92, 242, 99, 179, 144, 42, 41, 166, 201, 63, 18, 16, 222, 230, 28, 163, 101, 194, 241, 106, 255, 13, 36, 32, 130, 70, 7, 161, 229, 171, 205, 183, 21, 235, 95, 176, 145, 50, 73, 191, 200, 215, 98, 58, 77, 164, 236, 47, 175, 112, 212, 129, 190, 192, 227, 55, 125, 79, 238, 86, 208, 148, 76]","[4, 6, 15, 18, 28, 31, 32, 42, 47, 49, 55, 71, 74, 78, 79, 89, 118, 125, 128, 139, 144, 145, 164, 170, 176, 184, 188, 196, 200, 207, 208, 212, 222, 226, 227, 235, 236, 243]"
|
|
||||||
10,"[182, 73, 5, 39, 24, 235, 225, 81, 15, 153, 123, 80, 93, 46, 247, 91, 69, 236, 185, 212, 184, 42, 135, 168, 54, 19, 230, 72, 76, 211, 180, 97, 86, 40, 109, 51, 63, 206, 37, 115, 34, 100, 102, 158, 237, 103, 137, 251, 234, 6, 204, 68, 117, 29, 244, 49, 111, 165, 85, 224, 161, 146, 90, 176, 10, 56, 28, 189, 213, 202, 245, 178, 58, 50, 220, 107, 129, 243, 171, 195, 13, 122, 66, 149, 33, 166, 20, 104, 144, 105, 45, 89, 2, 152, 193, 98, 145, 38, 155, 229, 120, 191, 163, 143, 57, 95, 113, 92, 17, 154, 12, 41, 106, 7, 132, 84, 227, 133, 249, 157, 83, 164, 101, 118, 179, 170, 71, 138]","[112, 85, 138, 202, 102, 142, 224, 178, 25, 30, 52, 78, 21, 187, 239, 126, 20, 164, 58, 190, 149, 250, 92, 24, 12, 69, 0, 1, 235, 55, 242, 89, 182, 118, 97, 39, 73, 99, 124, 135, 79, 32, 22, 162, 207, 157, 184, 122, 16, 84, 180, 192, 76, 222, 19, 230, 57, 133, 81, 160, 27, 109, 174, 11, 150, 134, 255, 98, 199, 188, 43, 245, 218, 147, 179, 183, 63, 238, 132, 146, 232, 111, 165, 244, 60, 166, 34, 101, 181, 220, 44, 74, 143, 38, 113, 154, 13, 42, 47, 23, 171, 29, 223, 66, 7, 186, 231, 177, 153, 159, 221, 53, 83, 35, 167, 4, 129, 252, 50, 61, 214, 201, 205, 2, 217, 213, 5, 41]","[2, 5, 7, 12, 13, 19, 20, 24, 29, 34, 38, 39, 41, 42, 50, 57, 58, 63, 66, 69, 73, 76, 81, 83, 84, 85, 89, 92, 97, 98, 101, 102, 109, 111, 113, 118, 122, 129, 132, 133, 135, 138, 143, 146, 149, 153, 154, 157, 164, 165, 166, 171, 178, 179, 180, 182, 184, 202, 213, 220, 224, 230, 235, 244, 245]"
|
|
||||||
11,"[42, 176, 29, 59, 122, 184, 171, 137, 82, 136, 41, 80, 95, 5, 187, 32, 138, 68, 185, 43, 75, 49, 224, 88, 69, 111, 196, 247, 126, 117, 56, 105, 98, 37, 153, 54, 87, 114, 19, 180, 238, 93, 90, 223, 165, 124, 227, 241, 125, 210, 128, 242, 116, 120, 40, 55, 228, 26, 45, 218, 21, 107, 132, 119, 57, 189, 199, 27, 141, 97, 83, 109, 127, 164, 231, 17, 76, 209, 215, 159, 172, 84, 65, 1, 73, 143, 30, 202, 252, 94, 103, 52, 96, 170, 86, 130, 154, 4, 8, 134, 174, 79, 102, 175, 91, 200, 216, 36, 9, 63, 192, 207, 62, 225, 167, 77, 35, 155, 100, 212, 129, 115, 240, 149, 12, 16, 255, 123]","[117, 134, 8, 87, 41, 32, 19, 131, 196, 83, 54, 84, 4, 80, 238, 252, 176, 242, 17, 149, 46, 225, 102, 18, 76, 42, 55, 229, 200, 108, 228, 115, 151, 164, 231, 171, 1, 159, 23, 170, 167, 88, 180, 45, 98, 59, 215, 68, 216, 77, 130, 66, 12, 158, 114, 205, 3, 166, 187, 35, 99, 165, 29, 195, 182, 129, 125, 185, 30, 27, 120, 21, 213, 36, 147, 85, 234, 143, 97, 123, 109, 189, 86, 190, 153, 81, 9, 74, 255, 16, 126, 70, 155, 184, 82, 207, 6, 140, 223, 227, 208, 241, 79, 162, 69, 127, 212, 247, 132, 128, 44, 191, 91, 107, 217, 111, 26, 47, 177, 100, 93, 37, 110, 235, 202, 137, 138, 73]","[1, 4, 8, 9, 12, 16, 17, 19, 21, 26, 27, 29, 30, 32, 35, 36, 37, 41, 42, 45, 54, 55, 59, 68, 69, 73, 76, 77, 79, 80, 82, 83, 84, 86, 87, 88, 91, 93, 97, 98, 100, 102, 107, 109, 111, 114, 115, 117, 120, 123, 125, 126, 127, 128, 129, 130, 132, 134, 137, 138, 143, 149, 153, 155, 159, 164, 165, 167, 170, 171, 176, 180, 184, 185, 187, 189, 196, 200, 202, 207, 212, 215, 216, 223, 225, 227, 228, 231, 238, 241, 242, 247, 252, 255]"
|
|
||||||
12,"[127, 104, 110, 26, 245, 157, 167, 84, 78, 210, 59, 41, 1, 123, 181, 85, 168, 69, 86, 138, 30, 94, 36, 8, 50, 77, 228, 19, 23, 63, 103, 216, 173, 152, 82, 202, 112, 139, 254, 250, 121, 47, 195, 243, 241, 109, 51, 183, 61, 65, 193, 122, 133, 180, 194, 158, 175, 182, 42, 10, 12, 171, 64, 154, 108, 252, 170, 113, 177, 48, 53, 115, 197, 4, 219, 99, 239, 159, 226, 211, 143, 235, 32, 20, 222, 169, 229, 15, 43, 106, 129, 221, 233, 33, 11, 114, 7, 18, 144, 204, 253, 119, 49, 0, 45, 187, 67, 9, 98, 29, 255, 125, 27, 232, 142, 134, 68, 101, 205, 70, 97, 223, 225, 185, 176, 72, 136, 107]","[32, 37, 144, 113, 109, 179, 87, 176, 108, 107, 152, 129, 254, 170, 211, 169, 174, 246, 42, 171, 187, 41, 99, 225, 11, 186, 85, 64, 63, 101, 136, 68, 159, 57, 19, 95, 156, 104, 224, 189, 110, 103, 138, 219, 117, 131, 229, 172, 163, 72, 194, 123, 3, 112, 21, 165, 139, 202, 177, 185, 60, 40, 197, 52, 88, 114, 124, 226, 73, 150, 216, 164, 133, 175, 118, 44, 50, 10, 121, 127, 77, 234, 46, 1, 151, 206, 115, 245, 241, 36, 223, 147, 56, 205, 61, 28, 94, 180, 141, 166, 83, 247, 132, 240, 192, 157, 204, 74, 193, 217, 86, 196, 30, 243, 78, 98, 53, 120, 49, 119, 250, 238, 236, 48, 232, 13, 65, 182]","[1, 10, 11, 19, 30, 32, 36, 41, 42, 48, 49, 50, 53, 61, 63, 64, 65, 68, 72, 77, 78, 85, 86, 94, 98, 99, 101, 103, 104, 107, 108, 109, 110, 112, 113, 114, 115, 119, 121, 123, 127, 129, 133, 136, 138, 139, 144, 152, 157, 159, 169, 170, 171, 175, 176, 177, 180, 182, 185, 187, 193, 194, 197, 202, 204, 205, 211, 216, 219, 223, 225, 226, 229, 232, 241, 243, 245, 250, 254]"
|
|
||||||
13,"[76, 5, 56, 132, 218, 246, 20, 237, 225, 62, 23, 58, 175, 177, 34, 183, 12, 67, 199, 164, 117, 93, 174, 171, 255, 130, 244, 36, 0, 249, 112, 4, 11, 221, 111, 113, 137, 253, 90, 166, 95, 241, 7, 138, 88, 63, 180, 185, 213, 26, 220, 208, 210, 105, 242, 8, 124, 158, 92, 9, 181, 135, 103, 35, 81, 106, 97, 94, 27, 204, 155, 205, 184, 32, 142, 17, 121, 89, 141, 245, 53, 151, 82, 98, 45, 233, 6, 198, 71, 195, 128, 161, 109, 226, 172, 129, 86, 201, 157, 217, 19, 99, 70, 122, 165, 28, 46, 209, 29, 131, 116, 57, 152, 176, 43, 230, 178, 119, 194, 40, 30, 190, 232, 143, 51, 48, 169, 216]","[51, 110, 203, 45, 249, 0, 136, 122, 217, 176, 111, 213, 244, 4, 67, 63, 5, 147, 191, 177, 81, 80, 104, 134, 130, 65, 133, 212, 181, 155, 94, 56, 28, 185, 208, 106, 88, 161, 233, 105, 226, 169, 12, 222, 251, 18, 82, 20, 142, 184, 135, 250, 204, 90, 151, 35, 178, 1, 202, 91, 159, 128, 86, 139, 163, 174, 187, 32, 26, 148, 112, 7, 164, 179, 49, 158, 83, 96, 69, 29, 123, 61, 168, 165, 68, 107, 166, 216, 193, 232, 113, 8, 138, 71, 225, 93, 223, 170, 237, 255, 132, 17, 57, 129, 53, 72, 197, 171, 24, 76, 221, 201, 2, 209, 99, 66, 73, 175, 162, 97, 23, 37, 200, 241, 101, 218, 150, 108]","[0, 4, 5, 7, 8, 12, 17, 20, 23, 26, 28, 29, 32, 35, 45, 51, 53, 56, 57, 63, 67, 71, 76, 81, 82, 86, 88, 90, 93, 94, 97, 99, 105, 106, 111, 112, 113, 122, 128, 129, 130, 132, 135, 138, 142, 151, 155, 158, 161, 164, 165, 166, 169, 171, 174, 175, 176, 177, 178, 181, 184, 185, 201, 204, 208, 209, 213, 216, 217, 218, 221, 225, 226, 232, 233, 237, 241, 244, 249, 255]"
|
|
||||||
14,"[82, 156, 200, 53, 194, 3, 80, 8, 120, 141, 179, 84, 1, 208, 79, 148, 125, 130, 63, 127, 62, 146, 248, 152, 215, 210, 74, 135, 242, 83, 162, 115, 47, 164, 233, 108, 134, 68, 13, 67, 45, 173, 54, 126, 251, 195, 16, 123, 250, 186, 128, 255, 168, 90, 157, 119, 136, 227, 36, 38, 104, 238, 99, 244, 185, 247, 236, 29, 206, 165, 226, 225, 132, 60, 103, 77, 142, 216, 55, 32, 253, 159, 129, 91, 205, 232, 163, 78, 150, 50, 193, 167, 37, 81, 71, 145, 188, 66, 26, 114, 211, 76, 196, 106, 4, 30, 202, 237, 151, 110, 51, 2, 154, 94, 46, 109, 11, 143, 92, 21, 241, 97, 124, 219, 166, 221, 34, 137]","[132, 216, 245, 247, 222, 210, 190, 235, 18, 227, 215, 208, 187, 90, 68, 197, 94, 121, 95, 50, 52, 64, 217, 49, 230, 183, 127, 77, 125, 239, 115, 72, 103, 62, 250, 139, 9, 176, 111, 8, 154, 136, 74, 2, 143, 225, 54, 87, 141, 129, 60, 155, 170, 99, 23, 148, 126, 53, 171, 147, 97, 145, 163, 45, 140, 84, 110, 168, 29, 195, 101, 67, 251, 137, 91, 73, 173, 223, 81, 243, 82, 119, 175, 167, 252, 92, 93, 32, 79, 221, 22, 51, 112, 185, 12, 130, 241, 138, 233, 192, 142, 219, 104, 46, 254, 248, 229, 150, 169, 232, 1, 198, 212, 57, 63, 59, 238, 105, 213, 13, 158, 134, 3, 240, 135, 36, 128, 76]","[1, 2, 3, 8, 13, 29, 32, 36, 45, 46, 50, 51, 53, 54, 60, 62, 63, 67, 68, 74, 76, 77, 79, 81, 82, 84, 90, 91, 92, 94, 97, 99, 103, 104, 110, 115, 119, 125, 126, 127, 128, 129, 130, 132, 134, 135, 136, 137, 141, 142, 143, 145, 148, 150, 154, 163, 167, 168, 173, 185, 195, 208, 210, 215, 216, 219, 221, 225, 227, 232, 233, 238, 241, 247, 248, 250, 251]"
|
|
||||||
15,"[245, 178, 15, 66, 12, 133, 241, 198, 186, 107, 210, 199, 62, 55, 194, 190, 92, 175, 127, 76, 22, 161, 126, 173, 101, 88, 211, 129, 204, 136, 3, 5, 98, 187, 213, 157, 89, 205, 226, 32, 94, 160, 233, 197, 123, 193, 214, 182, 74, 150, 121, 240, 0, 50, 181, 31, 58, 17, 177, 172, 128, 69, 73, 16, 159, 102, 28, 83, 23, 147, 59, 20, 11, 195, 222, 46, 243, 119, 248, 41, 218, 217, 65, 224, 149, 131, 191, 34, 207, 228, 143, 54, 44, 48, 132, 184, 171, 154, 114, 103, 67, 206, 29, 180, 10, 120, 219, 118, 87, 1, 153, 163, 14, 100, 42, 231, 75, 237, 189, 232, 24, 125, 202, 93, 148, 52, 104, 167]","[42, 139, 183, 54, 228, 208, 238, 146, 177, 22, 73, 32, 231, 132, 151, 57, 148, 154, 122, 218, 240, 6, 194, 0, 134, 28, 128, 76, 15, 119, 166, 232, 173, 165, 17, 244, 193, 131, 120, 121, 205, 3, 104, 46, 1, 147, 78, 11, 49, 62, 30, 60, 214, 140, 13, 69, 10, 143, 171, 115, 108, 101, 178, 58, 40, 124, 251, 220, 235, 103, 59, 82, 160, 102, 107, 249, 48, 174, 118, 185, 230, 74, 44, 229, 210, 138, 7, 213, 5, 94, 136, 87, 239, 190, 180, 20, 27, 182, 162, 142, 167, 252, 129, 172, 31, 170, 159, 243, 181, 222, 184, 158, 197, 155, 92, 211, 242, 90, 163, 88, 175, 187, 24, 186, 123, 81, 80, 133]","[0, 1, 3, 5, 10, 11, 15, 17, 20, 22, 24, 28, 31, 32, 42, 44, 46, 48, 54, 58, 59, 62, 69, 73, 74, 76, 87, 88, 92, 94, 101, 102, 103, 104, 107, 118, 119, 120, 121, 123, 128, 129, 131, 132, 133, 136, 143, 147, 148, 154, 159, 160, 163, 167, 171, 172, 173, 175, 177, 178, 180, 181, 182, 184, 186, 187, 190, 193, 194, 197, 205, 210, 211, 213, 214, 218, 222, 228, 231, 232, 240, 243]"
|
|
||||||
16,"[240, 13, 120, 4, 65, 20, 250, 254, 84, 22, 220, 105, 121, 95, 112, 46, 8, 150, 238, 99, 114, 171, 182, 55, 41, 122, 90, 6, 60, 23, 44, 235, 64, 50, 123, 162, 133, 30, 205, 125, 29, 115, 109, 151, 199, 40, 159, 212, 241, 37, 2, 88, 80, 74, 106, 131, 253, 0, 32, 86, 153, 113, 39, 7, 58, 208, 179, 14, 28, 19, 201, 166, 203, 149, 25, 27, 107, 119, 219, 152, 76, 18, 140, 12, 234, 138, 168, 144, 216, 164, 67, 239, 173, 102, 198, 130, 89, 81, 177, 77, 62, 26, 53, 213, 48, 132, 129, 193, 135, 35, 10, 214, 194, 137, 127, 170, 209, 101, 185, 141, 233, 15, 190, 21, 71, 229, 202, 155]","[242, 78, 19, 137, 224, 99, 152, 55, 113, 143, 24, 135, 183, 12, 129, 114, 211, 188, 172, 248, 30, 91, 40, 254, 16, 49, 25, 90, 212, 175, 119, 163, 8, 142, 117, 170, 6, 164, 180, 182, 171, 26, 22, 246, 160, 85, 186, 157, 147, 138, 220, 84, 107, 223, 64, 39, 5, 74, 11, 35, 201, 29, 9, 145, 149, 177, 41, 2, 87, 57, 249, 118, 82, 14, 151, 44, 232, 210, 100, 215, 66, 205, 47, 236, 96, 7, 219, 227, 140, 167, 176, 228, 15, 38, 95, 109, 0, 72, 120, 251, 124, 75, 127, 203, 200, 3, 54, 101, 144, 68, 46, 93, 71, 63, 125, 4, 89, 115, 92, 165, 158, 18, 13, 181, 148, 130, 146, 42]","[0, 2, 4, 6, 7, 8, 12, 13, 14, 15, 18, 19, 22, 25, 26, 29, 30, 35, 39, 40, 41, 44, 46, 55, 64, 71, 74, 84, 89, 90, 95, 99, 101, 107, 109, 113, 114, 115, 119, 120, 125, 127, 129, 130, 135, 137, 138, 140, 144, 149, 151, 152, 164, 170, 171, 177, 182, 201, 203, 205, 212, 219, 220, 254]"
|
|
||||||
17,"[225, 235, 155, 37, 56, 40, 237, 163, 191, 211, 0, 93, 43, 107, 122, 134, 52, 165, 171, 145, 19, 164, 194, 101, 94, 9, 173, 88, 31, 124, 240, 183, 97, 90, 76, 147, 72, 35, 137, 2, 13, 140, 128, 241, 115, 6, 22, 158, 50, 223, 192, 119, 41, 184, 196, 175, 25, 58, 33, 74, 89, 62, 215, 248, 96, 16, 98, 143, 12, 160, 121, 104, 68, 80, 17, 84, 53, 92, 182, 130, 153, 133, 148, 114, 23, 185, 159, 154, 78, 100, 87, 73, 47, 49, 108, 34, 5, 109, 149, 157, 129, 169, 28, 127, 233, 126, 193, 57, 207, 125, 206, 189, 249, 71, 4, 199, 63, 113, 131, 46, 142, 91, 112, 51, 170, 39, 77, 64]","[58, 84, 239, 219, 30, 54, 179, 194, 78, 52, 183, 236, 85, 45, 12, 167, 213, 253, 177, 14, 165, 38, 237, 61, 29, 116, 226, 83, 171, 48, 225, 67, 146, 32, 60, 182, 188, 251, 36, 103, 70, 228, 123, 75, 181, 69, 222, 235, 162, 221, 53, 3, 223, 135, 113, 8, 66, 152, 140, 174, 178, 180, 111, 19, 127, 156, 108, 232, 244, 15, 241, 195, 187, 231, 74, 27, 7, 56, 240, 202, 24, 93, 220, 184, 173, 79, 11, 41, 216, 106, 31, 217, 198, 26, 203, 238, 33, 141, 160, 218, 20, 247, 72, 252, 229, 155, 166, 18, 161, 118, 126, 139, 10, 68, 51, 63, 212, 151, 132, 21, 62, 159, 59, 208, 193, 176, 168, 86]","[12, 19, 31, 33, 41, 51, 52, 53, 56, 58, 62, 63, 68, 72, 74, 78, 84, 93, 108, 113, 126, 127, 140, 155, 159, 160, 165, 171, 173, 182, 183, 184, 193, 194, 223, 225, 235, 237, 240, 241]"
|
|
||||||
18,"[67, 52, 148, 72, 183, 24, 175, 149, 31, 252, 93, 95, 171, 36, 105, 140, 138, 27, 104, 50, 219, 5, 159, 160, 135, 124, 185, 141, 32, 89, 184, 218, 34, 133, 18, 58, 119, 11, 38, 189, 143, 198, 78, 91, 29, 121, 188, 162, 158, 221, 103, 240, 62, 42, 204, 26, 6, 68, 60, 90, 44, 139, 63, 165, 254, 94, 194, 45, 116, 115, 152, 2, 110, 250, 157, 241, 209, 215, 41, 51, 40, 19, 156, 123, 107, 12, 128, 101, 231, 224, 208, 192, 161, 33, 39, 21, 48, 168, 75, 69, 22, 76, 245, 179, 0, 132, 117, 70, 200, 213, 79, 74, 106, 114, 30, 230, 56, 13, 239, 120, 113, 233, 109, 57, 155, 81, 237, 23]","[72, 40, 133, 181, 24, 153, 179, 187, 232, 90, 51, 225, 152, 115, 13, 254, 188, 158, 218, 30, 175, 138, 107, 71, 32, 19, 5, 114, 228, 79, 184, 219, 91, 0, 6, 62, 239, 233, 54, 48, 183, 140, 68, 69, 159, 155, 148, 162, 207, 9, 120, 34, 87, 29, 75, 38, 17, 56, 76, 246, 104, 176, 142, 78, 147, 25, 211, 247, 241, 44, 11, 80, 251, 177, 221, 99, 16, 50, 204, 88, 61, 165, 199, 112, 116, 89, 171, 123, 121, 197, 226, 65, 42, 168, 41, 136, 173, 31, 36, 198, 244, 119, 150, 97, 58, 193, 7, 190, 45, 93, 216, 2, 110, 101, 57, 74, 94, 185, 70, 83, 245, 126, 39, 27, 195, 157, 18, 161]","[0, 2, 5, 6, 11, 13, 18, 19, 24, 27, 29, 30, 31, 32, 34, 36, 38, 39, 40, 41, 42, 44, 45, 48, 50, 51, 56, 57, 58, 62, 68, 69, 70, 72, 74, 75, 76, 78, 79, 89, 90, 91, 93, 94, 101, 104, 107, 110, 114, 115, 116, 119, 120, 121, 123, 133, 138, 140, 148, 152, 155, 157, 158, 159, 161, 162, 165, 168, 171, 175, 179, 183, 184, 185, 188, 198, 204, 218, 219, 221, 233, 239, 241, 245, 254]"
|
|
||||||
19,"[48, 72, 149, 243, 127, 11, 222, 132, 158, 183, 90, 191, 179, 74, 186, 87, 26, 232, 42, 20, 43, 201, 193, 190, 167, 122, 199, 161, 92, 86, 100, 235, 54, 130, 57, 231, 224, 123, 139, 140, 65, 152, 254, 83, 49, 192, 202, 221, 62, 197, 73, 156, 27, 189, 113, 160, 252, 40, 157, 207, 106, 103, 217, 94, 150, 47, 169, 219, 71, 69, 151, 204, 203, 10, 15, 182, 31, 29, 39, 107, 8, 118, 24, 177, 226, 155, 25, 81, 110, 97, 131, 168, 214, 143, 237, 115, 55, 23, 176, 144, 244, 22, 12, 227, 93, 185, 223, 95, 4, 56, 76, 121, 208, 109, 129, 146, 242, 88, 145, 17, 5, 6, 18, 236, 163, 80, 200, 141]","[4, 121, 99, 115, 201, 233, 194, 193, 116, 59, 202, 216, 234, 25, 236, 21, 104, 42, 0, 228, 87, 7, 192, 47, 165, 119, 197, 17, 127, 106, 35, 81, 39, 235, 254, 231, 154, 161, 3, 248, 95, 175, 79, 60, 184, 52, 217, 67, 187, 251, 54, 133, 212, 98, 245, 205, 132, 75, 103, 224, 138, 229, 50, 73, 163, 239, 230, 11, 172, 183, 112, 92, 13, 253, 151, 180, 221, 26, 91, 126, 86, 9, 150, 93, 232, 120, 88, 38, 55, 178, 20, 46, 185, 243, 2, 16, 252, 199, 44, 134, 241, 237, 109, 177, 174, 108, 142, 15, 36, 63, 51, 209, 22, 100, 215, 255, 62, 34, 48, 89, 28, 139, 196, 168, 43, 94, 223, 162]","[4, 11, 15, 17, 20, 22, 25, 26, 39, 42, 43, 47, 48, 54, 55, 62, 73, 81, 86, 87, 88, 92, 93, 94, 95, 100, 103, 106, 109, 115, 121, 127, 132, 139, 150, 151, 161, 163, 168, 177, 183, 185, 192, 193, 197, 199, 201, 202, 217, 221, 223, 224, 231, 232, 235, 236, 237, 243, 252, 254]"
|
|
||||||
20,"[8, 91, 14, 177, 25, 233, 95, 5, 166, 157, 80, 232, 114, 2, 187, 230, 74, 186, 92, 229, 33, 131, 161, 32, 31, 68, 127, 196, 171, 7, 222, 48, 113, 165, 59, 86, 141, 243, 96, 149, 181, 21, 146, 77, 81, 29, 200, 99, 122, 204, 40, 13, 198, 39, 94, 58, 47, 71, 102, 185, 223, 98, 88, 155, 97, 119, 49, 11, 61, 55, 132, 234, 135, 46, 116, 70, 138, 35, 192, 6, 208, 214, 199, 51, 150, 154, 103, 37, 241, 190, 43, 87, 160, 117, 26, 23, 101, 184, 17, 137, 189, 209, 175, 129, 56, 20, 89, 78, 44, 42, 125, 213, 227, 15, 140, 191, 179, 69, 124, 133, 93, 134, 105, 79, 254, 168, 130, 83]","[219, 23, 154, 109, 111, 99, 68, 187, 150, 241, 179, 166, 201, 220, 128, 31, 216, 81, 71, 63, 112, 67, 144, 181, 39, 238, 153, 191, 106, 196, 232, 0, 57, 157, 16, 91, 95, 80, 198, 158, 160, 192, 25, 48, 141, 59, 66, 180, 252, 228, 170, 124, 3, 113, 188, 182, 58, 172, 211, 247, 114, 240, 88, 45, 231, 225, 131, 69, 152, 34, 72, 130, 183, 121, 123, 4, 235, 35, 167, 50, 87, 138, 115, 92, 75, 205, 32, 155, 8, 210, 49, 173, 118, 119, 239, 202, 27, 215, 60, 133, 11, 104, 56, 37, 178, 229, 62, 40, 214, 47, 156, 46, 2, 105, 186, 185, 122, 206, 64, 253, 7, 107, 168, 129, 135, 255, 147, 190]","[2, 7, 8, 11, 23, 25, 31, 32, 35, 37, 39, 40, 46, 47, 48, 49, 56, 58, 59, 68, 69, 71, 80, 81, 87, 88, 91, 92, 95, 99, 105, 113, 114, 119, 122, 124, 129, 130, 131, 133, 135, 138, 141, 150, 154, 155, 157, 160, 166, 168, 179, 181, 185, 186, 187, 190, 191, 192, 196, 198, 214, 229, 232, 241]"
|
|
||||||
21,"[155, 142, 202, 88, 19, 121, 2, 216, 28, 132, 124, 23, 187, 237, 207, 169, 89, 204, 98, 160, 240, 173, 120, 41, 227, 49, 0, 172, 38, 182, 59, 232, 103, 54, 134, 222, 91, 69, 37, 122, 26, 238, 33, 20, 236, 146, 221, 171, 43, 74, 159, 24, 133, 253, 193, 163, 78, 174, 210, 176, 13, 178, 229, 144, 138, 96, 55, 139, 235, 211, 183, 115, 203, 101, 81, 112, 255, 228, 175, 252, 53, 27, 197, 212, 114, 136, 199, 51, 12, 84, 108, 177, 107, 249, 10, 82, 48, 57, 165, 179, 40, 198, 166, 102, 191, 99, 190, 123, 76, 164, 147, 224, 145, 68, 77, 194, 42, 22, 129, 135, 109, 247, 32, 94, 73, 161, 185, 60]","[50, 116, 218, 37, 100, 194, 107, 212, 31, 12, 213, 167, 4, 228, 204, 169, 155, 30, 156, 102, 94, 206, 91, 226, 14, 232, 105, 121, 227, 98, 56, 18, 170, 185, 178, 166, 8, 149, 162, 58, 158, 202, 73, 97, 69, 172, 99, 223, 147, 234, 211, 32, 33, 145, 131, 214, 5, 200, 229, 0, 140, 19, 254, 103, 137, 208, 123, 207, 62, 217, 118, 49, 80, 219, 109, 101, 237, 184, 46, 195, 65, 72, 183, 153, 24, 28, 23, 242, 54, 113, 110, 90, 151, 16, 47, 1, 230, 83, 87, 15, 78, 187, 245, 251, 177, 17, 64, 7, 240, 157, 129, 10, 128, 89, 114, 29, 67, 55, 86, 63, 246, 25, 92, 179, 70, 96, 44, 205]","[0, 10, 12, 19, 23, 24, 28, 32, 33, 37, 49, 54, 55, 69, 73, 78, 89, 91, 94, 96, 98, 99, 101, 102, 103, 107, 109, 114, 121, 123, 129, 145, 147, 155, 166, 169, 172, 177, 178, 179, 183, 185, 187, 194, 202, 204, 207, 211, 212, 227, 228, 229, 232, 237, 240]"
|
|
||||||
22,"[132, 44, 226, 229, 117, 145, 14, 169, 177, 1, 195, 235, 71, 21, 99, 101, 184, 32, 178, 139, 134, 173, 74, 93, 66, 18, 26, 180, 73, 67, 54, 196, 191, 162, 157, 10, 128, 107, 222, 38, 144, 142, 53, 102, 237, 233, 137, 59, 39, 97, 200, 230, 82, 171, 160, 212, 158, 49, 114, 8, 77, 94, 165, 141, 122, 188, 154, 4, 198, 143, 9, 55, 13, 105, 106, 227, 210, 2, 80, 95, 203, 201, 118, 152, 252, 187, 156, 116, 125, 199, 151, 197, 243, 79, 228, 185, 170, 96, 121, 84, 186, 214, 78, 64, 127, 255, 253, 164, 98, 103, 100, 3, 146, 48, 211, 30, 75, 245, 155, 153, 92, 81, 22, 7, 194, 27, 206, 131]","[187, 19, 217, 89, 130, 212, 56, 140, 53, 23, 230, 196, 181, 38, 168, 170, 70, 189, 244, 147, 191, 242, 247, 225, 183, 203, 17, 24, 224, 133, 164, 119, 1, 25, 219, 83, 64, 21, 37, 72, 62, 234, 40, 91, 251, 74, 128, 141, 204, 178, 0, 77, 7, 249, 154, 30, 245, 193, 127, 213, 51, 161, 28, 250, 12, 63, 118, 107, 68, 129, 175, 166, 208, 75, 205, 92, 120, 60, 190, 109, 221, 36, 155, 246, 210, 5, 14, 220, 135, 228, 248, 115, 113, 43, 34, 20, 88, 145, 136, 138, 100, 42, 45, 253, 11, 47, 134, 29, 15, 6, 240, 48, 55, 182, 69, 46, 61, 104, 159, 126, 99, 39, 124, 238, 194, 27, 96, 3]","[1, 3, 7, 14, 21, 27, 30, 38, 39, 48, 53, 55, 64, 74, 75, 77, 92, 96, 99, 100, 107, 118, 127, 128, 134, 141, 145, 154, 155, 164, 170, 178, 187, 191, 194, 196, 203, 210, 212, 228, 230, 245, 253]"
|
|
||||||
23,"[25, 37, 51, 69, 36, 237, 182, 163, 29, 144, 211, 39, 139, 204, 32, 126, 250, 56, 114, 111, 156, 30, 55, 72, 179, 124, 14, 203, 88, 110, 38, 18, 58, 170, 174, 148, 42, 57, 138, 87, 91, 207, 108, 66, 62, 184, 175, 90, 122, 149, 15, 172, 140, 92, 26, 193, 153, 28, 159, 160, 217, 173, 120, 221, 116, 191, 46, 2, 98, 161, 113, 115, 83, 198, 24, 65, 251, 240, 76, 209, 150, 195, 249, 186, 134, 162, 21, 5, 143, 4, 226, 100, 127, 52, 254, 44, 80, 215, 97, 107, 155, 82, 73, 197, 8, 225, 192, 45, 10, 81, 243, 189, 54, 164, 40, 19, 137, 158, 129, 68, 70, 53, 231, 176, 196, 165, 136, 43]","[155, 179, 247, 45, 85, 26, 131, 110, 4, 149, 103, 180, 43, 210, 19, 143, 18, 50, 46, 240, 120, 62, 67, 76, 113, 181, 172, 162, 156, 92, 126, 69, 107, 48, 56, 221, 170, 29, 164, 207, 129, 96, 183, 127, 111, 23, 253, 205, 97, 226, 90, 88, 161, 231, 25, 157, 132, 37, 39, 136, 40, 153, 100, 254, 160, 191, 198, 204, 223, 6, 0, 30, 163, 193, 151, 174, 190, 237, 108, 51, 24, 245, 73, 98, 251, 82, 38, 182, 91, 138, 49, 242, 79, 14, 200, 72, 173, 186, 93, 145, 133, 249, 197, 232, 57, 150, 134, 166, 8, 68, 189, 54, 227, 52, 235, 16, 58, 211, 80, 219, 139, 15, 70, 114, 158, 1, 81, 5]","[4, 5, 8, 14, 15, 18, 19, 24, 25, 26, 29, 30, 37, 38, 39, 40, 43, 45, 46, 51, 52, 54, 56, 57, 58, 62, 68, 69, 70, 72, 73, 76, 80, 81, 82, 88, 90, 91, 92, 97, 98, 100, 107, 108, 110, 111, 113, 114, 120, 126, 127, 129, 134, 136, 138, 139, 143, 149, 150, 153, 155, 156, 158, 160, 161, 162, 163, 164, 170, 172, 173, 174, 179, 182, 186, 189, 191, 193, 197, 198, 204, 207, 211, 221, 226, 231, 237, 240, 249, 251, 254]"
|
|
||||||
24,"[142, 130, 105, 108, 209, 10, 177, 52, 106, 33, 126, 4, 5, 118, 44, 24, 47, 39, 190, 81, 125, 104, 249, 57, 86, 28, 124, 62, 13, 234, 14, 150, 189, 148, 58, 87, 240, 43, 22, 80, 245, 12, 115, 244, 121, 171, 159, 90, 102, 247, 131, 163, 218, 120, 196, 61, 141, 70, 139, 220, 56, 117, 93, 100, 162, 6, 133, 37, 53, 25, 137, 122, 94, 119, 7, 111, 151, 188, 98, 84, 164, 213, 26, 67, 233, 59, 11, 16, 223, 101, 204, 210, 224, 250, 216, 54, 74, 36, 46, 91, 49, 252, 170, 158, 76, 32, 138, 198, 0, 73, 181, 60, 161, 9, 95, 174, 232, 173, 77, 254, 132, 21, 85, 144, 55, 140, 194, 68]","[5, 94, 118, 244, 186, 15, 138, 101, 198, 164, 209, 204, 57, 180, 21, 56, 182, 217, 93, 80, 223, 205, 140, 183, 59, 250, 150, 91, 72, 192, 137, 45, 4, 87, 120, 16, 58, 126, 194, 134, 30, 35, 151, 73, 26, 195, 13, 153, 33, 83, 95, 224, 191, 55, 131, 75, 129, 254, 113, 0, 202, 188, 90, 156, 46, 96, 201, 108, 218, 216, 144, 103, 53, 251, 11, 184, 111, 125, 88, 61, 104, 25, 114, 84, 36, 24, 8, 127, 106, 48, 231, 196, 234, 133, 2, 64, 77, 197, 68, 132, 67, 161, 154, 181, 149, 42, 115, 155, 121, 74, 245, 6, 52, 167, 38, 34, 158, 78, 28, 100, 208, 82, 81, 232, 105, 10, 62, 43]","[0, 4, 5, 6, 10, 11, 13, 16, 21, 24, 25, 26, 28, 33, 36, 43, 46, 52, 53, 55, 56, 57, 58, 59, 61, 62, 67, 68, 73, 74, 77, 80, 81, 84, 87, 90, 91, 93, 94, 95, 100, 101, 104, 105, 106, 108, 111, 115, 118, 120, 121, 125, 126, 131, 132, 133, 137, 138, 140, 144, 150, 151, 158, 161, 164, 181, 188, 194, 196, 198, 204, 209, 216, 218, 223, 224, 232, 234, 244, 245, 250, 254]"
|
|
||||||
25,"[24, 28, 167, 253, 5, 86, 156, 224, 164, 97, 66, 87, 171, 95, 124, 99, 240, 137, 14, 200, 209, 210, 185, 198, 148, 234, 25, 9, 64, 108, 184, 59, 30, 121, 149, 195, 150, 130, 117, 89, 191, 93, 225, 84, 161, 199, 16, 172, 173, 182, 63, 162, 21, 113, 189, 54, 70, 129, 19, 128, 165, 49, 180, 114, 122, 177, 220, 231, 135, 141, 69, 250, 73, 105, 212, 232, 183, 136, 140, 160, 96, 205, 211, 27, 35, 88, 252, 103, 203, 100, 174, 127, 197, 223, 42, 46, 222, 236, 77, 85, 106, 43, 82, 41, 152, 10, 18, 153, 65, 229, 142, 76, 36, 45, 22, 192, 169, 34, 71, 217, 90, 79, 201, 111, 33, 179, 2, 1]","[157, 21, 175, 8, 58, 25, 135, 173, 180, 128, 129, 171, 179, 48, 90, 237, 122, 189, 187, 228, 77, 207, 114, 30, 63, 18, 95, 152, 22, 155, 247, 250, 5, 225, 165, 20, 137, 185, 218, 126, 248, 36, 72, 118, 242, 96, 233, 142, 17, 98, 146, 54, 106, 205, 42, 34, 198, 147, 10, 252, 140, 74, 73, 161, 202, 193, 226, 209, 254, 133, 145, 7, 102, 59, 158, 255, 177, 41, 60, 195, 156, 83, 105, 84, 2, 125, 87, 220, 43, 70, 206, 52, 19, 213, 32, 243, 49, 111, 181, 82, 76, 124, 188, 251, 33, 236, 44, 66, 191, 127, 183, 184, 16, 182, 11, 62, 27, 28, 134, 210, 245, 154, 78, 196, 249, 232, 108, 115]","[2, 5, 10, 16, 18, 19, 21, 22, 25, 27, 28, 30, 33, 34, 36, 41, 42, 43, 49, 54, 59, 63, 66, 70, 73, 76, 77, 82, 84, 87, 90, 95, 96, 105, 106, 108, 111, 114, 122, 124, 127, 128, 129, 135, 137, 140, 142, 152, 156, 161, 165, 171, 173, 177, 179, 180, 182, 183, 184, 185, 189, 191, 195, 198, 205, 209, 210, 220, 225, 232, 236, 250, 252]"
|
|
||||||
26,"[43, 89, 58, 116, 35, 166, 155, 190, 75, 19, 14, 90, 8, 12, 42, 147, 96, 39, 93, 123, 29, 245, 127, 34, 134, 163, 241, 18, 197, 192, 11, 66, 235, 234, 229, 132, 118, 145, 33, 194, 223, 198, 176, 172, 57, 152, 76, 1, 13, 218, 68, 253, 140, 113, 105, 15, 74, 246, 128, 44, 6, 122, 250, 86, 72, 32, 3, 148, 84, 171, 121, 104, 45, 177, 5, 71, 125, 219, 249, 21, 151, 178, 181, 238, 109, 187, 142, 99, 150, 203, 54, 85, 213, 17, 63, 211, 62, 102, 156, 195, 82, 205, 65, 252, 95, 111, 168, 38, 164, 46, 25, 226, 88, 56, 107, 40, 108, 199, 16, 83, 196, 70, 153, 91, 188, 126, 10, 144]","[67, 175, 70, 227, 244, 2, 49, 59, 26, 29, 139, 6, 77, 4, 157, 36, 43, 167, 173, 101, 172, 25, 98, 88, 203, 150, 155, 42, 54, 185, 144, 28, 226, 178, 99, 39, 252, 176, 34, 3, 187, 100, 10, 1, 18, 5, 241, 214, 192, 19, 180, 79, 204, 159, 27, 196, 237, 13, 75, 81, 137, 108, 201, 212, 149, 218, 16, 184, 110, 86, 37, 51, 160, 142, 40, 87, 60, 165, 170, 230, 50, 221, 135, 89, 217, 223, 174, 200, 125, 140, 46, 111, 106, 158, 191, 181, 20, 208, 240, 228, 48, 35, 128, 107, 236, 235, 82, 71, 182, 31, 8, 143, 138, 246, 194, 0, 162, 78, 24, 231, 136, 220, 242, 65, 163, 216, 11, 73]","[1, 3, 5, 6, 8, 10, 11, 13, 16, 18, 19, 25, 29, 34, 35, 39, 40, 42, 43, 46, 54, 65, 70, 71, 75, 82, 86, 88, 89, 99, 107, 108, 111, 125, 128, 140, 142, 144, 150, 155, 163, 172, 176, 178, 181, 187, 192, 194, 196, 203, 218, 223, 226, 235, 241, 246, 252]"
|
|
||||||
27,"[113, 186, 51, 131, 184, 25, 224, 238, 253, 121, 204, 201, 87, 3, 202, 15, 163, 84, 203, 82, 55, 37, 1, 150, 172, 67, 206, 161, 61, 83, 148, 60, 237, 49, 31, 234, 24, 166, 134, 77, 124, 22, 104, 177, 152, 114, 162, 158, 23, 78, 102, 75, 189, 249, 226, 105, 2, 69, 180, 254, 157, 58, 181, 153, 229, 70, 164, 27, 59, 56, 174, 191, 41, 250, 182, 47, 178, 97, 213, 179, 12, 199, 194, 72, 146, 140, 111, 44, 221, 240, 255, 96, 214, 252, 123, 193, 143, 95, 248, 232, 159, 73, 175, 195, 39, 40, 91, 200, 116, 88, 17, 43, 168, 144, 223, 64, 137, 90, 212, 149, 130, 133, 188, 38, 71, 100, 230, 20]","[147, 36, 25, 228, 20, 111, 120, 184, 165, 75, 188, 35, 217, 189, 70, 114, 45, 15, 177, 16, 104, 65, 24, 23, 44, 238, 94, 30, 105, 195, 163, 87, 102, 204, 159, 166, 97, 154, 182, 124, 5, 214, 221, 183, 67, 113, 220, 93, 109, 131, 91, 3, 203, 199, 146, 71, 142, 4, 66, 148, 175, 193, 82, 230, 74, 19, 49, 200, 108, 96, 73, 149, 253, 106, 140, 56, 38, 69, 81, 122, 179, 99, 255, 144, 245, 78, 224, 169, 153, 46, 89, 186, 47, 229, 12, 248, 254, 137, 37, 158, 205, 234, 201, 173, 246, 202, 6, 85, 170, 150, 90, 168, 152, 22, 134, 222, 161, 232, 247, 180, 32, 100, 55, 194, 125, 164, 50, 112]","[3, 12, 15, 20, 22, 23, 24, 25, 37, 38, 44, 47, 49, 55, 56, 67, 69, 70, 71, 73, 75, 78, 82, 87, 90, 91, 96, 97, 100, 102, 104, 105, 111, 113, 114, 124, 131, 134, 137, 140, 144, 146, 148, 149, 150, 152, 153, 158, 159, 161, 163, 164, 166, 168, 175, 177, 179, 180, 182, 184, 186, 188, 189, 193, 194, 195, 199, 200, 201, 202, 203, 204, 214, 221, 224, 229, 230, 232, 234, 238, 248, 253, 254, 255]"
|
|
||||||
28,"[48, 238, 196, 250, 226, 236, 129, 187, 130, 169, 167, 94, 110, 122, 138, 114, 56, 165, 184, 14, 220, 18, 149, 132, 45, 46, 71, 252, 108, 237, 76, 69, 73, 180, 245, 34, 163, 171, 195, 159, 41, 248, 151, 51, 131, 228, 9, 103, 16, 158, 241, 235, 12, 64, 136, 77, 81, 142, 58, 135, 203, 70, 198, 125, 234, 4, 75, 80, 90, 5, 182, 153, 23, 208, 112, 183, 244, 47, 50, 86, 141, 100, 38, 162, 115, 178, 113, 26, 200, 177, 22, 19, 249, 150, 83, 33, 181, 223, 209, 35, 140, 253, 215, 144, 102, 246, 85, 118, 13, 20, 105, 55, 123, 119, 225, 239, 172, 63, 72, 44, 15, 188, 166, 65, 157, 43, 201, 78]","[139, 52, 132, 164, 53, 102, 149, 214, 147, 3, 144, 163, 244, 67, 58, 205, 15, 203, 33, 200, 22, 46, 157, 57, 204, 105, 27, 92, 35, 175, 245, 77, 10, 54, 178, 253, 187, 138, 241, 66, 88, 34, 101, 216, 64, 32, 254, 255, 26, 236, 169, 252, 135, 104, 70, 30, 81, 198, 73, 61, 24, 121, 158, 192, 39, 9, 165, 196, 197, 21, 50, 160, 191, 145, 184, 123, 207, 224, 99, 93, 136, 112, 249, 199, 155, 231, 85, 250, 185, 190, 94, 186, 68, 44, 56, 237, 242, 180, 209, 28, 113, 118, 124, 19, 177, 251, 173, 60, 167, 233, 40, 218, 116, 153, 14, 188, 129, 159, 225, 48, 193, 170, 95, 195, 74, 211, 161, 43]","[9, 14, 15, 19, 22, 26, 33, 34, 35, 43, 44, 46, 48, 50, 56, 58, 64, 70, 73, 77, 81, 85, 94, 102, 105, 112, 113, 118, 123, 129, 132, 135, 136, 138, 144, 149, 153, 157, 158, 159, 163, 165, 167, 169, 177, 178, 180, 184, 187, 188, 195, 196, 198, 200, 203, 209, 225, 236, 237, 241, 244, 245, 249, 250, 252, 253]"
|
|
||||||
29,"[217, 50, 160, 234, 240, 53, 162, 13, 84, 171, 1, 245, 0, 67, 138, 238, 198, 71, 27, 173, 28, 132, 24, 47, 255, 101, 176, 82, 252, 150, 151, 66, 106, 226, 222, 81, 70, 10, 79, 48, 211, 152, 126, 131, 207, 202, 94, 181, 122, 105, 83, 38, 17, 212, 178, 58, 188, 29, 148, 64, 135, 118, 100, 104, 12, 117, 95, 57, 113, 200, 14, 169, 22, 8, 5, 158, 233, 232, 194, 179, 85, 25, 154, 149, 16, 90, 140, 201, 223, 196, 184, 2, 129, 103, 157, 86, 144, 93, 96, 187, 30, 239, 143, 37, 120, 36, 237, 220, 130, 209, 210, 190, 180, 119, 197, 251, 254, 121, 205, 65, 192, 213, 107, 111, 75, 136, 49, 97]","[67, 193, 39, 161, 246, 15, 158, 175, 54, 63, 28, 180, 96, 224, 184, 140, 111, 113, 59, 192, 204, 194, 249, 160, 238, 32, 166, 51, 84, 243, 10, 75, 247, 80, 202, 124, 186, 13, 103, 201, 72, 248, 1, 190, 97, 200, 93, 90, 242, 198, 79, 153, 30, 207, 255, 241, 139, 203, 228, 99, 214, 235, 106, 16, 41, 128, 183, 86, 58, 212, 0, 221, 208, 42, 218, 87, 55, 4, 151, 73, 254, 250, 122, 231, 71, 19, 145, 85, 244, 24, 112, 206, 174, 22, 138, 101, 225, 215, 88, 92, 150, 2, 14, 25, 177, 102, 155, 47, 62, 20, 74, 252, 134, 45, 147, 125, 240, 98, 53, 8, 209, 130, 229, 107, 156, 169, 196, 109]","[0, 1, 2, 8, 10, 13, 14, 16, 22, 24, 25, 28, 30, 47, 53, 58, 67, 71, 75, 79, 84, 85, 86, 90, 93, 96, 97, 101, 103, 106, 107, 111, 113, 122, 130, 138, 140, 150, 151, 158, 160, 169, 180, 184, 190, 192, 194, 196, 198, 200, 201, 202, 207, 209, 212, 238, 240, 252, 254, 255]"
|
|
||||||
30,"[91, 89, 254, 244, 22, 229, 57, 178, 175, 124, 187, 46, 82, 201, 81, 165, 49, 43, 139, 8, 222, 156, 243, 172, 132, 159, 205, 88, 3, 138, 16, 193, 116, 20, 214, 144, 208, 241, 52, 13, 167, 2, 12, 182, 207, 253, 196, 230, 38, 92, 56, 4, 17, 169, 29, 11, 195, 247, 40, 173, 255, 102, 125, 202, 204, 100, 237, 210, 110, 251, 18, 197, 71, 34, 250, 10, 21, 15, 80, 137, 134, 161, 37, 79, 109, 184, 26, 65, 74, 51, 112, 85, 60, 185, 25, 93, 223, 5, 142, 234, 6, 203, 36, 150, 106, 168, 32, 105, 114, 141, 14, 158, 33, 9, 59, 231, 28, 216, 236, 68, 115, 145, 129, 240, 95, 180, 45, 119]","[81, 189, 225, 86, 113, 64, 168, 62, 129, 22, 84, 145, 56, 112, 229, 103, 108, 132, 41, 191, 98, 59, 65, 167, 138, 73, 51, 159, 11, 28, 176, 158, 100, 165, 74, 208, 245, 38, 219, 150, 164, 3, 6, 152, 200, 90, 94, 232, 50, 154, 171, 174, 239, 102, 151, 221, 27, 126, 34, 30, 178, 179, 139, 17, 131, 63, 36, 5, 123, 141, 52, 244, 235, 10, 210, 172, 121, 31, 220, 201, 144, 99, 248, 125, 8, 24, 107, 215, 49, 42, 39, 162, 190, 161, 228, 14, 58, 116, 160, 70, 188, 209, 43, 253, 55, 170, 227, 230, 207, 237, 78, 66, 157, 68, 12, 18, 242, 163, 47, 236, 7, 224, 46, 184, 197, 203, 1, 198]","[3, 5, 6, 8, 10, 11, 12, 14, 17, 18, 22, 28, 34, 36, 38, 43, 46, 49, 51, 52, 56, 59, 65, 68, 74, 81, 100, 102, 112, 116, 125, 129, 132, 138, 139, 141, 144, 145, 150, 158, 159, 161, 165, 167, 168, 172, 178, 184, 197, 201, 203, 207, 208, 210, 229, 230, 236, 237, 244, 253]"
|
|
||||||
31,"[94, 211, 43, 174, 255, 68, 1, 254, 0, 183, 198, 160, 64, 79, 164, 38, 97, 167, 220, 70, 35, 87, 54, 76, 75, 10, 213, 148, 212, 133, 22, 56, 14, 85, 197, 141, 236, 137, 202, 244, 31, 184, 143, 182, 93, 72, 168, 52, 50, 130, 246, 46, 17, 120, 203, 139, 81, 101, 151, 229, 116, 9, 95, 221, 102, 3, 201, 19, 176, 173, 59, 65, 108, 227, 224, 129, 53, 241, 172, 121, 251, 30, 234, 44, 84, 34, 159, 214, 2, 6, 24, 126, 88, 125, 112, 175, 189, 29, 163, 13, 240, 92, 186, 181, 238, 57, 4, 71, 122, 131, 178, 144, 158, 250, 61, 247, 216, 135, 231, 41, 170, 82, 98, 40, 217, 199, 243, 195]","[44, 23, 96, 80, 81, 0, 100, 85, 24, 90, 152, 210, 179, 63, 71, 143, 89, 97, 123, 28, 170, 27, 107, 32, 178, 101, 248, 41, 197, 253, 193, 221, 243, 192, 99, 74, 228, 75, 52, 124, 47, 154, 66, 142, 153, 43, 102, 137, 38, 149, 139, 191, 204, 215, 234, 6, 8, 196, 70, 106, 88, 111, 36, 25, 249, 15, 200, 211, 168, 150, 128, 105, 17, 254, 255, 78, 226, 86, 180, 83, 91, 138, 77, 68, 103, 218, 185, 206, 30, 140, 232, 209, 5, 60, 239, 122, 87, 203, 18, 16, 231, 219, 212, 235, 230, 49, 159, 108, 146, 229, 241, 233, 169, 165, 190, 9, 42, 245, 164, 37, 46, 252, 171, 136, 118, 237, 217, 194]","[0, 6, 9, 17, 24, 30, 38, 41, 43, 44, 46, 52, 68, 70, 71, 75, 81, 85, 87, 88, 97, 101, 102, 108, 122, 137, 139, 143, 159, 164, 168, 170, 178, 197, 203, 211, 212, 217, 221, 229, 231, 234, 241, 243, 254, 255]"
|
|
||||||
32,"[250, 139, 20, 180, 73, 192, 45, 242, 170, 184, 114, 98, 228, 134, 113, 130, 178, 123, 56, 161, 253, 9, 102, 186, 110, 84, 30, 59, 94, 3, 44, 33, 19, 5, 187, 26, 150, 203, 104, 215, 108, 181, 244, 243, 148, 46, 13, 37, 103, 223, 87, 202, 149, 214, 15, 82, 191, 23, 6, 117, 251, 229, 50, 67, 36, 60, 52, 120, 107, 85, 121, 81, 128, 7, 152, 158, 38, 43, 217, 183, 175, 249, 163, 95, 124, 116, 92, 35, 18, 236, 157, 80, 162, 112, 8, 90, 255, 125, 63, 173, 254, 99, 198, 126, 232, 118, 68, 88, 140, 89, 189, 194, 32, 10, 25, 119, 145, 34, 39, 234, 61, 166, 100, 218, 40, 168, 156, 83]","[181, 100, 237, 192, 35, 252, 14, 111, 215, 129, 219, 7, 77, 235, 9, 207, 125, 86, 170, 201, 245, 169, 69, 72, 105, 92, 42, 155, 16, 55, 224, 101, 231, 157, 177, 17, 28, 171, 64, 66, 248, 74, 0, 227, 172, 62, 199, 106, 79, 205, 216, 126, 145, 70, 222, 122, 6, 114, 39, 31, 1, 134, 226, 61, 141, 220, 144, 189, 160, 133, 110, 193, 127, 137, 27, 158, 115, 41, 63, 75, 230, 131, 135, 221, 103, 179, 246, 143, 73, 178, 2, 206, 254, 29, 211, 210, 53, 147, 167, 138, 54, 195, 113, 33, 24, 194, 214, 132, 123, 247, 255, 109, 150, 165, 15, 76, 240, 57, 51, 93, 146, 153, 228, 202, 161, 238, 213, 78]","[6, 7, 9, 15, 33, 35, 39, 61, 63, 73, 92, 100, 103, 110, 113, 114, 123, 125, 126, 134, 145, 150, 157, 158, 161, 170, 178, 181, 189, 192, 194, 202, 214, 215, 228, 254, 255]"
|
|
||||||
33,"[54, 182, 21, 237, 130, 154, 133, 138, 144, 220, 62, 3, 188, 231, 198, 213, 48, 209, 66, 178, 164, 14, 129, 131, 83, 233, 73, 156, 30, 151, 58, 94, 176, 86, 125, 17, 91, 45, 147, 205, 16, 247, 81, 162, 57, 63, 107, 23, 110, 253, 7, 120, 174, 67, 89, 194, 26, 19, 27, 68, 15, 72, 168, 226, 146, 5, 215, 199, 113, 59, 111, 163, 87, 245, 181, 218, 104, 126, 22, 179, 56, 10, 11, 189, 0, 42, 169, 202, 187, 75, 224, 210, 24, 141, 177, 193, 158, 225, 98, 76, 248, 88, 82, 135, 20, 186, 171, 219, 53, 201, 142, 175, 127, 35, 40, 249, 92, 32, 132, 80, 208, 115, 191, 84, 90, 4, 109, 97]","[143, 58, 8, 76, 173, 87, 241, 213, 54, 180, 136, 156, 107, 193, 38, 147, 82, 75, 68, 30, 174, 28, 59, 78, 245, 72, 185, 135, 69, 191, 138, 92, 225, 150, 45, 226, 234, 109, 81, 130, 190, 52, 62, 90, 247, 103, 211, 154, 37, 108, 46, 53, 157, 22, 146, 84, 220, 73, 91, 224, 232, 79, 231, 26, 121, 2, 229, 116, 66, 228, 182, 152, 55, 153, 210, 160, 218, 131, 246, 127, 149, 238, 70, 235, 250, 114, 117, 183, 165, 196, 33, 199, 163, 12, 142, 178, 115, 113, 16, 243, 64, 99, 105, 171, 10, 132, 63, 244, 44, 118, 96, 161, 122, 255, 40, 134, 188, 39, 110, 200, 187, 14, 219, 41, 42, 0, 221, 24]","[0, 10, 14, 16, 22, 24, 26, 30, 40, 42, 45, 53, 54, 58, 59, 62, 63, 66, 68, 72, 73, 75, 76, 81, 82, 84, 87, 90, 91, 92, 107, 109, 110, 113, 115, 127, 130, 131, 132, 135, 138, 142, 146, 147, 154, 156, 163, 171, 174, 178, 182, 187, 188, 191, 193, 199, 210, 213, 218, 219, 220, 224, 225, 226, 231, 245, 247]"
|
|
||||||
34,"[118, 74, 66, 206, 224, 64, 238, 170, 37, 113, 27, 82, 105, 67, 63, 134, 24, 35, 81, 40, 42, 189, 55, 214, 87, 204, 34, 232, 144, 159, 31, 195, 116, 91, 59, 123, 152, 28, 72, 254, 175, 6, 162, 45, 234, 223, 176, 245, 76, 211, 142, 241, 120, 61, 137, 210, 36, 166, 225, 71, 124, 22, 209, 103, 92, 203, 133, 10, 83, 213, 228, 236, 14, 136, 46, 255, 68, 197, 243, 149, 54, 160, 193, 191, 77, 168, 218, 49, 114, 230, 32, 147, 125, 121, 172, 151, 226, 178, 16, 110, 251, 235, 84, 132, 252, 192, 180, 97, 38, 39, 237, 102, 58, 139, 13, 246, 78, 70, 150, 108, 248, 85, 33, 4, 26, 143, 201, 186]","[28, 235, 117, 142, 130, 194, 61, 31, 144, 180, 40, 71, 59, 233, 66, 21, 118, 16, 110, 195, 148, 46, 18, 139, 58, 55, 143, 22, 211, 214, 231, 38, 197, 187, 186, 226, 113, 155, 241, 87, 45, 175, 13, 83, 39, 115, 42, 108, 6, 78, 93, 185, 8, 164, 68, 137, 149, 203, 163, 212, 74, 136, 179, 98, 34, 229, 170, 123, 84, 188, 204, 27, 49, 33, 10, 152, 120, 107, 26, 3, 191, 166, 111, 159, 95, 97, 134, 54, 252, 72, 238, 151, 19, 213, 12, 132, 245, 99, 218, 184, 30, 43, 102, 217, 63, 36, 244, 224, 206, 167, 192, 62, 4, 236, 82, 60, 24, 70, 92, 64, 234, 190, 77, 14, 56, 219, 37, 222]","[4, 6, 10, 13, 14, 16, 22, 24, 26, 27, 28, 31, 33, 34, 36, 37, 38, 39, 40, 42, 45, 46, 49, 54, 55, 58, 59, 61, 63, 64, 66, 68, 70, 71, 72, 74, 77, 78, 82, 83, 84, 87, 92, 97, 102, 108, 110, 113, 118, 120, 123, 132, 134, 136, 137, 139, 142, 143, 144, 149, 151, 152, 159, 166, 170, 175, 180, 186, 191, 192, 195, 197, 203, 204, 206, 211, 213, 214, 218, 224, 226, 234, 235, 236, 238, 241, 245, 252]"
|
|
||||||
35,"[22, 111, 13, 56, 148, 208, 202, 10, 47, 174, 14, 131, 192, 99, 254, 212, 103, 130, 24, 40, 100, 233, 124, 253, 105, 118, 133, 195, 119, 138, 216, 38, 140, 106, 6, 127, 21, 53, 28, 242, 4, 101, 80, 123, 48, 180, 117, 31, 190, 160, 17, 179, 59, 178, 139, 75, 102, 169, 1, 250, 246, 41, 36, 193, 172, 5, 96, 128, 222, 228, 25, 211, 63, 196, 98, 205, 241, 74, 125, 30, 165, 168, 94, 9, 170, 232, 217, 82, 81, 91, 78, 69, 86, 177, 29, 199, 219, 46, 3, 8, 184, 161, 152, 23, 116, 113, 115, 70, 244, 92, 164, 88, 110, 19, 84, 76, 32, 66, 181, 173, 166, 231, 155, 104, 52, 159, 223, 15]","[220, 79, 56, 237, 34, 19, 202, 55, 70, 214, 43, 215, 41, 241, 11, 177, 16, 109, 9, 163, 63, 136, 154, 246, 124, 120, 206, 100, 49, 204, 247, 2, 213, 188, 145, 108, 168, 167, 229, 72, 67, 64, 232, 42, 208, 185, 187, 198, 35, 81, 95, 221, 165, 218, 201, 226, 242, 51, 200, 111, 10, 44, 240, 26, 175, 155, 129, 183, 248, 12, 150, 190, 141, 59, 126, 23, 238, 210, 116, 182, 33, 32, 93, 54, 224, 207, 130, 217, 77, 176, 252, 114, 4, 227, 132, 60, 45, 179, 0, 37, 73, 105, 191, 255, 157, 235, 83, 135, 239, 189, 244, 203, 234, 76, 174, 166, 223, 71, 162, 112, 144, 50, 97, 158, 27, 7, 230, 58]","[4, 9, 10, 19, 23, 32, 41, 56, 59, 63, 70, 76, 81, 100, 105, 111, 116, 124, 130, 155, 165, 166, 168, 174, 177, 179, 190, 202, 208, 217, 223, 232, 241, 242, 244, 246]"
|
|
||||||
36,"[45, 64, 25, 225, 224, 111, 127, 9, 185, 107, 13, 68, 237, 23, 216, 32, 72, 190, 202, 100, 240, 5, 17, 200, 160, 244, 71, 51, 35, 136, 118, 170, 254, 95, 201, 149, 236, 43, 153, 159, 38, 196, 58, 75, 192, 155, 252, 151, 138, 2, 18, 89, 233, 49, 122, 203, 142, 70, 179, 229, 66, 162, 57, 186, 222, 53, 113, 231, 146, 246, 211, 125, 166, 86, 182, 22, 143, 248, 253, 131, 117, 54, 30, 114, 173, 139, 226, 59, 154, 126, 78, 48, 215, 243, 50, 187, 65, 56, 21, 37, 133, 110, 1, 195, 27, 115, 130, 164, 123, 74, 26, 157, 145, 234, 47, 106, 92, 239, 230, 24, 19, 242, 208, 93, 193, 12, 39, 171]","[91, 145, 33, 196, 35, 20, 203, 79, 26, 184, 59, 88, 72, 193, 25, 39, 105, 24, 46, 215, 217, 65, 201, 110, 186, 225, 198, 250, 144, 239, 125, 19, 120, 75, 127, 149, 77, 222, 10, 53, 218, 138, 92, 113, 213, 171, 38, 178, 192, 182, 100, 23, 154, 168, 226, 152, 32, 43, 190, 70, 64, 132, 212, 161, 208, 58, 86, 183, 179, 48, 130, 45, 2, 244, 34, 129, 31, 22, 200, 68, 37, 90, 162, 18, 51, 112, 156, 232, 139, 160, 27, 180, 123, 111, 4, 204, 253, 89, 74, 142, 93, 249, 47, 214, 211, 6, 67, 119, 153, 21, 243, 252, 3, 254, 187, 95, 185, 155, 57, 126, 216, 150, 151, 81, 195, 122, 56, 9]","[2, 9, 18, 19, 21, 22, 23, 24, 25, 26, 27, 32, 35, 37, 38, 39, 43, 45, 47, 48, 51, 53, 56, 57, 58, 59, 64, 65, 68, 70, 72, 74, 75, 86, 89, 92, 93, 95, 100, 110, 111, 113, 122, 123, 125, 126, 127, 130, 138, 139, 142, 145, 149, 151, 153, 154, 155, 160, 162, 171, 179, 182, 185, 186, 187, 190, 192, 193, 195, 196, 200, 201, 203, 208, 211, 215, 216, 222, 225, 226, 239, 243, 244, 252, 253, 254]"
|
|
||||||
37,"[90, 131, 196, 213, 45, 24, 38, 142, 147, 158, 118, 190, 40, 76, 245, 3, 170, 100, 37, 126, 140, 6, 246, 129, 194, 181, 111, 5, 85, 73, 46, 167, 54, 211, 89, 87, 74, 117, 183, 161, 15, 7, 136, 1, 200, 239, 19, 180, 207, 124, 16, 47, 27, 224, 187, 145, 144, 98, 165, 152, 134, 184, 235, 159, 55, 34, 219, 4, 30, 164, 252, 202, 191, 41, 56, 214, 21, 249, 238, 229, 23, 78, 67, 75, 139, 60, 250, 226, 110, 17, 18, 59, 8, 2, 137, 209, 189, 77, 72, 102, 119, 132, 188, 148, 61, 91, 69, 113, 66, 174, 208, 22, 58, 168, 70, 153, 138, 169, 146, 84, 48, 35, 123, 221, 106, 29, 151, 154]","[110, 3, 108, 117, 7, 181, 18, 66, 139, 90, 16, 248, 5, 241, 46, 124, 118, 19, 27, 40, 170, 196, 24, 69, 36, 245, 188, 61, 153, 246, 134, 78, 14, 45, 253, 98, 50, 71, 229, 112, 8, 59, 202, 140, 38, 119, 232, 79, 194, 159, 144, 228, 32, 180, 148, 77, 189, 213, 60, 238, 127, 158, 41, 255, 192, 96, 215, 131, 55, 242, 76, 161, 226, 254, 176, 102, 6, 73, 126, 129, 221, 193, 47, 164, 239, 37, 191, 80, 186, 113, 1, 165, 123, 141, 233, 172, 72, 166, 101, 68, 49, 184, 136, 135, 97, 197, 2, 89, 74, 63, 216, 137, 250, 54, 155, 84, 0, 212, 252, 114, 67, 111, 30, 185, 230, 244, 171, 138]","[1, 2, 3, 5, 6, 7, 8, 16, 18, 19, 24, 27, 30, 37, 38, 40, 41, 45, 46, 47, 54, 55, 59, 60, 61, 66, 67, 69, 72, 73, 74, 76, 77, 78, 84, 89, 90, 98, 102, 110, 111, 113, 117, 118, 119, 123, 124, 126, 129, 131, 134, 136, 137, 138, 139, 140, 144, 148, 153, 158, 159, 161, 164, 165, 170, 180, 181, 184, 188, 189, 191, 194, 196, 202, 213, 221, 226, 229, 238, 239, 245, 246, 250, 252]"
|
|
||||||
38,"[68, 42, 242, 215, 126, 155, 33, 181, 21, 125, 87, 102, 173, 60, 164, 174, 36, 131, 143, 63, 34, 86, 80, 58, 207, 239, 222, 9, 24, 202, 175, 128, 228, 187, 133, 151, 19, 92, 224, 30, 98, 0, 61, 201, 203, 116, 50, 113, 190, 14, 53, 49, 56, 191, 106, 22, 172, 109, 132, 205, 122, 140, 152, 195, 188, 107, 135, 197, 17, 81, 123, 245, 176, 97, 120, 213, 184, 100, 27, 5, 54, 161, 226, 158, 130, 31, 199, 232, 96, 229, 55, 112, 160, 10, 146, 101, 193, 99, 117, 51, 48, 104, 88, 7, 129, 77, 139, 37, 217, 94, 124, 119, 142, 66, 8, 238, 29, 74, 157, 4, 20, 134, 41, 148, 166, 103, 247, 147]","[83, 184, 29, 42, 91, 229, 253, 117, 49, 13, 205, 85, 218, 208, 9, 183, 122, 101, 189, 67, 14, 37, 76, 94, 48, 17, 62, 149, 61, 159, 170, 75, 44, 90, 142, 164, 249, 7, 47, 126, 139, 132, 216, 10, 173, 26, 25, 254, 89, 70, 248, 154, 18, 68, 198, 155, 69, 163, 103, 32, 79, 228, 217, 2, 241, 45, 121, 204, 255, 144, 71, 141, 224, 145, 60, 210, 236, 230, 213, 35, 30, 11, 78, 225, 15, 169, 22, 64, 252, 219, 178, 54, 251, 186, 168, 8, 151, 95, 28, 74, 63, 200, 220, 143, 55, 99, 112, 114, 156, 92, 110, 96, 223, 100, 171, 136, 209, 240, 129, 21, 84, 59, 3, 82, 40, 177, 138, 88]","[7, 8, 9, 10, 14, 17, 21, 22, 29, 30, 37, 42, 48, 49, 54, 55, 60, 61, 63, 68, 74, 88, 92, 94, 96, 99, 100, 101, 103, 112, 117, 122, 126, 129, 132, 139, 142, 143, 151, 155, 164, 173, 184, 205, 213, 217, 224, 228, 229]"
|
|
||||||
39,"[253, 41, 93, 243, 136, 238, 188, 85, 212, 55, 91, 61, 75, 116, 66, 81, 248, 120, 159, 207, 111, 6, 139, 164, 186, 12, 24, 181, 123, 112, 69, 50, 138, 189, 39, 29, 68, 163, 227, 146, 214, 187, 60, 18, 162, 228, 153, 23, 137, 210, 182, 99, 98, 155, 87, 203, 0, 209, 144, 173, 26, 96, 4, 179, 217, 74, 7, 208, 221, 104, 114, 180, 10, 77, 70, 40, 67, 82, 28, 191, 178, 252, 130, 21, 175, 165, 201, 51, 117, 32, 218, 34, 83, 25, 16, 128, 31, 194, 171, 9, 36, 193, 202, 109, 48, 127, 236, 73, 255, 13, 241, 240, 232, 115, 254, 172, 157, 101, 72, 251, 35, 131, 76, 237, 147, 79, 233, 121]","[0, 143, 97, 32, 13, 85, 181, 228, 114, 218, 17, 38, 72, 186, 23, 252, 12, 69, 1, 238, 197, 88, 116, 99, 81, 75, 10, 248, 173, 100, 115, 92, 198, 162, 237, 70, 167, 255, 193, 43, 172, 241, 156, 188, 149, 203, 184, 199, 21, 145, 242, 253, 26, 66, 15, 20, 146, 120, 147, 7, 61, 98, 179, 30, 196, 180, 57, 102, 229, 240, 121, 125, 76, 91, 174, 42, 243, 50, 127, 158, 68, 141, 117, 207, 110, 175, 164, 122, 227, 190, 39, 3, 126, 47, 206, 166, 89, 101, 35, 140, 134, 123, 132, 239, 29, 194, 33, 73, 235, 195, 65, 182, 189, 64, 109, 246, 220, 159, 254, 202, 103, 19, 25, 60, 58, 78, 124, 170]","[0, 7, 10, 12, 13, 21, 23, 25, 26, 29, 32, 35, 39, 50, 60, 61, 66, 68, 69, 70, 72, 73, 75, 76, 81, 85, 91, 98, 99, 101, 109, 114, 115, 116, 117, 120, 121, 123, 127, 146, 147, 159, 162, 164, 172, 173, 175, 179, 180, 181, 182, 186, 188, 189, 193, 194, 202, 203, 207, 218, 227, 228, 237, 238, 240, 241, 243, 248, 252, 253, 254, 255]"
|
|
||||||
40,"[255, 12, 158, 117, 69, 26, 154, 159, 219, 109, 70, 245, 136, 49, 127, 167, 128, 124, 94, 32, 152, 96, 142, 112, 46, 92, 139, 48, 18, 161, 236, 134, 57, 45, 53, 211, 192, 133, 30, 52, 17, 223, 71, 50, 232, 213, 33, 61, 156, 40, 4, 103, 146, 137, 121, 25, 1, 63, 244, 87, 175, 165, 64, 157, 11, 180, 107, 222, 37, 31, 101, 99, 191, 42, 148, 183, 65, 56, 111, 77, 5, 58, 227, 108, 78, 179, 166, 212, 120, 125, 238, 215, 131, 2, 171, 89, 116, 234, 114, 47, 36, 196, 151, 93, 230, 9, 155, 68, 15, 59, 119, 110, 35, 226, 126, 178, 23, 209, 170, 16, 248, 22, 144, 203, 143, 186, 100, 206]","[85, 8, 108, 182, 146, 245, 64, 250, 174, 25, 172, 127, 168, 90, 179, 100, 76, 177, 249, 229, 73, 18, 228, 50, 116, 55, 231, 6, 199, 158, 113, 96, 82, 52, 132, 66, 19, 247, 242, 107, 80, 20, 236, 214, 176, 43, 138, 150, 15, 163, 209, 44, 79, 84, 86, 218, 21, 41, 67, 180, 173, 81, 104, 160, 101, 221, 224, 165, 35, 195, 200, 185, 243, 110, 14, 26, 34, 219, 189, 122, 204, 181, 115, 207, 239, 233, 24, 106, 202, 190, 156, 232, 112, 246, 147, 71, 136, 91, 140, 192, 118, 17, 28, 119, 208, 123, 164, 213, 241, 135, 54, 9, 70, 210, 36, 13, 145, 38, 46, 244, 222, 184, 149, 153, 75, 141, 154, 87]","[9, 15, 17, 18, 25, 26, 35, 36, 46, 50, 52, 64, 70, 71, 87, 96, 100, 101, 107, 108, 110, 112, 116, 119, 127, 136, 146, 154, 156, 158, 165, 179, 180, 192, 209, 213, 219, 222, 232, 236, 244, 245]"
|
|
||||||
41,"[83, 142, 66, 130, 202, 11, 13, 139, 92, 100, 214, 21, 245, 176, 210, 71, 189, 235, 74, 9, 227, 226, 58, 122, 247, 104, 81, 194, 110, 20, 161, 249, 157, 150, 143, 121, 173, 204, 118, 133, 107, 10, 95, 8, 237, 37, 85, 2, 126, 75, 137, 233, 234, 28, 218, 219, 138, 148, 171, 47, 119, 170, 144, 31, 33, 30, 162, 223, 127, 147, 211, 236, 206, 63, 117, 35, 101, 67, 229, 106, 64, 158, 91, 141, 39, 222, 196, 89, 129, 19, 187, 14, 205, 169, 25, 99, 178, 42, 1, 77, 78, 132, 69, 193, 114, 61, 55, 164, 116, 41, 23, 242, 96, 108, 38, 84, 182, 238, 52, 22, 241, 215, 51, 217, 212, 140, 252, 70]","[138, 185, 225, 196, 101, 129, 76, 108, 40, 106, 78, 128, 216, 195, 182, 189, 198, 171, 158, 130, 148, 21, 163, 126, 56, 1, 215, 74, 71, 154, 223, 54, 34, 240, 10, 233, 247, 234, 96, 70, 91, 169, 241, 0, 141, 88, 239, 160, 67, 63, 16, 18, 19, 159, 61, 58, 104, 173, 222, 205, 95, 111, 203, 238, 186, 103, 17, 150, 83, 85, 123, 212, 153, 110, 69, 48, 208, 105, 15, 31, 62, 59, 92, 80, 55, 84, 190, 2, 52, 115, 114, 109, 226, 64, 29, 42, 13, 152, 143, 107, 155, 236, 99, 132, 20, 142, 33, 100, 246, 51, 140, 35, 214, 177, 122, 37, 237, 219, 166, 11, 184, 176, 38, 180, 200, 66, 161, 97]","[1, 2, 10, 11, 13, 19, 20, 21, 31, 33, 35, 37, 38, 42, 51, 52, 55, 58, 61, 63, 64, 66, 67, 69, 70, 71, 74, 78, 83, 84, 85, 91, 92, 95, 96, 99, 100, 101, 104, 106, 107, 108, 110, 114, 122, 126, 129, 130, 132, 138, 140, 141, 142, 143, 148, 150, 158, 161, 169, 171, 173, 176, 182, 189, 196, 205, 212, 214, 215, 219, 222, 223, 226, 233, 234, 236, 237, 238, 241, 247]"
|
|
||||||
42,"[26, 67, 91, 13, 45, 255, 131, 76, 211, 176, 126, 18, 237, 97, 15, 124, 128, 88, 66, 89, 30, 65, 83, 149, 236, 144, 160, 23, 79, 207, 94, 203, 197, 7, 218, 161, 205, 42, 80, 72, 107, 22, 209, 60, 172, 247, 157, 81, 99, 179, 95, 82, 139, 53, 216, 188, 70, 204, 12, 19, 213, 167, 104, 175, 59, 186, 222, 29, 116, 86, 181, 148, 38, 156, 168, 110, 184, 194, 130, 164, 122, 34, 252, 192, 78, 137, 242, 56, 74, 198, 193, 84, 75, 47, 2, 85, 240, 35, 163, 152, 63, 25, 61, 103, 234, 180, 182, 62, 173, 132, 100, 28, 151, 31, 190, 155, 206, 73, 231, 166, 158, 111, 127, 165, 229, 191, 93, 50]","[95, 224, 84, 10, 143, 18, 91, 223, 104, 42, 140, 221, 179, 32, 186, 144, 236, 216, 164, 200, 247, 109, 110, 29, 85, 210, 2, 241, 71, 252, 19, 23, 135, 234, 211, 150, 168, 66, 159, 92, 141, 4, 48, 240, 245, 127, 3, 61, 57, 129, 125, 250, 130, 182, 38, 230, 166, 36, 83, 94, 111, 207, 249, 22, 72, 193, 116, 58, 12, 45, 112, 167, 74, 215, 190, 103, 24, 98, 138, 30, 147, 183, 16, 56, 152, 99, 217, 59, 96, 148, 142, 79, 105, 194, 254, 222, 47, 41, 9, 160, 205, 60, 26, 25, 35, 73, 49, 7, 123, 184, 65, 177, 227, 176, 87, 82, 67, 76, 237, 128, 6, 77, 157, 149, 131, 62, 97, 220]","[2, 7, 12, 18, 19, 22, 23, 25, 26, 29, 30, 35, 38, 42, 45, 47, 56, 59, 60, 61, 62, 65, 66, 67, 72, 73, 74, 76, 79, 82, 83, 84, 85, 91, 94, 95, 97, 99, 103, 104, 110, 111, 116, 127, 128, 130, 131, 144, 148, 149, 152, 157, 160, 164, 166, 167, 168, 176, 179, 182, 184, 186, 190, 193, 194, 205, 207, 211, 216, 222, 234, 236, 237, 240, 247, 252]"
|
|
||||||
43,"[132, 30, 65, 191, 88, 168, 60, 170, 142, 146, 53, 250, 75, 143, 155, 57, 100, 4, 208, 243, 56, 246, 157, 120, 113, 206, 77, 21, 121, 255, 166, 227, 117, 219, 40, 89, 147, 254, 198, 144, 45, 141, 134, 164, 108, 232, 63, 245, 150, 66, 221, 48, 222, 0, 44, 167, 55, 244, 138, 26, 46, 3, 10, 237, 145, 54, 223, 95, 105, 11, 42, 160, 188, 27, 71, 24, 131, 51, 230, 76, 185, 62, 127, 156, 91, 234, 178, 13, 172, 73, 50, 86, 165, 87, 28, 96, 129, 217, 158, 114, 111, 8, 242, 136, 7, 201, 2, 41, 133, 104, 52, 94, 203, 162, 9, 124, 173, 37, 139, 70, 194, 115, 43, 161, 163, 205, 204, 72]","[46, 31, 3, 220, 117, 1, 206, 77, 164, 126, 169, 200, 194, 223, 80, 0, 137, 144, 246, 11, 92, 15, 29, 209, 138, 202, 100, 129, 51, 99, 27, 216, 147, 5, 177, 131, 244, 240, 163, 104, 124, 96, 2, 132, 112, 24, 36, 66, 114, 247, 190, 97, 182, 91, 253, 18, 186, 72, 207, 239, 153, 47, 183, 249, 233, 61, 155, 17, 192, 32, 184, 60, 205, 214, 39, 168, 154, 122, 69, 90, 217, 151, 48, 57, 235, 19, 176, 16, 135, 152, 119, 59, 102, 236, 67, 89, 224, 110, 128, 158, 162, 187, 62, 238, 157, 40, 52, 73, 23, 30, 230, 231, 136, 65, 83, 212, 56, 252, 133, 118, 225, 20, 21, 181, 215, 78, 120, 197]","[0, 2, 3, 11, 21, 24, 27, 30, 40, 46, 48, 51, 52, 56, 57, 60, 62, 65, 66, 72, 73, 77, 89, 91, 96, 100, 104, 114, 117, 120, 124, 129, 131, 132, 133, 136, 138, 144, 147, 155, 157, 158, 162, 163, 164, 168, 194, 205, 206, 217, 223, 230, 244, 246]"
|
|
||||||
44,"[89, 250, 21, 160, 71, 68, 188, 26, 192, 87, 54, 72, 162, 210, 254, 82, 149, 204, 77, 85, 75, 69, 63, 1, 35, 114, 62, 45, 109, 38, 224, 132, 147, 20, 172, 191, 70, 86, 84, 176, 27, 140, 99, 50, 226, 13, 64, 238, 104, 112, 74, 235, 105, 125, 164, 159, 40, 227, 234, 7, 228, 121, 128, 96, 156, 198, 163, 56, 37, 4, 44, 146, 73, 173, 168, 81, 80, 25, 52, 208, 16, 60, 245, 55, 138, 0, 53, 6, 102, 221, 95, 136, 24, 177, 142, 211, 233, 215, 33, 43, 110, 48, 17, 122, 83, 120, 248, 246, 103, 100, 223, 29, 141, 167, 115, 155, 135, 65, 126, 137, 134, 31, 219, 184, 255, 46, 212, 32]","[7, 12, 224, 207, 2, 178, 126, 25, 116, 166, 56, 124, 102, 113, 30, 85, 91, 94, 232, 101, 1, 123, 183, 180, 244, 212, 181, 78, 225, 36, 150, 93, 127, 10, 185, 76, 8, 67, 48, 249, 200, 128, 100, 218, 205, 238, 108, 66, 243, 240, 221, 158, 174, 246, 11, 51, 251, 217, 97, 253, 237, 168, 220, 165, 61, 175, 43, 190, 139, 39, 194, 28, 131, 72, 154, 192, 195, 161, 3, 230, 59, 199, 219, 104, 187, 6, 201, 197, 164, 233, 148, 34, 235, 15, 152, 50, 241, 19, 9, 18, 5, 88, 68, 86, 92, 73, 118, 26, 242, 119, 90, 42, 170, 142, 203, 186, 193, 236, 214, 35, 80, 89, 64, 252, 111, 22, 38, 206]","[1, 6, 7, 25, 26, 35, 38, 43, 48, 50, 56, 64, 68, 72, 73, 80, 85, 86, 89, 100, 102, 104, 126, 128, 142, 164, 168, 192, 212, 219, 221, 224, 233, 235, 238, 246]"
|
|
||||||
45,"[74, 9, 207, 177, 24, 175, 211, 229, 87, 222, 143, 252, 7, 134, 132, 5, 0, 250, 204, 59, 151, 201, 187, 155, 58, 118, 223, 67, 139, 146, 137, 88, 232, 172, 210, 153, 10, 77, 208, 178, 72, 39, 157, 46, 47, 221, 57, 82, 214, 159, 56, 119, 63, 168, 51, 79, 35, 125, 180, 103, 99, 147, 169, 26, 94, 148, 127, 171, 224, 212, 138, 251, 34, 198, 161, 135, 102, 238, 89, 20, 195, 209, 216, 239, 122, 234, 247, 140, 236, 182, 14, 116, 48, 129, 249, 206, 37, 170, 81, 131, 150, 245, 18, 235, 141, 106, 244, 144, 183, 154, 130, 33, 228, 52, 42, 101, 40, 230, 184, 97, 112, 133, 117, 191, 124, 50, 126, 108]","[25, 171, 74, 148, 207, 177, 237, 143, 59, 57, 150, 123, 35, 70, 40, 37, 19, 139, 144, 228, 122, 132, 155, 94, 211, 89, 138, 80, 160, 72, 226, 81, 14, 244, 252, 219, 233, 206, 126, 247, 20, 255, 184, 5, 198, 117, 169, 214, 120, 168, 108, 39, 232, 127, 216, 251, 182, 239, 51, 115, 201, 170, 79, 110, 209, 175, 88, 134, 26, 16, 87, 235, 50, 130, 77, 92, 161, 188, 137, 102, 31, 180, 124, 55, 56, 157, 222, 199, 119, 86, 32, 145, 61, 0, 68, 186, 42, 78, 142, 146, 34, 82, 103, 125, 221, 204, 128, 212, 97, 196, 141, 147, 241, 250, 236, 49, 191, 249, 99, 245, 44, 58, 7, 116, 159, 224, 63, 43]","[0, 5, 7, 14, 20, 26, 34, 35, 37, 39, 40, 42, 50, 51, 56, 57, 58, 59, 63, 72, 74, 77, 79, 81, 82, 87, 88, 89, 94, 97, 99, 102, 103, 108, 116, 117, 119, 122, 124, 125, 126, 127, 130, 132, 134, 137, 138, 139, 141, 143, 144, 146, 147, 148, 150, 155, 157, 159, 161, 168, 169, 170, 171, 175, 177, 180, 182, 184, 191, 198, 201, 204, 206, 207, 209, 211, 212, 214, 216, 221, 222, 224, 228, 232, 235, 236, 239, 244, 245, 247, 249, 250, 251, 252]"
|
|
||||||
46,"[25, 113, 164, 213, 27, 166, 238, 202, 149, 130, 30, 197, 127, 143, 174, 24, 240, 63, 89, 126, 139, 175, 26, 173, 35, 237, 169, 115, 16, 241, 234, 88, 68, 154, 137, 8, 190, 97, 110, 21, 224, 236, 170, 140, 124, 232, 13, 81, 57, 9, 228, 181, 94, 233, 69, 204, 7, 47, 18, 17, 128, 201, 251, 105, 243, 43, 176, 199, 117, 39, 46, 112, 100, 101, 82, 239, 3, 151, 62, 203, 98, 158, 119, 72, 231, 247, 210, 198, 165, 150, 83, 65, 152, 193, 12, 48, 96, 93, 15, 219, 19, 29, 131, 206, 54, 85, 230, 148, 211, 45, 120, 160, 122, 207, 161, 133, 147, 36, 156, 185, 6, 10, 90, 53, 87, 220, 1, 168]","[92, 179, 14, 39, 41, 63, 174, 148, 45, 21, 18, 182, 212, 40, 8, 197, 237, 66, 46, 126, 116, 79, 195, 5, 149, 137, 167, 43, 36, 95, 205, 246, 85, 211, 127, 163, 60, 133, 221, 47, 83, 42, 24, 156, 180, 164, 252, 183, 123, 124, 188, 19, 192, 208, 44, 161, 110, 51, 122, 86, 112, 11, 111, 218, 236, 177, 228, 17, 49, 58, 157, 125, 254, 239, 59, 186, 26, 229, 233, 73, 106, 37, 120, 217, 90, 10, 16, 215, 169, 87, 235, 64, 105, 159, 3, 6, 166, 160, 70, 154, 191, 220, 189, 81, 171, 115, 76, 196, 118, 253, 241, 200, 68, 74, 88, 248, 91, 1, 141, 72, 245, 146, 32, 34, 25, 244, 98, 53]","[1, 3, 6, 8, 10, 16, 17, 18, 19, 21, 24, 25, 26, 36, 39, 43, 45, 46, 47, 53, 63, 68, 72, 81, 83, 85, 87, 88, 90, 98, 105, 110, 112, 115, 120, 122, 124, 126, 127, 133, 137, 148, 149, 154, 156, 160, 161, 164, 166, 169, 174, 197, 211, 220, 228, 233, 236, 237, 239, 241]"
|
|
||||||
47,"[223, 221, 49, 78, 141, 190, 66, 147, 10, 34, 58, 134, 136, 244, 250, 65, 108, 181, 91, 44, 178, 0, 88, 85, 224, 100, 29, 98, 127, 39, 245, 219, 177, 242, 198, 77, 251, 8, 180, 193, 128, 133, 15, 42, 68, 73, 76, 225, 90, 164, 110, 195, 145, 56, 246, 131, 236, 35, 53, 95, 204, 212, 214, 18, 211, 20, 14, 87, 231, 158, 72, 184, 43, 57, 249, 59, 149, 218, 107, 240, 47, 167, 192, 227, 123, 109, 4, 50, 121, 142, 74, 96, 122, 202, 186, 169, 124, 111, 33, 137, 156, 126, 45, 89, 84, 38, 93, 37, 60, 28, 206, 187, 201, 171, 146, 222, 5, 116, 25, 30, 26, 54, 154, 41, 165, 159, 252, 148]","[41, 137, 138, 116, 65, 221, 240, 193, 202, 159, 36, 52, 120, 10, 68, 241, 156, 84, 71, 251, 227, 39, 5, 180, 28, 178, 66, 233, 177, 76, 165, 158, 34, 44, 190, 79, 123, 216, 128, 33, 219, 45, 72, 140, 184, 22, 176, 59, 56, 214, 194, 147, 38, 143, 26, 49, 0, 109, 89, 19, 98, 54, 247, 25, 69, 75, 64, 1, 42, 136, 108, 179, 226, 175, 80, 232, 231, 85, 201, 81, 58, 27, 228, 250, 87, 17, 154, 55, 14, 130, 30, 218, 131, 51, 127, 20, 198, 205, 107, 244, 187, 141, 53, 126, 167, 246, 96, 134, 62, 90, 225, 73, 245, 206, 100, 43, 70, 35, 78, 168, 212, 121, 252, 162, 220, 145, 47, 197]","[0, 5, 10, 14, 20, 25, 26, 28, 30, 33, 34, 35, 38, 39, 41, 42, 43, 44, 45, 47, 49, 53, 54, 56, 58, 59, 65, 66, 68, 72, 73, 76, 78, 84, 85, 87, 89, 90, 96, 98, 100, 107, 108, 109, 116, 121, 123, 126, 127, 128, 131, 134, 136, 137, 141, 145, 147, 154, 156, 158, 159, 165, 167, 177, 178, 180, 184, 187, 190, 193, 198, 201, 202, 206, 212, 214, 218, 219, 221, 225, 227, 231, 240, 244, 245, 246, 250, 251, 252]"
|
|
||||||
48,"[199, 71, 243, 12, 84, 29, 115, 128, 255, 7, 31, 175, 155, 49, 188, 39, 165, 180, 60, 97, 50, 206, 189, 85, 4, 65, 116, 185, 126, 46, 23, 114, 159, 2, 75, 101, 152, 63, 25, 88, 141, 147, 28, 229, 87, 125, 57, 173, 19, 54, 251, 77, 135, 113, 26, 191, 211, 86, 138, 208, 235, 66, 253, 151, 170, 131, 240, 194, 48, 62, 51, 55, 139, 80, 37, 160, 237, 91, 45, 92, 190, 148, 187, 98, 34, 58, 78, 216, 0, 119, 82, 111, 109, 1, 213, 90, 17, 79, 145, 18, 212, 112, 238, 94, 171, 225, 83, 169, 164, 89, 153, 27, 69, 42, 33, 100, 105, 195, 30, 32, 96, 76, 149, 120, 110, 202, 186, 95]","[93, 185, 78, 251, 128, 153, 4, 237, 16, 69, 148, 204, 233, 224, 84, 60, 213, 252, 126, 81, 196, 255, 244, 113, 31, 71, 98, 243, 77, 229, 19, 219, 49, 163, 138, 190, 159, 100, 183, 18, 23, 54, 51, 27, 234, 228, 231, 7, 58, 35, 2, 14, 130, 73, 162, 123, 157, 67, 175, 179, 248, 12, 253, 86, 217, 24, 85, 116, 68, 53, 74, 109, 95, 189, 182, 239, 250, 43, 191, 211, 26, 235, 207, 104, 197, 34, 240, 133, 142, 227, 174, 29, 42, 76, 247, 117, 226, 1, 202, 110, 83, 145, 63, 135, 212, 90, 170, 147, 129, 88, 242, 176, 38, 173, 57, 225, 55, 177, 156, 201, 238, 39, 3, 194, 152, 64, 120, 112]","[1, 2, 4, 7, 12, 18, 19, 23, 26, 27, 29, 31, 34, 39, 42, 49, 51, 54, 55, 57, 58, 60, 63, 69, 71, 76, 77, 78, 83, 84, 85, 86, 88, 90, 95, 98, 100, 109, 110, 112, 113, 116, 120, 126, 128, 135, 138, 145, 147, 148, 152, 153, 159, 170, 173, 175, 185, 189, 190, 191, 194, 202, 211, 212, 213, 225, 229, 235, 237, 238, 240, 243, 251, 253, 255]"
|
|
||||||
49,"[208, 165, 238, 245, 104, 110, 220, 233, 95, 70, 122, 60, 117, 244, 47, 135, 76, 154, 216, 97, 0, 180, 204, 255, 7, 181, 211, 75, 100, 72, 91, 49, 144, 124, 38, 232, 65, 251, 230, 137, 241, 50, 20, 193, 174, 80, 162, 224, 57, 142, 133, 253, 1, 202, 147, 148, 158, 191, 43, 41, 126, 85, 13, 27, 82, 222, 196, 182, 30, 68, 221, 141, 112, 134, 212, 2, 44, 113, 74, 176, 153, 52, 164, 9, 78, 48, 189, 121, 219, 84, 198, 105, 73, 163, 6, 127, 28, 225, 194, 145, 66, 160, 123, 203, 16, 71, 35, 101, 23, 33, 188, 90, 125, 237, 139, 63, 99, 40, 24, 128, 118, 248, 214, 69, 231, 177, 161, 39]","[237, 213, 89, 171, 0, 14, 161, 113, 85, 23, 198, 202, 110, 126, 172, 230, 244, 86, 50, 33, 155, 201, 60, 197, 95, 13, 196, 241, 5, 143, 163, 165, 49, 160, 131, 97, 53, 133, 106, 79, 231, 81, 117, 39, 121, 100, 246, 167, 1, 168, 26, 127, 66, 219, 224, 253, 195, 71, 139, 41, 211, 142, 235, 159, 10, 188, 124, 158, 254, 57, 20, 65, 73, 137, 189, 242, 16, 148, 226, 112, 52, 27, 58, 51, 22, 56, 122, 7, 218, 30, 11, 42, 150, 225, 250, 228, 68, 38, 144, 236, 40, 180, 104, 153, 99, 47, 233, 248, 61, 177, 2, 204, 232, 96, 193, 245, 6, 69, 107, 185, 88, 67, 216, 15, 173, 90, 72, 102]","[0, 1, 2, 6, 7, 13, 16, 20, 23, 27, 30, 33, 38, 39, 40, 41, 47, 49, 50, 52, 57, 60, 65, 66, 68, 69, 71, 72, 73, 85, 90, 95, 97, 99, 100, 104, 110, 112, 113, 117, 121, 122, 124, 126, 127, 133, 137, 139, 142, 144, 148, 153, 158, 160, 161, 163, 165, 177, 180, 188, 189, 193, 196, 198, 202, 204, 211, 216, 219, 224, 225, 230, 231, 232, 233, 237, 241, 244, 245, 248, 253]"
|
|
||||||
50,"[249, 29, 241, 55, 181, 41, 44, 67, 25, 149, 192, 228, 191, 164, 143, 96, 0, 112, 105, 193, 92, 220, 206, 119, 141, 30, 211, 214, 32, 196, 226, 243, 117, 85, 132, 78, 39, 59, 24, 238, 147, 171, 16, 5, 204, 110, 185, 202, 46, 144, 95, 199, 54, 186, 80, 73, 43, 120, 34, 62, 47, 135, 146, 205, 82, 166, 51, 72, 114, 231, 244, 248, 21, 33, 69, 131, 198, 79, 137, 174, 1, 45, 180, 18, 187, 63, 170, 107, 40, 179, 176, 50, 200, 88, 175, 203, 98, 35, 124, 84, 224, 13, 86, 28, 127, 14, 142, 115, 74, 128, 37, 104, 253, 77, 208, 100, 102, 157, 91, 81, 217, 255, 15, 168, 136, 195, 17, 230]","[213, 208, 193, 55, 98, 76, 28, 223, 180, 241, 69, 146, 152, 65, 226, 120, 53, 158, 99, 103, 170, 41, 111, 84, 62, 187, 175, 116, 154, 77, 96, 25, 26, 137, 91, 171, 2, 224, 173, 27, 86, 206, 5, 119, 32, 161, 85, 33, 68, 236, 23, 139, 196, 217, 13, 6, 14, 167, 124, 144, 88, 174, 172, 233, 219, 106, 133, 108, 44, 30, 122, 100, 190, 205, 129, 57, 74, 238, 29, 132, 189, 166, 239, 127, 204, 20, 228, 11, 63, 254, 186, 168, 56, 24, 47, 230, 0, 94, 207, 177, 34, 40, 73, 15, 3, 184, 66, 81, 147, 249, 50, 67, 220, 235, 141, 51, 181, 107, 102, 214, 198, 72, 78, 130, 80, 110, 8, 203]","[0, 5, 13, 14, 15, 24, 25, 28, 29, 30, 32, 33, 34, 40, 41, 44, 47, 50, 51, 55, 62, 63, 67, 69, 72, 73, 74, 77, 78, 80, 81, 84, 85, 86, 88, 91, 96, 98, 100, 102, 107, 110, 119, 120, 124, 127, 132, 137, 141, 144, 146, 147, 166, 168, 170, 171, 174, 175, 180, 181, 186, 187, 193, 196, 198, 203, 204, 205, 206, 208, 214, 217, 220, 224, 226, 228, 230, 238, 241, 249]"
|
|
||||||
51,"[71, 12, 53, 20, 90, 244, 87, 51, 238, 24, 126, 109, 11, 129, 40, 191, 103, 154, 167, 124, 43, 72, 41, 56, 99, 84, 42, 57, 235, 39, 215, 115, 199, 212, 149, 182, 61, 97, 22, 165, 125, 221, 185, 106, 138, 35, 63, 189, 1, 140, 116, 184, 83, 234, 159, 172, 176, 195, 78, 48, 174, 59, 114, 45, 194, 246, 218, 197, 29, 60, 26, 128, 204, 150, 156, 2, 14, 5, 69, 3, 8, 47, 33, 81, 4, 190, 148, 161, 229, 52, 21, 153, 111, 119, 163, 139, 117, 94, 67, 36, 232, 102, 9, 7, 101, 50, 188, 137, 132, 198, 44, 136, 216, 224, 170, 6, 169, 92, 219, 134, 177, 0, 118, 183, 88, 211, 147, 104]","[252, 212, 160, 236, 209, 109, 202, 143, 255, 105, 228, 106, 156, 54, 62, 229, 58, 82, 223, 139, 68, 49, 136, 126, 253, 254, 187, 52, 217, 129, 124, 19, 41, 99, 247, 190, 91, 173, 33, 40, 220, 18, 46, 115, 231, 107, 164, 240, 64, 29, 59, 83, 76, 70, 86, 179, 26, 237, 55, 251, 79, 174, 137, 71, 180, 225, 207, 241, 37, 140, 153, 192, 81, 51, 80, 93, 61, 16, 171, 249, 60, 155, 245, 53, 210, 121, 222, 43, 244, 130, 34, 213, 122, 248, 108, 10, 158, 239, 196, 98, 205, 185, 32, 238, 57, 120, 227, 145, 191, 21, 193, 141, 165, 206, 150, 89, 38, 250, 135, 24, 200, 157, 168, 0, 87, 39, 66, 214]","[0, 21, 24, 26, 29, 33, 39, 40, 41, 43, 51, 52, 53, 57, 59, 60, 61, 71, 81, 83, 87, 99, 106, 109, 115, 124, 126, 129, 136, 137, 139, 140, 150, 153, 156, 165, 174, 185, 190, 191, 212, 229, 238, 244]"
|
|
||||||
52,"[207, 108, 194, 18, 182, 186, 195, 117, 223, 222, 92, 132, 235, 40, 127, 107, 189, 228, 31, 101, 119, 97, 28, 42, 116, 155, 204, 93, 241, 89, 21, 149, 27, 252, 14, 202, 104, 206, 34, 163, 136, 177, 248, 209, 133, 237, 78, 190, 130, 16, 10, 70, 175, 11, 167, 60, 20, 148, 32, 154, 231, 110, 173, 103, 105, 114, 17, 123, 94, 251, 232, 214, 152, 96, 210, 122, 12, 64, 215, 75, 196, 69, 197, 98, 166, 63, 48, 128, 198, 181, 54, 184, 255, 246, 76, 1, 67, 147, 211, 52, 65, 129, 139, 137, 242, 143, 36, 102, 8, 22, 55, 200, 15, 165, 77, 185, 134, 58, 82, 39, 205, 124, 45, 157, 26, 191, 106, 49]","[115, 42, 104, 175, 26, 56, 86, 210, 65, 164, 233, 91, 228, 97, 223, 38, 47, 46, 10, 155, 102, 126, 108, 177, 133, 254, 161, 122, 227, 63, 176, 90, 83, 106, 225, 142, 78, 71, 22, 234, 206, 55, 64, 229, 17, 185, 103, 209, 128, 21, 93, 198, 131, 28, 25, 245, 179, 199, 189, 19, 62, 188, 138, 100, 134, 213, 201, 51, 193, 44, 75, 121, 214, 23, 170, 148, 12, 50, 3, 6, 238, 187, 68, 24, 207, 178, 89, 15, 72, 84, 145, 35, 172, 45, 251, 58, 31, 190, 174, 162, 252, 53, 4, 147, 1, 57, 80, 74, 76, 135, 79, 29, 219, 224, 95, 249, 0, 212, 194, 220, 73, 87, 173, 69, 247, 60, 248, 125]","[1, 10, 12, 15, 17, 21, 22, 26, 28, 31, 42, 45, 55, 58, 60, 63, 64, 65, 69, 75, 76, 78, 89, 93, 97, 102, 103, 104, 106, 108, 122, 128, 133, 134, 147, 148, 155, 173, 175, 177, 185, 189, 190, 194, 198, 206, 207, 209, 210, 214, 223, 228, 248, 251, 252]"
|
|
||||||
53,"[218, 189, 239, 222, 104, 165, 160, 85, 49, 72, 166, 177, 97, 130, 164, 147, 84, 172, 156, 106, 204, 255, 60, 51, 212, 221, 25, 196, 87, 79, 183, 149, 86, 200, 138, 181, 241, 113, 76, 224, 24, 73, 245, 63, 96, 159, 3, 48, 11, 137, 182, 57, 142, 112, 140, 31, 80, 95, 89, 211, 82, 23, 206, 227, 75, 167, 152, 151, 194, 173, 146, 135, 58, 64, 157, 213, 55, 192, 199, 188, 249, 56, 240, 186, 207, 145, 161, 148, 35, 108, 32, 78, 105, 66, 17, 109, 2, 118, 228, 4, 44, 65, 27, 125, 169, 208, 93, 10, 126, 6, 28, 36, 33, 144, 14, 155, 179, 232, 59, 154, 94, 237, 229, 54, 226, 114, 53, 201]","[201, 181, 225, 249, 65, 183, 159, 20, 162, 171, 212, 121, 241, 138, 38, 207, 194, 191, 30, 210, 143, 216, 45, 218, 245, 158, 50, 197, 209, 118, 66, 18, 117, 135, 125, 234, 48, 157, 43, 146, 92, 19, 208, 119, 86, 0, 128, 93, 29, 8, 104, 46, 74, 172, 17, 177, 75, 202, 87, 122, 79, 99, 195, 12, 69, 133, 186, 156, 80, 180, 51, 149, 170, 175, 124, 35, 68, 193, 243, 169, 107, 251, 2, 228, 15, 206, 63, 78, 23, 132, 10, 98, 154, 56, 40, 205, 3, 247, 178, 163, 14, 164, 88, 176, 130, 248, 114, 190, 81, 196, 55, 185, 33, 145, 165, 24, 214, 53, 82, 37, 96, 21, 221, 230, 77, 103, 168, 153]","[2, 3, 10, 14, 17, 23, 24, 33, 35, 48, 51, 53, 55, 56, 63, 65, 66, 75, 78, 79, 80, 82, 86, 87, 93, 96, 104, 114, 118, 125, 130, 135, 138, 145, 146, 149, 154, 156, 157, 159, 164, 165, 169, 172, 177, 181, 183, 186, 194, 196, 201, 206, 207, 208, 212, 218, 221, 228, 241, 245, 249]"
|
|
||||||
54,"[26, 21, 179, 2, 178, 236, 125, 167, 175, 150, 160, 218, 153, 1, 133, 91, 136, 112, 227, 67, 119, 97, 84, 131, 243, 65, 43, 188, 156, 108, 46, 33, 177, 142, 5, 115, 171, 211, 207, 121, 85, 10, 76, 149, 134, 203, 47, 198, 22, 73, 144, 27, 100, 143, 70, 234, 37, 32, 122, 140, 20, 225, 217, 190, 111, 71, 34, 135, 127, 75, 223, 226, 186, 185, 154, 99, 39, 53, 102, 132, 29, 101, 106, 15, 83, 244, 189, 155, 214, 36, 230, 251, 183, 77, 204, 38, 18, 42, 170, 197, 161, 195, 0, 69, 128, 90, 61, 123, 235, 16, 93, 107, 216, 68, 252, 95, 191, 166, 7, 172, 41, 253, 9, 199, 62, 221, 8, 120]","[5, 43, 162, 166, 67, 134, 12, 214, 13, 137, 149, 181, 188, 39, 7, 109, 139, 41, 237, 80, 210, 128, 60, 99, 154, 21, 100, 165, 204, 61, 226, 152, 83, 224, 192, 48, 88, 253, 55, 73, 132, 160, 126, 174, 219, 178, 11, 247, 122, 96, 143, 74, 69, 78, 133, 118, 10, 163, 213, 175, 177, 225, 25, 233, 199, 236, 6, 53, 54, 216, 52, 235, 56, 14, 158, 205, 92, 58, 142, 117, 108, 249, 114, 200, 66, 62, 120, 63, 238, 57, 153, 197, 230, 155, 182, 18, 2, 0, 141, 22, 27, 107, 159, 26, 82, 85, 90, 202, 30, 33, 16, 176, 211, 157, 76, 242, 189, 35, 196, 59, 40, 203, 91, 215, 212, 254, 187, 190]","[0, 2, 5, 7, 10, 16, 18, 21, 22, 26, 27, 33, 39, 41, 43, 53, 61, 62, 67, 69, 73, 76, 83, 85, 90, 91, 99, 100, 107, 108, 120, 122, 128, 132, 133, 134, 142, 143, 149, 153, 154, 155, 160, 166, 175, 177, 178, 188, 189, 190, 197, 199, 203, 204, 211, 214, 216, 225, 226, 230, 235, 236, 253]"
|
|
||||||
55,"[248, 222, 131, 31, 201, 184, 74, 197, 154, 88, 137, 226, 82, 217, 255, 171, 9, 6, 219, 57, 161, 145, 155, 100, 234, 106, 77, 191, 12, 37, 199, 229, 200, 85, 118, 22, 69, 47, 90, 177, 214, 183, 60, 46, 38, 20, 249, 81, 196, 190, 34, 108, 56, 231, 241, 114, 211, 54, 243, 207, 78, 232, 136, 61, 45, 179, 95, 27, 185, 122, 125, 192, 83, 59, 166, 18, 63, 208, 151, 170, 93, 147, 14, 115, 101, 120, 104, 167, 41, 29, 159, 25, 92, 213, 209, 216, 124, 17, 33, 188, 19, 10, 75, 181, 228, 142, 50, 66, 253, 238, 130, 0, 4, 113, 133, 84, 202, 21, 76, 129, 237, 138, 99, 91, 148, 127, 52, 157]","[166, 73, 172, 182, 45, 212, 186, 13, 226, 129, 195, 216, 134, 144, 47, 198, 162, 107, 219, 202, 165, 206, 220, 128, 109, 79, 35, 2, 126, 152, 239, 210, 121, 96, 204, 29, 39, 193, 163, 132, 247, 41, 249, 3, 189, 80, 36, 93, 88, 178, 175, 43, 141, 28, 22, 52, 174, 103, 11, 253, 177, 230, 223, 67, 5, 140, 252, 65, 15, 111, 55, 4, 112, 98, 68, 17, 246, 237, 218, 23, 217, 176, 34, 26, 100, 155, 250, 203, 48, 72, 51, 105, 145, 97, 173, 149, 236, 139, 54, 8, 214, 62, 194, 42, 241, 233, 85, 58, 242, 19, 180, 168, 151, 119, 244, 254, 225, 169, 49, 32, 238, 187, 24, 117, 59, 158, 84, 156]","[4, 17, 19, 22, 29, 34, 41, 45, 47, 52, 54, 59, 84, 85, 88, 93, 100, 129, 145, 151, 155, 166, 177, 202, 214, 216, 217, 219, 226, 237, 238, 241, 249, 253]"
|
|
||||||
56,"[87, 195, 76, 243, 23, 130, 241, 205, 215, 107, 117, 236, 252, 201, 79, 29, 103, 193, 210, 203, 80, 175, 143, 221, 174, 45, 2, 37, 36, 65, 116, 154, 5, 32, 240, 16, 225, 250, 124, 147, 25, 11, 151, 44, 242, 59, 19, 227, 249, 46, 224, 198, 128, 163, 92, 74, 142, 96, 12, 213, 83, 145, 49, 52, 105, 155, 67, 72, 35, 207, 159, 10, 56, 177, 211, 42, 248, 62, 123, 17, 78, 84, 244, 81, 218, 57, 104, 41, 73, 208, 8, 39, 66, 38, 141, 48, 157, 223, 217, 149, 136, 125, 133, 101, 111, 183, 43, 194, 22, 132, 34, 68, 165, 138, 184, 171, 127, 122, 181, 94, 20, 70, 53, 173, 232, 18, 89, 14]","[112, 105, 200, 252, 167, 71, 80, 99, 201, 203, 89, 182, 117, 243, 223, 225, 177, 79, 62, 173, 38, 110, 130, 86, 220, 57, 210, 239, 77, 207, 83, 24, 43, 124, 116, 133, 34, 114, 152, 135, 214, 254, 139, 233, 49, 184, 67, 199, 134, 150, 101, 171, 151, 174, 0, 216, 1, 44, 248, 98, 186, 149, 215, 168, 36, 48, 120, 95, 178, 96, 32, 190, 88, 118, 228, 229, 163, 192, 121, 21, 81, 146, 156, 108, 8, 61, 137, 53, 218, 230, 60, 23, 40, 250, 142, 14, 52, 31, 54, 56, 45, 5, 123, 245, 68, 27, 13, 249, 29, 138, 74, 176, 136, 155, 10, 238, 170, 213, 191, 232, 104, 158, 69, 59, 97, 47, 166, 189]","[5, 8, 10, 14, 23, 29, 32, 34, 36, 38, 43, 44, 45, 48, 49, 52, 53, 56, 57, 59, 62, 67, 68, 74, 79, 80, 81, 83, 89, 96, 101, 104, 105, 116, 117, 123, 124, 130, 133, 136, 138, 142, 149, 151, 155, 163, 171, 173, 174, 177, 184, 201, 203, 207, 210, 213, 215, 218, 223, 225, 232, 243, 248, 249, 250, 252]"
|
|
||||||
57,"[122, 222, 51, 117, 128, 41, 133, 136, 30, 149, 140, 37, 130, 47, 38, 107, 218, 48, 177, 84, 132, 213, 112, 36, 220, 67, 164, 90, 243, 26, 144, 254, 101, 61, 211, 118, 18, 57, 45, 99, 46, 234, 215, 111, 221, 131, 89, 188, 25, 141, 127, 71, 54, 148, 150, 85, 153, 207, 72, 102, 251, 171, 170, 96, 119, 92, 44, 185, 87, 236, 248, 50, 55, 80, 239, 192, 21, 70, 91, 125, 28, 223, 10, 142, 155, 20, 249, 181, 79, 9, 233, 199, 235, 83, 53, 43, 24, 88, 156, 166, 73, 81, 151, 16, 143, 98, 172, 174, 137, 78, 175, 49, 31, 56, 8, 193, 93, 3, 216, 86, 110, 252, 202, 146, 162, 6, 255, 109]","[73, 173, 100, 98, 99, 253, 86, 40, 164, 245, 171, 128, 5, 12, 218, 201, 63, 53, 181, 160, 229, 68, 153, 19, 147, 179, 233, 158, 212, 85, 197, 91, 42, 97, 0, 113, 39, 69, 204, 194, 64, 29, 228, 249, 200, 106, 183, 241, 220, 76, 244, 89, 15, 10, 105, 242, 232, 255, 166, 94, 108, 152, 9, 135, 95, 122, 22, 23, 170, 163, 189, 208, 178, 203, 167, 18, 214, 165, 168, 141, 58, 49, 154, 65, 44, 247, 33, 190, 92, 2, 77, 226, 195, 224, 180, 134, 60, 14, 117, 75, 32, 188, 74, 62, 126, 121, 4, 206, 37, 192, 176, 116, 198, 82, 34, 139, 110, 28, 182, 236, 26, 25, 54, 6, 38, 205, 84, 115]","[6, 9, 10, 18, 25, 26, 28, 37, 38, 44, 49, 53, 54, 73, 84, 85, 86, 89, 91, 92, 98, 99, 110, 117, 122, 128, 141, 153, 164, 166, 170, 171, 181, 188, 192, 218, 220, 233, 236, 249, 255]"
|
|
||||||
58,"[98, 26, 62, 141, 81, 214, 132, 127, 158, 160, 15, 195, 103, 153, 61, 56, 254, 228, 181, 152, 213, 130, 203, 197, 43, 251, 220, 71, 164, 2, 165, 194, 146, 46, 94, 76, 240, 5, 140, 65, 83, 134, 167, 151, 29, 52, 120, 136, 105, 128, 119, 112, 122, 217, 216, 123, 173, 116, 25, 107, 115, 74, 55, 226, 21, 60, 87, 96, 243, 176, 182, 20, 8, 200, 38, 121, 91, 129, 66, 64, 7, 222, 183, 187, 84, 209, 117, 114, 180, 199, 40, 168, 14, 33, 193, 102, 82, 13, 99, 162, 3, 234, 67, 31, 238, 19, 157, 93, 161, 17, 241, 9, 172, 125, 166, 135, 126, 24, 139, 48, 111, 92, 225, 106, 150, 50, 49, 95]","[189, 209, 166, 52, 130, 210, 78, 186, 216, 151, 107, 6, 95, 251, 93, 21, 17, 191, 22, 18, 87, 79, 164, 212, 106, 69, 84, 147, 105, 201, 56, 66, 127, 195, 31, 114, 185, 141, 10, 88, 99, 125, 197, 158, 57, 211, 145, 46, 167, 16, 104, 61, 118, 0, 204, 174, 225, 25, 113, 172, 128, 206, 24, 182, 26, 41, 171, 215, 40, 190, 50, 165, 74, 221, 110, 98, 1, 7, 250, 67, 27, 229, 47, 238, 112, 162, 157, 188, 246, 102, 83, 123, 135, 153, 9, 144, 239, 120, 134, 150, 187, 160, 176, 237, 208, 2, 129, 39, 217, 220, 60, 11, 76, 80, 228, 152, 33, 103, 91, 173, 82, 94, 117, 68, 81, 131, 5, 96]","[2, 5, 7, 9, 17, 21, 24, 25, 26, 31, 33, 40, 46, 50, 52, 56, 60, 61, 66, 67, 74, 76, 81, 82, 83, 84, 87, 91, 93, 94, 95, 96, 98, 99, 102, 103, 105, 106, 107, 112, 114, 117, 120, 123, 125, 127, 128, 129, 130, 134, 135, 141, 150, 151, 152, 153, 157, 158, 160, 162, 164, 165, 166, 167, 172, 173, 176, 182, 187, 195, 197, 209, 216, 217, 220, 225, 228, 238, 251]"
|
|
||||||
59,"[205, 159, 8, 189, 121, 29, 225, 76, 99, 6, 149, 37, 150, 216, 88, 203, 127, 58, 211, 93, 190, 16, 241, 47, 81, 195, 11, 129, 71, 178, 40, 122, 154, 89, 21, 20, 146, 96, 69, 73, 214, 39, 160, 38, 255, 219, 4, 84, 196, 184, 87, 209, 162, 103, 166, 42, 155, 2, 74, 238, 169, 145, 142, 168, 98, 163, 75, 35, 234, 179, 1, 67, 41, 111, 56, 107, 62, 23, 77, 152, 36, 104, 181, 172, 217, 156, 125, 180, 198, 210, 165, 183, 212, 64, 27, 100, 0, 220, 236, 106, 177, 112, 148, 113, 230, 199, 153, 24, 5, 90, 201, 60, 82, 19, 101, 65, 186, 18, 192, 171, 120, 229, 114, 70, 12, 237, 108, 30]","[49, 203, 179, 153, 67, 25, 93, 0, 40, 89, 218, 146, 70, 9, 120, 138, 104, 31, 41, 96, 80, 76, 106, 195, 69, 141, 197, 10, 130, 121, 215, 20, 65, 201, 187, 82, 237, 183, 226, 160, 191, 101, 139, 103, 102, 21, 100, 71, 108, 238, 16, 4, 44, 239, 174, 202, 241, 184, 37, 229, 214, 55, 131, 190, 231, 99, 36, 244, 161, 118, 17, 50, 39, 95, 137, 219, 177, 148, 149, 216, 129, 30, 233, 33, 147, 29, 236, 74, 230, 87, 19, 181, 88, 144, 166, 143, 105, 134, 196, 73, 27, 75, 18, 164, 109, 204, 113, 222, 59, 176, 245, 135, 171, 162, 119, 34, 156, 35, 173, 8, 167, 5, 248, 90, 126, 47, 81, 136]","[0, 4, 5, 8, 16, 18, 19, 20, 21, 27, 29, 30, 35, 36, 37, 39, 40, 41, 47, 65, 67, 69, 70, 71, 73, 74, 75, 76, 81, 82, 87, 88, 89, 90, 93, 96, 99, 100, 101, 103, 104, 106, 108, 113, 120, 121, 129, 146, 148, 149, 153, 156, 160, 162, 166, 171, 177, 179, 181, 183, 184, 190, 195, 196, 201, 203, 214, 216, 219, 229, 230, 236, 237, 238, 241]"
|
|
||||||
60,"[162, 122, 131, 126, 97, 205, 36, 83, 163, 15, 52, 232, 1, 93, 156, 200, 72, 245, 176, 189, 64, 207, 241, 77, 172, 65, 151, 228, 67, 40, 90, 21, 128, 130, 142, 124, 28, 68, 250, 174, 51, 179, 54, 31, 27, 248, 158, 238, 210, 20, 127, 252, 191, 16, 177, 8, 82, 63, 78, 84, 96, 199, 0, 37, 53, 29, 193, 114, 129, 19, 35, 111, 226, 13, 88, 14, 203, 227, 103, 175, 201, 17, 33, 121, 157, 55, 133, 94, 18, 143, 89, 188, 212, 148, 185, 71, 221, 135, 100, 106, 184, 236, 60, 87, 164, 233, 42, 117, 108, 239, 242, 196, 102, 46, 47, 118, 73, 186, 110, 182, 69, 149, 224, 198, 217, 234, 99, 225]","[31, 130, 4, 99, 187, 88, 213, 103, 205, 162, 93, 117, 192, 200, 225, 155, 236, 151, 242, 252, 181, 253, 89, 165, 7, 66, 10, 85, 37, 223, 204, 87, 77, 20, 138, 21, 241, 65, 0, 72, 13, 232, 206, 212, 108, 51, 133, 42, 18, 122, 209, 69, 142, 12, 248, 60, 16, 196, 239, 110, 136, 102, 164, 17, 221, 73, 55, 174, 226, 134, 220, 234, 27, 54, 145, 52, 158, 63, 71, 15, 104, 100, 157, 105, 149, 215, 36, 70, 128, 57, 97, 120, 129, 189, 111, 179, 78, 148, 34, 107, 240, 67, 19, 44, 47, 175, 40, 53, 125, 94, 182, 201, 191, 127, 245, 80, 166, 171, 118, 14, 217, 203, 186, 84, 227, 82, 83, 32]","[0, 13, 14, 15, 16, 17, 18, 19, 20, 21, 27, 31, 36, 37, 40, 42, 47, 51, 52, 53, 54, 55, 60, 63, 65, 67, 69, 71, 72, 73, 77, 78, 82, 83, 84, 87, 88, 89, 93, 94, 97, 99, 100, 102, 103, 108, 110, 111, 117, 118, 122, 127, 128, 129, 130, 133, 142, 148, 149, 151, 157, 158, 162, 164, 174, 175, 179, 182, 186, 189, 191, 196, 200, 201, 203, 205, 212, 217, 221, 225, 226, 227, 232, 234, 236, 239, 241, 242, 245, 248, 252]"
|
|
||||||
61,"[247, 207, 102, 61, 8, 6, 189, 186, 219, 213, 148, 200, 98, 248, 129, 35, 109, 174, 240, 69, 106, 160, 146, 40, 48, 107, 224, 208, 175, 90, 122, 123, 209, 149, 62, 126, 110, 236, 85, 9, 68, 217, 121, 137, 24, 212, 50, 65, 203, 71, 187, 56, 76, 178, 222, 158, 156, 30, 143, 99, 54, 112, 111, 7, 130, 125, 183, 100, 53, 152, 227, 46, 211, 170, 167, 241, 5, 182, 173, 74, 23, 171, 80, 242, 243, 124, 27, 39, 221, 72, 190, 89, 153, 141, 215, 206, 231, 3, 0, 66, 17, 169, 44, 16, 25, 131, 210, 2, 154, 97, 192, 163, 179, 84, 63, 28, 14, 57, 250, 114, 161, 92, 191, 249, 75, 188, 103, 15]","[57, 235, 242, 204, 22, 77, 120, 15, 103, 183, 11, 222, 213, 124, 149, 115, 189, 107, 186, 36, 30, 17, 80, 239, 65, 44, 122, 50, 234, 54, 10, 87, 49, 26, 164, 152, 248, 176, 162, 39, 190, 203, 3, 27, 221, 137, 148, 105, 142, 109, 205, 66, 179, 0, 91, 14, 214, 151, 175, 254, 136, 6, 206, 131, 128, 73, 58, 19, 16, 38, 99, 42, 31, 225, 158, 188, 90, 134, 112, 219, 174, 171, 194, 25, 249, 212, 211, 228, 43, 60, 64, 102, 62, 146, 130, 97, 156, 53, 56, 170, 8, 29, 143, 55, 250, 100, 68, 106, 84, 138, 215, 180, 236, 198, 224, 231, 165, 114, 76, 226, 207, 35, 1, 246, 144, 23, 7, 126]","[0, 3, 6, 7, 8, 14, 15, 16, 17, 23, 25, 27, 30, 35, 39, 44, 50, 53, 54, 56, 57, 62, 65, 66, 68, 76, 80, 84, 90, 97, 99, 100, 102, 103, 106, 107, 109, 112, 114, 122, 124, 126, 130, 131, 137, 143, 146, 148, 149, 152, 156, 158, 170, 171, 174, 175, 179, 183, 186, 188, 189, 190, 203, 206, 207, 211, 212, 213, 215, 219, 221, 222, 224, 231, 236, 242, 248, 249, 250]"
|
|
||||||
62,"[208, 179, 190, 88, 105, 214, 166, 177, 2, 126, 157, 170, 51, 225, 158, 140, 142, 108, 80, 21, 143, 175, 40, 106, 81, 95, 115, 230, 69, 189, 249, 181, 94, 124, 121, 62, 103, 114, 9, 137, 152, 162, 90, 198, 195, 125, 82, 217, 71, 112, 244, 7, 8, 98, 30, 53, 49, 155, 251, 13, 97, 58, 18, 15, 20, 202, 64, 79, 70, 42, 240, 47, 39, 14, 73, 222, 224, 160, 24, 92, 104, 25, 72, 60, 44, 57, 133, 45, 43, 91, 176, 68, 29, 174, 83, 46, 3, 247, 17, 169, 110, 229, 220, 250, 149, 22, 194, 54, 116, 138, 163, 135, 172, 210, 241, 35, 151, 159, 215, 76, 48, 231, 66, 243, 197, 93, 218, 128]","[70, 38, 131, 66, 245, 25, 210, 61, 45, 173, 121, 34, 206, 23, 164, 150, 27, 51, 252, 106, 240, 204, 40, 32, 109, 226, 148, 2, 203, 238, 77, 134, 58, 190, 132, 144, 241, 179, 65, 230, 207, 158, 111, 123, 196, 198, 102, 172, 193, 175, 156, 36, 221, 153, 15, 98, 76, 8, 149, 220, 188, 222, 41, 53, 236, 182, 49, 95, 69, 214, 30, 233, 139, 39, 225, 84, 191, 119, 228, 26, 211, 19, 4, 89, 129, 120, 244, 5, 217, 199, 87, 146, 189, 22, 99, 154, 212, 64, 74, 93, 201, 213, 94, 200, 247, 163, 169, 197, 31, 162, 138, 168, 101, 14, 209, 85, 44, 42, 56, 78, 242, 232, 60, 127, 103, 80, 110, 12]","[2, 8, 14, 15, 22, 25, 30, 39, 40, 42, 44, 45, 49, 51, 53, 58, 60, 64, 66, 69, 70, 76, 80, 93, 94, 95, 98, 103, 106, 110, 121, 138, 149, 158, 162, 163, 169, 172, 175, 179, 189, 190, 197, 198, 210, 214, 217, 220, 222, 225, 230, 240, 241, 244, 247]"
|
|
||||||
63,"[242, 44, 174, 36, 133, 90, 72, 124, 95, 223, 182, 157, 16, 58, 85, 57, 167, 180, 196, 225, 220, 68, 156, 203, 6, 191, 15, 76, 117, 148, 131, 108, 70, 104, 13, 150, 123, 240, 151, 55, 132, 201, 89, 173, 98, 199, 237, 127, 186, 206, 253, 20, 140, 244, 61, 22, 27, 79, 38, 137, 49, 210, 154, 92, 97, 75, 112, 142, 252, 32, 159, 135, 51, 169, 39, 3, 200, 181, 62, 218, 195, 83, 143, 205, 19, 96, 26, 202, 236, 18, 145, 43, 184, 2, 197, 65, 121, 109, 229, 107, 221, 190, 47, 73, 8, 93, 35, 41, 129, 155, 211, 31, 146, 23, 78, 136, 149, 50, 81, 217, 162, 100, 185, 34, 77, 130, 71, 110]","[75, 136, 163, 146, 228, 117, 112, 154, 184, 78, 77, 164, 63, 242, 43, 62, 200, 193, 202, 93, 35, 87, 123, 20, 122, 26, 220, 14, 131, 165, 148, 185, 142, 172, 2, 8, 197, 237, 130, 240, 222, 1, 253, 86, 182, 196, 76, 84, 135, 36, 18, 73, 90, 114, 236, 143, 103, 229, 214, 167, 181, 232, 129, 157, 124, 221, 15, 34, 223, 206, 44, 38, 169, 16, 195, 89, 175, 141, 115, 6, 159, 186, 155, 210, 145, 180, 219, 42, 32, 156, 56, 137, 246, 11, 92, 65, 127, 201, 107, 108, 51, 91, 191, 179, 70, 58, 31, 10, 160, 94, 48, 3, 104, 208, 85, 27, 133, 194, 121, 71, 13, 151, 68, 96, 9, 162, 203, 41]","[2, 3, 6, 8, 13, 15, 16, 18, 20, 26, 27, 31, 32, 34, 35, 36, 38, 41, 43, 44, 51, 58, 62, 65, 68, 70, 71, 73, 75, 76, 77, 78, 85, 89, 90, 92, 93, 96, 104, 107, 108, 112, 117, 121, 123, 124, 127, 129, 130, 131, 133, 135, 136, 137, 142, 143, 145, 146, 148, 151, 154, 155, 156, 157, 159, 162, 167, 169, 180, 181, 182, 184, 185, 186, 191, 195, 196, 197, 200, 201, 202, 203, 206, 210, 220, 221, 223, 229, 236, 237, 240, 242, 253]"
|
|
||||||
64,"[122, 85, 222, 137, 183, 115, 151, 4, 68, 161, 72, 254, 251, 216, 13, 248, 141, 60, 153, 110, 90, 9, 191, 185, 36, 109, 174, 98, 166, 46, 38, 41, 242, 233, 203, 130, 143, 80, 17, 78, 99, 255, 34, 39, 89, 158, 40, 220, 120, 214, 69, 223, 45, 84, 202, 81, 65, 5, 207, 170, 201, 199, 239, 77, 209, 43, 25, 107, 11, 96, 102, 194, 240, 140, 2, 200, 111, 27, 87, 52, 1, 234, 6, 18, 227, 197, 21, 83, 106, 173, 231, 148, 70, 213, 159, 128, 224, 252, 103, 75, 145, 236, 92, 243, 135, 178, 226, 117, 86, 19, 55, 112, 14, 119, 57, 79, 28, 238, 94, 168, 164, 132, 229, 88, 190, 249, 37, 108]","[217, 163, 75, 35, 27, 242, 148, 124, 202, 137, 11, 254, 83, 28, 142, 67, 206, 55, 115, 86, 66, 48, 172, 245, 213, 98, 214, 65, 49, 77, 68, 127, 32, 39, 46, 237, 90, 9, 78, 203, 31, 149, 88, 225, 110, 246, 91, 160, 208, 135, 114, 20, 171, 192, 250, 167, 244, 164, 221, 113, 190, 226, 247, 29, 50, 57, 3, 232, 122, 52, 118, 168, 151, 14, 139, 117, 25, 21, 169, 64, 37, 58, 154, 188, 74, 161, 255, 182, 8, 10, 36, 56, 72, 165, 84, 136, 178, 17, 215, 22, 7, 26, 159, 183, 16, 235, 41, 177, 180, 179, 143, 43, 23, 44, 107, 71, 191, 195, 193, 85, 76, 62, 105, 59, 131, 243, 1, 150]","[1, 9, 11, 14, 17, 21, 25, 27, 28, 36, 37, 39, 41, 43, 46, 52, 55, 57, 65, 68, 72, 75, 77, 78, 83, 84, 85, 86, 88, 90, 98, 107, 110, 115, 117, 122, 135, 137, 143, 148, 151, 159, 161, 164, 168, 178, 183, 190, 191, 202, 203, 213, 214, 226, 242, 243, 254, 255]"
|
|
||||||
65,"[46, 31, 41, 76, 252, 144, 214, 109, 7, 193, 153, 99, 137, 170, 1, 140, 94, 149, 100, 221, 198, 56, 2, 136, 61, 119, 51, 60, 133, 254, 247, 224, 161, 82, 45, 199, 26, 255, 75, 85, 156, 248, 102, 101, 70, 229, 215, 12, 25, 241, 71, 186, 173, 192, 132, 62, 27, 36, 80, 239, 98, 96, 223, 246, 110, 121, 208, 207, 106, 95, 131, 124, 22, 57, 163, 164, 18, 35, 44, 23, 167, 81, 118, 225, 234, 103, 90, 33, 204, 177, 78, 114, 120, 151, 55, 104, 49, 77, 138, 185, 200, 92, 37, 157, 253, 128, 171, 29, 152, 211, 216, 145, 84, 218, 4, 3, 190, 182, 139, 11, 123, 117, 168, 19, 179, 107, 48, 162]","[83, 173, 51, 85, 217, 58, 250, 56, 69, 40, 8, 18, 57, 5, 129, 87, 147, 222, 188, 30, 144, 89, 184, 12, 59, 139, 43, 66, 15, 32, 201, 49, 31, 1, 218, 172, 181, 186, 228, 211, 74, 73, 153, 94, 127, 109, 195, 154, 75, 108, 236, 171, 206, 194, 116, 21, 166, 125, 10, 233, 246, 191, 62, 167, 65, 101, 151, 3, 148, 175, 170, 23, 105, 67, 149, 152, 248, 117, 24, 64, 46, 63, 130, 229, 237, 119, 251, 121, 252, 13, 86, 19, 213, 182, 238, 254, 92, 138, 189, 197, 111, 203, 126, 42, 240, 230, 68, 190, 204, 45, 38, 16, 88, 47, 11, 176, 20, 122, 25, 123, 17, 247, 209, 115, 220, 96, 192, 22]","[1, 3, 11, 12, 18, 19, 22, 23, 25, 31, 45, 46, 49, 51, 56, 57, 62, 75, 85, 92, 94, 96, 101, 109, 117, 119, 121, 123, 138, 139, 144, 149, 151, 152, 153, 167, 170, 171, 173, 182, 186, 190, 192, 204, 211, 218, 229, 246, 247, 248, 252, 254]"
|
|
||||||
66,"[89, 218, 185, 164, 114, 157, 141, 104, 148, 60, 234, 5, 73, 128, 26, 43, 171, 93, 91, 62, 244, 146, 100, 94, 109, 117, 34, 65, 90, 231, 124, 83, 14, 155, 125, 228, 127, 33, 163, 25, 129, 181, 174, 31, 106, 103, 32, 173, 84, 39, 241, 15, 169, 145, 233, 3, 123, 119, 160, 86, 120, 133, 247, 121, 2, 180, 229, 215, 107, 210, 116, 137, 110, 208, 152, 11, 205, 147, 80, 156, 194, 97, 52, 151, 59, 201, 167, 162, 4, 102, 144, 58, 245, 64, 252, 158, 55, 200, 239, 42, 198, 132, 16, 204, 46, 219, 143, 226, 192, 56, 206, 48, 214, 36, 191, 131, 150, 177, 136, 130, 232, 196, 24, 35, 225, 7, 72, 111]","[130, 92, 249, 210, 134, 95, 226, 102, 141, 247, 155, 64, 220, 26, 80, 160, 232, 159, 245, 229, 248, 191, 166, 183, 208, 136, 145, 129, 25, 42, 36, 132, 218, 217, 51, 231, 93, 164, 196, 176, 146, 152, 137, 239, 185, 66, 241, 215, 49, 103, 135, 175, 94, 110, 48, 13, 120, 44, 99, 128, 4, 7, 233, 173, 214, 234, 131, 158, 167, 197, 124, 83, 252, 91, 97, 23, 228, 148, 157, 27, 35, 165, 238, 121, 65, 168, 212, 52, 106, 211, 174, 89, 24, 127, 209, 72, 206, 243, 151, 180, 161, 111, 114, 235, 117, 73, 60, 162, 5, 169, 56, 242, 12, 79, 223, 195, 78, 201, 216, 34, 138, 33, 43, 3, 107, 59, 119, 171]","[3, 4, 5, 7, 24, 25, 26, 33, 34, 35, 36, 42, 43, 48, 52, 56, 59, 60, 64, 65, 72, 73, 80, 83, 89, 91, 93, 94, 97, 102, 103, 106, 107, 110, 111, 114, 117, 119, 120, 121, 124, 127, 128, 129, 130, 131, 132, 136, 137, 141, 145, 146, 148, 151, 152, 155, 157, 158, 160, 162, 164, 167, 169, 171, 173, 174, 180, 185, 191, 196, 201, 206, 208, 210, 214, 215, 218, 226, 228, 229, 231, 232, 233, 234, 239, 241, 245, 247, 252]"
|
|
||||||
67,"[79, 188, 139, 172, 53, 200, 153, 31, 60, 61, 156, 145, 215, 30, 90, 255, 248, 144, 147, 87, 182, 131, 196, 163, 243, 114, 120, 104, 242, 160, 224, 138, 107, 42, 164, 73, 99, 191, 125, 43, 25, 216, 94, 86, 148, 178, 194, 8, 212, 170, 204, 74, 82, 195, 254, 197, 106, 155, 158, 231, 228, 180, 124, 113, 143, 173, 140, 17, 58, 103, 12, 190, 189, 48, 203, 136, 27, 223, 142, 67, 51, 57, 102, 141, 64, 151, 121, 209, 133, 11, 183, 134, 177, 240, 137, 171, 72, 55, 16, 165, 135, 3, 47, 222, 122, 146, 205, 201, 33, 202, 18, 83, 157, 59, 95, 34, 52, 226, 97, 2, 29, 207, 150, 213, 192, 41, 44, 229]","[17, 159, 164, 181, 74, 34, 213, 121, 29, 43, 111, 75, 11, 152, 182, 0, 238, 195, 62, 145, 220, 243, 21, 41, 137, 202, 149, 201, 35, 154, 130, 97, 234, 98, 168, 70, 224, 215, 241, 94, 136, 138, 87, 68, 200, 40, 61, 49, 48, 175, 244, 245, 252, 187, 19, 210, 160, 6, 106, 249, 36, 88, 63, 14, 167, 96, 208, 38, 101, 163, 214, 89, 10, 132, 193, 194, 117, 123, 82, 144, 65, 1, 23, 166, 156, 12, 90, 4, 242, 248, 115, 24, 211, 196, 169, 165, 217, 126, 39, 16, 71, 229, 157, 46, 251, 198, 188, 66, 32, 15, 204, 93, 206, 100, 47, 236, 58, 255, 79, 172, 84, 127, 104, 221, 103, 83, 223, 51]","[11, 12, 16, 17, 29, 34, 41, 43, 47, 48, 51, 58, 61, 74, 79, 82, 83, 87, 90, 94, 97, 103, 104, 106, 121, 136, 137, 138, 144, 145, 156, 157, 160, 163, 164, 165, 172, 182, 188, 194, 195, 196, 200, 201, 202, 204, 213, 215, 223, 224, 229, 242, 243, 248, 255]"
|
|
||||||
68,"[27, 62, 111, 247, 156, 113, 97, 18, 215, 13, 160, 151, 199, 230, 167, 47, 28, 214, 40, 253, 59, 212, 61, 148, 218, 107, 140, 70, 172, 76, 185, 95, 126, 57, 246, 78, 254, 65, 25, 96, 71, 16, 94, 80, 15, 171, 72, 19, 10, 24, 3, 115, 188, 183, 118, 128, 31, 92, 180, 251, 63, 58, 60, 26, 30, 141, 198, 6, 133, 55, 211, 179, 0, 243, 74, 177, 64, 223, 112, 77, 75, 21, 184, 157, 125, 67, 101, 201, 209, 145, 36, 106, 81, 120, 1, 52, 9, 164, 105, 213, 196, 82, 135, 129, 149, 7, 200, 222, 226, 186, 91, 165, 237, 136, 20, 166, 161, 8, 147, 38, 207, 37, 234, 104, 150, 83, 73, 49]","[153, 55, 216, 33, 195, 82, 167, 173, 36, 245, 24, 23, 189, 102, 194, 249, 54, 241, 233, 86, 14, 110, 196, 87, 22, 57, 124, 13, 227, 207, 32, 170, 137, 242, 109, 41, 90, 97, 205, 2, 66, 9, 193, 56, 134, 44, 247, 121, 11, 203, 117, 231, 120, 230, 8, 221, 187, 60, 138, 159, 68, 176, 17, 101, 69, 127, 160, 174, 100, 103, 246, 12, 88, 228, 81, 204, 218, 220, 71, 99, 79, 64, 107, 212, 59, 53, 122, 51, 156, 191, 98, 236, 182, 244, 108, 89, 225, 154, 208, 52, 163, 84, 200, 85, 190, 112, 229, 143, 92, 133, 50, 29, 146, 238, 37, 226, 210, 206, 39, 178, 34, 43, 46, 123, 5, 147, 181, 38]","[8, 9, 13, 24, 36, 37, 38, 52, 55, 57, 59, 60, 64, 71, 81, 82, 92, 97, 101, 107, 112, 120, 133, 147, 156, 160, 167, 196, 200, 207, 212, 218, 226, 230, 246, 247]"
|
|
||||||
69,"[73, 21, 161, 32, 5, 130, 48, 78, 43, 11, 102, 186, 172, 46, 213, 188, 204, 98, 180, 22, 82, 142, 248, 83, 138, 202, 207, 25, 37, 167, 151, 71, 80, 219, 227, 86, 109, 252, 245, 45, 36, 169, 147, 175, 242, 247, 50, 157, 103, 193, 235, 222, 9, 59, 85, 89, 214, 53, 114, 20, 35, 170, 81, 143, 146, 243, 26, 68, 91, 65, 54, 218, 16, 236, 116, 174, 123, 154, 234, 145, 208, 69, 49, 215, 122, 126, 72, 87, 128, 239, 155, 33, 7, 115, 141, 93, 231, 56, 62, 41, 183, 144, 95, 199, 230, 88, 75, 139, 182, 197, 63, 134, 150, 195, 246, 162, 12, 163, 149, 15, 55, 100, 132, 189, 112, 232, 38, 28]","[63, 221, 61, 189, 252, 198, 14, 146, 246, 242, 41, 175, 140, 170, 116, 158, 135, 204, 186, 80, 156, 173, 219, 215, 128, 117, 169, 55, 110, 240, 159, 144, 58, 45, 120, 171, 51, 190, 137, 200, 168, 213, 253, 53, 142, 99, 207, 193, 96, 153, 67, 39, 81, 69, 4, 11, 161, 86, 237, 91, 68, 40, 78, 203, 233, 65, 244, 228, 172, 113, 187, 12, 194, 184, 202, 25, 88, 148, 62, 84, 208, 9, 178, 238, 220, 160, 214, 47, 147, 195, 15, 44, 212, 138, 76, 201, 209, 19, 35, 196, 255, 174, 192, 254, 199, 155, 56, 176, 90, 105, 133, 102, 104, 106, 21, 29, 118, 49, 234, 179, 166, 241, 82, 230, 151, 43, 239, 26]","[9, 11, 12, 15, 21, 25, 26, 35, 41, 43, 45, 49, 53, 55, 56, 62, 63, 65, 68, 69, 78, 80, 81, 82, 86, 88, 91, 102, 116, 128, 138, 142, 144, 146, 147, 151, 155, 161, 169, 170, 172, 174, 175, 186, 189, 193, 195, 199, 202, 204, 207, 208, 213, 214, 215, 219, 230, 234, 239, 242, 246, 252]"
|
|
||||||
70,"[95, 195, 213, 163, 33, 172, 98, 70, 4, 56, 83, 232, 227, 21, 89, 119, 5, 94, 197, 127, 223, 130, 108, 156, 199, 205, 63, 75, 206, 198, 170, 61, 136, 80, 160, 154, 147, 161, 209, 74, 228, 171, 186, 238, 13, 182, 251, 29, 179, 242, 178, 233, 104, 84, 46, 219, 49, 48, 105, 194, 114, 116, 222, 151, 153, 100, 117, 188, 193, 77, 14, 200, 248, 244, 102, 234, 62, 26, 162, 246, 87, 22, 67, 166, 155, 212, 236, 96, 68, 176, 34, 167, 134, 247, 144, 66, 69, 208, 53, 60, 164, 1, 73, 42, 132, 91, 216, 252, 231, 28, 146, 86, 52, 128, 8, 9, 131, 175, 183, 241, 250, 174, 152, 191, 12, 215, 220, 221]","[91, 81, 20, 122, 61, 21, 25, 112, 175, 113, 183, 255, 30, 108, 198, 121, 95, 168, 68, 163, 233, 82, 33, 160, 181, 232, 87, 169, 164, 230, 243, 127, 26, 2, 70, 226, 34, 74, 220, 193, 253, 155, 170, 117, 49, 139, 172, 240, 178, 162, 84, 100, 199, 4, 231, 53, 196, 40, 222, 57, 77, 156, 133, 63, 79, 105, 119, 130, 94, 215, 58, 48, 144, 75, 116, 10, 188, 97, 17, 206, 12, 132, 88, 131, 64, 227, 182, 234, 161, 96, 128, 15, 153, 76, 29, 194, 147, 9, 114, 83, 251, 51, 135, 89, 252, 6, 171, 213, 248, 52, 124, 24, 28, 242, 32, 145, 72, 99, 3, 179, 80, 134, 78, 67, 157, 46, 92, 152]","[4, 9, 12, 21, 26, 28, 29, 33, 34, 46, 48, 49, 52, 53, 61, 63, 67, 68, 70, 74, 75, 77, 80, 83, 84, 87, 89, 91, 94, 95, 96, 100, 105, 108, 114, 116, 117, 119, 127, 128, 130, 131, 132, 134, 144, 147, 152, 153, 155, 156, 160, 161, 162, 163, 164, 170, 171, 172, 175, 178, 179, 182, 183, 188, 193, 194, 198, 199, 206, 213, 215, 220, 222, 227, 231, 232, 233, 234, 242, 248, 251, 252]"
|
|
||||||
71,"[5, 50, 214, 206, 113, 163, 116, 245, 165, 186, 88, 228, 155, 89, 175, 2, 230, 212, 193, 108, 107, 48, 130, 176, 111, 106, 187, 125, 69, 117, 147, 172, 83, 208, 61, 1, 233, 136, 96, 58, 160, 44, 252, 36, 105, 41, 213, 149, 62, 164, 132, 255, 99, 153, 229, 118, 253, 34, 197, 224, 123, 151, 104, 145, 35, 95, 53, 30, 169, 64, 33, 236, 171, 237, 27, 205, 85, 66, 167, 238, 135, 65, 177, 76, 184, 77, 242, 221, 115, 146, 216, 217, 120, 98, 3, 200, 49, 114, 144, 159, 60, 188, 240, 195, 109, 192, 174, 43, 202, 196, 86, 198, 189, 78, 18, 156, 170, 215, 124, 25, 220, 45, 219, 127, 57, 227, 103, 21]","[225, 131, 209, 152, 230, 79, 47, 239, 63, 0, 238, 168, 224, 22, 134, 232, 162, 164, 218, 163, 173, 252, 254, 204, 251, 38, 27, 122, 26, 81, 190, 6, 9, 59, 108, 34, 3, 171, 32, 90, 31, 8, 28, 23, 75, 74, 228, 165, 247, 101, 56, 210, 113, 37, 84, 60, 241, 223, 212, 120, 226, 183, 112, 61, 11, 208, 148, 25, 245, 109, 5, 87, 44, 72, 243, 42, 203, 40, 118, 211, 19, 166, 138, 144, 234, 39, 244, 69, 52, 180, 248, 29, 194, 80, 124, 1, 151, 205, 193, 188, 64, 201, 58, 246, 7, 179, 4, 150, 149, 174, 92, 68, 161, 100, 200, 71, 160, 86, 130, 250, 50, 35, 97, 17, 91, 231, 126, 85]","[1, 3, 5, 25, 27, 34, 35, 44, 50, 58, 60, 61, 64, 69, 85, 86, 108, 109, 113, 118, 120, 124, 130, 144, 149, 151, 160, 163, 164, 165, 171, 174, 188, 193, 200, 205, 208, 212, 224, 228, 230, 238, 245, 252]"
|
|
||||||
72,"[221, 136, 42, 53, 109, 1, 51, 103, 113, 25, 33, 141, 10, 148, 250, 76, 175, 32, 94, 81, 213, 115, 118, 72, 219, 48, 74, 58, 142, 34, 150, 100, 220, 245, 46, 55, 95, 230, 167, 60, 7, 71, 105, 41, 82, 50, 3, 101, 56, 181, 247, 216, 145, 18, 196, 21, 47, 215, 203, 31, 114, 149, 17, 6, 16, 176, 228, 57, 116, 65, 238, 164, 61, 102, 119, 123, 191, 155, 170, 59, 84, 218, 9, 198, 15, 224, 29, 144, 184, 93, 67, 45, 192, 157, 234, 77, 152, 129, 173, 134, 19, 232, 49, 241, 147, 98, 174, 197, 26, 223, 24, 252, 185, 107, 227, 68, 117, 13, 200, 194, 190, 211, 14, 205, 231, 235, 70, 137]","[164, 99, 107, 117, 211, 251, 114, 26, 40, 97, 84, 60, 138, 147, 77, 230, 233, 98, 177, 13, 54, 110, 213, 245, 25, 195, 85, 42, 129, 156, 0, 113, 122, 142, 9, 170, 92, 46, 8, 18, 29, 246, 49, 243, 212, 141, 15, 192, 56, 191, 35, 95, 250, 108, 159, 23, 79, 100, 204, 154, 131, 165, 244, 182, 232, 180, 200, 111, 45, 22, 66, 36, 72, 4, 83, 139, 7, 158, 144, 102, 183, 160, 128, 223, 91, 229, 228, 58, 217, 241, 21, 106, 215, 197, 80, 253, 186, 133, 239, 136, 187, 17, 220, 234, 103, 20, 169, 61, 52, 155, 12, 39, 10, 43, 126, 64, 193, 227, 90, 73, 34, 14, 63, 71, 38, 189, 19, 132]","[7, 9, 10, 13, 14, 15, 17, 18, 19, 21, 25, 26, 29, 34, 42, 45, 46, 49, 56, 58, 60, 61, 71, 72, 77, 84, 95, 98, 100, 102, 103, 107, 113, 114, 117, 129, 136, 141, 142, 144, 147, 155, 164, 170, 191, 192, 197, 200, 211, 213, 215, 220, 223, 227, 228, 230, 232, 234, 241, 245, 250]"
|
|
||||||
73,"[168, 98, 228, 244, 44, 78, 37, 23, 50, 73, 254, 90, 134, 215, 25, 84, 156, 20, 95, 127, 190, 143, 30, 39, 140, 234, 123, 13, 86, 210, 62, 2, 14, 173, 255, 225, 132, 151, 180, 24, 224, 105, 96, 38, 76, 160, 233, 16, 128, 150, 223, 51, 120, 226, 0, 15, 252, 48, 110, 251, 113, 40, 213, 47, 185, 19, 9, 89, 43, 131, 35, 77, 183, 216, 109, 181, 83, 21, 157, 125, 31, 27, 107, 52, 201, 172, 235, 102, 74, 1, 179, 97, 207, 58, 218, 56, 117, 81, 69, 246, 115, 85, 64, 28, 135, 243, 142, 170, 196, 227, 68, 220, 169, 103, 4, 212, 6, 10, 71, 144, 198, 54, 8, 200, 22, 250, 82, 165]","[26, 106, 42, 72, 61, 116, 167, 18, 119, 49, 108, 205, 9, 193, 48, 30, 96, 240, 194, 80, 91, 94, 155, 232, 149, 146, 209, 25, 27, 121, 137, 148, 192, 7, 71, 236, 218, 63, 50, 239, 253, 93, 118, 126, 234, 136, 66, 2, 104, 175, 127, 76, 11, 10, 81, 223, 115, 88, 231, 235, 198, 31, 237, 154, 44, 37, 186, 220, 164, 147, 204, 65, 130, 5, 156, 191, 166, 124, 230, 162, 3, 212, 189, 57, 141, 77, 90, 62, 23, 99, 196, 17, 55, 153, 46, 8, 150, 73, 159, 35, 32, 70, 161, 210, 98, 68, 195, 217, 168, 170, 188, 176, 199, 202, 242, 145, 182, 177, 34, 86, 238, 139, 214, 92, 203, 114, 36, 171]","[2, 8, 9, 10, 23, 25, 27, 30, 31, 35, 37, 44, 48, 50, 62, 68, 71, 73, 76, 77, 81, 86, 90, 96, 98, 115, 127, 150, 156, 168, 170, 196, 198, 210, 212, 218, 220, 223, 234, 235]"
|
|
||||||
74,"[197, 1, 2, 190, 229, 221, 236, 194, 147, 241, 143, 171, 233, 13, 129, 77, 141, 179, 228, 40, 74, 19, 8, 248, 186, 174, 166, 182, 136, 79, 175, 90, 119, 115, 109, 111, 130, 67, 33, 242, 14, 206, 97, 112, 184, 18, 108, 224, 20, 22, 50, 45, 103, 76, 59, 71, 235, 159, 231, 86, 253, 148, 107, 56, 25, 142, 55, 24, 162, 146, 127, 84, 131, 94, 93, 38, 4, 200, 161, 126, 43, 213, 134, 208, 188, 7, 150, 240, 102, 11, 101, 28, 114, 113, 100, 21, 31, 211, 110, 57, 61, 44, 15, 106, 201, 48, 170, 10, 157, 124, 132, 87, 180, 123, 9, 125, 16, 244, 243, 133, 196, 98, 178, 23, 153, 138, 251, 128]","[100, 140, 126, 98, 249, 116, 119, 107, 185, 222, 214, 21, 17, 139, 55, 217, 33, 171, 136, 213, 204, 133, 161, 151, 64, 195, 70, 76, 131, 79, 158, 7, 188, 111, 197, 88, 172, 190, 108, 179, 180, 194, 174, 175, 101, 37, 61, 60, 102, 134, 248, 200, 62, 103, 239, 130, 146, 16, 6, 170, 152, 141, 150, 144, 74, 159, 233, 147, 90, 173, 39, 25, 94, 241, 122, 168, 71, 22, 177, 99, 142, 96, 225, 236, 156, 4, 181, 231, 11, 85, 84, 221, 18, 53, 124, 12, 105, 243, 245, 209, 2, 13, 66, 196, 169, 57, 251, 38, 40, 148, 93, 115, 54, 29, 43, 83, 252, 91, 52, 206, 125, 234, 67, 201, 220, 95, 41, 238]","[2, 4, 7, 11, 13, 16, 18, 21, 22, 25, 33, 38, 40, 43, 55, 57, 61, 67, 71, 74, 76, 79, 84, 90, 93, 94, 98, 100, 101, 102, 103, 107, 108, 111, 115, 119, 124, 125, 126, 130, 131, 133, 134, 136, 141, 142, 146, 147, 148, 150, 159, 161, 170, 171, 174, 175, 179, 180, 188, 190, 194, 196, 197, 200, 201, 206, 213, 221, 231, 233, 236, 241, 243, 248, 251]"
|
|
||||||
75,"[77, 51, 29, 200, 57, 32, 28, 188, 96, 113, 137, 204, 146, 235, 242, 93, 84, 56, 109, 63, 153, 219, 50, 171, 232, 41, 72, 251, 2, 227, 148, 58, 76, 21, 118, 14, 175, 173, 139, 40, 121, 39, 147, 230, 182, 144, 215, 174, 60, 180, 165, 0, 143, 16, 43, 222, 192, 34, 151, 30, 199, 103, 130, 190, 85, 150, 91, 228, 53, 178, 95, 217, 168, 97, 98, 189, 142, 193, 194, 163, 123, 117, 234, 47, 71, 141, 129, 102, 161, 248, 124, 152, 250, 54, 229, 19, 249, 164, 114, 156, 253, 3, 233, 18, 176, 197, 212, 223, 64, 67, 145, 70, 112, 52, 101, 115, 158, 44, 20, 255, 211, 23, 237, 87, 61, 8, 46, 83]","[38, 29, 220, 104, 206, 165, 246, 221, 126, 173, 222, 223, 148, 116, 53, 41, 43, 114, 241, 228, 23, 120, 129, 162, 59, 224, 57, 122, 156, 45, 183, 111, 54, 187, 215, 40, 58, 235, 35, 227, 247, 83, 219, 163, 20, 137, 239, 185, 3, 101, 196, 194, 108, 32, 67, 117, 8, 75, 204, 179, 86, 167, 55, 121, 39, 21, 152, 88, 177, 110, 84, 31, 47, 19, 142, 68, 65, 181, 184, 95, 202, 230, 205, 232, 42, 172, 89, 123, 132, 87, 158, 24, 7, 125, 138, 107, 73, 96, 208, 214, 70, 195, 237, 243, 105, 72, 143, 85, 66, 154, 115, 131, 186, 99, 134, 124, 25, 211, 1, 171, 36, 74, 113, 146, 189, 151, 119, 79]","[3, 8, 19, 20, 21, 23, 29, 32, 39, 40, 41, 43, 47, 53, 54, 57, 58, 67, 70, 72, 83, 84, 85, 87, 95, 96, 101, 113, 114, 115, 117, 121, 123, 124, 129, 137, 142, 143, 146, 148, 151, 152, 156, 158, 163, 165, 171, 173, 189, 194, 204, 211, 215, 219, 222, 223, 227, 228, 230, 232, 235, 237]"
|
|
||||||
76,"[73, 143, 50, 223, 38, 182, 76, 201, 119, 172, 142, 149, 65, 101, 140, 79, 35, 33, 47, 229, 165, 234, 215, 192, 173, 102, 136, 219, 78, 174, 111, 153, 162, 159, 122, 220, 178, 34, 147, 13, 133, 239, 245, 80, 68, 124, 171, 237, 110, 87, 199, 6, 154, 92, 69, 21, 17, 95, 99, 186, 37, 112, 202, 49, 134, 151, 22, 26, 39, 148, 118, 168, 98, 43, 139, 54, 114, 120, 10, 189, 167, 75, 81, 224, 66, 246, 163, 193, 9, 108, 161, 179, 51, 117, 197, 41, 24, 30, 255, 225, 194, 209, 125, 82, 19, 5, 187, 141, 230, 127, 100, 253, 218, 93, 184, 181, 158, 8, 106, 115, 7, 175, 105, 40, 48, 150, 14, 83]","[117, 72, 229, 209, 62, 173, 160, 170, 66, 221, 204, 138, 128, 41, 153, 25, 146, 145, 76, 102, 124, 123, 36, 32, 231, 185, 230, 166, 177, 11, 159, 19, 137, 254, 197, 196, 217, 144, 95, 251, 210, 38, 15, 20, 89, 135, 208, 129, 13, 140, 222, 106, 28, 199, 63, 86, 33, 121, 82, 47, 220, 184, 51, 85, 55, 49, 42, 171, 228, 107, 31, 61, 52, 104, 162, 195, 23, 237, 188, 53, 205, 136, 249, 12, 247, 167, 50, 172, 88, 201, 83, 59, 35, 9, 39, 1, 176, 130, 126, 116, 8, 40, 206, 29, 68, 0, 243, 200, 80, 152, 30, 227, 101, 198, 60, 216, 98, 4, 75, 100, 235, 139, 178, 232, 48, 103, 65, 161]","[8, 9, 13, 19, 30, 33, 35, 38, 39, 40, 41, 47, 48, 49, 50, 51, 65, 66, 68, 75, 76, 80, 82, 83, 95, 98, 100, 101, 102, 106, 117, 124, 136, 139, 140, 153, 159, 161, 162, 167, 171, 172, 173, 178, 184, 197, 199, 201, 209, 220, 229, 230, 237]"
|
|
||||||
77,"[160, 254, 249, 32, 122, 201, 152, 18, 132, 120, 242, 225, 141, 230, 7, 146, 199, 180, 127, 216, 12, 128, 30, 211, 179, 197, 17, 62, 137, 60, 98, 45, 80, 25, 207, 110, 241, 86, 50, 87, 54, 185, 8, 36, 40, 3, 150, 131, 186, 135, 237, 49, 23, 212, 229, 187, 74, 205, 82, 165, 58, 59, 156, 183, 22, 76, 130, 193, 178, 94, 107, 114, 47, 102, 245, 70, 15, 92, 63, 19, 196, 56, 41, 111, 208, 108, 52, 35, 190, 159, 123, 194, 144, 9, 75, 85, 239, 97, 153, 99, 168, 158, 188, 118, 95, 121, 84, 170, 33, 31, 105, 24, 145, 106, 147, 67, 214, 164, 21, 103, 10, 129, 115, 139, 5, 231, 133, 161]","[4, 206, 201, 87, 86, 191, 89, 110, 6, 84, 96, 16, 195, 111, 116, 12, 103, 1, 194, 72, 62, 31, 200, 213, 19, 51, 91, 240, 182, 25, 237, 225, 26, 56, 43, 8, 234, 70, 123, 104, 215, 98, 94, 55, 57, 217, 250, 3, 28, 150, 199, 204, 85, 196, 63, 90, 37, 223, 121, 38, 53, 125, 138, 142, 224, 46, 143, 140, 202, 236, 136, 151, 246, 177, 128, 17, 174, 15, 61, 255, 166, 18, 58, 164, 197, 109, 100, 227, 169, 157, 95, 222, 216, 209, 184, 112, 171, 71, 108, 176, 238, 11, 154, 42, 189, 252, 115, 13, 30, 134, 60, 124, 44, 167, 47, 173, 251, 119, 221, 78, 214, 27, 229, 178, 160, 253, 65, 79]","[3, 8, 12, 15, 17, 18, 19, 25, 30, 31, 47, 56, 58, 60, 62, 63, 70, 84, 85, 86, 87, 94, 95, 98, 103, 108, 110, 111, 115, 121, 123, 128, 150, 160, 164, 178, 194, 196, 197, 199, 201, 214, 216, 225, 229, 237]"
|
|
||||||
78,"[167, 43, 198, 217, 90, 94, 132, 183, 236, 172, 98, 195, 118, 121, 19, 30, 93, 235, 12, 201, 81, 4, 29, 148, 64, 2, 20, 27, 77, 86, 46, 17, 207, 152, 140, 14, 214, 245, 166, 71, 48, 175, 241, 44, 229, 184, 185, 219, 153, 124, 63, 95, 161, 155, 196, 133, 56, 209, 141, 34, 149, 151, 82, 38, 247, 85, 128, 96, 54, 25, 160, 120, 108, 127, 92, 252, 39, 254, 58, 68, 146, 249, 21, 41, 53, 80, 234, 136, 173, 142, 22, 18, 50, 224, 170, 104, 130, 197, 251, 176, 208, 206, 139, 47, 72, 116, 114, 194, 189, 212, 61, 177, 156, 144, 154, 200, 40, 119, 1, 76, 51, 15, 135, 158, 70, 138, 218, 230]","[152, 37, 128, 234, 23, 219, 28, 108, 254, 31, 19, 36, 173, 231, 217, 154, 122, 30, 235, 106, 248, 58, 212, 33, 26, 155, 123, 135, 158, 216, 16, 90, 4, 71, 15, 180, 185, 86, 238, 196, 85, 65, 195, 104, 161, 214, 151, 107, 13, 126, 125, 247, 134, 236, 43, 96, 119, 245, 177, 109, 98, 87, 11, 47, 174, 178, 60, 181, 78, 226, 127, 191, 81, 64, 82, 242, 80, 164, 75, 246, 250, 102, 156, 218, 77, 194, 241, 160, 171, 232, 157, 150, 175, 130, 120, 103, 51, 101, 53, 144, 118, 39, 94, 2, 32, 227, 167, 210, 52, 153, 48, 179, 211, 42, 3, 131, 69, 129, 165, 132, 67, 20, 139, 0, 184, 62, 46, 73]","[2, 4, 15, 19, 20, 30, 39, 43, 46, 47, 48, 51, 53, 58, 64, 71, 77, 80, 81, 82, 85, 86, 90, 94, 96, 98, 104, 108, 118, 119, 120, 127, 128, 130, 132, 135, 139, 144, 151, 152, 153, 154, 155, 156, 158, 160, 161, 167, 173, 175, 177, 184, 185, 194, 195, 196, 212, 214, 217, 218, 219, 234, 235, 236, 241, 245, 247, 254]"
|
|
||||||
79,"[71, 63, 10, 149, 82, 115, 8, 20, 162, 142, 199, 120, 218, 214, 210, 174, 42, 163, 105, 197, 116, 238, 50, 55, 154, 26, 16, 134, 220, 117, 64, 194, 87, 157, 173, 213, 180, 196, 2, 241, 123, 60, 159, 183, 83, 225, 191, 107, 126, 90, 39, 27, 224, 86, 146, 53, 219, 138, 128, 170, 91, 156, 187, 85, 176, 160, 54, 236, 45, 179, 204, 141, 161, 145, 30, 21, 195, 244, 182, 110, 245, 52, 114, 235, 240, 51, 246, 32, 155, 175, 88, 84, 7, 136, 234, 168, 131, 73, 229, 100, 254, 185, 217, 221, 148, 121, 216, 206, 188, 43, 34, 76, 125, 144, 253, 104, 94, 66, 230, 99, 127, 227, 208, 97, 178, 232, 31, 250]","[63, 5, 208, 115, 39, 132, 21, 113, 154, 180, 210, 231, 44, 146, 179, 186, 36, 238, 165, 243, 200, 84, 254, 61, 149, 51, 34, 241, 62, 20, 189, 91, 2, 82, 118, 100, 137, 198, 105, 121, 70, 174, 14, 83, 71, 148, 246, 69, 116, 150, 89, 136, 76, 46, 57, 98, 177, 250, 43, 32, 197, 104, 45, 52, 194, 68, 159, 18, 96, 54, 139, 8, 25, 224, 245, 42, 90, 242, 253, 126, 171, 12, 151, 216, 161, 114, 211, 67, 201, 176, 170, 128, 199, 26, 109, 135, 244, 124, 223, 234, 66, 58, 190, 162, 131, 214, 140, 73, 187, 41, 173, 202, 182, 252, 125, 64, 196, 193, 10, 95, 9, 13, 147, 178, 106, 217, 50, 205]","[2, 8, 10, 20, 21, 26, 32, 34, 39, 42, 43, 45, 50, 51, 52, 54, 63, 64, 66, 71, 73, 76, 82, 83, 84, 90, 91, 100, 104, 105, 114, 115, 116, 121, 125, 126, 128, 131, 136, 146, 148, 149, 154, 159, 161, 162, 170, 173, 174, 176, 178, 179, 180, 182, 187, 194, 196, 197, 199, 208, 210, 214, 216, 217, 224, 234, 238, 241, 244, 245, 246, 250, 253, 254]"
|
|
||||||
80,"[53, 193, 196, 87, 121, 49, 167, 74, 113, 168, 14, 126, 100, 52, 96, 133, 132, 172, 80, 68, 7, 111, 22, 243, 42, 95, 166, 94, 12, 173, 142, 93, 128, 191, 23, 57, 101, 3, 106, 141, 207, 158, 189, 175, 13, 112, 183, 150, 127, 34, 29, 198, 194, 103, 24, 26, 82, 56, 225, 9, 105, 240, 222, 115, 20, 8, 138, 174, 206, 90, 62, 186, 97, 31, 204, 131, 159, 137, 170, 226, 114, 129, 61, 209, 60, 180, 212, 47, 162, 125, 169, 30, 233, 55, 165, 6, 184, 35, 199, 255, 134, 107, 5, 152, 215, 177, 66, 37, 120, 21, 241, 124, 251, 148, 41, 192, 157, 40, 201, 118, 64, 220, 236, 117, 69, 135, 77, 164]","[150, 172, 131, 62, 194, 134, 63, 82, 117, 189, 127, 253, 207, 185, 170, 69, 7, 46, 232, 105, 193, 241, 43, 51, 159, 101, 141, 96, 164, 31, 87, 209, 125, 166, 100, 199, 129, 118, 240, 21, 66, 169, 48, 238, 97, 60, 55, 247, 59, 0, 12, 165, 177, 204, 93, 58, 50, 61, 47, 49, 126, 111, 64, 128, 179, 231, 57, 157, 220, 65, 77, 80, 78, 99, 115, 144, 39, 244, 138, 135, 52, 90, 27, 25, 23, 103, 20, 73, 10, 130, 29, 124, 94, 227, 212, 175, 158, 113, 95, 230, 14, 234, 222, 120, 252, 206, 1, 190, 140, 30, 3, 237, 24, 36, 42, 112, 152, 226, 74, 167, 201, 215, 40, 168, 22, 191, 34, 9]","[3, 7, 9, 12, 14, 20, 21, 22, 23, 24, 29, 30, 31, 34, 40, 42, 47, 49, 52, 55, 57, 60, 61, 62, 64, 66, 69, 74, 77, 80, 82, 87, 90, 93, 94, 95, 96, 97, 100, 101, 103, 105, 111, 112, 113, 115, 117, 118, 120, 124, 125, 126, 127, 128, 129, 131, 134, 135, 138, 141, 150, 152, 157, 158, 159, 164, 165, 166, 167, 168, 169, 170, 172, 175, 177, 189, 191, 193, 194, 199, 201, 204, 206, 207, 209, 212, 215, 220, 222, 226, 240, 241]"
|
|
||||||
81,"[122, 218, 255, 78, 64, 244, 88, 99, 101, 183, 35, 93, 20, 236, 56, 157, 70, 19, 60, 77, 11, 85, 196, 197, 69, 90, 195, 135, 202, 113, 118, 172, 213, 224, 29, 72, 249, 25, 108, 112, 58, 75, 240, 94, 221, 92, 175, 91, 252, 128, 38, 203, 200, 247, 27, 208, 250, 41, 169, 48, 121, 239, 166, 46, 179, 206, 205, 152, 146, 97, 13, 22, 192, 164, 162, 80, 76, 156, 26, 168, 187, 248, 142, 12, 52, 161, 237, 253, 109, 233, 3, 31, 151, 133, 114, 18, 23, 130, 153, 149, 4, 147, 126, 137, 176, 180, 106, 7, 10, 74, 243, 30, 131, 150, 0, 51, 117, 98, 220, 125, 198, 217, 228, 160, 193, 144, 134, 158]","[14, 211, 147, 6, 186, 27, 134, 155, 77, 37, 65, 34, 150, 10, 4, 2, 115, 195, 218, 138, 73, 213, 222, 180, 187, 235, 193, 229, 89, 233, 91, 54, 214, 209, 153, 46, 17, 103, 83, 234, 129, 161, 249, 239, 237, 196, 41, 171, 64, 253, 85, 11, 181, 84, 90, 245, 159, 59, 21, 170, 177, 116, 104, 148, 208, 45, 119, 183, 28, 133, 61, 248, 51, 29, 48, 108, 143, 5, 96, 52, 106, 165, 136, 42, 13, 39, 75, 250, 242, 125, 216, 95, 87, 44, 212, 200, 189, 123, 107, 122, 162, 207, 190, 74, 231, 240, 128, 226, 230, 141, 78, 247, 243, 58, 36, 174, 191, 157, 168, 68, 63, 97, 23, 50, 164, 194, 252, 20]","[4, 10, 11, 13, 20, 23, 27, 29, 41, 46, 48, 51, 52, 58, 64, 74, 75, 77, 78, 85, 90, 91, 97, 106, 108, 122, 125, 128, 133, 134, 147, 150, 153, 157, 161, 162, 164, 168, 180, 183, 187, 193, 195, 196, 200, 208, 213, 218, 233, 237, 239, 240, 243, 247, 248, 249, 250, 252, 253]"
|
|
||||||
82,"[158, 147, 85, 53, 35, 103, 30, 155, 111, 36, 139, 87, 91, 28, 220, 186, 239, 164, 75, 203, 191, 146, 222, 79, 125, 250, 167, 223, 27, 54, 58, 196, 80, 229, 26, 76, 136, 83, 50, 177, 120, 182, 157, 32, 121, 254, 143, 245, 71, 246, 23, 230, 199, 106, 37, 130, 131, 151, 96, 137, 231, 134, 107, 135, 174, 127, 38, 13, 59, 179, 189, 145, 208, 184, 166, 154, 141, 17, 9, 110, 16, 227, 34, 39, 1, 224, 202, 4, 77, 228, 5, 212, 185, 68, 140, 180, 93, 104, 232, 144, 49, 152, 29, 173, 101, 81, 194, 205, 129, 210, 124, 176, 86, 74, 25, 117, 66, 178, 61, 201, 119, 219, 200, 181, 112, 97, 198, 42]","[92, 218, 55, 208, 193, 59, 189, 213, 34, 150, 177, 20, 159, 15, 71, 247, 237, 238, 10, 172, 18, 41, 225, 102, 72, 161, 198, 73, 184, 134, 126, 96, 86, 248, 245, 22, 58, 62, 115, 23, 240, 28, 69, 209, 109, 255, 136, 99, 204, 44, 215, 42, 64, 70, 3, 13, 162, 217, 85, 214, 234, 45, 90, 207, 12, 88, 25, 82, 169, 164, 43, 246, 95, 104, 63, 84, 108, 80, 7, 79, 49, 78, 127, 21, 91, 24, 107, 180, 188, 142, 165, 253, 226, 235, 224, 103, 168, 61, 222, 32, 233, 101, 6, 187, 35, 216, 190, 128, 39, 163, 170, 192, 154, 166, 30, 242, 132, 46, 116, 33, 52, 89, 98, 236, 243, 17, 195, 183]","[13, 17, 23, 25, 28, 30, 32, 34, 35, 39, 42, 49, 58, 59, 61, 71, 79, 80, 85, 86, 91, 96, 101, 103, 104, 107, 127, 134, 136, 154, 164, 166, 177, 180, 184, 189, 198, 208, 222, 224, 245, 246]"
|
|
||||||
83,"[9, 111, 16, 236, 44, 121, 177, 93, 132, 151, 213, 6, 109, 203, 230, 201, 66, 181, 168, 13, 18, 30, 202, 216, 183, 199, 176, 200, 105, 42, 173, 245, 161, 188, 195, 102, 71, 144, 97, 26, 49, 197, 250, 61, 184, 94, 27, 77, 76, 209, 226, 90, 98, 170, 101, 7, 92, 75, 52, 186, 22, 62, 45, 59, 122, 85, 108, 56, 24, 215, 223, 115, 210, 175, 74, 155, 82, 158, 140, 19, 46, 193, 67, 80, 214, 10, 43, 156, 130, 243, 189, 1, 221, 206, 91, 2, 48, 73, 113, 137, 128, 169, 225, 222, 36, 60, 229, 47, 191, 55, 136, 127, 134, 153, 100, 119, 172, 57, 83, 159, 244, 248, 192, 187, 4, 25, 190, 240]","[235, 36, 128, 147, 184, 100, 204, 92, 93, 197, 170, 90, 236, 158, 183, 222, 209, 17, 48, 22, 26, 127, 20, 132, 176, 233, 200, 150, 55, 195, 8, 210, 216, 243, 81, 45, 155, 144, 129, 98, 203, 37, 213, 113, 140, 159, 181, 101, 31, 30, 18, 24, 99, 238, 122, 27, 164, 62, 16, 208, 248, 167, 207, 199, 85, 1, 231, 118, 228, 225, 19, 82, 148, 104, 49, 135, 121, 191, 25, 202, 67, 54, 11, 111, 77, 226, 91, 255, 189, 240, 68, 201, 149, 120, 245, 151, 142, 125, 44, 143, 6, 86, 206, 133, 174, 40, 7, 247, 76, 41, 33, 47, 187, 97, 160, 168, 145, 250, 108, 130, 241, 136, 52, 94, 105, 61, 66, 230]","[1, 6, 7, 16, 18, 19, 22, 24, 25, 26, 27, 30, 36, 44, 45, 47, 48, 49, 52, 55, 61, 62, 66, 67, 76, 77, 82, 85, 90, 91, 92, 93, 94, 97, 98, 100, 101, 105, 108, 111, 113, 121, 122, 127, 128, 130, 132, 136, 140, 144, 151, 155, 158, 159, 168, 170, 176, 181, 183, 184, 187, 189, 191, 195, 197, 199, 200, 201, 202, 203, 206, 209, 210, 213, 216, 222, 225, 226, 230, 236, 240, 243, 245, 248, 250]"
|
|
||||||
84,"[83, 105, 15, 77, 189, 213, 46, 190, 134, 234, 254, 73, 232, 151, 81, 1, 37, 85, 164, 93, 16, 143, 54, 228, 176, 247, 118, 203, 221, 86, 168, 103, 80, 101, 43, 11, 3, 194, 90, 69, 210, 67, 237, 49, 45, 162, 96, 34, 76, 79, 208, 204, 242, 58, 13, 51, 22, 147, 124, 60, 121, 135, 82, 52, 249, 193, 173, 200, 112, 230, 158, 160, 115, 71, 119, 215, 250, 88, 114, 53, 41, 236, 84, 231, 206, 59, 122, 175, 74, 97, 255, 21, 156, 104, 169, 141, 110, 233, 92, 212, 146, 64, 78, 133, 205, 48, 33, 12, 0, 35, 226, 14, 132, 187, 61, 216, 5, 106, 155, 152, 163, 4, 201, 20, 120, 24, 27, 128]","[51, 22, 185, 127, 203, 246, 237, 133, 31, 54, 193, 93, 72, 145, 83, 52, 82, 107, 249, 61, 220, 25, 216, 104, 59, 1, 34, 197, 149, 23, 135, 123, 78, 236, 180, 224, 74, 40, 163, 189, 157, 9, 192, 15, 117, 222, 18, 29, 144, 207, 119, 122, 114, 150, 183, 24, 199, 8, 154, 5, 85, 254, 204, 241, 103, 165, 247, 194, 43, 96, 223, 158, 140, 97, 177, 2, 7, 116, 187, 162, 75, 118, 239, 109, 242, 46, 244, 131, 186, 188, 12, 206, 99, 47, 210, 53, 68, 91, 77, 124, 218, 10, 20, 35, 14, 184, 232, 173, 49, 86, 128, 87, 84, 4, 80, 178, 175, 67, 198, 3, 16, 221, 255, 160, 228, 182, 234, 201]","[1, 3, 4, 5, 12, 14, 15, 16, 20, 22, 24, 34, 35, 43, 46, 49, 51, 52, 53, 54, 59, 61, 67, 74, 77, 78, 80, 82, 83, 84, 85, 86, 93, 96, 97, 103, 104, 114, 118, 119, 122, 124, 128, 133, 135, 158, 160, 162, 163, 173, 175, 187, 189, 193, 194, 201, 203, 204, 206, 210, 216, 221, 228, 232, 234, 236, 237, 242, 247, 249, 254, 255]"
|
|
||||||
85,"[92, 30, 150, 106, 205, 143, 89, 62, 231, 63, 86, 119, 5, 66, 204, 209, 167, 28, 108, 147, 23, 69, 153, 162, 164, 14, 55, 90, 88, 29, 118, 148, 171, 180, 145, 253, 238, 215, 41, 199, 61, 75, 9, 202, 131, 242, 31, 227, 72, 240, 217, 109, 73, 239, 140, 241, 2, 194, 77, 95, 24, 195, 48, 187, 50, 160, 121, 96, 100, 65, 82, 128, 222, 44, 154, 243, 146, 101, 93, 105, 110, 193, 183, 250, 225, 78, 151, 18, 47, 22, 11, 117, 216, 188, 49, 26, 36, 0, 33, 6, 85, 179, 184, 57, 255, 114, 220, 247, 53, 173, 102, 125, 172, 12, 126, 139, 221, 133, 141, 212, 165, 149, 7, 3, 144, 59, 35, 191]","[63, 90, 116, 243, 184, 136, 226, 139, 216, 199, 10, 84, 95, 182, 132, 97, 147, 89, 138, 160, 221, 83, 77, 66, 236, 141, 194, 140, 131, 146, 49, 206, 61, 87, 47, 75, 45, 54, 249, 125, 150, 119, 190, 208, 235, 81, 59, 9, 52, 15, 166, 126, 48, 164, 153, 134, 26, 38, 204, 3, 8, 17, 240, 121, 112, 36, 103, 0, 102, 172, 250, 238, 218, 6, 113, 124, 60, 76, 71, 193, 41, 14, 91, 154, 178, 186, 144, 98, 101, 88, 253, 223, 50, 108, 133, 123, 174, 185, 156, 152, 201, 31, 167, 237, 127, 4, 114, 255, 227, 85, 148, 162, 30, 33, 248, 62, 96, 177, 22, 209, 58, 57, 73, 183, 173, 11, 247, 217]","[0, 3, 6, 9, 11, 14, 22, 26, 30, 31, 33, 36, 41, 47, 48, 49, 50, 57, 59, 61, 62, 63, 66, 73, 75, 77, 85, 88, 89, 90, 95, 96, 101, 102, 108, 114, 119, 121, 125, 126, 131, 133, 139, 140, 141, 144, 146, 147, 148, 150, 153, 154, 160, 162, 164, 167, 172, 173, 183, 184, 193, 194, 199, 204, 209, 216, 217, 221, 227, 238, 240, 243, 247, 250, 253, 255]"
|
|
||||||
86,"[226, 15, 208, 173, 64, 66, 110, 202, 33, 112, 74, 225, 177, 252, 105, 80, 47, 246, 182, 248, 16, 7, 237, 189, 25, 19, 1, 140, 104, 222, 79, 26, 94, 203, 97, 126, 174, 160, 238, 48, 188, 96, 151, 135, 103, 196, 124, 198, 84, 82, 136, 159, 27, 115, 253, 109, 90, 199, 190, 86, 53, 101, 169, 142, 167, 77, 68, 43, 168, 17, 31, 83, 114, 236, 205, 11, 234, 59, 42, 98, 23, 176, 197, 55, 215, 217, 134, 184, 113, 207, 49, 30, 214, 32, 57, 209, 5, 243, 228, 156, 254, 186, 133, 181, 85, 144, 128, 163, 155, 146, 200, 158, 171, 162, 10, 250, 121, 120, 12, 220, 40, 152, 130, 111, 249, 212, 193, 178]","[229, 140, 47, 176, 150, 73, 134, 89, 118, 157, 44, 163, 209, 18, 92, 217, 234, 192, 101, 222, 202, 223, 31, 63, 164, 75, 102, 53, 48, 83, 211, 219, 179, 119, 96, 187, 232, 180, 141, 122, 21, 28, 195, 153, 72, 186, 206, 9, 129, 145, 81, 148, 233, 65, 242, 91, 210, 0, 52, 19, 178, 116, 196, 212, 221, 43, 137, 247, 58, 239, 138, 62, 146, 131, 35, 54, 95, 41, 175, 112, 204, 29, 34, 143, 236, 45, 240, 218, 144, 226, 139, 15, 6, 166, 20, 167, 106, 213, 99, 14, 121, 4, 1, 165, 189, 93, 151, 39, 154, 231, 36, 184, 100, 255, 161, 181, 56, 149, 37, 185, 147, 22, 17, 194, 252, 49, 70, 224]","[1, 15, 17, 19, 31, 43, 47, 48, 49, 53, 83, 96, 101, 112, 121, 134, 140, 144, 146, 151, 163, 167, 176, 178, 181, 184, 186, 189, 196, 202, 209, 212, 217, 222, 226, 234, 236, 252]"
|
|
||||||
87,"[232, 96, 28, 45, 44, 19, 24, 129, 49, 27, 111, 109, 237, 50, 245, 17, 60, 200, 90, 145, 91, 54, 120, 73, 81, 59, 101, 115, 103, 139, 174, 180, 110, 67, 221, 61, 218, 143, 196, 104, 173, 80, 225, 22, 147, 235, 108, 86, 236, 113, 185, 164, 247, 171, 201, 211, 242, 43, 222, 74, 135, 75, 202, 83, 48, 138, 176, 51, 119, 11, 5, 165, 224, 182, 189, 220, 253, 188, 41, 217, 46, 118, 249, 56, 149, 57, 14, 152, 126, 64, 47, 114, 15, 199, 121, 216, 195, 223, 102, 35, 97, 32, 95, 85, 107, 36, 175, 58, 166, 4, 156, 153, 230, 18, 133, 89, 78, 100, 243, 132, 227, 0, 42, 20, 183, 226, 167, 112]","[194, 230, 112, 78, 30, 52, 32, 90, 27, 42, 71, 61, 191, 83, 164, 202, 208, 29, 56, 176, 25, 130, 68, 64, 118, 66, 123, 167, 21, 147, 209, 174, 210, 190, 163, 155, 206, 224, 221, 158, 10, 117, 166, 110, 105, 82, 192, 50, 254, 143, 70, 6, 33, 136, 92, 31, 236, 138, 3, 24, 170, 44, 131, 88, 151, 89, 93, 36, 146, 18, 228, 172, 231, 133, 28, 137, 182, 195, 248, 94, 19, 240, 76, 65, 127, 212, 237, 62, 187, 197, 139, 2, 162, 60, 108, 116, 213, 34, 140, 53, 100, 46, 169, 252, 159, 196, 38, 72, 7, 128, 26, 47, 156, 200, 111, 1, 91, 80, 253, 148, 40, 79, 142, 20, 141, 23, 178, 106]","[18, 19, 20, 24, 27, 28, 32, 36, 42, 44, 46, 47, 50, 56, 60, 61, 64, 78, 80, 83, 89, 90, 91, 100, 108, 110, 111, 112, 118, 133, 138, 139, 143, 147, 156, 164, 166, 167, 174, 176, 182, 195, 196, 200, 202, 221, 224, 230, 236, 237, 253]"
|
|
||||||
88,"[163, 37, 59, 83, 95, 242, 175, 250, 67, 185, 135, 66, 228, 157, 53, 105, 60, 43, 180, 219, 164, 99, 78, 162, 179, 29, 82, 130, 30, 189, 36, 199, 57, 144, 26, 19, 155, 160, 87, 223, 126, 134, 64, 46, 0, 159, 24, 212, 55, 54, 100, 85, 48, 96, 65, 178, 10, 6, 149, 109, 112, 153, 169, 4, 232, 137, 139, 9, 237, 102, 16, 113, 1, 77, 90, 84, 238, 63, 49, 91, 146, 123, 25, 121, 94, 235, 131, 211, 114, 214, 174, 45, 209, 231, 129, 103, 72, 142, 172, 246, 251, 61, 190, 215, 218, 140, 208, 161, 33, 191, 50, 197, 40, 88, 222, 220, 27, 31, 71, 234, 184, 230, 249, 217, 150, 76, 154, 115]","[212, 24, 21, 219, 152, 161, 229, 197, 18, 39, 168, 160, 189, 231, 232, 0, 98, 246, 132, 149, 209, 67, 77, 122, 173, 195, 140, 146, 102, 94, 105, 54, 66, 91, 109, 249, 221, 188, 139, 194, 162, 63, 182, 95, 57, 50, 199, 90, 52, 53, 25, 178, 154, 191, 96, 167, 130, 206, 86, 157, 253, 114, 254, 227, 183, 116, 71, 58, 185, 218, 126, 72, 4, 5, 43, 112, 135, 108, 151, 223, 73, 205, 169, 247, 234, 220, 186, 100, 87, 19, 20, 215, 147, 17, 193, 10, 175, 83, 103, 210, 192, 55, 222, 180, 136, 76, 142, 250, 2, 213, 30, 113, 107, 242, 163, 36, 14, 79, 137, 99, 104, 22, 64, 141, 29, 144, 251, 164]","[0, 4, 10, 19, 24, 25, 29, 30, 36, 43, 50, 53, 54, 55, 57, 63, 64, 66, 67, 71, 72, 76, 77, 83, 87, 90, 91, 94, 95, 96, 99, 100, 102, 103, 105, 109, 112, 113, 114, 126, 130, 135, 137, 139, 140, 142, 144, 146, 149, 154, 157, 160, 161, 162, 163, 164, 169, 175, 178, 180, 185, 189, 191, 197, 199, 209, 212, 215, 218, 219, 220, 222, 223, 231, 232, 234, 242, 246, 249, 250, 251]"
|
|
||||||
89,"[21, 132, 215, 96, 71, 240, 242, 193, 165, 154, 100, 158, 59, 75, 249, 79, 62, 81, 29, 9, 47, 144, 241, 112, 101, 129, 185, 27, 151, 16, 37, 84, 90, 92, 38, 78, 145, 152, 160, 203, 87, 167, 236, 122, 6, 1, 250, 220, 105, 124, 88, 103, 175, 52, 243, 33, 162, 178, 217, 164, 137, 174, 20, 76, 5, 189, 72, 91, 17, 210, 58, 104, 26, 148, 182, 173, 131, 39, 231, 65, 139, 70, 15, 150, 234, 36, 82, 63, 126, 186, 18, 48, 41, 206, 11, 163, 247, 128, 177, 55, 196, 40, 181, 111, 32, 223, 66, 89, 30, 183, 60, 28, 74, 190, 85, 255, 141, 212, 166, 0, 176, 146, 208, 50, 86, 93, 31, 169]","[148, 62, 51, 34, 16, 108, 91, 103, 169, 173, 207, 231, 102, 101, 63, 72, 183, 234, 163, 9, 115, 243, 52, 160, 5, 215, 50, 150, 6, 70, 180, 121, 134, 213, 0, 75, 158, 199, 87, 100, 96, 76, 241, 193, 170, 25, 21, 110, 248, 74, 190, 20, 88, 120, 165, 222, 112, 210, 64, 212, 58, 166, 217, 159, 39, 152, 90, 19, 26, 57, 98, 176, 253, 242, 85, 53, 139, 132, 200, 92, 136, 174, 137, 182, 71, 144, 79, 29, 227, 141, 240, 107, 17, 128, 93, 175, 95, 238, 109, 105, 36, 195, 89, 247, 143, 185, 77, 145, 187, 13, 154, 196, 84, 38, 122, 255, 48, 99, 179, 60, 22, 226, 205, 27, 116, 33, 124, 118]","[0, 5, 6, 9, 16, 17, 20, 21, 26, 27, 29, 33, 36, 38, 39, 48, 50, 52, 58, 60, 62, 63, 70, 71, 72, 74, 75, 76, 79, 84, 85, 87, 88, 89, 90, 91, 92, 93, 96, 100, 101, 103, 105, 112, 122, 124, 128, 132, 137, 139, 141, 144, 145, 148, 150, 152, 154, 158, 160, 163, 165, 166, 169, 173, 174, 175, 176, 182, 183, 185, 190, 193, 196, 210, 212, 215, 217, 231, 234, 240, 241, 242, 243, 247, 255]"
|
|
||||||
90,"[19, 173, 102, 86, 253, 56, 25, 135, 250, 126, 91, 137, 146, 45, 100, 209, 12, 1, 238, 225, 109, 208, 43, 184, 116, 93, 84, 89, 136, 92, 186, 166, 8, 99, 62, 159, 90, 32, 160, 153, 194, 37, 110, 83, 108, 80, 247, 192, 204, 72, 224, 185, 113, 81, 234, 33, 187, 44, 22, 123, 24, 76, 133, 164, 17, 2, 206, 47, 105, 107, 52, 36, 223, 176, 53, 236, 3, 71, 88, 249, 60, 245, 198, 4, 195, 151, 121, 175, 58, 154, 241, 213, 196, 205, 18, 57, 203, 248, 20, 42, 134, 49, 211, 95, 142, 230, 150, 15, 118, 233, 131, 190, 178, 28, 177, 39, 170, 148, 174, 183, 70, 179, 127, 104, 152, 157, 252, 54]","[59, 222, 183, 169, 127, 24, 158, 85, 204, 152, 115, 37, 136, 1, 47, 72, 198, 214, 252, 75, 8, 98, 148, 0, 248, 108, 167, 23, 49, 62, 182, 99, 199, 64, 225, 162, 41, 84, 22, 220, 137, 56, 65, 42, 7, 101, 6, 68, 138, 243, 126, 143, 34, 106, 18, 172, 224, 30, 102, 231, 55, 161, 170, 173, 97, 189, 200, 124, 219, 160, 44, 46, 146, 235, 73, 79, 70, 234, 40, 150, 33, 10, 216, 178, 88, 230, 129, 105, 166, 197, 19, 14, 109, 87, 63, 147, 181, 188, 130, 168, 236, 104, 195, 95, 144, 254, 51, 132, 128, 202, 26, 131, 134, 5, 193, 61, 107, 226, 60, 4, 17, 139, 89, 103, 58, 180, 39, 93]","[1, 4, 8, 17, 18, 19, 22, 24, 33, 37, 39, 42, 44, 47, 49, 56, 58, 60, 62, 70, 72, 84, 88, 89, 93, 95, 99, 102, 104, 105, 107, 108, 109, 126, 127, 131, 134, 136, 137, 146, 148, 150, 152, 160, 166, 170, 173, 178, 183, 195, 198, 204, 224, 225, 230, 234, 236, 248, 252]"
|
|
||||||
91,"[231, 211, 124, 114, 87, 47, 213, 139, 111, 176, 113, 103, 163, 20, 202, 174, 54, 53, 158, 19, 0, 29, 39, 179, 105, 6, 102, 92, 122, 226, 232, 162, 195, 233, 194, 67, 12, 91, 149, 65, 35, 27, 245, 48, 125, 108, 248, 96, 78, 64, 198, 30, 38, 212, 9, 109, 200, 132, 61, 217, 100, 94, 24, 76, 119, 218, 249, 222, 57, 237, 106, 28, 137, 210, 167, 135, 49, 225, 110, 221, 240, 214, 189, 14, 242, 123, 16, 21, 165, 197, 82, 68, 4, 26, 36, 42, 193, 80, 169, 55, 129, 89, 5, 33, 10, 227, 157, 229, 219, 187, 13, 126, 185, 160, 17, 152, 107, 171, 177, 164, 146, 25, 51, 70, 203, 97, 18, 134]","[149, 39, 217, 47, 10, 250, 233, 216, 110, 17, 213, 37, 67, 229, 26, 121, 164, 120, 102, 153, 48, 61, 249, 169, 27, 195, 14, 38, 174, 51, 45, 28, 176, 97, 244, 170, 254, 232, 80, 142, 139, 187, 9, 179, 163, 41, 103, 106, 210, 82, 202, 138, 167, 159, 194, 24, 251, 46, 15, 215, 171, 162, 86, 81, 150, 208, 214, 172, 184, 243, 52, 35, 11, 0, 70, 87, 8, 245, 13, 247, 131, 219, 207, 91, 115, 75, 33, 177, 96, 64, 132, 122, 211, 137, 130, 19, 22, 36, 218, 156, 178, 116, 114, 29, 49, 125, 57, 20, 168, 175, 95, 78, 212, 225, 73, 111, 124, 248, 12, 205, 99, 112, 152, 65, 222, 92, 113, 54]","[0, 9, 10, 12, 13, 14, 17, 19, 20, 24, 26, 27, 28, 29, 33, 35, 36, 38, 39, 47, 48, 49, 51, 54, 57, 61, 64, 65, 67, 70, 78, 80, 82, 87, 91, 92, 96, 97, 102, 103, 106, 110, 111, 113, 114, 122, 124, 125, 132, 137, 139, 149, 152, 162, 163, 164, 167, 169, 171, 174, 176, 177, 179, 187, 194, 195, 202, 210, 211, 212, 213, 214, 217, 218, 219, 222, 225, 229, 232, 233, 245, 248, 249]"
|
|
||||||
92,"[108, 79, 59, 203, 244, 151, 39, 102, 112, 219, 128, 35, 11, 235, 251, 153, 240, 12, 32, 125, 211, 164, 218, 3, 229, 139, 34, 16, 189, 253, 236, 127, 57, 143, 7, 114, 48, 146, 252, 72, 197, 18, 38, 93, 199, 176, 149, 0, 129, 109, 170, 115, 222, 180, 220, 183, 15, 56, 230, 2, 107, 106, 168, 100, 111, 152, 136, 210, 173, 30, 86, 88, 204, 123, 28, 166, 226, 137, 84, 9, 228, 26, 36, 67, 122, 82, 20, 117, 160, 174, 175, 22, 171, 154, 5, 55, 40, 165, 50, 184, 94, 52, 195, 144, 51, 121, 14, 198, 8, 196, 120, 187, 21, 233, 249, 163, 75, 140, 142, 110, 224, 97, 25, 178, 104, 132, 54, 73]","[241, 182, 195, 229, 235, 249, 174, 98, 142, 51, 233, 94, 97, 196, 7, 6, 106, 203, 186, 251, 175, 34, 11, 56, 42, 156, 171, 40, 140, 149, 14, 152, 180, 22, 36, 128, 215, 93, 198, 189, 204, 114, 20, 104, 243, 90, 109, 39, 116, 236, 177, 224, 139, 117, 38, 83, 253, 165, 153, 58, 162, 99, 154, 239, 143, 244, 252, 57, 66, 48, 81, 183, 50, 199, 70, 222, 88, 84, 62, 144, 138, 9, 5, 120, 173, 218, 54, 150, 202, 32, 16, 220, 193, 64, 67, 49, 105, 230, 212, 110, 100, 226, 107, 44, 238, 169, 194, 201, 181, 125, 21, 141, 2, 240, 4, 0, 146, 115, 59, 250, 46, 72, 65, 78, 221, 3, 37, 136]","[0, 2, 3, 5, 7, 9, 11, 14, 16, 20, 21, 22, 32, 34, 36, 38, 39, 40, 48, 50, 51, 54, 56, 57, 59, 67, 72, 84, 88, 93, 94, 97, 100, 104, 106, 107, 109, 110, 114, 115, 117, 120, 125, 128, 136, 139, 140, 142, 143, 144, 146, 149, 152, 153, 154, 165, 171, 173, 174, 175, 180, 183, 189, 195, 196, 198, 199, 203, 204, 218, 220, 222, 224, 226, 229, 230, 233, 235, 236, 240, 244, 249, 251, 252, 253]"
|
|
||||||
93,"[59, 142, 165, 194, 103, 159, 67, 84, 41, 116, 151, 157, 20, 222, 217, 135, 97, 60, 40, 29, 108, 6, 225, 22, 8, 68, 83, 224, 255, 209, 250, 182, 207, 58, 14, 130, 70, 66, 133, 39, 214, 77, 80, 132, 90, 45, 96, 153, 88, 65, 134, 120, 146, 232, 74, 49, 95, 124, 5, 156, 136, 223, 176, 198, 28, 181, 100, 127, 64, 10, 190, 203, 73, 240, 110, 38, 140, 188, 242, 93, 125, 75, 152, 158, 46, 197, 19, 117, 33, 178, 227, 115, 123, 101, 121, 251, 154, 236, 113, 37, 91, 162, 34, 53, 109, 216, 143, 48, 138, 106, 253, 50, 56, 89, 85, 184, 220, 51, 218, 185, 26, 206, 241, 183, 175, 169, 27, 245]","[214, 30, 120, 101, 162, 126, 167, 182, 193, 109, 96, 83, 90, 3, 146, 69, 179, 129, 84, 213, 68, 184, 40, 119, 206, 176, 152, 21, 192, 29, 150, 72, 144, 139, 220, 174, 180, 57, 191, 202, 2, 200, 246, 231, 199, 147, 195, 127, 161, 56, 170, 238, 80, 12, 27, 212, 141, 128, 92, 219, 243, 148, 164, 172, 241, 158, 64, 81, 208, 151, 14, 248, 115, 177, 88, 252, 181, 201, 215, 225, 15, 35, 7, 205, 60, 228, 85, 54, 42, 61, 10, 137, 6, 36, 18, 229, 67, 204, 169, 66, 47, 166, 132, 31, 65, 138, 253, 190, 91, 19, 48, 103, 232, 133, 226, 125, 16, 249, 32, 185, 49, 175, 244, 70, 247, 230, 183, 99]","[6, 10, 14, 19, 27, 29, 40, 48, 49, 56, 60, 64, 65, 66, 67, 68, 70, 80, 83, 84, 85, 88, 90, 91, 96, 101, 103, 109, 115, 120, 125, 127, 132, 133, 138, 146, 151, 152, 158, 162, 169, 175, 176, 181, 182, 183, 184, 185, 190, 206, 214, 220, 225, 232, 241, 253]"
|
|
||||||
94,"[76, 151, 57, 73, 22, 42, 24, 90, 186, 139, 200, 16, 128, 137, 160, 74, 165, 152, 38, 66, 21, 156, 205, 54, 198, 102, 81, 176, 122, 202, 61, 65, 159, 111, 167, 179, 226, 8, 109, 210, 10, 101, 146, 1, 185, 4, 112, 204, 192, 168, 115, 41, 108, 121, 64, 217, 119, 44, 103, 83, 241, 232, 148, 30, 55, 104, 140, 199, 173, 71, 120, 9, 93, 164, 123, 252, 191, 78, 97, 184, 117, 182, 126, 189, 197, 143, 194, 247, 37, 26, 222, 45, 15, 105, 216, 27, 62, 39, 246, 130, 153, 98, 47, 174, 135, 82, 35, 13, 234, 211, 40, 209, 195, 110, 154, 229, 17, 89, 106, 29, 99, 178, 113, 136, 212, 18, 129, 158]","[204, 55, 90, 175, 210, 196, 98, 149, 191, 31, 95, 239, 13, 110, 9, 236, 33, 82, 41, 67, 102, 136, 153, 217, 121, 85, 166, 74, 108, 123, 216, 176, 246, 158, 64, 180, 6, 198, 119, 186, 107, 147, 241, 131, 26, 179, 128, 111, 211, 230, 190, 206, 17, 15, 151, 89, 1, 178, 57, 187, 73, 104, 62, 30, 137, 24, 125, 78, 139, 240, 247, 189, 134, 45, 194, 60, 207, 27, 101, 99, 197, 10, 92, 35, 69, 22, 237, 229, 195, 212, 28, 58, 65, 8, 61, 109, 117, 168, 36, 159, 145, 251, 38, 129, 148, 182, 25, 11, 254, 97, 93, 66, 146, 192, 174, 126, 234, 29, 72, 209, 156, 81, 39, 37, 12, 83, 71, 48]","[1, 8, 9, 10, 13, 15, 17, 22, 24, 26, 27, 29, 30, 35, 37, 38, 39, 41, 45, 55, 57, 61, 62, 64, 65, 66, 71, 73, 74, 78, 81, 82, 83, 89, 90, 93, 97, 98, 99, 101, 102, 104, 108, 109, 110, 111, 117, 119, 121, 123, 126, 128, 129, 136, 137, 139, 146, 148, 151, 153, 156, 158, 159, 168, 174, 176, 178, 179, 182, 186, 189, 191, 192, 194, 195, 197, 198, 204, 209, 210, 211, 212, 216, 217, 229, 234, 241, 246, 247]"
|
|
||||||
95,"[50, 65, 223, 73, 121, 91, 127, 148, 45, 242, 95, 198, 253, 171, 169, 25, 135, 3, 166, 92, 98, 123, 207, 24, 149, 90, 180, 13, 31, 153, 68, 132, 79, 140, 40, 44, 4, 2, 17, 110, 168, 129, 165, 30, 14, 158, 155, 41, 118, 239, 226, 77, 164, 228, 23, 200, 111, 105, 144, 229, 170, 85, 194, 61, 214, 36, 141, 26, 234, 101, 197, 71, 106, 219, 16, 252, 125, 54, 67, 182, 213, 134, 227, 143, 255, 47, 160, 48, 43, 108, 39, 1, 230, 94, 42, 222, 206, 152, 52, 51, 20, 11, 5, 185, 245, 120, 57, 34, 83, 187, 174, 56, 89, 137, 231, 27, 145, 75, 193, 190, 66, 96, 115, 102, 32, 201, 37, 114]","[159, 37, 170, 232, 122, 210, 177, 7, 211, 126, 220, 111, 63, 49, 190, 85, 230, 56, 103, 247, 22, 185, 234, 218, 244, 129, 81, 120, 51, 239, 109, 1, 186, 149, 60, 199, 58, 21, 78, 46, 171, 4, 141, 45, 19, 31, 27, 11, 213, 175, 73, 44, 135, 235, 184, 254, 246, 193, 42, 204, 24, 139, 137, 173, 82, 121, 201, 95, 202, 150, 187, 71, 38, 225, 91, 28, 64, 221, 136, 145, 124, 216, 15, 76, 180, 14, 18, 90, 162, 9, 233, 147, 251, 167, 8, 10, 212, 181, 32, 203, 176, 87, 69, 86, 128, 29, 72, 237, 113, 189, 205, 156, 241, 39, 155, 243, 209, 238, 92, 83, 47, 179, 196, 74, 255, 101, 5, 114]","[1, 4, 5, 11, 14, 24, 27, 31, 32, 37, 39, 42, 44, 45, 47, 51, 56, 71, 73, 83, 85, 90, 91, 92, 95, 101, 111, 114, 120, 121, 129, 135, 137, 141, 145, 149, 155, 170, 171, 180, 185, 187, 190, 193, 201, 213, 230, 234, 239, 255]"
|
|
||||||
96,"[108, 104, 142, 134, 39, 173, 143, 83, 127, 158, 11, 151, 1, 50, 253, 113, 24, 150, 53, 17, 90, 224, 112, 124, 74, 98, 206, 160, 250, 156, 178, 120, 153, 137, 107, 129, 60, 35, 217, 176, 184, 70, 0, 12, 52, 101, 48, 86, 210, 201, 66, 19, 163, 223, 131, 194, 167, 152, 102, 6, 182, 226, 190, 29, 38, 96, 140, 87, 133, 193, 208, 166, 79, 27, 132, 13, 110, 236, 212, 192, 118, 203, 62, 149, 161, 23, 76, 85, 213, 43, 99, 130, 197, 46, 254, 97, 7, 205, 185, 81, 196, 111, 195, 92, 199, 123, 44, 80, 187, 128, 58, 22, 237, 200, 45, 57, 144, 247, 91, 221, 243, 51, 215, 68, 75, 69, 183, 64]","[83, 19, 171, 79, 167, 177, 207, 46, 229, 126, 10, 241, 64, 168, 238, 121, 115, 235, 85, 141, 231, 213, 89, 15, 82, 243, 230, 35, 5, 191, 189, 17, 12, 44, 119, 218, 156, 86, 215, 33, 67, 142, 219, 109, 129, 169, 249, 179, 93, 214, 2, 7, 61, 244, 140, 147, 111, 209, 22, 133, 103, 223, 37, 25, 41, 188, 114, 106, 248, 23, 53, 155, 31, 84, 175, 148, 6, 196, 255, 246, 154, 50, 87, 192, 228, 165, 232, 245, 34, 202, 159, 96, 158, 73, 63, 130, 138, 78, 247, 65, 220, 125, 80, 144, 240, 252, 150, 117, 206, 236, 16, 132, 18, 194, 47, 251, 174, 1, 94, 66, 105, 71, 99, 134, 239, 42, 13, 136]","[1, 6, 7, 12, 13, 17, 19, 22, 23, 35, 44, 46, 50, 53, 64, 66, 79, 80, 83, 85, 86, 87, 96, 99, 111, 129, 130, 132, 133, 134, 140, 142, 144, 150, 156, 158, 167, 192, 194, 196, 206, 213, 215, 223, 236, 243, 247]"
|
|
||||||
97,"[176, 194, 118, 22, 208, 218, 108, 242, 33, 62, 157, 211, 17, 89, 213, 214, 49, 178, 142, 103, 150, 141, 54, 171, 126, 229, 45, 201, 240, 99, 2, 97, 82, 162, 192, 102, 18, 121, 66, 190, 151, 12, 254, 138, 198, 137, 155, 173, 71, 179, 90, 0, 129, 147, 161, 223, 46, 215, 234, 57, 28, 125, 60, 196, 250, 177, 123, 225, 159, 205, 139, 243, 37, 25, 128, 120, 131, 24, 111, 8, 56, 42, 1, 85, 156, 72, 195, 92, 26, 93, 187, 36, 81, 64, 228, 226, 227, 53, 167, 59, 210, 52, 189, 83, 95, 116, 182, 135, 132, 79, 236, 9, 148, 237, 169, 55, 199, 222, 105, 20, 255, 140, 113, 217, 122, 124, 146, 143]","[142, 208, 205, 157, 71, 72, 200, 1, 143, 209, 99, 8, 117, 168, 82, 98, 171, 79, 24, 166, 223, 241, 238, 22, 60, 103, 158, 43, 89, 201, 118, 147, 235, 186, 178, 90, 56, 59, 64, 4, 104, 247, 192, 250, 86, 145, 130, 49, 88, 19, 167, 169, 196, 210, 95, 85, 17, 225, 189, 226, 148, 121, 154, 40, 84, 230, 3, 80, 122, 213, 73, 214, 92, 26, 227, 67, 216, 124, 57, 212, 231, 31, 202, 54, 179, 13, 141, 127, 245, 173, 164, 195, 44, 155, 170, 242, 239, 181, 18, 58, 183, 138, 132, 14, 110, 97, 37, 68, 101, 28, 87, 81, 111, 236, 215, 123, 135, 62, 172, 55, 180, 45, 187, 106, 93, 246, 129, 70]","[1, 8, 17, 18, 22, 24, 26, 28, 37, 45, 49, 54, 55, 56, 57, 59, 60, 62, 64, 71, 72, 79, 81, 82, 85, 89, 90, 92, 93, 95, 97, 99, 103, 111, 118, 121, 122, 123, 124, 129, 132, 135, 138, 141, 142, 143, 147, 148, 155, 157, 167, 169, 171, 173, 178, 179, 187, 189, 192, 195, 196, 201, 205, 208, 210, 213, 214, 215, 223, 225, 226, 227, 236, 242, 250]"
|
|
||||||
98,"[154, 94, 248, 0, 212, 211, 77, 235, 79, 140, 121, 245, 15, 46, 172, 76, 30, 162, 194, 198, 26, 142, 209, 243, 96, 21, 184, 135, 213, 218, 63, 42, 171, 208, 146, 220, 95, 20, 173, 25, 55, 199, 53, 253, 34, 227, 112, 226, 52, 192, 167, 221, 105, 223, 64, 126, 164, 152, 13, 204, 239, 117, 128, 123, 175, 185, 233, 91, 149, 75, 11, 205, 9, 85, 107, 81, 155, 153, 141, 203, 240, 165, 54, 148, 17, 33, 68, 39, 120, 86, 187, 237, 137, 150, 118, 252, 157, 136, 195, 45, 103, 202, 111, 206, 108, 181, 176, 83, 23, 210, 32, 62, 48, 109, 37, 166, 170, 124, 196, 36, 160, 127, 110, 100, 177, 169, 29, 130]","[96, 119, 233, 113, 48, 24, 166, 130, 50, 235, 5, 136, 245, 102, 201, 90, 72, 49, 164, 109, 175, 2, 36, 210, 58, 11, 240, 31, 131, 237, 198, 67, 171, 52, 196, 26, 129, 77, 236, 56, 203, 121, 139, 213, 209, 112, 182, 224, 81, 86, 84, 134, 0, 186, 168, 15, 162, 63, 16, 208, 98, 199, 123, 43, 178, 212, 71, 218, 107, 179, 38, 191, 78, 180, 126, 4, 219, 9, 153, 30, 118, 21, 29, 120, 7, 76, 145, 69, 185, 35, 99, 124, 13, 105, 32, 54, 184, 181, 189, 159, 174, 128, 110, 234, 33, 82, 195, 73, 249, 37, 204, 230, 248, 141, 202, 220, 44, 59, 194, 154, 143, 20, 3, 160, 148, 34, 74, 250]","[0, 9, 11, 13, 15, 20, 21, 26, 29, 30, 32, 33, 34, 36, 37, 48, 52, 54, 63, 76, 77, 81, 86, 96, 105, 107, 109, 110, 112, 118, 120, 121, 123, 124, 126, 128, 130, 136, 141, 148, 153, 154, 160, 162, 164, 166, 171, 175, 181, 184, 185, 194, 195, 196, 198, 199, 202, 203, 204, 208, 209, 210, 212, 213, 218, 220, 233, 235, 237, 240, 245, 248]"
|
|
||||||
99,"[107, 192, 88, 185, 186, 100, 34, 0, 22, 235, 239, 69, 64, 240, 21, 74, 232, 230, 175, 91, 189, 193, 13, 43, 247, 6, 85, 161, 57, 54, 231, 188, 31, 98, 11, 32, 217, 72, 200, 215, 199, 224, 10, 245, 152, 120, 253, 180, 182, 46, 196, 228, 243, 109, 187, 160, 70, 80, 65, 174, 241, 110, 158, 30, 112, 129, 146, 3, 76, 9, 208, 97, 254, 29, 165, 190, 227, 163, 149, 211, 134, 123, 204, 198, 221, 169, 82, 181, 203, 5, 168, 219, 154, 14, 170, 71, 155, 16, 130, 116, 73, 205, 195, 15, 103, 136, 118, 101, 58, 95, 191, 226, 89, 166, 17, 50, 59, 61, 66, 233, 83, 68, 183, 184, 63, 117, 99, 77]","[100, 229, 45, 88, 62, 52, 119, 22, 233, 91, 4, 27, 138, 24, 158, 48, 6, 255, 242, 145, 167, 41, 205, 28, 237, 47, 126, 139, 248, 85, 197, 30, 246, 14, 69, 58, 102, 65, 214, 225, 130, 131, 223, 107, 108, 251, 201, 37, 97, 204, 238, 78, 44, 73, 218, 86, 232, 244, 149, 216, 212, 38, 79, 92, 230, 196, 173, 2, 169, 179, 87, 156, 254, 54, 93, 206, 101, 143, 147, 89, 209, 252, 25, 104, 249, 127, 96, 81, 153, 114, 133, 36, 122, 241, 82, 150, 176, 171, 162, 98, 178, 210, 188, 181, 148, 42, 203, 164, 90, 213, 33, 144, 105, 175, 39, 70, 3, 222, 236, 240, 128, 111, 226, 51, 202, 56, 31, 113]","[3, 6, 14, 22, 30, 31, 54, 58, 65, 69, 70, 73, 82, 85, 88, 89, 91, 97, 98, 100, 101, 107, 130, 149, 158, 169, 175, 181, 188, 196, 203, 204, 205, 226, 230, 232, 233, 240, 241, 254]"
|
|
||||||
|
@ -1,805 +0,0 @@
|
|||||||
# Copyright (c) 2023 ETH Zurich.
|
|
||||||
# All rights reserved.
|
|
||||||
#
|
|
||||||
# Use of this source code is governed by a BSD-style license that can be
|
|
||||||
# found in the LICENSE file.
|
|
||||||
#
|
|
||||||
# The source code is adapted from the sorting source code written by
|
|
||||||
# Nils Blach.
|
|
||||||
#
|
|
||||||
# main author: Robert Gerstenberger
|
|
||||||
|
|
||||||
import os
|
|
||||||
import logging
|
|
||||||
import datetime
|
|
||||||
import json
|
|
||||||
import csv
|
|
||||||
from typing import Dict, List, Callable, Union
|
|
||||||
from graph_of_thoughts import controller, language_models, operations, prompter, parser
|
|
||||||
|
|
||||||
# This is a hack to also allow execution of this file from the examples directory
|
|
||||||
try:
|
|
||||||
from . import utils
|
|
||||||
except ImportError:
|
|
||||||
import utils
|
|
||||||
|
|
||||||
|
|
||||||
class SetIntersectionPrompter(prompter.Prompter):
|
|
||||||
"""
|
|
||||||
SetIntersectionPrompter provides the generation of prompts specific to the
|
|
||||||
set intersection example for the language models.
|
|
||||||
|
|
||||||
Inherits from the Prompter class and implements its abstract methods.
|
|
||||||
"""
|
|
||||||
|
|
||||||
intersection_prompt = """<Instruction> Find the intersection of two sets of numbers. Output only the set of numbers that are present in both sets, no additional text. </Instruction>
|
|
||||||
|
|
||||||
<Examples>
|
|
||||||
Input Set 1: [13, 16, 30, 6, 21, 7, 31, 15, 11, 1, 24, 10, 9, 3, 20, 8]
|
|
||||||
Input Set 2: [25, 24, 10, 4, 27, 0, 14, 12, 8, 2, 29, 20, 17, 19, 26, 23]
|
|
||||||
Output: [24, 10, 20, 8]
|
|
||||||
|
|
||||||
Input Set 1: [26, 40, 42, 57, 15, 31, 5, 32, 11, 4, 24, 28, 51, 54, 12, 22, 33, 35, 7, 13, 2, 59, 8, 23, 43, 16, 29, 55, 25, 63, 21, 18]
|
|
||||||
Input Set 2: [16, 60, 36, 48, 0, 15, 5, 19, 46, 24, 1, 6, 61, 10, 38, 53, 58, 9, 44, 14, 35, 63, 52, 20, 27, 17, 39, 47, 34, 56, 40, 59]
|
|
||||||
Output: [40, 15, 5, 24, 35, 59, 16, 63]
|
|
||||||
|
|
||||||
Input Set 1: [115, 61, 35, 103, 90, 117, 86, 44, 63, 45, 40, 30, 74, 33, 31, 1, 118, 48, 38, 0, 119, 51, 64, 78, 15, 121, 89, 101, 79, 69, 120, 29, 58, 50, 116, 11, 60, 12, 39, 95, 23, 2, 109, 84, 7, 43, 99, 98, 52, 70, 75, 102, 57, 19, 94, 36, 114, 88, 71, 56, 83, 6, 96, 107]
|
|
||||||
Input Set 2: [13, 35, 20, 96, 34, 18, 47, 127, 126, 9, 21, 16, 77, 22, 111, 122, 85, 73, 42, 105, 123, 15, 33, 59, 67, 57, 104, 8, 30, 89, 76, 12, 65, 84, 32, 40, 7, 100, 108, 50, 14, 28, 24, 53, 90, 17, 91, 81, 124, 63, 5, 46, 125, 93, 49, 66, 117, 37, 115, 113, 2, 106, 41, 72]
|
|
||||||
Output: [115, 35, 90, 117, 63, 40, 30, 33, 15, 89, 50, 12, 2, 84, 7, 57, 96]
|
|
||||||
</Examples>
|
|
||||||
|
|
||||||
Input Set 1: {set1}
|
|
||||||
Input Set 2: {set2}"""
|
|
||||||
|
|
||||||
intersection_prompt_cot = """<Instruction> Find the intersection of two sets of numbers. You can generate any intermediate solutions, but the final output should be the set of numbers that are present in both sets, prefixed with "Output: ". </Instruction>
|
|
||||||
|
|
||||||
<Approach>
|
|
||||||
To find the intersection of the two sets follow these steps:
|
|
||||||
1. Split the second input set of numbers into two to four subsets, each containing an equal number of elements from the original set (make sure they don't overlap).
|
|
||||||
2. For each subset find the set of numbers that are present in the subset and the first input set.
|
|
||||||
3. Merge the resulting sets into a single output set.
|
|
||||||
</Approach>
|
|
||||||
|
|
||||||
<Examples>
|
|
||||||
Input Set 1: [13, 16, 30, 6, 21, 7, 31, 15, 11, 1, 24, 10, 9, 3, 20, 8]
|
|
||||||
Input Set 2: [25, 24, 10, 4, 27, 0, 14, 12, 8, 2, 29, 20, 17, 19, 26, 23]
|
|
||||||
Subsets of Input Set 2:
|
|
||||||
[25, 24, 10, 4, 27, 0, 14, 12]
|
|
||||||
[8, 2, 29, 20, 17, 19, 26, 23]
|
|
||||||
Intersected Subsets with Input Set 1:
|
|
||||||
[24, 10]
|
|
||||||
[8, 20]
|
|
||||||
Output: [24, 10, 8, 20]
|
|
||||||
|
|
||||||
Input Set 1: [26, 40, 42, 57, 15, 31, 5, 32, 11, 4, 24, 28, 51, 54, 12, 22, 33, 35, 7, 13, 2, 59, 8, 23, 43, 16, 29, 55, 25, 63, 21, 18]
|
|
||||||
Input Set 2: [16, 60, 36, 48, 0, 15, 5, 19, 46, 24, 1, 6, 61, 10, 38, 53, 58, 9, 44, 14, 35, 63, 52, 20, 27, 17, 39, 47, 34, 56, 40, 59]
|
|
||||||
Subsets of Input Set 2:
|
|
||||||
[16, 60, 36, 48, 0, 15, 5, 19, 46, 24, 1, 6, 61, 10, 38, 53]
|
|
||||||
[58, 9, 44, 14, 35, 63, 52, 20, 27, 17, 39, 47, 34, 56, 40, 59]
|
|
||||||
Intersected Subsets with Input Set 1:
|
|
||||||
[16, 15, 5, 24]
|
|
||||||
[35, 63, 40, 59]
|
|
||||||
Output: [16, 15, 5, 24, 35, 63, 40, 59]
|
|
||||||
|
|
||||||
Input Set 1: [115, 61, 35, 103, 90, 117, 86, 44, 63, 45, 40, 30, 74, 33, 31, 1, 118, 48, 38, 0, 119, 51, 64, 78, 15, 121, 89, 101, 79, 69, 120, 29, 58, 50, 116, 11, 60, 12, 39, 95, 23, 2, 109, 84, 7, 43, 99, 98, 52, 70, 75, 102, 57, 19, 94, 36, 114, 88, 71, 56, 83, 6, 96, 107]
|
|
||||||
Input Set 2: [13, 35, 20, 96, 34, 18, 47, 127, 126, 9, 21, 16, 77, 22, 111, 122, 85, 73, 42, 105, 123, 15, 33, 59, 67, 57, 104, 8, 30, 89, 76, 12, 65, 84, 32, 40, 7, 100, 108, 50, 14, 28, 24, 53, 90, 17, 91, 81, 124, 63, 5, 46, 125, 93, 49, 66, 117, 37, 115, 113, 2, 106, 41, 72]
|
|
||||||
Subsets of Input Set 2:
|
|
||||||
[13, 35, 20, 96, 34, 18, 47, 127, 126, 9, 21, 16, 77, 22, 111, 122]
|
|
||||||
[85, 73, 42, 105, 123, 15, 33, 59, 67, 57, 104, 8, 30, 89, 76, 12]
|
|
||||||
[65, 84, 32, 40, 7, 100, 108, 50, 14, 28, 24, 53, 90, 17, 91, 81]
|
|
||||||
[124, 63, 5, 46, 125, 93, 49, 66, 117, 37, 115, 113, 2, 106, 41, 72]
|
|
||||||
Intersected Subsets with Input Set 1:
|
|
||||||
[35, 96]
|
|
||||||
[15, 33, 57, 30, 89, 12]
|
|
||||||
[84, 40, 7, 50, 90]
|
|
||||||
[63, 117, 115, 2]
|
|
||||||
Output: [35, 96, 15, 33, 57, 30, 89, 12, 84, 40, 7, 50, 90, 63, 117, 115, 2]
|
|
||||||
</Examples>
|
|
||||||
|
|
||||||
Input Set 1: {set1}
|
|
||||||
Input Set 2: {set2}"""
|
|
||||||
|
|
||||||
tot_improve_prompt = """<Instruction> The following three sets represent two sets and an intersection set of those two sets. The intersection set is not correct. Fix the intersection set so that it is correct.
|
|
||||||
Make sure that the numbers in the intersection set can be found in both input sets. </Instruction>
|
|
||||||
|
|
||||||
<Approach>
|
|
||||||
To fix the incorrectly intersection set follow these steps:
|
|
||||||
1. Check for each number in the incorrect intersection set, whether it can be found in both input sets. If not, remove that number from the intersection set.
|
|
||||||
2. Iterate through the second input set and check whether each number is already in the incorrect intersection set and if not, check whether that number can also be found in the first input set. If so, add that number to the intersection set.
|
|
||||||
</Approach>
|
|
||||||
|
|
||||||
<Examples>
|
|
||||||
Input Set 1: [13, 16, 30, 6, 21, 7, 31, 15, 11, 1, 24, 10, 9, 3, 20, 8]
|
|
||||||
Input Set 2: [25, 24, 10, 4, 27, 0, 14, 12, 8, 2, 29, 20, 17, 19, 26, 23]
|
|
||||||
Incorrect Intersection Set: [24, 20, 25]
|
|
||||||
Reason: The incorrect intersection set contains the number 25, which is not present in the first input set and is missing the numbers 10 and 8.
|
|
||||||
Output: [24, 10, 20, 8]
|
|
||||||
|
|
||||||
Input Set 1: [26, 40, 42, 57, 15, 31, 5, 32, 11, 4, 24, 28, 51, 54, 12, 22, 33, 35, 7, 13, 2, 59, 8, 23, 43, 16, 29, 55, 25, 63, 21, 18]
|
|
||||||
Input Set 2: [16, 60, 36, 48, 0, 15, 5, 19, 46, 24, 1, 6, 61, 10, 38, 53, 58, 9, 44, 14, 35, 63, 52, 20, 27, 17, 39, 47, 34, 56, 40, 59]
|
|
||||||
Incorrect Intersection Set: [57, 16, 15, 24, 35, 10, 40]
|
|
||||||
Reason: The incorrect intersection set contains the numbers 57, which is not present in the second input set, and 10, which is not present in the first input set, and is missing the numbers 5, 63 and 59.
|
|
||||||
Output: [16, 15, 5, 24, 35, 63, 40, 59]
|
|
||||||
|
|
||||||
Input Set 1: [115, 61, 35, 103, 90, 117, 86, 44, 63, 45, 40, 30, 74, 33, 31, 1, 118, 48, 38, 0, 119, 51, 64, 78, 15, 121, 89, 101, 79, 69, 120, 29, 58, 50, 116, 11, 60, 12, 39, 95, 23, 2, 109, 84, 7, 43, 99, 98, 52, 70, 75, 102, 57, 19, 94, 36, 114, 88, 71, 56, 83, 6, 96, 107]
|
|
||||||
Input Set 2: [13, 35, 20, 96, 34, 18, 47, 127, 126, 9, 21, 16, 77, 22, 111, 122, 85, 73, 42, 105, 123, 15, 33, 59, 67, 57, 104, 8, 30, 89, 76, 12, 65, 84, 32, 40, 7, 100, 108, 50, 14, 28, 24, 53, 90, 17, 91, 81, 124, 63, 5, 46, 125, 93, 49, 66, 117, 37, 115, 113, 2, 106, 41, 72]
|
|
||||||
Incorrect Intersection Set: [35, 96, 44, 15, 33, 57, 30, 50, 90, 119, 123, 63, 117, 115, 2]
|
|
||||||
Reason: The incorrect intersection set contains the numbers 44 and 119, which are not present in the second input set, and 123, which is not present in the first input set, and is missing the numbers 89, 12, 84, 40 and 7.
|
|
||||||
Output: [35, 96, 15, 33, 57, 30, 89, 12, 84, 40, 7, 50, 90, 63, 117, 115, 2]
|
|
||||||
</Examples>
|
|
||||||
|
|
||||||
Input Set 1: {set1}
|
|
||||||
Input Set 2: {set2}
|
|
||||||
Incorrect Intersection Set: {incorrect_intersection}
|
|
||||||
"""
|
|
||||||
|
|
||||||
got_split_prompt = """<Instruction> Split the following list of 128 numbers into 8 lists of 16 numbers each, the first list should contain the first 16 numbers, the second list the second 16 numbers, the third list the third 16 numbers and the fourth list the fourth 16 numbers, the fifth list the fifth 16 numbers and so on.
|
|
||||||
Only output the 8 lists in the following format without any additional text or thoughts!:
|
|
||||||
{{
|
|
||||||
"List 1": [13, 35, 20, 96, 34, 18, 47, ...],
|
|
||||||
"List 2": [127, 126, 9, 21, 16, 77, 22, ...],
|
|
||||||
"List 3": [111, 122, 85, 73, 42, 105, 123, ...],
|
|
||||||
"List 4": [15, 33, 59, 67, 57, 104, 8, ...],
|
|
||||||
"List 5": [30, 89, 76, 12, 65, 84, 32, ...],
|
|
||||||
"List 6": [40, 7, 100, 108, 50, 14, 28, ...],
|
|
||||||
"List 7": [24, 53, 90, 17, 91, 81, 124, ...],
|
|
||||||
"List 8": [63, 5, 46, 125, 93, 49, 66, ...]
|
|
||||||
}} </Instruction>
|
|
||||||
|
|
||||||
<Example>
|
|
||||||
Input: [124, 100, 176, 65, 28, 214, 201, 67, 127, 40, 193, 26, 212, 196, 12, 205, 39, 162, 190, 152, 186, 182, 125, 229, 114, 70, 189, 242, 155, 32, 47, 113, 99, 177, 57, 103, 156, 107, 226, 188, 31, 25, 49, 171, 17, 64, 254, 217, 35, 22, 248, 88, 251, 219, 84, 82, 167, 14, 116, 92, 105, 148, 119, 58, 13, 160, 81, 120, 74, 94, 173, 38, 101, 104, 91, 135, 208, 118, 136, 87, 75, 41, 197, 76, 249, 240, 86, 246, 231, 44, 29, 141, 241, 227, 239, 142, 222, 115, 129, 255, 216, 153, 0, 110, 36, 130, 6, 102, 209, 37, 143, 30, 51, 146, 71, 27, 181, 183, 2, 122, 77, 184, 56, 225, 108, 83, 106, 15]
|
|
||||||
Output:
|
|
||||||
{{
|
|
||||||
"List 1": [124, 100, 176, 65, 28, 214, 201, 67, 127, 40, 193, 26, 212, 196, 12, 205],
|
|
||||||
"List 2": [39, 162, 190, 152, 186, 182, 125, 229, 114, 70, 189, 242, 155, 32, 47, 113],
|
|
||||||
"List 3": [99, 177, 57, 103, 156, 107, 226, 188, 31, 25, 49, 171, 17, 64, 254, 217],
|
|
||||||
"List 4": [35, 22, 248, 88, 251, 219, 84, 82, 167, 14, 116, 92, 105, 148, 119, 58],
|
|
||||||
"List 5": [13, 160, 81, 120, 74, 94, 173, 38, 101, 104, 91, 135, 208, 118, 136, 87],
|
|
||||||
"List 6": [75, 41, 197, 76, 249, 240, 86, 246, 231, 44, 29, 141, 241, 227, 239, 142],
|
|
||||||
"List 7": [222, 115, 129, 255, 216, 153, 0, 110, 36, 130, 6, 102, 209, 37, 143, 30],
|
|
||||||
"List 8": [51, 146, 71, 27, 181, 183, 2, 122, 77, 184, 56, 225, 108, 83, 106, 15]
|
|
||||||
}}
|
|
||||||
</Example>
|
|
||||||
|
|
||||||
Input: {input}"""
|
|
||||||
|
|
||||||
got_merge_prompt = """<Instruction> Merge the following 2 lists into one list by appending the second list to the first list.
|
|
||||||
Only output the final list without any additional text or thoughts! </Instruction>
|
|
||||||
|
|
||||||
List 1: {input1}
|
|
||||||
List 2: {input2}
|
|
||||||
"""
|
|
||||||
|
|
||||||
def aggregation_prompt(self, state_dicts: List[Dict], **kwargs) -> str:
|
|
||||||
"""
|
|
||||||
Generate an aggregation prompt for the language model.
|
|
||||||
|
|
||||||
:param state_dicts: The thought states that should be aggregated.
|
|
||||||
:type state_dicts: List[Dict]
|
|
||||||
:param kwargs: Additional keyword arguments.
|
|
||||||
:return: The aggregation prompt.
|
|
||||||
:rtype: str
|
|
||||||
:raise AssertionError: If not exactly two thought states are provided.
|
|
||||||
"""
|
|
||||||
assert len(state_dicts) == 2, "Expected two states for aggregation prompt."
|
|
||||||
|
|
||||||
return self.got_merge_prompt.format(
|
|
||||||
input1=state_dicts[0]["current"],
|
|
||||||
input2=state_dicts[1]["current"],
|
|
||||||
)
|
|
||||||
|
|
||||||
def generate_prompt(
|
|
||||||
self,
|
|
||||||
num_branches: int,
|
|
||||||
set1: str,
|
|
||||||
set2: str,
|
|
||||||
current: str,
|
|
||||||
method: str,
|
|
||||||
**kwargs,
|
|
||||||
) -> str:
|
|
||||||
"""
|
|
||||||
Generate a generate prompt for the language model.
|
|
||||||
|
|
||||||
:param num_branches: The number of responses the prompt should ask the LM to generate.
|
|
||||||
:type num_branches: int
|
|
||||||
:param set1: First input set.
|
|
||||||
:type set1: str
|
|
||||||
:param set2: Second input set.
|
|
||||||
:type set2: str
|
|
||||||
:param current: Intermediate solution.
|
|
||||||
:type current: str
|
|
||||||
:param method: Method for which the generate prompt is generated.
|
|
||||||
:type method: str
|
|
||||||
:param kwargs: Additional keyword arguments.
|
|
||||||
:return: The generate prompt.
|
|
||||||
:rtype: str
|
|
||||||
:raise AssertionError: If the requested number of branches is not one.
|
|
||||||
"""
|
|
||||||
|
|
||||||
assert num_branches == 1, "Branching should be done via multiple requests."
|
|
||||||
if method.startswith("io"):
|
|
||||||
return self.intersection_prompt.format(set1=set1, set2=set2)
|
|
||||||
elif method.startswith("cot"):
|
|
||||||
return self.intersection_prompt_cot.format(set1=set1, set2=set2)
|
|
||||||
elif method.startswith("tot"):
|
|
||||||
if current is None or current == "":
|
|
||||||
return self.intersection_prompt.format(set1=set1, set2=set2)
|
|
||||||
return self.tot_improve_prompt.format(
|
|
||||||
set1=set1, set2=set2, incorrect_intersection=current
|
|
||||||
)
|
|
||||||
elif method.startswith("got"):
|
|
||||||
if kwargs["phase"] == 0:
|
|
||||||
return self.got_split_prompt.format(input=set2)
|
|
||||||
|
|
||||||
input_set = set2
|
|
||||||
if "subset" in kwargs and kwargs["subset"] != "":
|
|
||||||
input_set = kwargs["subset"]
|
|
||||||
|
|
||||||
return self.intersection_prompt.format(set1=set1, set2=input_set)
|
|
||||||
|
|
||||||
def improve_prompt(self, **kwargs) -> str:
|
|
||||||
"""
|
|
||||||
Generate an improve prompt for the language model.
|
|
||||||
|
|
||||||
:param kwargs: Additional keyword arguments.
|
|
||||||
:return: The improve prompt.
|
|
||||||
:rtype: str
|
|
||||||
"""
|
|
||||||
pass
|
|
||||||
|
|
||||||
def validation_prompt(self, **kwargs) -> str:
|
|
||||||
"""
|
|
||||||
Generate a validation prompt for the language model.
|
|
||||||
|
|
||||||
:param kwargs: Additional keyword arguments.
|
|
||||||
:return: The validation prompt.
|
|
||||||
:rtype: str
|
|
||||||
"""
|
|
||||||
pass
|
|
||||||
|
|
||||||
def score_prompt(self, state_dicts: List[Dict], **kwargs) -> str:
|
|
||||||
"""
|
|
||||||
Generate a score prompt for the language model.
|
|
||||||
|
|
||||||
:param state_dicts: The thought states that should be scored,
|
|
||||||
if more than one, they should be scored together.
|
|
||||||
:type state_dicts: List[Dict]
|
|
||||||
:param kwargs: Additional keyword arguments.
|
|
||||||
:return: The score prompt.
|
|
||||||
:rtype: str
|
|
||||||
"""
|
|
||||||
pass
|
|
||||||
|
|
||||||
|
|
||||||
class SetIntersectionParser(parser.Parser):
|
|
||||||
"""
|
|
||||||
SetIntersectionParser provides the parsing of language model reponses
|
|
||||||
specific to the set intersection example.
|
|
||||||
|
|
||||||
Inherits from the Parser class and implements its abstract methods.
|
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(self) -> None:
|
|
||||||
"""
|
|
||||||
Inits the response cache.
|
|
||||||
"""
|
|
||||||
self.cache = {}
|
|
||||||
|
|
||||||
def parse_aggregation_answer(
|
|
||||||
self, states: List[Dict], texts: List[str]
|
|
||||||
) -> Union[Dict, List[Dict]]:
|
|
||||||
"""
|
|
||||||
Parse the response from the language model for an aggregation prompt.
|
|
||||||
|
|
||||||
:param states: The thought states used to generate the prompt.
|
|
||||||
:type states: List[Dict]
|
|
||||||
:param texts: The responses to the prompt from the language model.
|
|
||||||
:type texts: List[str]
|
|
||||||
:return: The new thought states after parsing the respones from the language model.
|
|
||||||
:rtype: Union[Dict, List[Dict]]
|
|
||||||
:raise AssertionError: If not exactly two thought states are provided.
|
|
||||||
"""
|
|
||||||
|
|
||||||
assert len(states) == 2, "Expected two states for aggregation answer."
|
|
||||||
new_states = []
|
|
||||||
for text in texts:
|
|
||||||
answers = text.strip().split("\n")
|
|
||||||
if any(["Output" in answer for answer in answers]):
|
|
||||||
# cut elements until last output is found
|
|
||||||
for answer in reversed(answers):
|
|
||||||
if "Output" in answer:
|
|
||||||
answers = answers[answers.index(answer) :]
|
|
||||||
break
|
|
||||||
|
|
||||||
answers_stripped = [
|
|
||||||
answer for answer in answers if "[" in answer and "]" in answer
|
|
||||||
]
|
|
||||||
if len(answers_stripped) == 0:
|
|
||||||
for answer in answers:
|
|
||||||
answer = "[" + answer + "]"
|
|
||||||
try:
|
|
||||||
answer_converted = utils.string_to_list(answer)
|
|
||||||
if len(answer_converted) > 0:
|
|
||||||
answers_stripped.append(answer)
|
|
||||||
except:
|
|
||||||
pass
|
|
||||||
if len(answers_stripped) == 0:
|
|
||||||
logging.warning(
|
|
||||||
f"Could not parse aggregation answer: {text}. Returning empty list."
|
|
||||||
)
|
|
||||||
answer = "[]"
|
|
||||||
else:
|
|
||||||
answer = [
|
|
||||||
answer[answer.index("[") : answer.index("]") + 1]
|
|
||||||
for answer in answers_stripped
|
|
||||||
][0]
|
|
||||||
states = sorted(states, key=lambda x: x["part"])
|
|
||||||
merged_subsets = states[0]["subset"][:-1] + ", " + states[1]["subset"][1:]
|
|
||||||
new_state = states[0].copy()
|
|
||||||
new_state["current"] = answer
|
|
||||||
new_state["subset"] = merged_subsets
|
|
||||||
new_states.append(new_state)
|
|
||||||
return new_states
|
|
||||||
|
|
||||||
def parse_improve_answer(self, state: Dict, texts: List[str]) -> Dict:
|
|
||||||
"""
|
|
||||||
Parse the response from the language model for an improve prompt.
|
|
||||||
|
|
||||||
:param state: The thought state used to generate the prompt.
|
|
||||||
:type state: Dict
|
|
||||||
:param texts: The responses to the prompt from the language model.
|
|
||||||
:type texts: List[str]
|
|
||||||
:return: The new thought state after parsing the responses from the language model.
|
|
||||||
:rtype: Dict
|
|
||||||
"""
|
|
||||||
pass
|
|
||||||
|
|
||||||
def parse_generate_answer(self, state: Dict, texts: List[str]) -> List[Dict]:
|
|
||||||
"""
|
|
||||||
Parse the response from the language model for a generate prompt.
|
|
||||||
|
|
||||||
:param state: The thought state used to generate the prompt.
|
|
||||||
:type state: Dict
|
|
||||||
:param texts: The responses to the prompt from the language model.
|
|
||||||
:type texts: List[str]
|
|
||||||
:return: The new thought states after parsing the respones from the language model.
|
|
||||||
:rtype: List[Dict]
|
|
||||||
"""
|
|
||||||
new_states = []
|
|
||||||
for text in texts:
|
|
||||||
if state["method"].startswith("got") and state["phase"] == 0:
|
|
||||||
# We expect a json which contains the two lists named "List 1" and "List 2"
|
|
||||||
# cut everything until the opening bracket and everything after the closing bracket
|
|
||||||
|
|
||||||
try:
|
|
||||||
text = text[text.index("{") : text.index("}") + 1]
|
|
||||||
json_dict = json.loads(text)
|
|
||||||
if len(json_dict.keys()) != 8:
|
|
||||||
logging.warning(
|
|
||||||
f"Expected 8 lists in json, but found {len(json_dict.keys())}."
|
|
||||||
)
|
|
||||||
for key, value in json_dict.items():
|
|
||||||
if "List" not in key:
|
|
||||||
logging.warning(
|
|
||||||
f"Expected key to contain 'List', but found {key}."
|
|
||||||
)
|
|
||||||
continue
|
|
||||||
if not isinstance(value, list):
|
|
||||||
value = utils.string_to_list(value)
|
|
||||||
new_state = state.copy()
|
|
||||||
new_state["current"] = ""
|
|
||||||
new_state["subset"] = str(value)
|
|
||||||
new_state["phase"] = 1
|
|
||||||
new_state["part"] = key
|
|
||||||
new_states.append(new_state)
|
|
||||||
except Exception as e:
|
|
||||||
logging.error(
|
|
||||||
f"Could not parse step answer: {text}. Encountered exception: {e}"
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
answers = text.strip().split("\n")
|
|
||||||
answers = [
|
|
||||||
answer for answer in answers if "[" in answer and "]" in answer
|
|
||||||
]
|
|
||||||
if any(["Output" in answer for answer in answers]):
|
|
||||||
# cut elements until last output is found
|
|
||||||
for answer in reversed(answers):
|
|
||||||
if "Output" in answer:
|
|
||||||
answers = answers[answers.index(answer) :]
|
|
||||||
break
|
|
||||||
|
|
||||||
answers = [
|
|
||||||
answer[answer.index("[") : answer.index("]") + 1]
|
|
||||||
for answer in answers
|
|
||||||
]
|
|
||||||
if len(answers) == 0:
|
|
||||||
logging.warning(
|
|
||||||
f"Could not parse step answer: {text}. Returning empty list."
|
|
||||||
)
|
|
||||||
answer = "[]"
|
|
||||||
else:
|
|
||||||
if len(answers) > 1:
|
|
||||||
logging.warning(
|
|
||||||
f"Multiple answers found for step answer: {text}. Using the first one."
|
|
||||||
)
|
|
||||||
answer = answers[0]
|
|
||||||
|
|
||||||
new_state = state.copy()
|
|
||||||
new_state["current"] = answer
|
|
||||||
new_state["phase"] = 2
|
|
||||||
new_states.append(new_state)
|
|
||||||
return new_states
|
|
||||||
|
|
||||||
def parse_validation_answer(self, state: Dict, texts: List[str]) -> bool:
|
|
||||||
"""
|
|
||||||
Parse the response from the language model for a validation prompt.
|
|
||||||
|
|
||||||
:param state: The thought state used to generate the prompt.
|
|
||||||
:type state: Dict
|
|
||||||
:param texts: The responses to the prompt from the language model.
|
|
||||||
:type texts: List[str]
|
|
||||||
:return: Whether the thought state is valid or not.
|
|
||||||
:rtype: bool
|
|
||||||
"""
|
|
||||||
pass
|
|
||||||
|
|
||||||
def parse_score_answer(self, states: List[Dict], texts: List[str]) -> List[float]:
|
|
||||||
"""
|
|
||||||
Parse the response from the language model for a score prompt.
|
|
||||||
|
|
||||||
:param states: The thought states used to generate the prompt.
|
|
||||||
:type states: List[Dict]
|
|
||||||
:param texts: The responses to the prompt from the language model.
|
|
||||||
:type texts: List[str]
|
|
||||||
:return: The scores for the thought states.
|
|
||||||
:rtype: List[float]
|
|
||||||
"""
|
|
||||||
pass
|
|
||||||
|
|
||||||
|
|
||||||
def io() -> operations.GraphOfOperations:
|
|
||||||
"""
|
|
||||||
Generates the Graph of Operations for the IO method.
|
|
||||||
|
|
||||||
:return: Graph of Operations
|
|
||||||
:rtype: GraphOfOperations
|
|
||||||
"""
|
|
||||||
operations_graph = operations.GraphOfOperations()
|
|
||||||
|
|
||||||
operations_graph.append_operation(operations.Generate(1, 1))
|
|
||||||
operations_graph.append_operation(operations.Score(1, False, utils.num_errors))
|
|
||||||
operations_graph.append_operation(
|
|
||||||
operations.GroundTruth(utils.test_set_intersection)
|
|
||||||
)
|
|
||||||
|
|
||||||
return operations_graph
|
|
||||||
|
|
||||||
|
|
||||||
def cot() -> operations.GraphOfOperations:
|
|
||||||
"""
|
|
||||||
Generates the Graph of Operations for the CoT method.
|
|
||||||
|
|
||||||
:return: Graph of Operations
|
|
||||||
:rtype: GraphOfOperations
|
|
||||||
"""
|
|
||||||
operations_graph = operations.GraphOfOperations()
|
|
||||||
|
|
||||||
operations_graph.append_operation(operations.Generate(1, 1))
|
|
||||||
operations_graph.append_operation(operations.Score(1, False, utils.num_errors))
|
|
||||||
operations_graph.append_operation(
|
|
||||||
operations.GroundTruth(utils.test_set_intersection)
|
|
||||||
)
|
|
||||||
|
|
||||||
return operations_graph
|
|
||||||
|
|
||||||
|
|
||||||
def tot() -> operations.GraphOfOperations:
|
|
||||||
"""
|
|
||||||
Generates the Graph of Operations for the ToT method.
|
|
||||||
ToT uses a wider tree, where on each level there are more branches.
|
|
||||||
|
|
||||||
:return: Graph of Operations
|
|
||||||
:rtype: GraphOfOperations
|
|
||||||
"""
|
|
||||||
operations_graph = operations.GraphOfOperations()
|
|
||||||
|
|
||||||
operations_graph.append_operation(operations.Generate(1, 25))
|
|
||||||
operations_graph.append_operation(operations.Score(1, False, utils.num_errors))
|
|
||||||
op_1 = operations.KeepBestN(1, False)
|
|
||||||
operations_graph.append_operation(op_1)
|
|
||||||
|
|
||||||
for _ in range(3):
|
|
||||||
operations_graph.append_operation(operations.Generate(1, 25))
|
|
||||||
operations_graph.append_operation(operations.Score(1, False, utils.num_errors))
|
|
||||||
op_2 = operations.KeepBestN(1, False)
|
|
||||||
op_2.add_predecessor(op_1)
|
|
||||||
operations_graph.append_operation(op_2)
|
|
||||||
op_1 = op_2
|
|
||||||
|
|
||||||
operations_graph.append_operation(
|
|
||||||
operations.GroundTruth(utils.test_set_intersection)
|
|
||||||
)
|
|
||||||
|
|
||||||
return operations_graph
|
|
||||||
|
|
||||||
|
|
||||||
def tot2() -> operations.GraphOfOperations:
|
|
||||||
"""
|
|
||||||
Generates the Graph of Operations for the ToT2 method.
|
|
||||||
ToT2 uses a tree with more levels, but with fewer branches per level.
|
|
||||||
|
|
||||||
:return: Graph of Operations
|
|
||||||
:rtype: GraphOfOperations
|
|
||||||
"""
|
|
||||||
operations_graph = operations.GraphOfOperations()
|
|
||||||
|
|
||||||
operations_graph.append_operation(operations.Generate(1, 10))
|
|
||||||
operations_graph.append_operation(operations.Score(1, False, utils.num_errors))
|
|
||||||
op_1 = operations.KeepBestN(1, False)
|
|
||||||
operations_graph.append_operation(op_1)
|
|
||||||
|
|
||||||
for _ in range(8):
|
|
||||||
operations_graph.append_operation(operations.Generate(1, 10))
|
|
||||||
operations_graph.append_operation(operations.Score(1, False, utils.num_errors))
|
|
||||||
op_2 = operations.KeepBestN(1, False)
|
|
||||||
op_2.add_predecessor(op_1)
|
|
||||||
operations_graph.append_operation(op_2)
|
|
||||||
op_1 = op_2
|
|
||||||
|
|
||||||
operations_graph.append_operation(
|
|
||||||
operations.GroundTruth(utils.test_set_intersection)
|
|
||||||
)
|
|
||||||
|
|
||||||
return operations_graph
|
|
||||||
|
|
||||||
|
|
||||||
def got() -> operations.GraphOfOperations:
|
|
||||||
"""
|
|
||||||
Generates the Graph of Operations for the GoT method.
|
|
||||||
|
|
||||||
:return: Graph of Operations
|
|
||||||
:rtype: GraphOfOperations
|
|
||||||
"""
|
|
||||||
operations_graph = operations.GraphOfOperations()
|
|
||||||
|
|
||||||
plans = operations.Generate(1, 1)
|
|
||||||
operations_graph.append_operation(plans) # generate the sublists
|
|
||||||
solved_subsets = []
|
|
||||||
for i in range(1, 9):
|
|
||||||
list_id = f"List {i}"
|
|
||||||
sub_list = operations.Selector(
|
|
||||||
lambda thoughts, list_id=list_id: [
|
|
||||||
thought for thought in thoughts if thought.state["part"] == list_id
|
|
||||||
]
|
|
||||||
)
|
|
||||||
sub_list.add_predecessor(plans)
|
|
||||||
operations_graph.add_operation(sub_list)
|
|
||||||
intersected_subset = operations.Generate(1, 5)
|
|
||||||
intersected_subset.add_predecessor(sub_list)
|
|
||||||
operations_graph.add_operation(intersected_subset)
|
|
||||||
score_sub_list = operations.Score(1, False, utils.num_errors)
|
|
||||||
score_sub_list.add_predecessor(intersected_subset)
|
|
||||||
operations_graph.add_operation(score_sub_list)
|
|
||||||
keep_best_sub_list = operations.KeepBestN(1, False)
|
|
||||||
keep_best_sub_list.add_predecessor(score_sub_list)
|
|
||||||
operations_graph.add_operation(keep_best_sub_list)
|
|
||||||
|
|
||||||
solved_subsets.append(keep_best_sub_list)
|
|
||||||
|
|
||||||
aggregate_1 = operations.Aggregate(5)
|
|
||||||
aggregate_1.add_predecessor(solved_subsets[0])
|
|
||||||
aggregate_1.add_predecessor(solved_subsets[1])
|
|
||||||
operations_graph.add_operation(aggregate_1)
|
|
||||||
score_aggregate_1 = operations.Score(1, False, utils.num_errors)
|
|
||||||
score_aggregate_1.add_predecessor(aggregate_1)
|
|
||||||
operations_graph.add_operation(score_aggregate_1)
|
|
||||||
keep_best_aggregate_1 = operations.KeepBestN(1, False)
|
|
||||||
keep_best_aggregate_1.add_predecessor(score_aggregate_1)
|
|
||||||
operations_graph.add_operation(keep_best_aggregate_1)
|
|
||||||
|
|
||||||
aggregate_2 = operations.Aggregate(5)
|
|
||||||
aggregate_2.add_predecessor(solved_subsets[2])
|
|
||||||
aggregate_2.add_predecessor(solved_subsets[3])
|
|
||||||
operations_graph.add_operation(aggregate_2)
|
|
||||||
score_aggregate_2 = operations.Score(1, False, utils.num_errors)
|
|
||||||
score_aggregate_2.add_predecessor(aggregate_2)
|
|
||||||
operations_graph.add_operation(score_aggregate_2)
|
|
||||||
keep_best_aggregate_2 = operations.KeepBestN(1, False)
|
|
||||||
keep_best_aggregate_2.add_predecessor(score_aggregate_2)
|
|
||||||
operations_graph.add_operation(keep_best_aggregate_2)
|
|
||||||
|
|
||||||
aggregate_3 = operations.Aggregate(5)
|
|
||||||
aggregate_3.add_predecessor(solved_subsets[4])
|
|
||||||
aggregate_3.add_predecessor(solved_subsets[5])
|
|
||||||
operations_graph.add_operation(aggregate_3)
|
|
||||||
score_aggregate_3 = operations.Score(1, False, utils.num_errors)
|
|
||||||
score_aggregate_3.add_predecessor(aggregate_3)
|
|
||||||
operations_graph.add_operation(score_aggregate_3)
|
|
||||||
keep_best_aggregate_3 = operations.KeepBestN(1, False)
|
|
||||||
keep_best_aggregate_3.add_predecessor(score_aggregate_3)
|
|
||||||
operations_graph.add_operation(keep_best_aggregate_3)
|
|
||||||
|
|
||||||
aggregate_4 = operations.Aggregate(5)
|
|
||||||
aggregate_4.add_predecessor(solved_subsets[6])
|
|
||||||
aggregate_4.add_predecessor(solved_subsets[7])
|
|
||||||
operations_graph.add_operation(aggregate_4)
|
|
||||||
score_aggregate_4 = operations.Score(1, False, utils.num_errors)
|
|
||||||
score_aggregate_4.add_predecessor(aggregate_4)
|
|
||||||
operations_graph.add_operation(score_aggregate_4)
|
|
||||||
keep_best_aggregate_4 = operations.KeepBestN(1, False)
|
|
||||||
keep_best_aggregate_4.add_predecessor(score_aggregate_4)
|
|
||||||
operations_graph.add_operation(keep_best_aggregate_4)
|
|
||||||
|
|
||||||
aggregate_1_2 = operations.Aggregate(5)
|
|
||||||
aggregate_1_2.add_predecessor(keep_best_aggregate_1)
|
|
||||||
aggregate_1_2.add_predecessor(keep_best_aggregate_2)
|
|
||||||
operations_graph.add_operation(aggregate_1_2)
|
|
||||||
score_aggregate_1_2 = operations.Score(1, False, utils.num_errors)
|
|
||||||
score_aggregate_1_2.add_predecessor(aggregate_1_2)
|
|
||||||
operations_graph.add_operation(score_aggregate_1_2)
|
|
||||||
keep_best_aggregate_1_2 = operations.KeepBestN(1, False)
|
|
||||||
keep_best_aggregate_1_2.add_predecessor(score_aggregate_1_2)
|
|
||||||
operations_graph.add_operation(keep_best_aggregate_1_2)
|
|
||||||
|
|
||||||
aggregate_3_4 = operations.Aggregate(5)
|
|
||||||
aggregate_3_4.add_predecessor(keep_best_aggregate_3)
|
|
||||||
aggregate_3_4.add_predecessor(keep_best_aggregate_4)
|
|
||||||
operations_graph.add_operation(aggregate_3_4)
|
|
||||||
score_aggregate_3_4 = operations.Score(1, False, utils.num_errors)
|
|
||||||
score_aggregate_3_4.add_predecessor(aggregate_3_4)
|
|
||||||
operations_graph.add_operation(score_aggregate_3_4)
|
|
||||||
keep_best_aggregate_3_4 = operations.KeepBestN(1, False)
|
|
||||||
keep_best_aggregate_3_4.add_predecessor(score_aggregate_3_4)
|
|
||||||
operations_graph.add_operation(keep_best_aggregate_3_4)
|
|
||||||
|
|
||||||
final_aggregate = operations.Aggregate(5)
|
|
||||||
operations_graph.append_operation(final_aggregate)
|
|
||||||
operations_graph.append_operation(operations.Score(1, False, utils.num_errors))
|
|
||||||
keep_best_aggregate_final = operations.KeepBestN(1, False)
|
|
||||||
operations_graph.append_operation(keep_best_aggregate_final)
|
|
||||||
|
|
||||||
operations_graph.append_operation(
|
|
||||||
operations.GroundTruth(utils.test_set_intersection)
|
|
||||||
)
|
|
||||||
|
|
||||||
return operations_graph
|
|
||||||
|
|
||||||
|
|
||||||
def run(
|
|
||||||
data_ids: List[int],
|
|
||||||
methods: List[Callable[[], operations.GraphOfOperations]],
|
|
||||||
budget: float,
|
|
||||||
lm_name: str,
|
|
||||||
) -> float:
|
|
||||||
"""
|
|
||||||
Controller function that executes each specified method for each specified
|
|
||||||
sample while the budget is not exhausted.
|
|
||||||
|
|
||||||
:param data_ids: Indices of the sample to be run.
|
|
||||||
:type data_ids: List[int]
|
|
||||||
:param methods: List of functions to generate Graphs of Operations.
|
|
||||||
:type methods: Each function generates a Graph of Operation.
|
|
||||||
:param budget: Language model budget for the execution in dollars.
|
|
||||||
:type budget: float
|
|
||||||
:param lm_name: Name of the language model to be used.
|
|
||||||
:type lm_name: str
|
|
||||||
:return: Spent budget in dollars.
|
|
||||||
:rtype: float
|
|
||||||
"""
|
|
||||||
|
|
||||||
orig_budget = budget
|
|
||||||
data_path = os.path.join(os.path.dirname(__file__), "set_intersection_128.csv")
|
|
||||||
data = []
|
|
||||||
with open(data_path, "r") as f:
|
|
||||||
reader = csv.reader(f)
|
|
||||||
next(reader)
|
|
||||||
for row in reader:
|
|
||||||
data.append([int(row[0]), row[1], row[2], row[3]])
|
|
||||||
|
|
||||||
if data_ids is None or len(data_ids) == 0:
|
|
||||||
data_ids = list(range(len(data)))
|
|
||||||
selected_data = [data[i] for i in data_ids]
|
|
||||||
|
|
||||||
results_dir = os.path.join(os.path.dirname(__file__), "results")
|
|
||||||
|
|
||||||
if not os.path.exists(results_dir):
|
|
||||||
os.makedirs(results_dir)
|
|
||||||
timestamp = datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
|
|
||||||
extra_info = f"{lm_name}_{'-'.join([method.__name__ for method in methods])}"
|
|
||||||
folder_name = f"{extra_info}_{timestamp}"
|
|
||||||
results_folder = os.path.join(results_dir, folder_name)
|
|
||||||
os.makedirs(results_folder)
|
|
||||||
|
|
||||||
config = {
|
|
||||||
"data": selected_data,
|
|
||||||
"methods": [method.__name__ for method in methods],
|
|
||||||
"lm": lm_name,
|
|
||||||
"budget": budget,
|
|
||||||
}
|
|
||||||
with open(os.path.join(results_folder, "config.json"), "w") as f:
|
|
||||||
json.dump(config, f)
|
|
||||||
|
|
||||||
logging.basicConfig(
|
|
||||||
filename=os.path.join(results_folder, "log.log"),
|
|
||||||
filemode="w",
|
|
||||||
format="%(name)s - %(levelname)s - %(message)s",
|
|
||||||
level=logging.DEBUG,
|
|
||||||
)
|
|
||||||
|
|
||||||
for method in methods:
|
|
||||||
# create a results directory for the method
|
|
||||||
os.makedirs(os.path.join(results_folder, method.__name__))
|
|
||||||
|
|
||||||
for data in selected_data:
|
|
||||||
logging.info(f"Running data {data[0]}: {data[1]} {data[2]}")
|
|
||||||
if budget <= 0.0:
|
|
||||||
logging.error(
|
|
||||||
f"Budget has been depleted, stopping. Data {data[0]} has not been run."
|
|
||||||
)
|
|
||||||
break
|
|
||||||
for method in methods:
|
|
||||||
logging.info(f"Running method {method.__name__}")
|
|
||||||
logging.info(f"Budget left: {budget}")
|
|
||||||
if budget <= 0.0:
|
|
||||||
logging.error(
|
|
||||||
f"Budget has been depleted, stopping. Method {method.__name__} has not been run."
|
|
||||||
)
|
|
||||||
break
|
|
||||||
lm = language_models.ChatGPT(
|
|
||||||
os.path.join(
|
|
||||||
os.path.dirname(__file__),
|
|
||||||
"../../graph_of_thoughts/language_models/config.json",
|
|
||||||
),
|
|
||||||
model_name=lm_name,
|
|
||||||
cache=True,
|
|
||||||
)
|
|
||||||
operations_graph = method()
|
|
||||||
executor = controller.Controller(
|
|
||||||
lm,
|
|
||||||
operations_graph,
|
|
||||||
SetIntersectionPrompter(),
|
|
||||||
SetIntersectionParser(),
|
|
||||||
{
|
|
||||||
"set1": data[1],
|
|
||||||
"set2": data[2],
|
|
||||||
"result": data[3],
|
|
||||||
"current": "",
|
|
||||||
"phase": 0,
|
|
||||||
"method": method.__name__,
|
|
||||||
},
|
|
||||||
)
|
|
||||||
try:
|
|
||||||
executor.run()
|
|
||||||
except Exception as e:
|
|
||||||
logging.error(f"Exception: {e}")
|
|
||||||
path = os.path.join(
|
|
||||||
results_folder,
|
|
||||||
method.__name__,
|
|
||||||
f"{data[0]}.json",
|
|
||||||
)
|
|
||||||
executor.output_graph(path)
|
|
||||||
budget -= lm.cost
|
|
||||||
|
|
||||||
return orig_budget - budget
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
"""
|
|
||||||
Input(x) : a list of 128 numbers between 0 and 255 (inclusive)
|
|
||||||
Input(y) : a list of 128 numbers between 0 and 255 (inclusive)
|
|
||||||
Output(z) : a list of the intersection between x and y
|
|
||||||
Correct : z = intersection(x, y)
|
|
||||||
Input Example:
|
|
||||||
[115, 61, 35, 103, 90, 117, 86, 44, 63, 45, 40, 30, 74, 33, 31, 1, 118, 48, 38, 0, 119, 51, 64, 78, 15, 121, 89, 101, 79, 69, 120, 29, 58, 50, 116, 11, 60, 12, 39, 95, 23, 2, 109, 84, 7, 43, 99, 98, 52, 70, 75, 102, 57, 19, 94, 36, 114, 88, 71, 56, 83, 6, 96, 107]
|
|
||||||
[13, 35, 20, 96, 34, 18, 47, 127, 126, 9, 21, 16, 77, 22, 111, 122, 85, 73, 42, 105, 123, 15, 33, 59, 67, 57, 104, 8, 30, 89, 76, 12, 65, 84, 32, 40, 7, 100, 108, 50, 14, 28, 24, 53, 90, 17, 91, 81, 124, 63, 5, 46, 125, 93, 49, 66, 117, 37, 115, 113, 2, 106, 41, 72]
|
|
||||||
Output Example:
|
|
||||||
[115, 35, 90, 117, 63, 40, 30, 33, 15, 89, 50, 12, 2, 84, 7, 57, 96]
|
|
||||||
"""
|
|
||||||
|
|
||||||
budget = 25
|
|
||||||
samples = [item for item in range(0, 100)]
|
|
||||||
approaches = [io, cot, tot, tot2, got]
|
|
||||||
|
|
||||||
spent = run(samples, approaches, budget, "chatgpt")
|
|
||||||
|
|
||||||
logging.info(f"Spent {spent} out of {budget} budget.")
|
|
||||||
@ -1,99 +0,0 @@
|
|||||||
# Copyright (c) 2023 ETH Zurich.
|
|
||||||
# All rights reserved.
|
|
||||||
#
|
|
||||||
# Use of this source code is governed by a BSD-style license that can be
|
|
||||||
# found in the LICENSE file.
|
|
||||||
#
|
|
||||||
# The source code is adapted from the sorting source code written by
|
|
||||||
# Nils Blach.
|
|
||||||
#
|
|
||||||
# main author: Robert Gerstenberger
|
|
||||||
|
|
||||||
from typing import Dict, List, Set
|
|
||||||
|
|
||||||
|
|
||||||
def string_to_list(string: str) -> List[int]:
|
|
||||||
"""
|
|
||||||
Helper function to convert a list encoded inside a string into a Python
|
|
||||||
list object of integer elements.
|
|
||||||
|
|
||||||
:param string: Input string containing a list.
|
|
||||||
:type string: str
|
|
||||||
:return: List of integer elements.
|
|
||||||
:rtype: List[int]
|
|
||||||
:raise AssertionError: If input string does not contain a list.
|
|
||||||
"""
|
|
||||||
|
|
||||||
assert string[0] == "[" and string[-1] == "]", "String is not a list."
|
|
||||||
return [int(num) for num in string[1:-1].split(",")]
|
|
||||||
|
|
||||||
|
|
||||||
def string_to_set(string: str) -> Set[int]:
|
|
||||||
"""
|
|
||||||
Helper function to convert a list encoded inside a string into a Python
|
|
||||||
set object of integer elements.
|
|
||||||
|
|
||||||
:param string: Input string containing a list.
|
|
||||||
:type string: str
|
|
||||||
:return: Set of integer elements.
|
|
||||||
:rtype: Set[int]
|
|
||||||
:raise AssertionError: If input string does not contain a list.
|
|
||||||
"""
|
|
||||||
|
|
||||||
assert string[0] == "[" and string[-1] == "]", "String is not a list."
|
|
||||||
return {int(num) for num in string[1:-1].split(",")}
|
|
||||||
|
|
||||||
|
|
||||||
def test_set_intersection(state: Dict) -> bool:
|
|
||||||
"""
|
|
||||||
Function to test whether the final solution matches ground truth.
|
|
||||||
|
|
||||||
:param state: Thought state that represents the final solution.
|
|
||||||
:type state: Dict
|
|
||||||
:return: Returns whether the solution matches the ground truth.
|
|
||||||
:rtype: bool
|
|
||||||
"""
|
|
||||||
|
|
||||||
# convert string to list
|
|
||||||
try:
|
|
||||||
correct_list = string_to_list(state["result"])
|
|
||||||
sorted_list = sorted(string_to_list(state["current"]))
|
|
||||||
return sorted_list == correct_list
|
|
||||||
except:
|
|
||||||
return False
|
|
||||||
|
|
||||||
|
|
||||||
def num_errors(state: Dict) -> float:
|
|
||||||
"""
|
|
||||||
Function to locally count the number of errors that serves as a score.
|
|
||||||
|
|
||||||
:param state: Thought state to be scored.
|
|
||||||
:type state: Dict
|
|
||||||
:return: Number of errors.
|
|
||||||
:rtype: float
|
|
||||||
"""
|
|
||||||
|
|
||||||
try:
|
|
||||||
set1 = string_to_set(state["set1"])
|
|
||||||
set2 = string_to_set(state["set2"])
|
|
||||||
if "subset" in state and state["subset"] != "" and state["subset"] is not None:
|
|
||||||
set2 = string_to_set(state["subset"])
|
|
||||||
common = sorted(list(set1 & set2))
|
|
||||||
llm_solution = sorted(string_to_list(state["current"]))
|
|
||||||
num_errors = 0
|
|
||||||
common_idx = 0
|
|
||||||
llm_idx = 0
|
|
||||||
while common_idx < len(common) and llm_idx < len(llm_solution):
|
|
||||||
if common[common_idx] == llm_solution[llm_idx]:
|
|
||||||
common_idx += 1
|
|
||||||
llm_idx += 1
|
|
||||||
elif common[common_idx] < llm_solution[llm_idx]:
|
|
||||||
common_idx += 1
|
|
||||||
num_errors += 1
|
|
||||||
elif common[common_idx] > llm_solution[llm_idx]:
|
|
||||||
llm_idx += 1
|
|
||||||
num_errors += 1
|
|
||||||
num_errors += len(common) - common_idx + len(llm_solution) - llm_idx
|
|
||||||
return num_errors
|
|
||||||
except:
|
|
||||||
return 1000
|
|
||||||
@ -1,46 +0,0 @@
|
|||||||
# Sorting
|
|
||||||
|
|
||||||
The use case in this directory sorts the provided list of
|
|
||||||
numbers containing numbers from 0 to 9 (duplicates allowed).
|
|
||||||
We provide implementations of five different approaches for
|
|
||||||
32, 64 and 128 elements:
|
|
||||||
- IO
|
|
||||||
- Chain-of-Thought (CoT)
|
|
||||||
- Tree of Thought (ToT):
|
|
||||||
- ToT: wider tree, meaning more branches per level
|
|
||||||
- ToT2: tree with more levels, but fewer branches per level
|
|
||||||
- Graph of Thoughts (GoT):
|
|
||||||
- GoT: split into subarrays / sort / merge
|
|
||||||
|
|
||||||
## Data
|
|
||||||
|
|
||||||
We provide input files with 100 precomputed samples for each list
|
|
||||||
length: `sorting_<number of elements>.csv`.
|
|
||||||
|
|
||||||
## Execution
|
|
||||||
|
|
||||||
The files to execute the use case are called
|
|
||||||
`sorting_<number of elements>.py`. In the main body, one can select the
|
|
||||||
specific samples to be run (variable sample) and the approaches
|
|
||||||
(variable approaches). It is also possible to set a budget in dollars
|
|
||||||
(variable budget).
|
|
||||||
The input filename for the samples is currently hardcoded to
|
|
||||||
`sorting_<number of elements>.csv`, but can be updated in the function
|
|
||||||
`run`.
|
|
||||||
|
|
||||||
The Python scripts will create the directory `result`, if it is not
|
|
||||||
already present. In the 'result' directory, another directory is created
|
|
||||||
for each run: `{name of LLM}_{list of approaches}_{day}_{start time}`.
|
|
||||||
Inside each execution specific directory two files (`config.json`,
|
|
||||||
`log.log`) and a separate directory for each selected approach are
|
|
||||||
created. `config.json` contains the configuration of the run: input data,
|
|
||||||
selected approaches, name of the LLM, and the budget. `log.log` contains
|
|
||||||
the prompts and responses of the LLM as well as additional debug data.
|
|
||||||
The approach directories contain a separate json file for every sample
|
|
||||||
and the file contains the Graph Reasoning State (GRS) for that sample.
|
|
||||||
|
|
||||||
## Plot Data
|
|
||||||
|
|
||||||
Change the results directory in line 171 of `plot.py` and update the
|
|
||||||
length parameter in the subsequent line and run `python3 plot.py` to
|
|
||||||
plot your data.
|
|
||||||
@ -1,295 +0,0 @@
|
|||||||
# Sorting 32 Numbers - Prompts and Examples
|
|
||||||
## Prompt Templates
|
|
||||||
### GENERATE: split_prompt
|
|
||||||
Replace `{input}` with the input list of numbers to be split.
|
|
||||||
```
|
|
||||||
<Instruction> Split the following list of 32 numbers into 2 lists of 16 numbers each, the first list should contain the first 16 numbers and the second list the second 16 numbers.
|
|
||||||
Only output the final 2 lists in the following format without any additional text or thoughts!:
|
|
||||||
{{
|
|
||||||
"List 1": [3, 4, 3, 5, 7, 8, 1, ...],
|
|
||||||
"List 2": [2, 9, 2, 4, 7, 1, 5, ...]
|
|
||||||
}} </Instruction>
|
|
||||||
|
|
||||||
<Example>
|
|
||||||
Input: [9, 6, 7, 7, 2, 0, 2, 2, 3, 5, 0, 9, 2, 2, 4, 4, 5, 2, 5, 1, 2, 8, 3, 8, 3, 9, 6, 0, 4, 2, 2, 3]
|
|
||||||
Output:
|
|
||||||
{{
|
|
||||||
"List 1": [9, 6, 7, 7, 2, 0, 2, 2, 3, 5, 0, 9, 2, 2, 4, 4],
|
|
||||||
"List 2": [5, 2, 5, 1, 2, 8, 3, 8, 3, 9, 6, 0, 4, 2, 2, 3]
|
|
||||||
}}
|
|
||||||
</Example>
|
|
||||||
|
|
||||||
Input: {input}
|
|
||||||
```
|
|
||||||
|
|
||||||
### GENERATE: sort_prompt
|
|
||||||
Replace `{input}` with the input list of numbers to be sorted.
|
|
||||||
```
|
|
||||||
<Instruction> Sort the following list of numbers in ascending order. Output only the sorted list of numbers, no additional text. </Instruction>
|
|
||||||
|
|
||||||
<Examples>
|
|
||||||
Input: [5, 1, 0, 1, 2, 0, 4, 8, 1, 9, 5, 1, 3, 3, 9, 7]
|
|
||||||
Output: [0, 0, 1, 1, 1, 1, 2, 3, 3, 4, 5, 5, 7, 8, 9, 9]
|
|
||||||
|
|
||||||
Input: [3, 7, 0, 2, 8, 1, 2, 2, 2, 4, 7, 8, 5, 5, 3, 9, 4, 3, 5, 6, 6, 4, 4, 5, 2, 0, 9, 3, 3, 9, 2, 1]
|
|
||||||
Output: [0, 0, 1, 1, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 9]
|
|
||||||
|
|
||||||
Input: [4, 4, 9, 7, 9, 7, 0, 0, 4, 9, 1, 7, 9, 5, 8, 7, 5, 6, 3, 8, 6, 7, 5, 8, 5, 0, 6, 3, 7, 0, 5, 3, 7, 5, 2, 4, 4, 9, 0, 7, 8, 2, 7, 7, 7, 2, 1, 3, 9, 9, 7, 9, 6, 6, 4, 5, 4, 2, 0, 8, 9, 0, 2, 2]
|
|
||||||
Output: [0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9]
|
|
||||||
</Examples>
|
|
||||||
|
|
||||||
Input: {input}
|
|
||||||
```
|
|
||||||
|
|
||||||
### IMPROVE: improve_prompt
|
|
||||||
Replace `{input}` with the input list of numbers to be sorted and `{incorrectly_sorted}` with the incorrectly sorted list of numbers. `{length}` is the length of the input list.
|
|
||||||
```
|
|
||||||
<Instruction> The following two lists represent an unsorted list of numbers and a sorted variant of that list. The sorted variant is not correct. Fix the sorted variant so that it is correct.
|
|
||||||
Make sure that the output list is sorted in ascending order, has the same number of elements as the input list ({length}), and contains the same elements as the input list. </Instruction>
|
|
||||||
|
|
||||||
<Approach>
|
|
||||||
To fix the incorrectly sorted list follow these steps:
|
|
||||||
1. For each number from 0 to 9, compare the frequency of that number in the incorrectly sorted list to the frequency of that number in the input list.
|
|
||||||
2. Iterate through the incorrectly sorted list and add or remove numbers as needed to make the frequency of each number in the incorrectly sorted list match the frequency of that number in the input list.
|
|
||||||
</Approach>
|
|
||||||
|
|
||||||
<Examples>
|
|
||||||
Input: [3, 7, 0, 2, 8, 1, 2, 2, 2, 4, 7, 8, 5, 5, 3, 9]
|
|
||||||
Incorrectly Sorted: [0, 0, 0, 0, 0, 1, 2, 2, 3, 3, 4, 4, 4, 5, 5, 7, 7, 8, 8, 9, 9, 9, 9]
|
|
||||||
Reason: The incorrectly sorted list contains four extra 0s, two extra 4s and three extra 9s and is missing two 2s.
|
|
||||||
Output: [0, 1, 2, 2, 2, 2, 3, 3, 4, 5, 5, 7, 7, 8, 8, 9]
|
|
||||||
|
|
||||||
Input: [6, 4, 5, 7, 5, 6, 9, 7, 6, 9, 4, 6, 9, 8, 1, 9, 2, 4, 9, 0, 7, 6, 5, 6, 6, 2, 8, 3, 9, 5, 6, 1]
|
|
||||||
Incorrectly Sorted: [0, 1, 1, 2, 2, 3, 4, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 7, 7, 7, 8, 8, 9, 9, 9, 9, 9]
|
|
||||||
Reason: The incorrectly sorted list contains two extra 4s and is missing two 6s and one 9.
|
|
||||||
Output: [0, 1, 1, 2, 2, 3, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 8, 8, 9, 9, 9, 9, 9, 9]
|
|
||||||
|
|
||||||
Input: [4, 4, 9, 7, 9, 7, 0, 0, 4, 9, 1, 7, 9, 5, 8, 7, 5, 6, 3, 8, 6, 7, 5, 8, 5, 0, 6, 3, 7, 0, 5, 3, 7, 5, 2, 4, 4, 9, 0, 7, 8, 2, 7, 7, 7, 2, 1, 3, 9, 9, 7, 9, 6, 6, 4, 5, 4, 2, 0, 8, 9, 0, 2, 2]
|
|
||||||
Incorrectly Sorted: [0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 2, 2, 2, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 5, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9]
|
|
||||||
Reason: The incorrectly sorted list contains one extra 8 and is missing two 2s, one 3, three 4s, two 5s, one 6, six 7s and one 9.
|
|
||||||
Output: [0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9]
|
|
||||||
</Examples>
|
|
||||||
|
|
||||||
Input: {input}
|
|
||||||
Incorrectly Sorted: {incorrectly_sorted}
|
|
||||||
```
|
|
||||||
|
|
||||||
### AGGREGATE: aggregate_prompt
|
|
||||||
Replace `{input_list1}` and `{input_list2}` with the input lists of numbers to be merged. `{length}` is the length of each input list and `{length_combined}` is the length of the combined list.
|
|
||||||
```
|
|
||||||
<Instruction> Merge the following 2 sorted lists of length {length} each, into one sorted list of length {length_combined} using a merge sort style approach.
|
|
||||||
Only output the final merged list without any additional text or thoughts!:</Instruction>
|
|
||||||
|
|
||||||
<Approach>
|
|
||||||
To merge the two lists in a merge-sort style approach, foloow these steps:
|
|
||||||
1. Compare the first element of both lists.
|
|
||||||
2. Append the smaller element to the merged list and move to the next element in the list from which the smaller element came.
|
|
||||||
3. Repeat steps 1 and 2 until one of the lists is empty.
|
|
||||||
4. Append the remaining elements of the non-empty list to the merged list.
|
|
||||||
</Approach>
|
|
||||||
|
|
||||||
Merge the following two lists into one sorted list:
|
|
||||||
1: {input_list1}
|
|
||||||
2: {input_list2}
|
|
||||||
|
|
||||||
Merged list:
|
|
||||||
```
|
|
||||||
|
|
||||||
## Complete Example Prompts
|
|
||||||
### The GoO Summarised
|
|
||||||
1. Split the input list into two sub-lists of equal size (split prompt)
|
|
||||||
2. For each sub-list: Sort the sub-list (sort prompt) five times; score each sort attempt; keep the best
|
|
||||||
3. Merge the sorted sub-lists into one fully sorted list (merge prompt) 10 times; score each merge attempt; keep the best
|
|
||||||
4. Fix any potential mistakes in the sorted list (improve prompt) 10 times; score each improvement attempt; keep the best
|
|
||||||
|
|
||||||
### Input
|
|
||||||
```
|
|
||||||
[8, 7, 1, 1, 1, 1, 3, 3, 0, 9, 4, 1, 0, 2, 5, 1, 0, 5, 6, 7, 1, 4, 5, 9, 4, 6, 2, 5, 8, 6, 2, 6]
|
|
||||||
```
|
|
||||||
|
|
||||||
### Step 1 - Prompt
|
|
||||||
```
|
|
||||||
<Instruction> Split the following list of 32 numbers into 2 lists of 16 numbers each, the first list should contain the first 16 numbers and the second list the second 16 numbers.
|
|
||||||
Only output the final 2 lists in the following format without any additional text or thoughts!:
|
|
||||||
{
|
|
||||||
"List 1": [3, 4, 3, 5, 7, 8, 1, ...],
|
|
||||||
"List 2": [2, 9, 2, 4, 7, 1, 5, ...]
|
|
||||||
} </Instruction>
|
|
||||||
|
|
||||||
<Example>
|
|
||||||
Input: [9, 6, 7, 7, 2, 0, 2, 2, 3, 5, 0, 9, 2, 2, 4, 4, 5, 2, 5, 1, 2, 8, 3, 8, 3, 9, 6, 0, 4, 2, 2, 3]
|
|
||||||
Output:
|
|
||||||
{
|
|
||||||
"List 1": [9, 6, 7, 7, 2, 0, 2, 2, 3, 5, 0, 9, 2, 2, 4, 4],
|
|
||||||
"List 2": [5, 2, 5, 1, 2, 8, 3, 8, 3, 9, 6, 0, 4, 2, 2, 3]
|
|
||||||
}
|
|
||||||
</Example>
|
|
||||||
|
|
||||||
Input: [8, 7, 1, 1, 1, 1, 3, 3, 0, 9, 4, 1, 0, 2, 5, 1, 0, 5, 6, 7, 1, 4, 5, 9, 4, 6, 2, 5, 8, 6, 2, 6]
|
|
||||||
```
|
|
||||||
|
|
||||||
### Step 1 - Response
|
|
||||||
```
|
|
||||||
{
|
|
||||||
"List 1": [8, 7, 1, 1, 1, 1, 3, 3, 0, 9, 4, 1, 0, 2, 5, 1],
|
|
||||||
"List 2": [0, 5, 6, 7, 1, 4, 5, 9, 4, 6, 2, 5, 8, 6, 2, 6]
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### Step 2a - Prompt
|
|
||||||
```
|
|
||||||
<Instruction> Sort the following list of numbers in ascending order. Output only the sorted list of numbers, no additional text. </Instruction>
|
|
||||||
|
|
||||||
<Examples>
|
|
||||||
Input: [5, 1, 0, 1, 2, 0, 4, 8, 1, 9, 5, 1, 3, 3, 9, 7]
|
|
||||||
Output: [0, 0, 1, 1, 1, 1, 2, 3, 3, 4, 5, 5, 7, 8, 9, 9]
|
|
||||||
|
|
||||||
Input: [3, 7, 0, 2, 8, 1, 2, 2, 2, 4, 7, 8, 5, 5, 3, 9, 4, 3, 5, 6, 6, 4, 4, 5, 2, 0, 9, 3, 3, 9, 2, 1]
|
|
||||||
Output: [0, 0, 1, 1, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 9]
|
|
||||||
|
|
||||||
Input: [4, 4, 9, 7, 9, 7, 0, 0, 4, 9, 1, 7, 9, 5, 8, 7, 5, 6, 3, 8, 6, 7, 5, 8, 5, 0, 6, 3, 7, 0, 5, 3, 7, 5, 2, 4, 4, 9, 0, 7, 8, 2, 7, 7, 7, 2, 1, 3, 9, 9, 7, 9, 6, 6, 4, 5, 4, 2, 0, 8, 9, 0, 2, 2]
|
|
||||||
Output: [0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9]
|
|
||||||
</Examples>
|
|
||||||
|
|
||||||
Input: [8, 7, 1, 1, 1, 1, 3, 3, 0, 9, 4, 1, 0, 2, 5, 1]
|
|
||||||
```
|
|
||||||
|
|
||||||
### Step 2a - 5 Responses
|
|
||||||
```
|
|
||||||
1. [0, 0, 1, 1, 1, 1, 1, 1, 2, 3, 3, 4, 5, 7, 8, 9]
|
|
||||||
2. [0, 0, 1, 1, 1, 1, 1, 2, 3, 3, 4, 5, 7, 8, 9]
|
|
||||||
3. [0, 0, 1, 1, 1, 1, 1, 2, 3, 3, 4, 5, 7, 8, 9]
|
|
||||||
4. [0, 0, 1, 1, 1, 1, 1, 2, 3, 3, 4, 5, 7, 8, 9]
|
|
||||||
5. [0, 0, 1, 1, 1, 1, 1, 2, 3, 3, 4, 5, 7, 8, 9]
|
|
||||||
```
|
|
||||||
|
|
||||||
### Step 2b - Prompt
|
|
||||||
```
|
|
||||||
<Instruction> Sort the following list of numbers in ascending order. Output only the sorted list of numbers, no additional text. </Instruction>
|
|
||||||
|
|
||||||
<Examples>
|
|
||||||
Input: [5, 1, 0, 1, 2, 0, 4, 8, 1, 9, 5, 1, 3, 3, 9, 7]
|
|
||||||
Output: [0, 0, 1, 1, 1, 1, 2, 3, 3, 4, 5, 5, 7, 8, 9, 9]
|
|
||||||
|
|
||||||
Input: [3, 7, 0, 2, 8, 1, 2, 2, 2, 4, 7, 8, 5, 5, 3, 9, 4, 3, 5, 6, 6, 4, 4, 5, 2, 0, 9, 3, 3, 9, 2, 1]
|
|
||||||
Output: [0, 0, 1, 1, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 9]
|
|
||||||
|
|
||||||
Input: [4, 4, 9, 7, 9, 7, 0, 0, 4, 9, 1, 7, 9, 5, 8, 7, 5, 6, 3, 8, 6, 7, 5, 8, 5, 0, 6, 3, 7, 0, 5, 3, 7, 5, 2, 4, 4, 9, 0, 7, 8, 2, 7, 7, 7, 2, 1, 3, 9, 9, 7, 9, 6, 6, 4, 5, 4, 2, 0, 8, 9, 0, 2, 2]
|
|
||||||
Output: [0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9]
|
|
||||||
</Examples>
|
|
||||||
|
|
||||||
Input: [0, 5, 6, 7, 1, 4, 5, 9, 4, 6, 2, 5, 8, 6, 2, 6]
|
|
||||||
```
|
|
||||||
|
|
||||||
### Step 2b - 5 Responses
|
|
||||||
```
|
|
||||||
1. [0, 1, 2, 2, 4, 4, 5, 5, 5, 6, 6, 6, 7, 8, 9]
|
|
||||||
2. [0, 1, 2, 2, 4, 4, 5, 5, 5, 6, 6, 6, 7, 8, 9]
|
|
||||||
3. [0, 1, 2, 2, 4, 4, 5, 5, 5, 6, 6, 6, 7, 8, 9]
|
|
||||||
4. [0, 1, 2, 2, 4, 4, 5, 5, 5, 6, 6, 6, 6, 7, 8, 9]
|
|
||||||
5. [0, 1, 2, 2, 4, 4, 5, 5, 5, 6, 6, 6, 7, 8, 9]
|
|
||||||
```
|
|
||||||
|
|
||||||
### Step 3 - Prompt
|
|
||||||
```
|
|
||||||
<Instruction> Merge the following 2 sorted lists of length 16 each, into one sorted list of length 32 using a merge sort style approach.
|
|
||||||
Only output the final merged list without any additional text or thoughts!:</Instruction>
|
|
||||||
|
|
||||||
<Approach>
|
|
||||||
To merge the two lists in a merge-sort style approach, follow these steps:
|
|
||||||
1. Compare the first element of both lists.
|
|
||||||
2. Append the smaller element to the merged list and move to the next element in the list from which the smaller element came.
|
|
||||||
3. Repeat steps 1 and 2 until one of the lists is empty.
|
|
||||||
4. Append the remaining elements of the non-empty list to the merged list.
|
|
||||||
</Approach>
|
|
||||||
|
|
||||||
Merge the following two lists into one sorted list:
|
|
||||||
1: [0, 0, 1, 1, 1, 1, 1, 1, 2, 3, 3, 4, 5, 7, 8, 9]
|
|
||||||
2: [0, 1, 2, 2, 4, 4, 5, 5, 5, 6, 6, 6, 6, 7, 8, 9]
|
|
||||||
|
|
||||||
Merged list:
|
|
||||||
```
|
|
||||||
|
|
||||||
### Step 3 - 10 Responses
|
|
||||||
```
|
|
||||||
1. [0, 0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 3, 3, 4, 4, 4, 5, 5, 5, 6, 6, 6, 6, 7, 7, 8, 8, 9, 9]
|
|
||||||
2. [0, 0, 0, 1, 1, 1, 1, 1, 2, 2, 2, 3, 3, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 7, 7, 8, 8, 9, 9]
|
|
||||||
3. [0, 0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 3, 3, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 6, 7, 7, 8, 8, 9, 9]
|
|
||||||
4. [0, 0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 3, 3, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 6, 7, 7, 8, 8, 9]
|
|
||||||
5. [0, 0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 3, 3, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 6, 7, 7, 8, 8, 9, 9]
|
|
||||||
6. [0, 0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 3, 3, 4, 4, 4, 5, 5, 5, 6, 6, 6, 6, 7, 7, 8, 8, 9, 9]
|
|
||||||
7. [0, 0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 3, 3, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 6, 7, 7, 8, 8, 9, 9]
|
|
||||||
8. [0, 0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 3, 3, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 7, 7, 8, 8, 9, 9]
|
|
||||||
9. [0, 0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 3, 3, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 7, 7, 8, 8, 9, 9]
|
|
||||||
10. [0, 0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 3, 3, 4, 4, 4, 5, 5, 5, 6, 6, 6, 6, 7, 7, 8, 8, 9, 9]
|
|
||||||
```
|
|
||||||
|
|
||||||
### Step 4 - Prompt
|
|
||||||
```
|
|
||||||
<Instruction> The following two lists represent an unsorted list of numbers and a sorted variant of that list. The sorted variant is not correct. Fix the sorted variant so that it is correct.
|
|
||||||
Make sure that the output list is sorted in ascending order, has the same number of elements as the input list (32), and contains the same elements as the input list. </Instruction>
|
|
||||||
|
|
||||||
<Approach>
|
|
||||||
To fix the incorrectly sorted list follow these steps:
|
|
||||||
1. For each number from 0 to 9, compare the frequency of that number in the incorrectly sorted list to the frequency of that number in the input list.
|
|
||||||
2. Iterate through the incorrectly sorted list and add or remove numbers as needed to make the frequency of each number in the incorrectly sorted list match the frequency of that number in the input list.
|
|
||||||
</Approach>
|
|
||||||
|
|
||||||
<Examples>
|
|
||||||
Input: [3, 7, 0, 2, 8, 1, 2, 2, 2, 4, 7, 8, 5, 5, 3, 9]
|
|
||||||
Incorrectly Sorted: [0, 0, 0, 0, 0, 1, 2, 2, 3, 3, 4, 4, 4, 5, 5, 7, 7, 8, 8, 9, 9, 9, 9]
|
|
||||||
Reason: The incorrectly sorted list contains four extra 0s, two extra 4s and three extra 9s and is missing two 2s.
|
|
||||||
Output: [0, 1, 2, 2, 2, 2, 3, 3, 4, 5, 5, 7, 7, 8, 8, 9]
|
|
||||||
|
|
||||||
Input: [6, 4, 5, 7, 5, 6, 9, 7, 6, 9, 4, 6, 9, 8, 1, 9, 2, 4, 9, 0, 7, 6, 5, 6, 6, 2, 8, 3, 9, 5, 6, 1]
|
|
||||||
Incorrectly Sorted: [0, 1, 1, 2, 2, 3, 4, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 7, 7, 7, 8, 8, 9, 9, 9, 9, 9]
|
|
||||||
Reason: The incorrectly sorted list contains two extra 4s and is missing two 6s and one 9.
|
|
||||||
Output: [0, 1, 1, 2, 2, 3, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 8, 8, 9, 9, 9, 9, 9, 9]
|
|
||||||
|
|
||||||
Input: [4, 4, 9, 7, 9, 7, 0, 0, 4, 9, 1, 7, 9, 5, 8, 7, 5, 6, 3, 8, 6, 7, 5, 8, 5, 0, 6, 3, 7, 0, 5, 3, 7, 5, 2, 4, 4, 9, 0, 7, 8, 2, 7, 7, 7, 2, 1, 3, 9, 9, 7, 9, 6, 6, 4, 5, 4, 2, 0, 8, 9, 0, 2, 2]
|
|
||||||
Incorrectly Sorted: [0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 2, 2, 2, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 5, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9]
|
|
||||||
Reason: The incorrectly sorted list contains one extra 8 and is missing two 2s, one 3, three 4s, two 5s, one 6, six 7s and one 9.
|
|
||||||
Output: [0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9]
|
|
||||||
</Examples>
|
|
||||||
|
|
||||||
Input: [8, 7, 1, 1, 1, 1, 3, 3, 0, 9, 4, 1, 0, 2, 5, 1, 0, 5, 6, 7, 1, 4, 5, 9, 4, 6, 2, 5, 8, 6, 2, 6]
|
|
||||||
Incorrectly Sorted: [0, 0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 3, 3, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 7, 7, 8, 8, 9, 9]
|
|
||||||
```
|
|
||||||
|
|
||||||
### Step 4 - 10 Responses
|
|
||||||
```
|
|
||||||
1. Reason: The incorrectly sorted list is missing two 0s, one 3, one 4, one 6 and one 9. The incorrectly sorted list also contains one extra 1, one extra 2, and one extra 5.
|
|
||||||
|
|
||||||
Output: [0, 0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 3, 3, 4, 5, 5, 5, 6, 6, 6, 6, 7, 7, 8, 8, 9, 9]
|
|
||||||
|
|
||||||
Note: The output list has the same number of elements as the input list, is sorted in ascending order, and contains the same elements as the input list.
|
|
||||||
|
|
||||||
2. Output: [0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 3, 3, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 6, 7, 7, 8, 8, 9, 9]
|
|
||||||
|
|
||||||
3. Output: [0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 4, 4, 4, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 7, 7, 7, 8, 8, 8, 9, 9, 9]
|
|
||||||
|
|
||||||
4. Reason: The incorrectly sorted list is missing two 9s and one 0.
|
|
||||||
Output: [0, 0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 3, 3, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 7, 7, 8, 8, 9, 9, 9, 9]
|
|
||||||
|
|
||||||
5. Reason: The incorrectly sorted list is missing one 0, one 3, one 4, four 6s, and one 9.
|
|
||||||
|
|
||||||
Output: [0, 0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 3, 3, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 8, 8, 8, 9, 9, 9, 9]
|
|
||||||
|
|
||||||
6. Output: [0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 6, 7, 7, 7, 7, 8, 8, 8, 9, 9, 9, 9]
|
|
||||||
|
|
||||||
7. Output: [0, 0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 7, 7, 8, 8, 9, 9]
|
|
||||||
|
|
||||||
8. Reason: The incorrectly sorted list is missing three 0s, two 3s, two 4s, one 5, two 6s, and one 9.
|
|
||||||
Output: [0, 0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 7, 7, 8, 8, 9]
|
|
||||||
|
|
||||||
9. Reason: The incorrectly sorted list is missing one 0, three 2s, two 3s, two 4s, three 5s, two 6s, one 7, and one 9.
|
|
||||||
Output: [0, 0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 7, 8, 8, 9, 9]
|
|
||||||
|
|
||||||
10. Reason: The incorrectly sorted list is missing three 1s, one 3, one 4, two 5s, one 6, one 7, one 8, and one 9.
|
|
||||||
Output: [0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 3, 3, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 7, 7, 8, 8, 9, 9]
|
|
||||||
```
|
|
||||||
|
|
||||||
### Final Result
|
|
||||||
```
|
|
||||||
[0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 3, 3, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 7, 7, 8, 8, 9, 9]
|
|
||||||
```
|
|
||||||
@ -1,186 +0,0 @@
|
|||||||
# Copyright (c) 2023 ETH Zurich.
|
|
||||||
# All rights reserved.
|
|
||||||
#
|
|
||||||
# Use of this source code is governed by a BSD-style license that can be
|
|
||||||
# found in the LICENSE file.
|
|
||||||
#
|
|
||||||
# main author: Nils Blach
|
|
||||||
# contributions: Robert Gerstenberger
|
|
||||||
|
|
||||||
import json
|
|
||||||
import os
|
|
||||||
import matplotlib.pyplot as plt
|
|
||||||
|
|
||||||
|
|
||||||
def get_complete_results(base_directory):
|
|
||||||
results_complete = {}
|
|
||||||
for folder_name in os.listdir(base_directory):
|
|
||||||
folder_path = os.path.join(base_directory, folder_name)
|
|
||||||
if os.path.isdir(folder_path):
|
|
||||||
results_complete[folder_name] = []
|
|
||||||
for file_name in os.listdir(folder_path):
|
|
||||||
if file_name.endswith(".json"):
|
|
||||||
file_path = os.path.join(folder_path, file_name)
|
|
||||||
with open(file_path, "r") as f:
|
|
||||||
data = json.load(f)
|
|
||||||
results_complete[folder_name].append(
|
|
||||||
{"key": int(file_name.split(".")[0]), "data": data}
|
|
||||||
)
|
|
||||||
for key in results_complete.keys():
|
|
||||||
results_complete[key] = sorted(
|
|
||||||
results_complete[key], key=lambda x: x["key"]
|
|
||||||
)
|
|
||||||
return results_complete
|
|
||||||
|
|
||||||
|
|
||||||
def get_final_scores(results_complete):
|
|
||||||
scores = {}
|
|
||||||
for method in results_complete.keys():
|
|
||||||
scores[method] = []
|
|
||||||
for result in results_complete[method]:
|
|
||||||
score = 100
|
|
||||||
solved = False
|
|
||||||
cost = 1
|
|
||||||
prompt_tokens = 0
|
|
||||||
completion_tokens = 0
|
|
||||||
for op in result["data"]:
|
|
||||||
if "operation" in op and op["operation"] == "ground_truth_evaluator":
|
|
||||||
try:
|
|
||||||
score = min(op["scores"])
|
|
||||||
solved = any(op["problem_solved"])
|
|
||||||
except:
|
|
||||||
continue
|
|
||||||
if "cost" in op:
|
|
||||||
cost = op["cost"]
|
|
||||||
prompt_tokens = op["prompt_tokens"]
|
|
||||||
completion_tokens = op["completion_tokens"]
|
|
||||||
scores[method].append(
|
|
||||||
[result["key"], score, solved, prompt_tokens, completion_tokens, cost]
|
|
||||||
)
|
|
||||||
scores[method] = sorted(scores[method], key=lambda x: x[0])
|
|
||||||
return scores
|
|
||||||
|
|
||||||
|
|
||||||
def get_plotting_data(base_directory):
|
|
||||||
results_complete = get_complete_results(base_directory)
|
|
||||||
scores = get_final_scores(results_complete)
|
|
||||||
results_plotting = {
|
|
||||||
method: {
|
|
||||||
"scores": [x[1] for x in scores[method]],
|
|
||||||
"solved": sum([1 for x in scores[method] if x[2]]),
|
|
||||||
"costs": [x[5] for x in scores[method]],
|
|
||||||
}
|
|
||||||
for method in scores.keys()
|
|
||||||
}
|
|
||||||
return results_plotting
|
|
||||||
|
|
||||||
|
|
||||||
def plot_results(
|
|
||||||
results,
|
|
||||||
methods_order=["io", "cot", "tot", "tot2", "got"],
|
|
||||||
model="GPT-3.5",
|
|
||||||
length=32,
|
|
||||||
y_lower=0,
|
|
||||||
cost_upper=0.0,
|
|
||||||
display_solved=True,
|
|
||||||
annotation_offset=0,
|
|
||||||
display_left_ylabel=False,
|
|
||||||
display_right_ylabel=False,
|
|
||||||
):
|
|
||||||
methods_order = [method for method in methods_order if method in results]
|
|
||||||
# Extract scores based on the order
|
|
||||||
scores_ordered = [
|
|
||||||
[
|
|
||||||
min(score, length)
|
|
||||||
for score in results[method]["scores"]
|
|
||||||
if score != 100 and score != 300
|
|
||||||
]
|
|
||||||
for method in methods_order
|
|
||||||
]
|
|
||||||
total_costs = [sum(results[method]["costs"]) for method in methods_order]
|
|
||||||
|
|
||||||
# Create figure and axis
|
|
||||||
fig, ax = plt.subplots(dpi=150, figsize=(2.5, 5))
|
|
||||||
|
|
||||||
# Create boxplots
|
|
||||||
positions = range(1, len(methods_order) + 1)
|
|
||||||
ax.boxplot(scores_ordered, positions=positions)
|
|
||||||
|
|
||||||
fig_fontsize = 12
|
|
||||||
|
|
||||||
# Set the ticks and labels
|
|
||||||
method_labels = ["IO", "CoT", "ToT", "ToT2", "GoT"]
|
|
||||||
plt.yticks(fontsize=fig_fontsize)
|
|
||||||
ax.set_xticks(range(1, len(methods_order) + 1))
|
|
||||||
ax.set_xticks(range(1, len(methods_order) + 1))
|
|
||||||
ax.set_xticklabels(method_labels, fontsize=fig_fontsize)
|
|
||||||
|
|
||||||
y_upper = length
|
|
||||||
|
|
||||||
range_increase = 1
|
|
||||||
if display_solved:
|
|
||||||
if length < 48:
|
|
||||||
range_increase = 2
|
|
||||||
elif length < 96:
|
|
||||||
range_increase = 4
|
|
||||||
else:
|
|
||||||
range_increase = 8
|
|
||||||
|
|
||||||
ax.set_ylim(y_lower, y_upper + range_increase)
|
|
||||||
ax1_yticks = range(
|
|
||||||
y_lower, y_upper + 1, 2 if length < 48 else (4 if length < 96 else 8)
|
|
||||||
)
|
|
||||||
ax.set_yticks(ax1_yticks)
|
|
||||||
if display_left_ylabel:
|
|
||||||
ax.set_ylabel(f"#incorrectly sorted elements; the lower the better")
|
|
||||||
|
|
||||||
ax.set_title(f"{length} elements")
|
|
||||||
|
|
||||||
ax2 = ax.twinx()
|
|
||||||
ax2.bar(positions, total_costs, alpha=0.5, color="blue", label="Total Cost ($)")
|
|
||||||
ax2.yaxis.set_tick_params(colors="#1919ff", labelsize=fig_fontsize)
|
|
||||||
if cost_upper > 0:
|
|
||||||
ax2.set_ylim(0, cost_upper)
|
|
||||||
number_of_ticks = len(ax.get_yticks())
|
|
||||||
tick_interval = cost_upper / (number_of_ticks)
|
|
||||||
ax2_ticks = [tick_interval * i for i in range(number_of_ticks)]
|
|
||||||
|
|
||||||
# Set custom tick positions for ax2
|
|
||||||
ax2.set_yticks(ax2_ticks)
|
|
||||||
|
|
||||||
if display_right_ylabel:
|
|
||||||
ax2.set_ylabel(
|
|
||||||
"Total Cost ($); the lower the better",
|
|
||||||
color="#1919ff",
|
|
||||||
fontsize=fig_fontsize,
|
|
||||||
)
|
|
||||||
|
|
||||||
if display_solved:
|
|
||||||
annotation_height = y_upper + annotation_offset
|
|
||||||
count = 1
|
|
||||||
for method in methods_order:
|
|
||||||
if method not in results:
|
|
||||||
continue
|
|
||||||
solved = results[method]["solved"]
|
|
||||||
ax.text(
|
|
||||||
count,
|
|
||||||
annotation_height,
|
|
||||||
f"{solved}",
|
|
||||||
ha="center",
|
|
||||||
va="bottom",
|
|
||||||
fontsize=fig_fontsize,
|
|
||||||
)
|
|
||||||
count += 1
|
|
||||||
|
|
||||||
model = model.replace(".", "").replace("-", "").lower()
|
|
||||||
fig.savefig(f"sorting_{model}_{length}.pdf", bbox_inches="tight")
|
|
||||||
|
|
||||||
|
|
||||||
plot_results(
|
|
||||||
get_plotting_data("results/"),
|
|
||||||
length=32,
|
|
||||||
display_solved=True,
|
|
||||||
model="GPT-3.5",
|
|
||||||
display_left_ylabel=True,
|
|
||||||
display_right_ylabel=True,
|
|
||||||
)
|
|
||||||
@ -1,101 +0,0 @@
|
|||||||
ID,Unsorted,Sorted
|
|
||||||
0,"[0, 0, 5, 9, 0, 7, 9, 9, 1, 2, 6, 1, 1, 9, 0, 1, 3, 5, 2, 3, 5, 6, 0, 2, 7, 4, 6, 2, 9, 7, 5, 9]","[0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 4, 5, 5, 5, 5, 6, 6, 6, 7, 7, 7, 9, 9, 9, 9, 9, 9]"
|
|
||||||
1,"[5, 6, 2, 3, 1, 4, 9, 0, 5, 7, 0, 7, 1, 3, 2, 4, 5, 5, 6, 6, 3, 6, 4, 4, 2, 3, 7, 1, 7, 0, 2, 5]","[0, 0, 0, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 5, 6, 6, 6, 6, 7, 7, 7, 7, 9]"
|
|
||||||
2,"[9, 6, 7, 0, 7, 7, 4, 7, 9, 3, 6, 5, 0, 8, 1, 8, 6, 1, 5, 3, 3, 5, 3, 4, 2, 2, 4, 5, 8, 6, 0, 3]","[0, 0, 0, 1, 1, 2, 2, 3, 3, 3, 3, 3, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 7, 7, 7, 7, 8, 8, 8, 9, 9]"
|
|
||||||
3,"[8, 7, 1, 1, 1, 1, 3, 3, 0, 9, 4, 1, 0, 2, 5, 1, 0, 5, 6, 7, 1, 4, 5, 9, 4, 6, 2, 5, 8, 6, 2, 6]","[0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 3, 3, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 7, 7, 8, 8, 9, 9]"
|
|
||||||
4,"[5, 3, 9, 6, 3, 2, 9, 8, 2, 4, 2, 1, 8, 1, 8, 0, 0, 3, 7, 4, 4, 9, 9, 5, 4, 6, 5, 4, 3, 4, 1, 1]","[0, 0, 1, 1, 1, 1, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 5, 5, 5, 6, 6, 7, 8, 8, 8, 9, 9, 9, 9]"
|
|
||||||
5,"[4, 6, 9, 5, 8, 6, 5, 4, 0, 4, 2, 6, 3, 6, 8, 5, 5, 2, 3, 5, 0, 9, 8, 6, 5, 5, 5, 8, 1, 5, 8, 7]","[0, 0, 1, 2, 2, 3, 3, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 7, 8, 8, 8, 8, 8, 9, 9]"
|
|
||||||
6,"[9, 6, 4, 1, 9, 6, 3, 1, 1, 7, 3, 9, 2, 6, 2, 9, 9, 4, 2, 7, 4, 0, 2, 3, 1, 7, 9, 8, 3, 0, 7, 3]","[0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 3, 4, 4, 4, 6, 6, 6, 7, 7, 7, 7, 8, 9, 9, 9, 9, 9, 9]"
|
|
||||||
7,"[9, 2, 4, 7, 4, 8, 8, 4, 7, 5, 8, 9, 0, 1, 0, 0, 0, 0, 7, 3, 4, 7, 7, 7, 9, 8, 6, 3, 2, 6, 6, 8]","[0, 0, 0, 0, 0, 1, 2, 2, 3, 3, 4, 4, 4, 4, 5, 6, 6, 6, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 9, 9, 9]"
|
|
||||||
8,"[6, 6, 9, 1, 7, 3, 5, 9, 7, 9, 1, 4, 8, 2, 5, 7, 0, 8, 8, 8, 9, 8, 1, 0, 5, 6, 5, 9, 2, 1, 7, 7]","[0, 0, 1, 1, 1, 1, 2, 2, 3, 4, 5, 5, 5, 5, 6, 6, 6, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9]"
|
|
||||||
9,"[0, 3, 5, 9, 3, 1, 0, 2, 9, 9, 8, 1, 7, 6, 8, 1, 7, 1, 7, 1, 3, 2, 4, 2, 8, 4, 6, 8, 3, 2, 7, 0]","[0, 0, 0, 1, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 5, 6, 6, 7, 7, 7, 7, 8, 8, 8, 8, 9, 9, 9]"
|
|
||||||
10,"[5, 2, 2, 3, 5, 5, 6, 2, 6, 9, 9, 9, 9, 7, 5, 1, 4, 5, 0, 4, 0, 3, 6, 6, 8, 1, 2, 2, 2, 3, 7, 2]","[0, 0, 1, 1, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 4, 4, 5, 5, 5, 5, 5, 6, 6, 6, 6, 7, 7, 8, 9, 9, 9, 9]"
|
|
||||||
11,"[4, 1, 7, 1, 8, 8, 7, 5, 7, 1, 6, 1, 6, 1, 4, 4, 0, 0, 3, 3, 5, 6, 9, 3, 4, 9, 9, 2, 3, 0, 5, 5]","[0, 0, 0, 1, 1, 1, 1, 1, 2, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 7, 7, 7, 8, 8, 9, 9, 9]"
|
|
||||||
12,"[6, 6, 5, 3, 1, 7, 2, 8, 7, 2, 0, 5, 4, 5, 0, 5, 8, 0, 4, 8, 5, 9, 0, 0, 2, 3, 4, 3, 2, 0, 4, 4]","[0, 0, 0, 0, 0, 0, 1, 2, 2, 2, 2, 3, 3, 3, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 6, 6, 7, 7, 8, 8, 8, 9]"
|
|
||||||
13,"[7, 0, 9, 2, 0, 6, 5, 9, 3, 5, 6, 2, 0, 1, 4, 7, 5, 7, 0, 9, 8, 2, 1, 3, 4, 4, 8, 4, 2, 7, 9, 7]","[0, 0, 0, 0, 1, 1, 2, 2, 2, 2, 3, 3, 4, 4, 4, 4, 5, 5, 5, 6, 6, 7, 7, 7, 7, 7, 8, 8, 9, 9, 9, 9]"
|
|
||||||
14,"[3, 4, 7, 1, 9, 8, 3, 0, 6, 4, 4, 1, 9, 9, 9, 2, 8, 4, 7, 4, 7, 9, 7, 6, 9, 7, 2, 1, 4, 5, 3, 1]","[0, 1, 1, 1, 1, 2, 2, 3, 3, 3, 4, 4, 4, 4, 4, 4, 5, 6, 6, 7, 7, 7, 7, 7, 8, 8, 9, 9, 9, 9, 9, 9]"
|
|
||||||
15,"[8, 7, 0, 7, 8, 6, 3, 6, 6, 9, 8, 0, 3, 9, 3, 1, 7, 7, 4, 2, 5, 8, 2, 8, 7, 2, 1, 2, 6, 5, 5, 2]","[0, 0, 1, 1, 2, 2, 2, 2, 2, 3, 3, 3, 4, 5, 5, 5, 6, 6, 6, 6, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 9, 9]"
|
|
||||||
16,"[9, 7, 1, 8, 7, 9, 5, 0, 9, 0, 6, 4, 5, 7, 0, 9, 2, 5, 8, 4, 3, 9, 6, 4, 6, 1, 7, 2, 2, 5, 3, 0]","[0, 0, 0, 0, 1, 1, 2, 2, 2, 3, 3, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 7, 7, 7, 7, 8, 8, 9, 9, 9, 9, 9]"
|
|
||||||
17,"[7, 5, 1, 8, 1, 5, 9, 6, 0, 9, 8, 2, 1, 2, 7, 4, 5, 7, 3, 7, 8, 5, 1, 0, 4, 7, 8, 1, 4, 1, 4, 2]","[0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 3, 4, 4, 4, 4, 5, 5, 5, 5, 6, 7, 7, 7, 7, 7, 8, 8, 8, 8, 9, 9]"
|
|
||||||
18,"[4, 3, 6, 3, 7, 8, 5, 6, 5, 6, 8, 3, 6, 5, 5, 6, 0, 8, 4, 9, 2, 0, 4, 3, 2, 2, 5, 0, 8, 5, 2, 6]","[0, 0, 0, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 7, 8, 8, 8, 8, 9]"
|
|
||||||
19,"[9, 7, 8, 2, 6, 2, 9, 7, 6, 8, 1, 0, 6, 3, 5, 0, 3, 6, 2, 4, 2, 0, 6, 8, 3, 1, 9, 7, 7, 2, 6, 4]","[0, 0, 0, 1, 1, 2, 2, 2, 2, 2, 3, 3, 3, 4, 4, 5, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 8, 8, 8, 9, 9, 9]"
|
|
||||||
20,"[8, 7, 6, 4, 5, 6, 4, 0, 2, 1, 8, 0, 2, 0, 8, 1, 8, 4, 4, 9, 7, 6, 0, 9, 6, 6, 1, 2, 5, 4, 7, 3]","[0, 0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 4, 4, 4, 4, 4, 5, 5, 6, 6, 6, 6, 6, 7, 7, 7, 8, 8, 8, 8, 9, 9]"
|
|
||||||
21,"[6, 6, 4, 0, 7, 9, 5, 2, 0, 4, 2, 4, 8, 4, 0, 3, 7, 0, 4, 3, 1, 3, 0, 1, 9, 6, 7, 4, 9, 1, 3, 0]","[0, 0, 0, 0, 0, 0, 1, 1, 1, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 5, 6, 6, 6, 7, 7, 7, 8, 9, 9, 9]"
|
|
||||||
22,"[6, 8, 0, 2, 7, 7, 9, 7, 9, 0, 6, 6, 8, 9, 1, 4, 2, 9, 6, 6, 2, 0, 0, 7, 5, 2, 5, 1, 3, 5, 2, 6]","[0, 0, 0, 0, 1, 1, 2, 2, 2, 2, 2, 3, 4, 5, 5, 5, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 8, 8, 9, 9, 9, 9]"
|
|
||||||
23,"[8, 5, 0, 3, 6, 5, 6, 5, 4, 8, 0, 0, 7, 3, 9, 2, 9, 2, 0, 4, 1, 0, 5, 5, 8, 2, 8, 4, 4, 3, 5, 1]","[0, 0, 0, 0, 0, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 6, 6, 7, 8, 8, 8, 8, 9, 9]"
|
|
||||||
24,"[8, 0, 5, 9, 3, 9, 2, 1, 7, 2, 1, 9, 7, 4, 8, 4, 2, 2, 9, 2, 1, 7, 3, 4, 4, 3, 3, 0, 8, 1, 8, 6]","[0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 5, 6, 7, 7, 7, 8, 8, 8, 8, 9, 9, 9, 9]"
|
|
||||||
25,"[9, 8, 8, 9, 8, 1, 9, 6, 5, 8, 0, 8, 5, 5, 5, 4, 3, 5, 8, 4, 0, 3, 3, 6, 6, 1, 4, 7, 0, 7, 2, 8]","[0, 0, 0, 1, 1, 2, 3, 3, 3, 4, 4, 4, 5, 5, 5, 5, 5, 6, 6, 6, 7, 7, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9]"
|
|
||||||
26,"[3, 0, 9, 6, 0, 7, 0, 2, 7, 8, 8, 9, 7, 5, 6, 0, 2, 3, 4, 9, 6, 4, 8, 9, 3, 5, 6, 2, 0, 1, 5, 3]","[0, 0, 0, 0, 0, 1, 2, 2, 2, 3, 3, 3, 3, 4, 4, 5, 5, 5, 6, 6, 6, 6, 7, 7, 7, 8, 8, 8, 9, 9, 9, 9]"
|
|
||||||
27,"[6, 4, 9, 0, 5, 9, 0, 8, 5, 5, 9, 2, 6, 4, 8, 1, 9, 1, 3, 9, 4, 6, 6, 9, 7, 8, 8, 8, 9, 1, 1, 4]","[0, 0, 1, 1, 1, 1, 2, 3, 4, 4, 4, 4, 5, 5, 5, 6, 6, 6, 6, 7, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
28,"[3, 9, 6, 5, 2, 1, 2, 7, 6, 9, 6, 4, 7, 2, 9, 1, 6, 6, 1, 6, 4, 7, 1, 0, 5, 7, 6, 2, 2, 2, 4, 8]","[0, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 3, 4, 4, 4, 5, 5, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 8, 9, 9, 9]"
|
|
||||||
29,"[3, 9, 5, 1, 3, 9, 0, 8, 0, 1, 9, 5, 8, 7, 3, 1, 6, 6, 2, 2, 0, 2, 7, 2, 5, 5, 5, 1, 6, 6, 9, 0]","[0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 5, 5, 5, 5, 5, 6, 6, 6, 6, 7, 7, 8, 8, 9, 9, 9, 9]"
|
|
||||||
30,"[5, 1, 2, 2, 1, 9, 1, 7, 7, 1, 4, 1, 1, 3, 7, 0, 2, 0, 7, 8, 0, 1, 9, 2, 0, 4, 5, 6, 6, 1, 8, 9]","[0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 3, 4, 4, 5, 5, 6, 6, 7, 7, 7, 7, 8, 8, 9, 9, 9]"
|
|
||||||
31,"[4, 2, 6, 4, 7, 2, 5, 2, 8, 4, 9, 5, 7, 7, 3, 2, 5, 5, 0, 2, 2, 8, 2, 8, 2, 6, 5, 3, 3, 5, 8, 4]","[0, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 6, 6, 7, 7, 7, 8, 8, 8, 8, 9]"
|
|
||||||
32,"[6, 6, 2, 2, 9, 8, 7, 2, 5, 3, 5, 7, 1, 3, 4, 0, 3, 6, 9, 1, 6, 1, 0, 4, 1, 2, 6, 7, 7, 6, 9, 1]","[0, 0, 1, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 4, 4, 5, 5, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 8, 9, 9, 9]"
|
|
||||||
33,"[6, 3, 9, 8, 1, 4, 2, 1, 0, 0, 1, 3, 7, 9, 8, 1, 3, 1, 5, 3, 9, 7, 8, 9, 3, 9, 4, 0, 2, 7, 3, 9]","[0, 0, 0, 1, 1, 1, 1, 1, 2, 2, 3, 3, 3, 3, 3, 3, 4, 4, 5, 6, 7, 7, 7, 8, 8, 8, 9, 9, 9, 9, 9, 9]"
|
|
||||||
34,"[2, 6, 4, 5, 8, 1, 9, 3, 6, 3, 9, 7, 3, 3, 8, 4, 6, 7, 2, 7, 3, 2, 3, 3, 5, 5, 6, 4, 1, 6, 6, 2]","[1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 5, 5, 5, 6, 6, 6, 6, 6, 6, 7, 7, 7, 8, 8, 9, 9]"
|
|
||||||
35,"[6, 6, 0, 8, 0, 6, 8, 4, 9, 8, 3, 8, 8, 1, 7, 1, 5, 0, 9, 1, 9, 9, 8, 1, 5, 3, 2, 2, 7, 0, 5, 6]","[0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 3, 3, 4, 5, 5, 5, 6, 6, 6, 6, 7, 7, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9]"
|
|
||||||
36,"[8, 9, 1, 5, 7, 4, 2, 6, 2, 8, 8, 9, 3, 0, 6, 7, 7, 8, 6, 5, 8, 3, 3, 0, 1, 6, 4, 0, 8, 8, 1, 7]","[0, 0, 0, 1, 1, 1, 2, 2, 3, 3, 3, 4, 4, 5, 5, 6, 6, 6, 6, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 9, 9]"
|
|
||||||
37,"[9, 5, 1, 1, 3, 1, 0, 4, 4, 6, 1, 9, 4, 8, 0, 9, 4, 0, 5, 1, 9, 5, 6, 7, 9, 1, 2, 7, 1, 3, 9, 6]","[0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 2, 3, 3, 4, 4, 4, 4, 5, 5, 5, 6, 6, 6, 7, 7, 8, 9, 9, 9, 9, 9, 9]"
|
|
||||||
38,"[4, 1, 1, 8, 1, 7, 9, 1, 5, 5, 1, 9, 0, 8, 7, 7, 9, 0, 7, 0, 4, 9, 8, 5, 9, 6, 6, 2, 6, 5, 1, 1]","[0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 2, 4, 4, 5, 5, 5, 5, 6, 6, 6, 7, 7, 7, 7, 8, 8, 8, 9, 9, 9, 9, 9]"
|
|
||||||
39,"[6, 1, 6, 5, 7, 1, 6, 3, 8, 7, 6, 1, 3, 8, 6, 9, 3, 1, 1, 9, 4, 8, 3, 1, 9, 4, 5, 9, 5, 4, 1, 8]","[1, 1, 1, 1, 1, 1, 1, 3, 3, 3, 3, 4, 4, 4, 5, 5, 5, 6, 6, 6, 6, 6, 7, 7, 8, 8, 8, 8, 9, 9, 9, 9]"
|
|
||||||
40,"[2, 1, 6, 3, 9, 3, 8, 3, 2, 3, 9, 9, 5, 4, 3, 2, 1, 9, 3, 4, 3, 3, 5, 4, 4, 4, 9, 7, 8, 3, 0, 8]","[0, 1, 1, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 5, 5, 6, 7, 8, 8, 8, 9, 9, 9, 9, 9]"
|
|
||||||
41,"[6, 5, 6, 4, 3, 7, 3, 0, 6, 9, 1, 0, 1, 7, 0, 1, 8, 6, 5, 9, 5, 9, 4, 2, 5, 3, 8, 8, 8, 0, 9, 7]","[0, 0, 0, 0, 1, 1, 1, 2, 3, 3, 3, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 7, 7, 7, 8, 8, 8, 8, 9, 9, 9, 9]"
|
|
||||||
42,"[9, 2, 6, 8, 4, 0, 5, 0, 8, 6, 1, 6, 6, 6, 7, 6, 5, 2, 3, 4, 3, 9, 2, 9, 3, 2, 0, 0, 1, 4, 6, 6]","[0, 0, 0, 0, 1, 1, 2, 2, 2, 2, 3, 3, 3, 4, 4, 4, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 7, 8, 8, 9, 9, 9]"
|
|
||||||
43,"[7, 7, 7, 8, 7, 9, 4, 3, 5, 5, 1, 7, 7, 4, 0, 5, 6, 2, 7, 3, 2, 2, 3, 2, 0, 8, 6, 7, 8, 4, 6, 4]","[0, 0, 1, 2, 2, 2, 2, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 9]"
|
|
||||||
44,"[0, 1, 6, 5, 4, 6, 9, 3, 7, 0, 0, 1, 1, 6, 7, 1, 9, 9, 4, 5, 9, 1, 2, 2, 7, 5, 6, 1, 9, 1, 2, 4]","[0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 3, 4, 4, 4, 5, 5, 5, 6, 6, 6, 6, 7, 7, 7, 9, 9, 9, 9, 9]"
|
|
||||||
45,"[1, 8, 6, 5, 0, 6, 2, 4, 2, 2, 0, 8, 7, 6, 1, 3, 1, 9, 7, 5, 9, 8, 1, 8, 2, 7, 7, 3, 7, 7, 8, 7]","[0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 4, 5, 5, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 9, 9]"
|
|
||||||
46,"[4, 7, 0, 6, 4, 5, 0, 3, 0, 3, 4, 6, 5, 2, 6, 5, 5, 9, 7, 1, 0, 8, 0, 4, 9, 1, 3, 5, 1, 7, 2, 0]","[0, 0, 0, 0, 0, 0, 1, 1, 1, 2, 2, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 5, 6, 6, 6, 7, 7, 7, 8, 9, 9]"
|
|
||||||
47,"[9, 6, 0, 1, 8, 4, 9, 6, 3, 6, 0, 7, 0, 8, 9, 3, 6, 6, 9, 3, 2, 2, 2, 9, 4, 9, 3, 8, 1, 9, 6, 1]","[0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3, 3, 4, 4, 6, 6, 6, 6, 6, 6, 7, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
48,"[7, 3, 4, 8, 4, 2, 8, 3, 1, 0, 7, 3, 1, 9, 1, 8, 0, 1, 9, 8, 3, 2, 4, 1, 7, 3, 1, 3, 2, 0, 4, 0]","[0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 7, 7, 7, 8, 8, 8, 8, 9, 9]"
|
|
||||||
49,"[2, 3, 7, 6, 5, 7, 5, 2, 8, 6, 8, 3, 7, 2, 3, 8, 6, 5, 4, 5, 3, 6, 8, 3, 0, 9, 2, 1, 7, 5, 5, 8]","[0, 1, 2, 2, 2, 2, 3, 3, 3, 3, 3, 4, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 7, 7, 7, 7, 8, 8, 8, 8, 8, 9]"
|
|
||||||
50,"[4, 0, 9, 1, 1, 6, 5, 0, 6, 4, 8, 1, 6, 9, 6, 1, 8, 1, 3, 1, 1, 5, 8, 3, 2, 2, 9, 8, 4, 6, 6, 9]","[0, 0, 1, 1, 1, 1, 1, 1, 1, 2, 2, 3, 3, 4, 4, 4, 5, 5, 6, 6, 6, 6, 6, 6, 8, 8, 8, 8, 9, 9, 9, 9]"
|
|
||||||
51,"[1, 9, 8, 5, 8, 5, 2, 4, 1, 4, 8, 0, 7, 9, 0, 5, 0, 5, 6, 4, 6, 2, 2, 3, 4, 2, 8, 8, 6, 7, 8, 9]","[0, 0, 0, 1, 1, 2, 2, 2, 2, 3, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 7, 7, 8, 8, 8, 8, 8, 8, 9, 9, 9]"
|
|
||||||
52,"[3, 5, 2, 8, 8, 5, 4, 6, 0, 7, 2, 0, 2, 5, 2, 6, 0, 1, 8, 0, 9, 2, 8, 2, 8, 7, 3, 5, 6, 8, 0, 8]","[0, 0, 0, 0, 0, 1, 2, 2, 2, 2, 2, 2, 3, 3, 4, 5, 5, 5, 5, 6, 6, 6, 7, 7, 8, 8, 8, 8, 8, 8, 8, 9]"
|
|
||||||
53,"[2, 4, 1, 8, 9, 8, 2, 4, 0, 7, 9, 6, 1, 8, 1, 5, 6, 5, 8, 8, 7, 0, 4, 7, 2, 1, 9, 4, 4, 5, 5, 6]","[0, 0, 1, 1, 1, 1, 2, 2, 2, 4, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 7, 7, 7, 8, 8, 8, 8, 8, 9, 9, 9]"
|
|
||||||
54,"[4, 9, 4, 3, 2, 0, 9, 3, 9, 7, 7, 5, 9, 4, 3, 1, 6, 2, 5, 1, 9, 7, 2, 1, 9, 4, 6, 0, 7, 4, 9, 4]","[0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 4, 4, 4, 4, 5, 5, 6, 6, 7, 7, 7, 7, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
55,"[2, 3, 6, 7, 2, 2, 1, 6, 4, 0, 0, 9, 1, 6, 9, 1, 1, 2, 5, 1, 8, 1, 7, 1, 2, 1, 6, 0, 1, 6, 4, 1]","[0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 3, 4, 4, 5, 6, 6, 6, 6, 6, 7, 7, 8, 9, 9]"
|
|
||||||
56,"[4, 7, 7, 5, 3, 5, 9, 9, 3, 1, 4, 7, 8, 3, 4, 7, 7, 3, 3, 7, 0, 0, 2, 9, 6, 5, 3, 7, 3, 0, 1, 1]","[0, 0, 0, 1, 1, 1, 2, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 5, 5, 5, 6, 7, 7, 7, 7, 7, 7, 7, 8, 9, 9, 9]"
|
|
||||||
57,"[0, 9, 8, 9, 1, 3, 8, 0, 5, 7, 8, 5, 3, 4, 2, 5, 7, 8, 9, 6, 5, 1, 4, 5, 1, 3, 1, 8, 9, 2, 6, 9]","[0, 0, 1, 1, 1, 1, 2, 2, 3, 3, 3, 4, 4, 5, 5, 5, 5, 5, 6, 6, 7, 7, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9]"
|
|
||||||
58,"[6, 1, 5, 1, 3, 5, 4, 0, 1, 0, 6, 8, 9, 8, 1, 7, 6, 8, 7, 4, 6, 2, 9, 0, 8, 0, 7, 2, 6, 6, 7, 7]","[0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 3, 4, 4, 5, 5, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 8, 8, 8, 8, 9, 9]"
|
|
||||||
59,"[2, 4, 0, 5, 4, 0, 6, 6, 1, 7, 8, 1, 7, 3, 4, 0, 9, 2, 8, 8, 8, 1, 1, 1, 7, 3, 1, 6, 0, 9, 1, 3]","[0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 2, 2, 3, 3, 3, 4, 4, 4, 5, 6, 6, 6, 7, 7, 7, 8, 8, 8, 8, 9, 9]"
|
|
||||||
60,"[3, 8, 0, 9, 5, 6, 6, 8, 5, 2, 9, 4, 8, 2, 3, 5, 7, 6, 1, 7, 1, 4, 3, 9, 4, 5, 9, 5, 1, 9, 9, 9]","[0, 1, 1, 1, 2, 2, 3, 3, 3, 4, 4, 4, 5, 5, 5, 5, 5, 6, 6, 6, 7, 7, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
61,"[8, 8, 0, 2, 4, 3, 1, 9, 2, 9, 1, 1, 3, 7, 9, 9, 9, 1, 5, 2, 1, 3, 1, 5, 7, 7, 8, 2, 8, 8, 8, 2]","[0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 3, 3, 3, 4, 5, 5, 7, 7, 7, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9]"
|
|
||||||
62,"[5, 0, 7, 6, 2, 6, 6, 7, 0, 5, 6, 3, 9, 5, 4, 9, 0, 0, 4, 9, 0, 3, 1, 4, 7, 5, 9, 8, 6, 0, 7, 3]","[0, 0, 0, 0, 0, 0, 1, 2, 3, 3, 3, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 6, 7, 7, 7, 7, 8, 9, 9, 9, 9]"
|
|
||||||
63,"[0, 0, 4, 5, 1, 1, 1, 0, 2, 2, 5, 2, 5, 0, 0, 2, 1, 4, 6, 2, 0, 8, 8, 0, 5, 4, 3, 6, 8, 1, 0, 0]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 3, 4, 4, 4, 5, 5, 5, 5, 6, 6, 8, 8, 8]"
|
|
||||||
64,"[3, 7, 4, 1, 6, 1, 9, 8, 1, 9, 7, 8, 8, 4, 6, 4, 3, 0, 2, 0, 0, 7, 0, 8, 1, 5, 3, 5, 2, 2, 0, 6]","[0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 4, 5, 5, 6, 6, 6, 7, 7, 7, 8, 8, 8, 8, 9, 9]"
|
|
||||||
65,"[3, 3, 7, 3, 2, 5, 7, 1, 4, 7, 0, 7, 1, 2, 5, 9, 6, 9, 5, 5, 0, 3, 2, 4, 1, 2, 0, 5, 5, 0, 1, 8]","[0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 5, 5, 5, 5, 5, 5, 6, 7, 7, 7, 7, 8, 9, 9]"
|
|
||||||
66,"[9, 1, 9, 1, 3, 4, 4, 5, 0, 1, 9, 8, 0, 7, 1, 8, 7, 7, 6, 0, 0, 6, 1, 4, 4, 5, 8, 5, 4, 4, 4, 8]","[0, 0, 0, 0, 1, 1, 1, 1, 1, 3, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 6, 6, 7, 7, 7, 8, 8, 8, 8, 9, 9, 9]"
|
|
||||||
67,"[0, 7, 5, 5, 0, 3, 0, 7, 3, 9, 8, 3, 1, 2, 1, 5, 3, 6, 8, 7, 2, 9, 9, 6, 9, 2, 0, 1, 7, 9, 2, 0]","[0, 0, 0, 0, 0, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 5, 5, 5, 6, 6, 7, 7, 7, 7, 8, 8, 9, 9, 9, 9, 9]"
|
|
||||||
68,"[1, 1, 6, 1, 1, 6, 0, 5, 0, 0, 8, 2, 5, 9, 6, 7, 1, 3, 9, 0, 9, 7, 0, 0, 3, 2, 2, 4, 0, 3, 6, 5]","[0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 2, 2, 2, 3, 3, 3, 4, 5, 5, 5, 6, 6, 6, 6, 7, 7, 8, 9, 9, 9]"
|
|
||||||
69,"[3, 4, 2, 5, 5, 8, 4, 8, 5, 5, 8, 3, 4, 9, 9, 8, 4, 9, 9, 9, 9, 3, 9, 0, 5, 8, 3, 7, 8, 6, 1, 1]","[0, 1, 1, 2, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 5, 6, 7, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
70,"[6, 4, 5, 0, 8, 3, 9, 7, 0, 2, 0, 2, 8, 5, 8, 2, 6, 2, 1, 9, 4, 5, 3, 3, 4, 0, 5, 7, 7, 2, 4, 0]","[0, 0, 0, 0, 0, 1, 2, 2, 2, 2, 2, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 7, 7, 7, 8, 8, 8, 9, 9]"
|
|
||||||
71,"[0, 4, 8, 5, 2, 4, 6, 7, 2, 6, 2, 8, 3, 6, 6, 2, 0, 9, 7, 7, 5, 1, 0, 8, 6, 0, 7, 7, 0, 3, 2, 6]","[0, 0, 0, 0, 0, 1, 2, 2, 2, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 8, 8, 8, 9]"
|
|
||||||
72,"[5, 1, 7, 2, 4, 5, 0, 2, 7, 2, 2, 9, 7, 0, 5, 8, 5, 2, 2, 1, 2, 6, 9, 7, 8, 8, 4, 5, 1, 8, 2, 8]","[0, 0, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 4, 4, 5, 5, 5, 5, 5, 6, 7, 7, 7, 7, 8, 8, 8, 8, 8, 9, 9]"
|
|
||||||
73,"[8, 9, 8, 8, 3, 5, 5, 3, 6, 1, 3, 2, 6, 3, 1, 3, 3, 5, 1, 7, 6, 0, 9, 6, 8, 1, 1, 0, 9, 5, 5, 6]","[0, 0, 1, 1, 1, 1, 1, 2, 3, 3, 3, 3, 3, 3, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 7, 8, 8, 8, 8, 9, 9, 9]"
|
|
||||||
74,"[4, 3, 3, 0, 4, 7, 9, 4, 3, 5, 3, 5, 4, 2, 0, 2, 1, 5, 8, 8, 4, 4, 3, 4, 3, 0, 1, 6, 7, 3, 2, 8]","[0, 0, 0, 1, 1, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 6, 7, 7, 8, 8, 8, 9]"
|
|
||||||
75,"[7, 1, 9, 4, 7, 4, 0, 5, 0, 7, 2, 0, 8, 5, 8, 4, 2, 2, 5, 9, 2, 3, 8, 2, 7, 4, 6, 6, 8, 6, 3, 3]","[0, 0, 0, 1, 2, 2, 2, 2, 2, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 6, 6, 6, 7, 7, 7, 7, 8, 8, 8, 8, 9, 9]"
|
|
||||||
76,"[7, 3, 1, 7, 2, 0, 4, 5, 6, 5, 8, 3, 2, 8, 8, 2, 2, 5, 3, 9, 1, 3, 4, 9, 2, 7, 7, 3, 6, 8, 3, 7]","[0, 1, 1, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 4, 4, 5, 5, 5, 6, 6, 7, 7, 7, 7, 7, 8, 8, 8, 8, 9, 9]"
|
|
||||||
77,"[9, 4, 1, 3, 1, 3, 4, 8, 8, 8, 7, 2, 7, 9, 5, 0, 2, 6, 4, 8, 1, 3, 5, 4, 1, 8, 7, 4, 7, 7, 0, 8]","[0, 0, 1, 1, 1, 1, 2, 2, 3, 3, 3, 4, 4, 4, 4, 4, 5, 5, 6, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 9, 9]"
|
|
||||||
78,"[3, 5, 5, 9, 6, 6, 6, 2, 9, 0, 3, 0, 2, 1, 2, 6, 5, 8, 4, 8, 5, 9, 9, 5, 7, 0, 6, 8, 9, 3, 3, 5]","[0, 0, 0, 1, 2, 2, 2, 3, 3, 3, 3, 4, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 7, 8, 8, 8, 9, 9, 9, 9, 9]"
|
|
||||||
79,"[9, 5, 9, 0, 4, 5, 6, 3, 0, 1, 4, 3, 1, 5, 9, 9, 3, 2, 9, 5, 5, 2, 9, 9, 9, 7, 8, 9, 8, 0, 2, 8]","[0, 0, 0, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 5, 5, 5, 5, 5, 6, 7, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
80,"[9, 3, 1, 0, 5, 7, 6, 1, 8, 9, 5, 9, 1, 9, 9, 9, 9, 2, 2, 4, 3, 0, 6, 1, 9, 2, 1, 3, 5, 2, 5, 1]","[0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 4, 5, 5, 5, 5, 6, 6, 7, 8, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
81,"[8, 1, 1, 3, 0, 2, 8, 0, 9, 5, 3, 7, 7, 6, 8, 5, 2, 9, 3, 2, 6, 4, 2, 1, 7, 9, 4, 0, 1, 9, 3, 0]","[0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 7, 8, 8, 8, 9, 9, 9, 9]"
|
|
||||||
82,"[0, 1, 4, 1, 5, 7, 4, 9, 3, 7, 5, 3, 6, 5, 1, 4, 1, 2, 2, 0, 0, 7, 5, 3, 5, 4, 6, 2, 1, 4, 1, 3]","[0, 0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 6, 6, 7, 7, 7, 9]"
|
|
||||||
83,"[2, 2, 4, 0, 5, 7, 4, 5, 9, 5, 7, 5, 1, 5, 9, 5, 4, 1, 7, 9, 2, 4, 4, 1, 4, 3, 7, 9, 0, 6, 1, 1]","[0, 0, 1, 1, 1, 1, 1, 2, 2, 2, 3, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 6, 7, 7, 7, 7, 9, 9, 9, 9]"
|
|
||||||
84,"[2, 5, 4, 9, 4, 5, 3, 8, 1, 5, 0, 4, 5, 5, 7, 4, 6, 6, 7, 7, 1, 9, 6, 5, 8, 6, 3, 6, 8, 9, 0, 7]","[0, 0, 1, 1, 2, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 7, 7, 7, 7, 8, 8, 8, 9, 9, 9]"
|
|
||||||
85,"[0, 4, 3, 0, 6, 1, 1, 9, 2, 1, 3, 6, 5, 1, 2, 3, 0, 1, 6, 7, 8, 4, 1, 0, 2, 1, 2, 3, 4, 5, 6, 1]","[0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 5, 5, 6, 6, 6, 6, 7, 8, 9]"
|
|
||||||
86,"[4, 6, 0, 6, 5, 1, 7, 7, 0, 8, 0, 4, 6, 0, 8, 5, 0, 1, 8, 8, 0, 1, 0, 9, 4, 5, 5, 3, 0, 1, 9, 0]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 3, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 7, 7, 8, 8, 8, 8, 9, 9]"
|
|
||||||
87,"[6, 0, 1, 6, 6, 0, 7, 9, 2, 5, 8, 4, 5, 8, 1, 4, 8, 0, 6, 4, 0, 7, 1, 2, 3, 9, 4, 8, 7, 3, 8, 2]","[0, 0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 4, 4, 4, 4, 5, 5, 6, 6, 6, 6, 7, 7, 7, 8, 8, 8, 8, 8, 9, 9]"
|
|
||||||
88,"[5, 1, 5, 3, 7, 6, 5, 0, 8, 6, 6, 4, 6, 7, 2, 0, 1, 9, 5, 2, 4, 1, 3, 0, 4, 6, 7, 8, 0, 7, 2, 8]","[0, 0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 6, 7, 7, 7, 7, 8, 8, 8, 9]"
|
|
||||||
89,"[5, 2, 2, 3, 8, 5, 0, 0, 6, 7, 3, 3, 7, 7, 6, 6, 8, 2, 0, 1, 6, 3, 3, 9, 9, 9, 1, 9, 5, 6, 4, 2]","[0, 0, 0, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 3, 4, 5, 5, 5, 6, 6, 6, 6, 6, 7, 7, 7, 8, 8, 9, 9, 9, 9]"
|
|
||||||
90,"[3, 0, 3, 3, 0, 9, 8, 7, 3, 9, 7, 0, 4, 0, 9, 4, 3, 7, 5, 1, 9, 5, 6, 6, 6, 3, 6, 3, 2, 8, 6, 8]","[0, 0, 0, 0, 1, 2, 3, 3, 3, 3, 3, 3, 3, 4, 4, 5, 5, 6, 6, 6, 6, 6, 7, 7, 7, 8, 8, 8, 9, 9, 9, 9]"
|
|
||||||
91,"[2, 4, 6, 0, 9, 4, 3, 5, 7, 4, 8, 8, 8, 1, 8, 5, 8, 0, 2, 6, 2, 9, 4, 0, 0, 8, 5, 0, 5, 1, 2, 8]","[0, 0, 0, 0, 0, 1, 1, 2, 2, 2, 2, 3, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 7, 8, 8, 8, 8, 8, 8, 8, 9, 9]"
|
|
||||||
92,"[6, 5, 3, 4, 8, 2, 5, 9, 1, 3, 9, 6, 7, 4, 4, 7, 9, 7, 2, 2, 2, 8, 5, 8, 6, 3, 8, 5, 0, 8, 9, 1]","[0, 1, 1, 2, 2, 2, 2, 3, 3, 3, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 7, 7, 7, 8, 8, 8, 8, 8, 9, 9, 9, 9]"
|
|
||||||
93,"[3, 8, 3, 9, 8, 4, 0, 7, 3, 9, 9, 2, 6, 0, 3, 5, 1, 3, 9, 3, 2, 0, 8, 8, 7, 3, 6, 1, 5, 4, 7, 7]","[0, 0, 0, 1, 1, 2, 2, 3, 3, 3, 3, 3, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 7, 7, 8, 8, 8, 8, 9, 9, 9, 9]"
|
|
||||||
94,"[2, 7, 8, 3, 3, 8, 7, 9, 2, 8, 0, 6, 9, 5, 8, 4, 2, 8, 3, 6, 3, 4, 1, 4, 8, 5, 5, 6, 0, 0, 7, 6]","[0, 0, 0, 1, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 5, 5, 5, 6, 6, 6, 6, 7, 7, 7, 8, 8, 8, 8, 8, 8, 9, 9]"
|
|
||||||
95,"[8, 1, 9, 6, 1, 7, 1, 2, 6, 9, 0, 6, 0, 6, 8, 2, 3, 5, 8, 7, 9, 0, 9, 1, 7, 5, 4, 3, 7, 1, 8, 8]","[0, 0, 0, 1, 1, 1, 1, 1, 2, 2, 3, 3, 4, 5, 5, 6, 6, 6, 6, 7, 7, 7, 7, 8, 8, 8, 8, 8, 9, 9, 9, 9]"
|
|
||||||
96,"[3, 7, 6, 2, 0, 4, 8, 7, 1, 1, 1, 1, 8, 2, 4, 1, 4, 7, 4, 0, 8, 4, 9, 5, 0, 0, 6, 8, 3, 2, 1, 2]","[0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 4, 4, 4, 4, 4, 5, 6, 6, 7, 7, 7, 8, 8, 8, 8, 9]"
|
|
||||||
97,"[3, 1, 2, 8, 6, 7, 0, 2, 3, 1, 6, 7, 9, 3, 1, 9, 7, 4, 5, 5, 0, 7, 8, 3, 3, 1, 4, 4, 2, 1, 1, 5]","[0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 3, 3, 3, 3, 3, 4, 4, 4, 5, 5, 5, 6, 6, 7, 7, 7, 7, 8, 8, 9, 9]"
|
|
||||||
98,"[5, 7, 1, 7, 2, 4, 1, 8, 4, 6, 9, 2, 0, 2, 3, 9, 3, 9, 7, 8, 3, 1, 1, 3, 9, 8, 5, 0, 2, 7, 0, 2]","[0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 5, 5, 6, 7, 7, 7, 7, 8, 8, 8, 9, 9, 9, 9]"
|
|
||||||
99,"[9, 4, 4, 7, 2, 3, 9, 8, 8, 3, 9, 8, 7, 1, 7, 9, 5, 1, 6, 1, 6, 2, 0, 4, 7, 6, 4, 6, 3, 2, 6, 2]","[0, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 4, 4, 4, 4, 5, 6, 6, 6, 6, 6, 7, 7, 7, 7, 8, 8, 8, 9, 9, 9, 9]"
|
|
||||||
|
@ -1,732 +0,0 @@
|
|||||||
# Copyright (c) 2023 ETH Zurich.
|
|
||||||
# All rights reserved.
|
|
||||||
#
|
|
||||||
# Use of this source code is governed by a BSD-style license that can be
|
|
||||||
# found in the LICENSE file.
|
|
||||||
#
|
|
||||||
# main author: Nils Blach
|
|
||||||
|
|
||||||
import os
|
|
||||||
import logging
|
|
||||||
import datetime
|
|
||||||
import json
|
|
||||||
import csv
|
|
||||||
from typing import Dict, List, Callable, Union
|
|
||||||
from graph_of_thoughts import controller, language_models, operations, prompter, parser
|
|
||||||
|
|
||||||
# This is a hack to also allow execution of this file from the examples directory
|
|
||||||
try:
|
|
||||||
from . import utils
|
|
||||||
except ImportError:
|
|
||||||
import utils
|
|
||||||
|
|
||||||
|
|
||||||
class SortingPrompter(prompter.Prompter):
|
|
||||||
"""
|
|
||||||
SortingPrompter provides the generation of prompts specific to the sorting
|
|
||||||
example for the language models.
|
|
||||||
|
|
||||||
Inherits from the Prompter class and implements its abstract methods.
|
|
||||||
"""
|
|
||||||
|
|
||||||
sort_prompt = """<Instruction> Sort the following list of numbers in ascending order. Output only the sorted list of numbers, no additional text. </Instruction>
|
|
||||||
|
|
||||||
<Examples>
|
|
||||||
Input: [5, 1, 0, 1, 2, 0, 4, 8, 1, 9, 5, 1, 3, 3, 9, 7]
|
|
||||||
Output: [0, 0, 1, 1, 1, 1, 2, 3, 3, 4, 5, 5, 7, 8, 9, 9]
|
|
||||||
|
|
||||||
Input: [3, 7, 0, 2, 8, 1, 2, 2, 2, 4, 7, 8, 5, 5, 3, 9, 4, 3, 5, 6, 6, 4, 4, 5, 2, 0, 9, 3, 3, 9, 2, 1]
|
|
||||||
Output: [0, 0, 1, 1, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 9]
|
|
||||||
|
|
||||||
Input: [4, 4, 9, 7, 9, 7, 0, 0, 4, 9, 1, 7, 9, 5, 8, 7, 5, 6, 3, 8, 6, 7, 5, 8, 5, 0, 6, 3, 7, 0, 5, 3, 7, 5, 2, 4, 4, 9, 0, 7, 8, 2, 7, 7, 7, 2, 1, 3, 9, 9, 7, 9, 6, 6, 4, 5, 4, 2, 0, 8, 9, 0, 2, 2]
|
|
||||||
Output: [0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9]
|
|
||||||
</Examples>
|
|
||||||
|
|
||||||
Input: {input}"""
|
|
||||||
|
|
||||||
sort_prompt_cot = """<Instruction> Sort the following list of numbers in ascending order. You can generate any intermediate lists, but the final output should be the sorted list of numbers, prefixed with "Output: ". </Instruction>
|
|
||||||
|
|
||||||
<Approach>
|
|
||||||
To sort the list of numbers follow these steps:
|
|
||||||
1. Split the list of numbers into two to four unsorted sublists, each containing an equal number of elements from the original list (make sure they don't overlap).
|
|
||||||
2. Sort each of the unsorted sublists.
|
|
||||||
3. Merge the sorted sublists into a single sorted list using the merging algorithm from merge sort.
|
|
||||||
</Approach>
|
|
||||||
|
|
||||||
<Examples>
|
|
||||||
Input: [4, 5, 3, 3, 7, 3, 0, 5, 0, 2, 8, 0, 2, 1, 6, 9]
|
|
||||||
Unsorted Subarrays:
|
|
||||||
[4, 5, 3, 3, 7, 3, 0, 5]
|
|
||||||
[0, 2, 8, 0, 2, 1, 6, 9]
|
|
||||||
Sorted Subarrays:
|
|
||||||
[0, 3, 3, 3, 4, 5, 5, 7]
|
|
||||||
[0, 0, 1, 2, 2, 6, 8, 9]
|
|
||||||
Output: [0, 0, 0, 1, 2, 2, 3, 3, 3, 4, 5, 5, 6, 7, 8, 9]
|
|
||||||
|
|
||||||
Input: [6, 4, 5, 7, 5, 6, 9, 7, 6, 9, 4, 6, 9, 8, 1, 9, 2, 4, 9, 0, 7, 6, 5, 6, 6, 2, 8, 3, 9, 5, 6, 1]
|
|
||||||
Unsorted Subarrays:
|
|
||||||
[6, 4, 5, 7, 5, 6, 9, 7, 6, 9, 4, 6, 9, 8, 1, 9]
|
|
||||||
[2, 4, 9, 0, 7, 6, 5, 6, 6, 2, 8, 3, 9, 5, 6, 1]
|
|
||||||
Sorted Subarrays:
|
|
||||||
[1, 4, 4, 5, 5, 6, 6, 6, 6, 7, 7, 8, 9, 9, 9, 9]
|
|
||||||
[0, 1, 2, 2, 3, 4, 5, 5, 6, 6, 6, 6, 7, 8, 9, 9]
|
|
||||||
Output: [0, 1, 1, 2, 2, 3, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 8, 8, 9, 9, 9, 9, 9, 9]
|
|
||||||
|
|
||||||
Input: [3, 7, 0, 2, 8, 1, 2, 2, 2, 4, 7, 8, 5, 5, 3, 9, 4, 3, 5, 6, 6, 4, 4, 5, 2, 0, 9, 3, 3, 9, 2, 1, 9, 3, 1, 8, 1, 8, 6, 0, 1, 6, 1, 7, 4, 4, 6, 3, 3, 7, 9, 3, 6, 0, 3, 4, 5, 6, 6, 9, 9, 9, 7, 3]
|
|
||||||
Unsorted Subarrays:
|
|
||||||
[3, 7, 0, 2, 8, 1, 2, 2, 2, 4, 7, 8, 5, 5, 3, 9]
|
|
||||||
[4, 3, 5, 6, 6, 4, 4, 5, 2, 0, 9, 3, 3, 9, 2, 1]
|
|
||||||
[9, 3, 1, 8, 1, 8, 6, 0, 1, 6, 1, 7, 4, 4, 6, 3]
|
|
||||||
[3, 7, 9, 3, 6, 0, 3, 4, 5, 6, 6, 9, 9, 9, 7, 3]
|
|
||||||
Sorted Subarrays:
|
|
||||||
[0, 1, 2, 2, 2, 2, 3, 3, 4, 5, 5, 7, 7, 8, 8, 9]
|
|
||||||
[0, 1, 2, 2, 3, 3, 3, 4, 4, 4, 5, 5, 6, 6, 9, 9]
|
|
||||||
[0, 1, 1, 1, 1, 3, 3, 4, 4, 6, 6, 6, 7, 8, 8, 9]
|
|
||||||
[0, 3, 3, 3, 3, 4, 5, 6, 6, 6, 7, 7, 9, 9, 9, 9]
|
|
||||||
Output: [0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9]
|
|
||||||
</Examples>
|
|
||||||
|
|
||||||
Input: {input}"""
|
|
||||||
|
|
||||||
tot_improve_prompt = """<Instruction> The following two lists represent an unsorted list of numbers and a sorted variant of that list. The sorted variant is not correct. Fix the sorted variant so that it is correct.
|
|
||||||
Make sure that the output list is sorted in ascending order, has the same number of elements as the input list ({length}), and contains the same elements as the input list. </Instruction>
|
|
||||||
|
|
||||||
<Approach>
|
|
||||||
To fix the incorrectly sorted list follow these steps:
|
|
||||||
1. For each number from 0 to 9, compare the frequency of that number in the incorrectly sorted list to the frequency of that number in the input list.
|
|
||||||
2. Iterate through the incorrectly sorted list and add or remove numbers as needed to make the frequency of each number in the incorrectly sorted list match the frequency of that number in the input list.
|
|
||||||
</Approach>
|
|
||||||
|
|
||||||
<Examples>
|
|
||||||
Input: [3, 7, 0, 2, 8, 1, 2, 2, 2, 4, 7, 8, 5, 5, 3, 9]
|
|
||||||
Incorrectly Sorted: [0, 0, 0, 0, 0, 1, 2, 2, 3, 3, 4, 4, 4, 5, 5, 7, 7, 8, 8, 9, 9, 9, 9]
|
|
||||||
Reason: The incorrectly sorted list contains four extra 0s, two extra 4s and three extra 9s and is missing two 2s.
|
|
||||||
Output: [0, 1, 2, 2, 2, 2, 3, 3, 4, 5, 5, 7, 7, 8, 8, 9]
|
|
||||||
|
|
||||||
Input: [6, 4, 5, 7, 5, 6, 9, 7, 6, 9, 4, 6, 9, 8, 1, 9, 2, 4, 9, 0, 7, 6, 5, 6, 6, 2, 8, 3, 9, 5, 6, 1]
|
|
||||||
Incorrectly Sorted: [0, 1, 1, 2, 2, 3, 4, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 7, 7, 7, 8, 8, 9, 9, 9, 9, 9]
|
|
||||||
Reason: The incorrectly sorted list contains two extra 4s and is missing two 6s and one 9.
|
|
||||||
Output: [0, 1, 1, 2, 2, 3, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 8, 8, 9, 9, 9, 9, 9, 9]
|
|
||||||
|
|
||||||
Input: [4, 4, 9, 7, 9, 7, 0, 0, 4, 9, 1, 7, 9, 5, 8, 7, 5, 6, 3, 8, 6, 7, 5, 8, 5, 0, 6, 3, 7, 0, 5, 3, 7, 5, 2, 4, 4, 9, 0, 7, 8, 2, 7, 7, 7, 2, 1, 3, 9, 9, 7, 9, 6, 6, 4, 5, 4, 2, 0, 8, 9, 0, 2, 2]
|
|
||||||
Incorrectly Sorted: [0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 2, 2, 2, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 5, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9]
|
|
||||||
Reason: The incorrectly sorted list contains one extra 8 and is missing two 2s, one 3, three 4s, two 5s, one 6, six 7s and one 9.
|
|
||||||
Output: [0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9]
|
|
||||||
</Examples>
|
|
||||||
|
|
||||||
Input: {input}
|
|
||||||
Incorrectly Sorted: {incorrectly_sorted}
|
|
||||||
"""
|
|
||||||
|
|
||||||
got_split_prompt = """<Instruction> Split the following list of 32 numbers into 2 lists of 16 numbers each, the first list should contain the first 16 numbers and the second list the second 16 numbers.
|
|
||||||
Only output the final 2 lists in the following format without any additional text or thoughts!:
|
|
||||||
{{
|
|
||||||
"List 1": [3, 4, 3, 5, 7, 8, 1, ...],
|
|
||||||
"List 2": [2, 9, 2, 4, 7, 1, 5, ...]
|
|
||||||
}} </Instruction>
|
|
||||||
|
|
||||||
<Example>
|
|
||||||
Input: [9, 6, 7, 7, 2, 0, 2, 2, 3, 5, 0, 9, 2, 2, 4, 4, 5, 2, 5, 1, 2, 8, 3, 8, 3, 9, 6, 0, 4, 2, 2, 3]
|
|
||||||
Output:
|
|
||||||
{{
|
|
||||||
"List 1": [9, 6, 7, 7, 2, 0, 2, 2, 3, 5, 0, 9, 2, 2, 4, 4],
|
|
||||||
"List 2": [5, 2, 5, 1, 2, 8, 3, 8, 3, 9, 6, 0, 4, 2, 2, 3]
|
|
||||||
}}
|
|
||||||
</Example>
|
|
||||||
|
|
||||||
Input: {input}"""
|
|
||||||
|
|
||||||
got_merge_prompt = """<Instruction> Merge the following 2 sorted lists of length {length1} each, into one sorted list of length {length2} using a merge sort style approach.
|
|
||||||
Only output the final merged list without any additional text or thoughts!:</Instruction>
|
|
||||||
|
|
||||||
<Approach>
|
|
||||||
To merge the two lists in a merge-sort style approach, follow these steps:
|
|
||||||
1. Compare the first element of both lists.
|
|
||||||
2. Append the smaller element to the merged list and move to the next element in the list from which the smaller element came.
|
|
||||||
3. Repeat steps 1 and 2 until one of the lists is empty.
|
|
||||||
4. Append the remaining elements of the non-empty list to the merged list.
|
|
||||||
</Approach>
|
|
||||||
|
|
||||||
Merge the following two lists into one sorted list:
|
|
||||||
1: {input1}
|
|
||||||
2: {input2}
|
|
||||||
|
|
||||||
Merged list:
|
|
||||||
"""
|
|
||||||
|
|
||||||
def aggregation_prompt(self, state_dicts: List[Dict], **kwargs) -> str:
|
|
||||||
"""
|
|
||||||
Generate an aggregation prompt for the language model.
|
|
||||||
|
|
||||||
:param state_dicts: The thought states that should be aggregated.
|
|
||||||
:type state_dicts: List[Dict]
|
|
||||||
:param kwargs: Additional keyword arguments.
|
|
||||||
:return: The aggregation prompt.
|
|
||||||
:rtype: str
|
|
||||||
:raise AssertionError: If not exactly two thought states are provided.
|
|
||||||
"""
|
|
||||||
assert len(state_dicts) == 2, "Expected two states for aggregation prompt."
|
|
||||||
len_input1 = len(utils.string_to_list(state_dicts[0]["current"]))
|
|
||||||
len_input2 = len(utils.string_to_list(state_dicts[1]["current"]))
|
|
||||||
if len_input1 == len_input2:
|
|
||||||
length = len_input1
|
|
||||||
elif len_input1 + len_input2 - 32 <= 16:
|
|
||||||
length = 16
|
|
||||||
else:
|
|
||||||
length = 32
|
|
||||||
|
|
||||||
return self.got_merge_prompt.format(
|
|
||||||
input1=state_dicts[0]["current"],
|
|
||||||
input2=state_dicts[1]["current"],
|
|
||||||
length1=length,
|
|
||||||
length2=length * 2,
|
|
||||||
)
|
|
||||||
|
|
||||||
def generate_prompt(
|
|
||||||
self, num_branches: int, original: str, current: str, method: str, **kwargs
|
|
||||||
) -> str:
|
|
||||||
"""
|
|
||||||
Generate a generate prompt for the language model.
|
|
||||||
|
|
||||||
:param num_branches: The number of responses the prompt should ask the LM to generate.
|
|
||||||
:type num_branches: int
|
|
||||||
:param original: Input list of numbers.
|
|
||||||
:type original: str
|
|
||||||
:param current: Intermediate solution.
|
|
||||||
:type current: str
|
|
||||||
:param method: Method for which the generate prompt is generated.
|
|
||||||
:type method: str
|
|
||||||
:param kwargs: Additional keyword arguments.
|
|
||||||
:return: The generate prompt.
|
|
||||||
:rtype: str
|
|
||||||
:raise AssertionError: If the requested number of branches is not one.
|
|
||||||
"""
|
|
||||||
|
|
||||||
if current is None or current == "":
|
|
||||||
input = original
|
|
||||||
else:
|
|
||||||
input = current
|
|
||||||
if method.startswith("io"):
|
|
||||||
return self.sort_prompt.format(input=input)
|
|
||||||
elif method.startswith("cot"):
|
|
||||||
return self.sort_prompt_cot.format(input=input)
|
|
||||||
elif method.startswith("tot"):
|
|
||||||
if current is None or current == "":
|
|
||||||
return self.sort_prompt.format(input=input)
|
|
||||||
return self.tot_improve_prompt.format(
|
|
||||||
input=original,
|
|
||||||
incorrectly_sorted=current,
|
|
||||||
length=len(utils.string_to_list(original)),
|
|
||||||
)
|
|
||||||
elif method.startswith("got"):
|
|
||||||
if current is None or current == "":
|
|
||||||
return self.got_split_prompt.format(input=input)
|
|
||||||
# if current is just a sublist of the original input, return the split prompt
|
|
||||||
if kwargs["phase"] == 1:
|
|
||||||
return self.sort_prompt.format(input=current)
|
|
||||||
|
|
||||||
if (
|
|
||||||
"unsorted_sublist" in kwargs
|
|
||||||
and kwargs["unsorted_sublist"] != ""
|
|
||||||
and len(kwargs["unsorted_sublist"]) < len(original) - 5
|
|
||||||
):
|
|
||||||
original = kwargs["unsorted_sublist"]
|
|
||||||
|
|
||||||
return self.tot_improve_prompt.format(
|
|
||||||
input=original,
|
|
||||||
incorrectly_sorted=current,
|
|
||||||
length=len(utils.string_to_list(original)),
|
|
||||||
)
|
|
||||||
|
|
||||||
def improve_prompt(self, **kwargs) -> str:
|
|
||||||
"""
|
|
||||||
Generate an improve prompt for the language model.
|
|
||||||
|
|
||||||
:param kwargs: Additional keyword arguments.
|
|
||||||
:return: The improve prompt.
|
|
||||||
:rtype: str
|
|
||||||
"""
|
|
||||||
pass
|
|
||||||
|
|
||||||
def validation_prompt(self, **kwargs) -> str:
|
|
||||||
"""
|
|
||||||
Generate a validation prompt for the language model.
|
|
||||||
|
|
||||||
:param kwargs: Additional keyword arguments.
|
|
||||||
:return: The validation prompt.
|
|
||||||
:rtype: str
|
|
||||||
"""
|
|
||||||
pass
|
|
||||||
|
|
||||||
def score_prompt(self, state_dicts: List[Dict], **kwargs) -> str:
|
|
||||||
"""
|
|
||||||
Generate a score prompt for the language model.
|
|
||||||
|
|
||||||
:param state_dicts: The thought states that should be scored,
|
|
||||||
if more than one, they should be scored together.
|
|
||||||
:type state_dicts: List[Dict]
|
|
||||||
:param kwargs: Additional keyword arguments.
|
|
||||||
:return: The score prompt.
|
|
||||||
:rtype: str
|
|
||||||
"""
|
|
||||||
pass
|
|
||||||
|
|
||||||
|
|
||||||
class SortingParser(parser.Parser):
|
|
||||||
"""
|
|
||||||
SortingParser provides the parsing of language model reponses specific to
|
|
||||||
the sorting example.
|
|
||||||
|
|
||||||
Inherits from the Parser class and implements its abstract methods.
|
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(self) -> None:
|
|
||||||
"""
|
|
||||||
Inits the response cache.
|
|
||||||
"""
|
|
||||||
self.cache = {}
|
|
||||||
|
|
||||||
def parse_aggregation_answer(
|
|
||||||
self, states: List[Dict], texts: List[str]
|
|
||||||
) -> Union[Dict, List[Dict]]:
|
|
||||||
"""
|
|
||||||
Parse the response from the language model for an aggregation prompt.
|
|
||||||
|
|
||||||
:param states: The thought states used to generate the prompt.
|
|
||||||
:type states: List[Dict]
|
|
||||||
:param texts: The responses to the prompt from the language model.
|
|
||||||
:type texts: List[str]
|
|
||||||
:return: The new thought states after parsing the respones from the language model.
|
|
||||||
:rtype: Union[Dict, List[Dict]]
|
|
||||||
:raise AssertionError: If not exactly two thought states are provided.
|
|
||||||
"""
|
|
||||||
|
|
||||||
assert len(states) == 2, "Expected two states for aggregation answer."
|
|
||||||
new_states = []
|
|
||||||
for text in texts:
|
|
||||||
answers = text.strip().split("\n")
|
|
||||||
if any(["Output" in answer for answer in answers]):
|
|
||||||
# cut elements until last output is found
|
|
||||||
for answer in reversed(answers):
|
|
||||||
if "Output" in answer:
|
|
||||||
answers = answers[answers.index(answer) :]
|
|
||||||
break
|
|
||||||
|
|
||||||
answers_stripped = [
|
|
||||||
answer for answer in answers if "[" in answer and "]" in answer
|
|
||||||
]
|
|
||||||
if len(answers_stripped) == 0:
|
|
||||||
for answer in answers:
|
|
||||||
answer = "[" + answer + "]"
|
|
||||||
try:
|
|
||||||
answer_converted = utils.string_to_list(answer)
|
|
||||||
if len(answer_converted) > 0:
|
|
||||||
answers_stripped.append(answer)
|
|
||||||
except:
|
|
||||||
pass
|
|
||||||
if len(answers_stripped) == 0:
|
|
||||||
logging.warning(
|
|
||||||
f"Could not parse aggregation answer: {text}. Returning empty list."
|
|
||||||
)
|
|
||||||
answer = "[]"
|
|
||||||
else:
|
|
||||||
answer = [
|
|
||||||
answer[answer.index("[") : answer.index("]") + 1]
|
|
||||||
for answer in answers_stripped
|
|
||||||
][0]
|
|
||||||
states = sorted(states, key=lambda x: x["part"])
|
|
||||||
merged_unsorted_sublists = (
|
|
||||||
states[0]["unsorted_sublist"][:-1]
|
|
||||||
+ ", "
|
|
||||||
+ states[1]["unsorted_sublist"][1:]
|
|
||||||
)
|
|
||||||
new_state = states[0].copy()
|
|
||||||
new_state["current"] = answer
|
|
||||||
new_state["unsorted_sublist"] = merged_unsorted_sublists
|
|
||||||
new_states.append(new_state)
|
|
||||||
return new_states
|
|
||||||
|
|
||||||
def parse_generate_answer(self, state: Dict, texts: List[str]) -> List[Dict]:
|
|
||||||
"""
|
|
||||||
Parse the response from the language model for a generate prompt.
|
|
||||||
|
|
||||||
:param state: The thought state used to generate the prompt.
|
|
||||||
:type state: Dict
|
|
||||||
:param texts: The responses to the prompt from the language model.
|
|
||||||
:type texts: List[str]
|
|
||||||
:return: The new thought states after parsing the respones from the language model.
|
|
||||||
:rtype: List[Dict]
|
|
||||||
"""
|
|
||||||
new_states = []
|
|
||||||
for text in texts:
|
|
||||||
if state["method"] == "got" and state["current"] == "":
|
|
||||||
# We expect a json which contains the four lists named "List 1" to "List 4"
|
|
||||||
# cut everything until the opening bracket and everything after the closing bracket
|
|
||||||
try:
|
|
||||||
text = text[text.index("{") : text.index("}") + 1]
|
|
||||||
json_dict = json.loads(text)
|
|
||||||
if len(json_dict.keys()) != 2:
|
|
||||||
logging.warning(
|
|
||||||
f"Expected 2 lists in json, but found {len(json_dict.keys())}."
|
|
||||||
)
|
|
||||||
for key, value in json_dict.items():
|
|
||||||
if "List" not in key:
|
|
||||||
logging.warning(
|
|
||||||
f"Expected key to contain 'List', but found {key}."
|
|
||||||
)
|
|
||||||
continue
|
|
||||||
if not isinstance(value, list):
|
|
||||||
value = utils.string_to_list(value)
|
|
||||||
new_state = state.copy()
|
|
||||||
new_state["current"] = str(value)
|
|
||||||
new_state["unsorted_sublist"] = str(value)
|
|
||||||
new_state["phase"] = 1
|
|
||||||
new_state["part"] = key
|
|
||||||
new_states.append(new_state)
|
|
||||||
except Exception as e:
|
|
||||||
logging.error(
|
|
||||||
f"Could not parse step answer: {text}. Encountered exception: {e}"
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
answers = text.strip().split("\n")
|
|
||||||
answers = [
|
|
||||||
answer for answer in answers if "[" in answer and "]" in answer
|
|
||||||
]
|
|
||||||
if any(["Output" in answer for answer in answers]):
|
|
||||||
# cut elements until last output is found
|
|
||||||
for answer in reversed(answers):
|
|
||||||
if "Output" in answer:
|
|
||||||
answers = answers[answers.index(answer) :]
|
|
||||||
break
|
|
||||||
|
|
||||||
answers = [
|
|
||||||
answer[answer.index("[") : answer.index("]") + 1]
|
|
||||||
for answer in answers
|
|
||||||
]
|
|
||||||
if len(answers) == 0:
|
|
||||||
logging.warning(
|
|
||||||
f"Could not parse step answer: {text}. Returning empty list."
|
|
||||||
)
|
|
||||||
answer = "[]"
|
|
||||||
else:
|
|
||||||
if len(answers) > 1:
|
|
||||||
logging.warning(
|
|
||||||
f"Multiple answers found for step answer: {text}. Using the first one."
|
|
||||||
)
|
|
||||||
answer = answers[0]
|
|
||||||
|
|
||||||
new_state = state.copy()
|
|
||||||
new_state["current"] = answer
|
|
||||||
new_state["phase"] = 2
|
|
||||||
new_states.append(new_state)
|
|
||||||
return new_states
|
|
||||||
|
|
||||||
def parse_improve_answer(self, state: Dict, texts: List[str]) -> Dict:
|
|
||||||
"""
|
|
||||||
Parse the response from the language model for an improve prompt.
|
|
||||||
|
|
||||||
:param state: The thought state used to generate the prompt.
|
|
||||||
:type state: Dict
|
|
||||||
:param texts: The responses to the prompt from the language model.
|
|
||||||
:type texts: List[str]
|
|
||||||
:return: The new thought state after parsing the responses from the language model.
|
|
||||||
:rtype: Dict
|
|
||||||
"""
|
|
||||||
pass
|
|
||||||
|
|
||||||
def parse_validation_answer(self, state: Dict, texts: List[str]) -> bool:
|
|
||||||
"""
|
|
||||||
Parse the response from the language model for a validation prompt.
|
|
||||||
|
|
||||||
:param state: The thought state used to generate the prompt.
|
|
||||||
:type state: Dict
|
|
||||||
:param texts: The responses to the prompt from the language model.
|
|
||||||
:type texts: List[str]
|
|
||||||
:return: Whether the thought state is valid or not.
|
|
||||||
:rtype: bool
|
|
||||||
"""
|
|
||||||
pass
|
|
||||||
|
|
||||||
def parse_score_answer(self, states: List[Dict], texts: List[str]) -> List[float]:
|
|
||||||
"""
|
|
||||||
Parse the response from the language model for a score prompt.
|
|
||||||
|
|
||||||
:param states: The thought states used to generate the prompt.
|
|
||||||
:type states: List[Dict]
|
|
||||||
:param texts: The responses to the prompt from the language model.
|
|
||||||
:type texts: List[str]
|
|
||||||
:return: The scores for the thought states.
|
|
||||||
:rtype: List[float]
|
|
||||||
"""
|
|
||||||
pass
|
|
||||||
|
|
||||||
|
|
||||||
def io() -> operations.GraphOfOperations:
|
|
||||||
"""
|
|
||||||
Generates the Graph of Operations for the IO method.
|
|
||||||
|
|
||||||
:return: Graph of Operations
|
|
||||||
:rtype: GraphOfOperations
|
|
||||||
"""
|
|
||||||
operations_graph = operations.GraphOfOperations()
|
|
||||||
|
|
||||||
operations_graph.append_operation(operations.Generate(1, 1))
|
|
||||||
operations_graph.append_operation(operations.Score(1, False, utils.num_errors))
|
|
||||||
operations_graph.append_operation(operations.GroundTruth(utils.test_sorting))
|
|
||||||
|
|
||||||
return operations_graph
|
|
||||||
|
|
||||||
|
|
||||||
def cot() -> operations.GraphOfOperations:
|
|
||||||
"""
|
|
||||||
Generates the Graph of Operations for the CoT method.
|
|
||||||
|
|
||||||
:return: Graph of Operations
|
|
||||||
:rtype: GraphOfOperations
|
|
||||||
"""
|
|
||||||
operations_graph = operations.GraphOfOperations()
|
|
||||||
|
|
||||||
operations_graph.append_operation(operations.Generate(1, 1))
|
|
||||||
operations_graph.append_operation(operations.Score(1, False, utils.num_errors))
|
|
||||||
operations_graph.append_operation(operations.GroundTruth(utils.test_sorting))
|
|
||||||
|
|
||||||
return operations_graph
|
|
||||||
|
|
||||||
|
|
||||||
def tot() -> operations.GraphOfOperations:
|
|
||||||
"""
|
|
||||||
Generates the Graph of Operations for the ToT method.
|
|
||||||
ToT uses a wider tree, where on each level there are more branches.
|
|
||||||
|
|
||||||
:return: Graph of Operations
|
|
||||||
:rtype: GraphOfOperations
|
|
||||||
"""
|
|
||||||
operations_graph = operations.GraphOfOperations()
|
|
||||||
|
|
||||||
operations_graph.append_operation(operations.Generate(1, 20))
|
|
||||||
operations_graph.append_operation(operations.Score(1, False, utils.num_errors))
|
|
||||||
keep_best_1 = operations.KeepBestN(1, False)
|
|
||||||
operations_graph.append_operation(keep_best_1)
|
|
||||||
|
|
||||||
for _ in range(1):
|
|
||||||
operations_graph.append_operation(operations.Generate(1, 20))
|
|
||||||
operations_graph.append_operation(operations.Score(1, False, utils.num_errors))
|
|
||||||
keep_best_2 = operations.KeepBestN(1, False)
|
|
||||||
keep_best_2.add_predecessor(keep_best_1)
|
|
||||||
operations_graph.append_operation(keep_best_2)
|
|
||||||
keep_best_1 = keep_best_2
|
|
||||||
|
|
||||||
operations_graph.append_operation(operations.KeepBestN(1, False))
|
|
||||||
operations_graph.append_operation(operations.GroundTruth(utils.test_sorting))
|
|
||||||
|
|
||||||
return operations_graph
|
|
||||||
|
|
||||||
|
|
||||||
def tot2() -> operations.GraphOfOperations:
|
|
||||||
"""
|
|
||||||
Generates the Graph of Operations for the ToT2 method.
|
|
||||||
ToT2 uses a tree with more levels, but with fewer branches per level.
|
|
||||||
|
|
||||||
:return: Graph of Operations
|
|
||||||
:rtype: GraphOfOperations
|
|
||||||
"""
|
|
||||||
operations_graph = operations.GraphOfOperations()
|
|
||||||
|
|
||||||
operations_graph.append_operation(operations.Generate(1, 10))
|
|
||||||
operations_graph.append_operation(operations.Score(1, False, utils.num_errors))
|
|
||||||
keep_best_1 = operations.KeepBestN(1, False)
|
|
||||||
operations_graph.append_operation(keep_best_1)
|
|
||||||
|
|
||||||
for _ in range(2):
|
|
||||||
operations_graph.append_operation(operations.Generate(1, 10))
|
|
||||||
operations_graph.append_operation(operations.Score(1, False, utils.num_errors))
|
|
||||||
keep_best_2 = operations.KeepBestN(1, False)
|
|
||||||
keep_best_2.add_predecessor(keep_best_1)
|
|
||||||
operations_graph.append_operation(keep_best_2)
|
|
||||||
keep_best_1 = keep_best_2
|
|
||||||
|
|
||||||
operations_graph.append_operation(operations.KeepBestN(1, False))
|
|
||||||
operations_graph.append_operation(operations.GroundTruth(utils.test_sorting))
|
|
||||||
|
|
||||||
return operations_graph
|
|
||||||
|
|
||||||
|
|
||||||
def got() -> operations.GraphOfOperations:
|
|
||||||
"""
|
|
||||||
Generates the Graph of Operations for the GoT method.
|
|
||||||
|
|
||||||
:return: Graph of Operations
|
|
||||||
:rtype: GraphOfOperations
|
|
||||||
"""
|
|
||||||
operations_graph = operations.GraphOfOperations()
|
|
||||||
|
|
||||||
plans = operations.Generate(1, 1)
|
|
||||||
operations_graph.append_operation(plans) # generate the sublists
|
|
||||||
for i in range(1, 3):
|
|
||||||
list_id = f"List {i}"
|
|
||||||
sub_list = operations.Selector(
|
|
||||||
lambda thoughts, list_id=list_id: [
|
|
||||||
thought for thought in thoughts if thought.state["part"] == list_id
|
|
||||||
]
|
|
||||||
)
|
|
||||||
sub_list.add_predecessor(plans)
|
|
||||||
operations_graph.add_operation(sub_list)
|
|
||||||
sort_sub_list = operations.Generate(1, 5)
|
|
||||||
sort_sub_list.add_predecessor(sub_list)
|
|
||||||
operations_graph.add_operation(sort_sub_list)
|
|
||||||
score_sub_list = operations.Score(1, False, utils.num_errors)
|
|
||||||
score_sub_list.add_predecessor(sort_sub_list)
|
|
||||||
operations_graph.add_operation(score_sub_list)
|
|
||||||
keep_best_sub_list = operations.KeepBestN(1, False)
|
|
||||||
keep_best_sub_list.add_predecessor(score_sub_list)
|
|
||||||
operations_graph.add_operation(keep_best_sub_list)
|
|
||||||
|
|
||||||
final_aggregate = operations.Aggregate(10)
|
|
||||||
operations_graph.append_operation(final_aggregate)
|
|
||||||
operations_graph.append_operation(operations.Score(1, False, utils.num_errors))
|
|
||||||
keep_best_aggregate_final = operations.KeepBestN(1, False)
|
|
||||||
operations_graph.append_operation(keep_best_aggregate_final)
|
|
||||||
|
|
||||||
operations_graph.append_operation(operations.Generate(1, 10))
|
|
||||||
score_aggr_3 = operations.Score(1, False, utils.num_errors)
|
|
||||||
score_aggr_3.add_predecessor(keep_best_aggregate_final)
|
|
||||||
operations_graph.append_operation(score_aggr_3)
|
|
||||||
operations_graph.append_operation(operations.KeepBestN(1, False))
|
|
||||||
|
|
||||||
operations_graph.append_operation(operations.GroundTruth(utils.test_sorting))
|
|
||||||
|
|
||||||
return operations_graph
|
|
||||||
|
|
||||||
|
|
||||||
def run(
|
|
||||||
data_ids: List[int],
|
|
||||||
methods: List[Callable[[], operations.GraphOfOperations]],
|
|
||||||
budget: float,
|
|
||||||
lm_name: str,
|
|
||||||
) -> float:
|
|
||||||
"""
|
|
||||||
Controller function that executes each specified method for each specified
|
|
||||||
sample while the budget is not exhausted.
|
|
||||||
|
|
||||||
:param data_ids: Indices of the sample to be run.
|
|
||||||
:type data_ids: List[int]
|
|
||||||
:param methods: List of functions to generate Graphs of Operations.
|
|
||||||
:type methods: Each function generates a Graph of Operation.
|
|
||||||
:param budget: Language model budget for the execution in dollars.
|
|
||||||
:type budget: float
|
|
||||||
:param lm_name: Name of the language model to be used.
|
|
||||||
:type lm_name: str
|
|
||||||
:return: Spent budget in dollars.
|
|
||||||
:rtype: float
|
|
||||||
"""
|
|
||||||
|
|
||||||
orig_budget = budget
|
|
||||||
data_path = os.path.join(os.path.dirname(__file__), "sorting_032.csv")
|
|
||||||
data = []
|
|
||||||
with open(data_path, "r") as f:
|
|
||||||
reader = csv.reader(f)
|
|
||||||
next(reader)
|
|
||||||
for row in reader:
|
|
||||||
data.append([int(row[0]), row[1], row[2]])
|
|
||||||
|
|
||||||
if data_ids is None or len(data_ids) == 0:
|
|
||||||
data_ids = list(range(len(data)))
|
|
||||||
selected_data = [data[i] for i in data_ids]
|
|
||||||
|
|
||||||
results_dir = os.path.join(os.path.dirname(__file__), "results")
|
|
||||||
|
|
||||||
if not os.path.exists(results_dir):
|
|
||||||
os.makedirs(results_dir)
|
|
||||||
timestamp = datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
|
|
||||||
extra_info = f"{lm_name}_{'-'.join([method.__name__ for method in methods])}"
|
|
||||||
folder_name = f"{extra_info}_{timestamp}"
|
|
||||||
results_folder = os.path.join(results_dir, folder_name)
|
|
||||||
os.makedirs(results_folder)
|
|
||||||
|
|
||||||
config = {
|
|
||||||
"data": selected_data,
|
|
||||||
"methods": [method.__name__ for method in methods],
|
|
||||||
"lm": lm_name,
|
|
||||||
"budget": budget,
|
|
||||||
}
|
|
||||||
with open(os.path.join(results_folder, "config.json"), "w") as f:
|
|
||||||
json.dump(config, f)
|
|
||||||
|
|
||||||
logging.basicConfig(
|
|
||||||
filename=os.path.join(results_folder, "log.log"),
|
|
||||||
filemode="w",
|
|
||||||
format="%(name)s - %(levelname)s - %(message)s",
|
|
||||||
level=logging.DEBUG,
|
|
||||||
)
|
|
||||||
|
|
||||||
for method in methods:
|
|
||||||
os.makedirs(os.path.join(results_folder, method.__name__))
|
|
||||||
|
|
||||||
for data in selected_data:
|
|
||||||
logging.info(f"Running data {data[0]}: {data[1]}")
|
|
||||||
if budget <= 0.0:
|
|
||||||
logging.error(
|
|
||||||
f"Budget has been depleted, stopping. Data {data[0]} has not been run."
|
|
||||||
)
|
|
||||||
break
|
|
||||||
for method in methods:
|
|
||||||
logging.info(f"Running method {method.__name__}")
|
|
||||||
logging.info(f"Budget left: {budget}")
|
|
||||||
if budget <= 0.0:
|
|
||||||
logging.error(
|
|
||||||
f"Budget has been depleted, stopping. Method {method.__name__} has not been run."
|
|
||||||
)
|
|
||||||
break
|
|
||||||
lm = language_models.ChatGPT(
|
|
||||||
os.path.join(
|
|
||||||
os.path.dirname(__file__),
|
|
||||||
"../../graph_of_thoughts/language_models/config.json",
|
|
||||||
),
|
|
||||||
model_name=lm_name,
|
|
||||||
cache=True,
|
|
||||||
)
|
|
||||||
operations_graph = method()
|
|
||||||
executor = controller.Controller(
|
|
||||||
lm,
|
|
||||||
operations_graph,
|
|
||||||
SortingPrompter(),
|
|
||||||
SortingParser(),
|
|
||||||
{
|
|
||||||
"original": data[1],
|
|
||||||
"current": "",
|
|
||||||
"phase": 0,
|
|
||||||
"method": method.__name__,
|
|
||||||
},
|
|
||||||
)
|
|
||||||
try:
|
|
||||||
executor.run()
|
|
||||||
except Exception as e:
|
|
||||||
logging.error(f"Exception: {e}")
|
|
||||||
path = os.path.join(
|
|
||||||
results_folder,
|
|
||||||
method.__name__,
|
|
||||||
f"{data[0]}.json",
|
|
||||||
)
|
|
||||||
executor.output_graph(path)
|
|
||||||
budget -= lm.cost
|
|
||||||
|
|
||||||
return orig_budget - budget
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
"""
|
|
||||||
Input (x) : an unordered list of 32 numbers between 0 and 9 (inclusive)
|
|
||||||
Output (y) : a sorted list of 32 numbers between 0 and 9 (inclusive)
|
|
||||||
Correct : y == sorted(x)
|
|
||||||
Input Example:
|
|
||||||
[0, 1, 9, 4, 2, 2, 0, 5, 1...]
|
|
||||||
Output Example:
|
|
||||||
[0, 0, 0, 0, 1, 1, 1, 1, 2...]
|
|
||||||
"""
|
|
||||||
budget = 30
|
|
||||||
samples = [item for item in range(0, 100)]
|
|
||||||
approaches = [io, cot, tot, tot2, got]
|
|
||||||
|
|
||||||
spent = run(samples, approaches, budget, "chatgpt")
|
|
||||||
|
|
||||||
logging.info(f"Spent {spent} out of {budget} budget.")
|
|
||||||
@ -1,101 +0,0 @@
|
|||||||
ID,Unsorted,Sorted
|
|
||||||
0,"[6, 3, 6, 5, 1, 2, 4, 3, 8, 0, 7, 8, 6, 4, 9, 5, 2, 4, 8, 4, 4, 4, 5, 6, 8, 4, 7, 7, 8, 9, 4, 9, 5, 4, 8, 4, 0, 5, 6, 9, 1, 2, 3, 6, 2, 0, 8, 1, 0, 7, 1, 2, 0, 7, 6, 9, 9, 9, 5, 6, 8, 3, 9, 0]","[0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
1,"[9, 1, 5, 7, 8, 5, 2, 5, 6, 9, 0, 3, 6, 9, 9, 5, 6, 5, 8, 6, 0, 1, 1, 8, 5, 3, 7, 9, 0, 2, 7, 2, 8, 1, 2, 5, 9, 3, 5, 0, 0, 3, 7, 5, 1, 1, 1, 8, 1, 2, 0, 8, 0, 7, 8, 1, 8, 2, 1, 0, 4, 7, 5, 2]","[0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9]"
|
|
||||||
2,"[6, 7, 0, 6, 8, 3, 0, 6, 1, 7, 8, 5, 5, 6, 1, 9, 6, 7, 2, 7, 2, 3, 3, 2, 0, 8, 4, 7, 1, 7, 2, 1, 4, 8, 6, 6, 6, 8, 9, 6, 0, 2, 9, 4, 8, 3, 3, 7, 9, 7, 6, 0, 3, 5, 9, 9, 2, 3, 2, 2, 3, 5, 6, 8]","[0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9]"
|
|
||||||
3,"[3, 4, 6, 1, 4, 2, 2, 2, 3, 0, 4, 3, 9, 9, 3, 8, 7, 2, 0, 6, 9, 0, 7, 5, 0, 6, 4, 7, 3, 5, 0, 3, 4, 0, 7, 1, 5, 1, 1, 2, 3, 9, 5, 4, 1, 6, 2, 0, 5, 7, 7, 0, 1, 6, 0, 5, 5, 7, 1, 4, 5, 7, 0, 8]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 9, 9, 9, 9]"
|
|
||||||
4,"[3, 5, 5, 8, 7, 3, 0, 2, 0, 1, 6, 3, 5, 6, 8, 2, 3, 7, 5, 1, 3, 6, 5, 8, 6, 7, 6, 5, 9, 8, 3, 9, 8, 1, 6, 1, 3, 3, 6, 9, 3, 8, 2, 6, 4, 2, 4, 4, 4, 1, 6, 2, 0, 0, 8, 0, 6, 4, 7, 9, 0, 5, 1, 4]","[0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9]"
|
|
||||||
5,"[9, 2, 1, 2, 4, 9, 5, 4, 3, 5, 3, 9, 9, 9, 5, 1, 2, 4, 8, 7, 5, 8, 6, 8, 1, 7, 6, 4, 7, 7, 1, 1, 4, 9, 3, 2, 4, 5, 2, 6, 0, 5, 6, 8, 0, 3, 1, 4, 3, 9, 0, 8, 7, 3, 1, 0, 9, 3, 0, 3, 8, 6, 1, 4]","[0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
6,"[5, 1, 6, 6, 3, 2, 9, 3, 5, 3, 9, 5, 1, 1, 2, 0, 2, 4, 9, 8, 9, 6, 0, 9, 2, 9, 8, 9, 5, 1, 1, 5, 5, 9, 9, 5, 7, 8, 3, 2, 0, 5, 4, 1, 7, 3, 8, 9, 2, 2, 3, 1, 8, 8, 4, 7, 8, 8, 3, 1, 5, 9, 0, 9]","[0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
7,"[7, 0, 5, 9, 3, 1, 9, 1, 5, 8, 9, 4, 6, 5, 3, 4, 3, 3, 6, 9, 9, 0, 5, 2, 5, 8, 2, 4, 4, 6, 8, 1, 6, 5, 5, 4, 9, 0, 5, 6, 4, 7, 4, 5, 7, 5, 2, 2, 5, 9, 3, 5, 9, 4, 6, 8, 4, 1, 1, 1, 6, 0, 8, 5]","[0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
8,"[7, 4, 9, 1, 0, 7, 1, 4, 3, 8, 3, 0, 8, 4, 7, 7, 0, 7, 3, 9, 7, 9, 6, 4, 4, 1, 5, 0, 5, 5, 8, 9, 9, 6, 9, 9, 9, 5, 6, 6, 3, 4, 0, 7, 7, 5, 4, 0, 6, 7, 3, 3, 6, 9, 4, 2, 5, 8, 2, 5, 1, 5, 6, 8]","[0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
9,"[2, 4, 9, 1, 8, 0, 6, 4, 5, 5, 3, 4, 8, 2, 1, 2, 2, 7, 4, 8, 0, 0, 8, 3, 9, 3, 2, 5, 5, 1, 6, 3, 5, 0, 5, 5, 4, 2, 9, 7, 5, 1, 5, 6, 2, 0, 5, 4, 9, 1, 9, 4, 3, 3, 1, 2, 6, 9, 6, 0, 9, 0, 4, 6]","[0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 7, 7, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
10,"[3, 4, 8, 8, 7, 9, 2, 3, 7, 2, 7, 2, 9, 0, 6, 3, 3, 6, 1, 1, 2, 1, 3, 4, 4, 6, 7, 6, 7, 5, 5, 1, 1, 7, 3, 5, 1, 0, 0, 6, 5, 8, 2, 6, 9, 9, 1, 3, 4, 3, 9, 7, 2, 5, 8, 4, 7, 2, 0, 6, 0, 4, 3, 3]","[0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 9, 9, 9, 9, 9]"
|
|
||||||
11,"[1, 5, 0, 4, 9, 1, 0, 1, 7, 9, 4, 8, 1, 0, 0, 8, 2, 2, 6, 0, 7, 7, 2, 5, 2, 7, 5, 0, 5, 9, 0, 5, 6, 3, 5, 2, 6, 1, 4, 2, 0, 3, 7, 3, 7, 8, 6, 4, 7, 4, 4, 0, 3, 1, 2, 0, 9, 2, 0, 8, 8, 0, 3, 8]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9]"
|
|
||||||
12,"[2, 6, 5, 0, 0, 6, 7, 8, 5, 6, 1, 8, 1, 6, 1, 3, 8, 3, 4, 3, 5, 6, 6, 2, 6, 4, 5, 8, 2, 3, 4, 4, 2, 5, 8, 4, 9, 6, 3, 8, 4, 4, 9, 5, 6, 6, 6, 0, 8, 5, 5, 0, 8, 4, 4, 5, 6, 8, 8, 4, 8, 7, 7, 5]","[0, 0, 0, 0, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9]"
|
|
||||||
13,"[5, 8, 0, 2, 3, 1, 7, 3, 4, 8, 8, 8, 7, 0, 0, 9, 4, 1, 7, 3, 3, 2, 4, 8, 8, 3, 0, 5, 1, 8, 8, 2, 0, 6, 8, 4, 9, 3, 7, 9, 8, 4, 8, 4, 6, 4, 0, 2, 7, 5, 5, 6, 8, 9, 3, 6, 6, 6, 9, 6, 0, 8, 5, 8]","[0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9]"
|
|
||||||
14,"[7, 0, 2, 5, 2, 9, 7, 1, 4, 9, 1, 6, 5, 4, 3, 1, 7, 2, 3, 5, 6, 4, 4, 6, 4, 8, 8, 1, 2, 7, 2, 9, 0, 3, 4, 9, 7, 2, 1, 9, 0, 0, 7, 8, 8, 6, 9, 5, 9, 5, 7, 5, 0, 2, 6, 0, 2, 2, 2, 9, 9, 4, 0, 0]","[0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
15,"[0, 8, 0, 4, 5, 9, 8, 6, 2, 8, 9, 5, 0, 1, 4, 2, 8, 5, 0, 2, 8, 2, 0, 2, 7, 6, 7, 0, 5, 2, 2, 9, 9, 7, 1, 9, 3, 6, 7, 8, 0, 5, 1, 6, 9, 9, 2, 8, 2, 1, 0, 8, 2, 8, 4, 5, 7, 1, 0, 7, 1, 6, 5, 7]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
16,"[3, 5, 0, 6, 4, 2, 0, 3, 9, 3, 3, 6, 7, 1, 4, 8, 6, 8, 1, 3, 6, 3, 5, 5, 6, 4, 6, 1, 9, 8, 0, 9, 8, 1, 9, 9, 0, 4, 5, 6, 2, 6, 4, 0, 9, 3, 1, 7, 8, 9, 8, 8, 2, 5, 0, 7, 1, 2, 7, 0, 4, 0, 3, 0]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
17,"[0, 5, 7, 5, 1, 0, 5, 5, 6, 5, 4, 4, 9, 9, 3, 3, 4, 9, 4, 6, 7, 6, 8, 9, 8, 0, 4, 6, 6, 5, 3, 7, 2, 2, 5, 4, 9, 2, 5, 1, 7, 4, 7, 1, 9, 4, 2, 0, 1, 2, 7, 3, 7, 5, 9, 9, 9, 0, 6, 3, 4, 3, 3, 2]","[0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
18,"[1, 4, 0, 5, 8, 1, 4, 0, 1, 1, 1, 5, 5, 5, 9, 0, 4, 5, 1, 8, 0, 4, 7, 0, 3, 5, 7, 7, 9, 6, 1, 3, 4, 6, 3, 5, 6, 6, 1, 1, 3, 5, 2, 0, 3, 4, 2, 0, 3, 1, 3, 9, 5, 2, 6, 4, 8, 9, 7, 0, 1, 6, 6, 8]","[0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 8, 8, 8, 8, 9, 9, 9, 9]"
|
|
||||||
19,"[1, 4, 5, 6, 7, 7, 1, 5, 5, 5, 5, 3, 0, 8, 3, 8, 1, 5, 1, 1, 6, 5, 3, 9, 0, 1, 5, 6, 1, 2, 4, 6, 4, 3, 1, 6, 9, 4, 6, 8, 4, 7, 9, 0, 9, 1, 2, 3, 8, 0, 3, 3, 2, 0, 0, 8, 9, 7, 8, 2, 0, 2, 2, 5]","[0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9]"
|
|
||||||
20,"[5, 5, 1, 5, 4, 7, 8, 4, 1, 7, 2, 5, 9, 7, 2, 6, 3, 5, 2, 4, 2, 1, 3, 8, 9, 6, 3, 9, 5, 4, 8, 1, 6, 6, 7, 3, 5, 4, 9, 6, 3, 4, 6, 7, 4, 4, 0, 4, 5, 3, 7, 9, 7, 8, 6, 8, 3, 6, 8, 3, 5, 1, 3, 7]","[0, 1, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9]"
|
|
||||||
21,"[8, 2, 7, 1, 2, 9, 0, 3, 5, 2, 2, 5, 3, 7, 6, 1, 2, 0, 6, 3, 2, 5, 5, 4, 2, 6, 7, 8, 4, 7, 1, 2, 8, 3, 2, 1, 6, 7, 5, 9, 9, 8, 8, 0, 4, 8, 6, 3, 6, 1, 9, 0, 2, 8, 6, 7, 3, 5, 7, 8, 7, 6, 5, 8]","[0, 0, 0, 0, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9]"
|
|
||||||
22,"[3, 1, 6, 6, 1, 8, 0, 3, 1, 7, 8, 1, 9, 1, 3, 9, 3, 7, 9, 0, 4, 4, 6, 1, 0, 5, 8, 3, 0, 9, 2, 2, 0, 1, 0, 8, 6, 8, 6, 9, 1, 5, 2, 8, 9, 4, 3, 6, 4, 8, 3, 3, 9, 3, 9, 0, 9, 7, 2, 0, 0, 8, 9, 2]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 6, 6, 6, 6, 6, 6, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
23,"[8, 2, 4, 6, 0, 6, 2, 2, 5, 0, 0, 3, 6, 7, 8, 4, 7, 6, 6, 6, 2, 9, 8, 4, 2, 3, 2, 2, 1, 3, 2, 5, 9, 7, 5, 2, 8, 3, 0, 5, 0, 4, 4, 8, 1, 6, 2, 4, 3, 7, 0, 1, 6, 2, 0, 7, 6, 4, 8, 4, 9, 9, 9, 3]","[0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9]"
|
|
||||||
24,"[5, 1, 2, 8, 8, 2, 9, 8, 7, 0, 3, 7, 2, 3, 3, 5, 4, 8, 4, 4, 1, 6, 5, 2, 3, 9, 5, 8, 8, 1, 9, 3, 6, 3, 1, 1, 6, 3, 7, 5, 5, 4, 0, 8, 4, 6, 4, 5, 3, 6, 9, 9, 9, 8, 1, 0, 1, 7, 7, 9, 9, 6, 0, 9]","[0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
25,"[8, 4, 5, 0, 8, 0, 2, 0, 2, 8, 0, 4, 1, 7, 9, 5, 5, 1, 5, 8, 2, 1, 1, 6, 9, 9, 4, 8, 7, 9, 1, 8, 6, 1, 2, 1, 5, 7, 3, 9, 9, 7, 0, 8, 8, 8, 3, 8, 7, 9, 7, 6, 9, 5, 2, 3, 1, 9, 5, 8, 7, 5, 9, 9]","[0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 3, 3, 3, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
26,"[8, 3, 6, 9, 6, 7, 3, 9, 6, 2, 8, 4, 1, 8, 3, 4, 0, 2, 0, 6, 0, 1, 9, 4, 5, 9, 3, 2, 1, 9, 8, 5, 2, 9, 8, 8, 7, 6, 3, 0, 5, 3, 8, 5, 3, 3, 6, 6, 7, 1, 7, 6, 1, 4, 1, 7, 4, 7, 5, 9, 4, 5, 4, 0]","[0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
27,"[5, 0, 7, 7, 9, 3, 3, 6, 7, 9, 7, 5, 9, 2, 9, 5, 3, 9, 5, 7, 0, 2, 7, 6, 9, 4, 0, 3, 0, 0, 7, 1, 2, 2, 1, 8, 7, 9, 3, 1, 3, 3, 0, 3, 2, 2, 1, 8, 9, 8, 9, 4, 7, 0, 6, 9, 5, 2, 9, 2, 4, 2, 0, 7]","[0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 5, 5, 5, 5, 5, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
28,"[6, 4, 7, 5, 8, 1, 7, 9, 6, 3, 5, 4, 4, 7, 9, 5, 6, 9, 7, 1, 0, 8, 0, 8, 5, 7, 3, 0, 4, 2, 8, 9, 0, 9, 2, 1, 7, 8, 1, 0, 3, 6, 5, 2, 2, 0, 8, 5, 5, 0, 9, 1, 4, 4, 7, 5, 2, 5, 1, 7, 5, 4, 5, 6]","[0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9]"
|
|
||||||
29,"[2, 1, 1, 7, 3, 3, 7, 3, 3, 7, 3, 1, 0, 6, 0, 3, 0, 2, 1, 8, 9, 5, 5, 1, 1, 5, 4, 8, 9, 3, 9, 3, 6, 7, 0, 1, 3, 8, 7, 8, 8, 2, 6, 7, 4, 7, 2, 9, 0, 7, 9, 4, 0, 1, 8, 9, 0, 3, 5, 3, 1, 9, 4, 0]","[0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
30,"[4, 2, 0, 8, 4, 7, 6, 6, 3, 9, 6, 0, 0, 7, 7, 5, 6, 3, 3, 7, 6, 9, 1, 4, 2, 7, 0, 6, 5, 9, 8, 6, 4, 6, 4, 9, 3, 2, 3, 0, 1, 5, 5, 4, 2, 7, 8, 2, 8, 1, 3, 6, 9, 6, 6, 5, 5, 4, 6, 6, 7, 8, 3, 7]","[0, 0, 0, 0, 0, 1, 1, 1, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9]"
|
|
||||||
31,"[9, 4, 6, 8, 3, 9, 0, 3, 4, 6, 8, 9, 4, 9, 0, 6, 4, 8, 9, 2, 0, 4, 5, 3, 4, 3, 1, 3, 2, 4, 3, 7, 5, 4, 5, 6, 5, 4, 3, 0, 6, 1, 9, 7, 5, 7, 0, 1, 4, 1, 4, 2, 6, 9, 0, 3, 0, 8, 3, 5, 7, 6, 6, 3]","[0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
32,"[9, 2, 3, 5, 3, 6, 0, 9, 2, 0, 7, 9, 7, 7, 5, 6, 5, 5, 5, 5, 3, 8, 0, 0, 1, 9, 8, 9, 5, 0, 0, 6, 5, 2, 0, 8, 7, 3, 3, 6, 7, 4, 2, 7, 1, 7, 6, 9, 4, 5, 8, 0, 8, 6, 5, 1, 3, 5, 2, 3, 4, 3, 6, 5]","[0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9]"
|
|
||||||
33,"[4, 8, 5, 4, 6, 2, 7, 7, 4, 1, 5, 5, 6, 0, 1, 1, 1, 3, 9, 3, 9, 6, 6, 5, 8, 2, 1, 3, 4, 2, 3, 8, 4, 5, 0, 6, 0, 4, 1, 5, 5, 2, 7, 0, 4, 8, 9, 5, 9, 4, 0, 5, 5, 8, 2, 7, 6, 2, 7, 1, 7, 9, 5, 3]","[0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9]"
|
|
||||||
34,"[2, 3, 7, 7, 9, 4, 0, 7, 7, 6, 5, 8, 7, 2, 2, 8, 6, 6, 5, 5, 4, 6, 5, 0, 8, 4, 1, 7, 4, 4, 3, 0, 8, 7, 3, 3, 5, 8, 1, 9, 1, 3, 3, 6, 7, 1, 4, 4, 6, 6, 1, 0, 5, 2, 0, 6, 4, 4, 8, 9, 5, 9, 4, 7]","[0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9]"
|
|
||||||
35,"[9, 7, 8, 8, 4, 3, 0, 9, 1, 4, 4, 8, 1, 8, 7, 9, 6, 4, 7, 9, 0, 4, 0, 7, 8, 9, 9, 7, 8, 0, 4, 5, 8, 8, 0, 3, 2, 0, 3, 9, 7, 4, 2, 4, 5, 1, 9, 3, 0, 5, 2, 5, 2, 5, 4, 4, 9, 6, 0, 4, 9, 4, 0, 9]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 6, 6, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
36,"[6, 5, 5, 1, 2, 2, 4, 3, 3, 5, 7, 3, 5, 7, 3, 2, 9, 1, 8, 2, 6, 2, 7, 6, 6, 2, 6, 0, 2, 2, 2, 0, 2, 7, 4, 9, 1, 9, 4, 4, 2, 4, 4, 4, 7, 3, 1, 6, 4, 2, 0, 6, 7, 5, 7, 9, 2, 1, 6, 7, 9, 3, 2, 5]","[0, 0, 0, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 8, 9, 9, 9, 9, 9]"
|
|
||||||
37,"[9, 3, 0, 1, 9, 0, 2, 7, 0, 5, 1, 7, 6, 2, 4, 4, 1, 6, 6, 7, 4, 8, 9, 3, 7, 3, 4, 0, 2, 5, 4, 6, 7, 3, 4, 2, 7, 0, 9, 7, 0, 7, 6, 5, 1, 3, 3, 4, 8, 0, 0, 7, 7, 0, 2, 0, 4, 2, 1, 4, 5, 5, 8, 2]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 9, 9, 9, 9]"
|
|
||||||
38,"[0, 7, 3, 0, 9, 2, 1, 9, 0, 5, 7, 0, 3, 0, 6, 5, 7, 5, 3, 4, 5, 9, 6, 1, 6, 8, 2, 2, 9, 3, 1, 6, 6, 7, 2, 9, 8, 9, 9, 9, 9, 2, 4, 1, 4, 2, 6, 0, 3, 8, 0, 3, 5, 0, 5, 1, 8, 0, 1, 2, 5, 8, 1, 7]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
39,"[1, 8, 8, 5, 4, 1, 1, 8, 7, 5, 6, 5, 7, 8, 3, 0, 1, 5, 7, 9, 3, 9, 6, 6, 2, 3, 3, 1, 5, 8, 9, 6, 4, 5, 7, 0, 2, 7, 2, 5, 3, 2, 3, 8, 1, 9, 0, 6, 7, 4, 6, 7, 4, 3, 6, 1, 7, 9, 6, 6, 0, 1, 4, 1]","[0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9]"
|
|
||||||
40,"[0, 7, 3, 9, 5, 8, 0, 4, 4, 8, 1, 9, 9, 4, 4, 2, 9, 3, 0, 0, 1, 2, 2, 2, 3, 8, 4, 6, 1, 0, 9, 9, 8, 1, 4, 2, 5, 9, 6, 6, 3, 2, 7, 5, 5, 7, 1, 3, 0, 6, 6, 4, 1, 2, 4, 9, 4, 7, 7, 2, 5, 5, 5, 6]","[0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
41,"[4, 6, 1, 2, 3, 0, 4, 6, 7, 3, 6, 8, 1, 2, 1, 8, 1, 5, 9, 8, 1, 7, 0, 6, 2, 7, 6, 2, 4, 3, 1, 6, 1, 3, 8, 5, 9, 5, 8, 6, 0, 3, 9, 4, 4, 2, 3, 0, 4, 3, 5, 3, 5, 9, 7, 0, 6, 5, 7, 4, 0, 1, 9, 6]","[0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9]"
|
|
||||||
42,"[3, 0, 2, 6, 0, 6, 6, 0, 7, 1, 5, 7, 6, 6, 6, 1, 3, 0, 9, 7, 6, 7, 8, 3, 0, 7, 5, 5, 6, 4, 4, 6, 0, 6, 7, 2, 8, 5, 4, 4, 5, 1, 1, 1, 6, 1, 0, 4, 1, 6, 4, 8, 7, 3, 6, 3, 1, 8, 7, 6, 0, 5, 2, 9]","[0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 9, 9]"
|
|
||||||
43,"[0, 4, 5, 8, 2, 5, 0, 5, 9, 0, 0, 3, 3, 3, 9, 0, 0, 4, 6, 1, 9, 8, 5, 0, 3, 7, 7, 4, 5, 6, 6, 0, 0, 7, 0, 9, 9, 3, 3, 0, 6, 7, 0, 4, 0, 4, 6, 0, 7, 8, 8, 0, 5, 2, 9, 3, 5, 0, 5, 7, 2, 1, 9, 1]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
44,"[1, 9, 9, 2, 0, 7, 9, 6, 2, 6, 7, 0, 6, 0, 3, 0, 6, 1, 8, 7, 3, 1, 5, 8, 4, 5, 5, 0, 9, 1, 4, 8, 0, 7, 2, 4, 6, 1, 8, 8, 2, 3, 2, 4, 4, 6, 0, 9, 5, 5, 6, 6, 5, 2, 2, 7, 1, 8, 6, 2, 3, 0, 1, 0]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9]"
|
|
||||||
45,"[8, 5, 9, 6, 7, 7, 6, 8, 7, 6, 2, 0, 2, 1, 4, 4, 3, 9, 5, 4, 1, 6, 8, 9, 6, 6, 0, 1, 7, 5, 4, 0, 1, 3, 8, 9, 5, 1, 7, 6, 0, 8, 0, 1, 5, 4, 1, 6, 0, 7, 1, 6, 3, 2, 3, 0, 0, 2, 0, 5, 7, 7, 2, 1]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 9, 9, 9, 9]"
|
|
||||||
46,"[8, 7, 0, 8, 5, 3, 1, 3, 7, 0, 4, 8, 5, 1, 5, 7, 9, 9, 1, 1, 5, 0, 1, 6, 1, 3, 8, 8, 1, 5, 0, 1, 5, 7, 9, 9, 5, 4, 3, 7, 1, 3, 4, 6, 5, 6, 6, 8, 6, 4, 8, 3, 8, 6, 3, 5, 0, 6, 0, 0, 6, 7, 6, 1]","[0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9]"
|
|
||||||
47,"[8, 4, 7, 3, 0, 3, 3, 1, 7, 8, 6, 3, 4, 6, 5, 1, 3, 8, 6, 7, 7, 8, 2, 1, 7, 0, 9, 3, 1, 0, 8, 1, 0, 6, 4, 0, 1, 9, 4, 6, 8, 4, 0, 7, 7, 3, 3, 7, 2, 8, 3, 8, 3, 3, 1, 9, 9, 1, 2, 8, 5, 0, 5, 0]","[0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 5, 5, 5, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9]"
|
|
||||||
48,"[5, 3, 2, 2, 5, 7, 3, 4, 8, 5, 8, 0, 1, 9, 6, 8, 9, 1, 2, 0, 0, 9, 0, 7, 7, 3, 8, 2, 4, 5, 0, 9, 9, 9, 8, 4, 5, 0, 9, 7, 1, 6, 6, 5, 1, 8, 5, 0, 5, 9, 0, 1, 0, 1, 4, 6, 0, 3, 6, 6, 6, 7, 6, 4]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
49,"[3, 1, 4, 7, 0, 5, 8, 5, 2, 9, 1, 4, 1, 0, 4, 1, 8, 6, 5, 1, 1, 6, 1, 0, 3, 0, 1, 3, 7, 8, 0, 9, 2, 3, 8, 3, 9, 0, 7, 3, 6, 2, 6, 1, 0, 5, 1, 9, 5, 3, 9, 6, 1, 2, 5, 0, 2, 1, 5, 9, 0, 4, 3, 3]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 7, 7, 7, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9]"
|
|
||||||
50,"[8, 2, 2, 5, 3, 1, 8, 8, 6, 3, 9, 9, 9, 8, 9, 4, 9, 4, 5, 7, 9, 1, 7, 4, 1, 5, 9, 6, 2, 6, 7, 1, 3, 2, 7, 5, 4, 3, 5, 2, 1, 7, 5, 2, 3, 1, 1, 3, 2, 8, 1, 3, 8, 5, 1, 6, 4, 5, 0, 1, 5, 1, 1, 8]","[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
51,"[3, 1, 9, 1, 7, 9, 6, 7, 4, 6, 2, 9, 9, 7, 2, 2, 5, 0, 0, 0, 3, 6, 6, 5, 7, 0, 0, 8, 4, 1, 1, 3, 4, 8, 6, 0, 0, 2, 7, 8, 7, 1, 3, 4, 8, 5, 2, 7, 9, 9, 5, 6, 9, 5, 7, 3, 8, 2, 8, 4, 1, 6, 8, 1]","[0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
52,"[8, 9, 0, 0, 6, 9, 9, 9, 5, 0, 9, 6, 1, 4, 6, 6, 0, 9, 9, 3, 1, 4, 1, 1, 8, 8, 6, 0, 0, 3, 1, 9, 9, 6, 4, 1, 4, 1, 8, 5, 9, 0, 2, 9, 8, 2, 9, 2, 2, 1, 0, 0, 1, 4, 2, 1, 6, 5, 7, 2, 6, 0, 6, 4]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 3, 3, 4, 4, 4, 4, 4, 4, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
53,"[2, 7, 6, 3, 6, 2, 8, 2, 6, 6, 1, 0, 5, 7, 1, 2, 4, 2, 8, 1, 4, 2, 5, 8, 3, 1, 8, 7, 9, 6, 0, 2, 4, 5, 7, 7, 7, 1, 8, 9, 6, 0, 0, 5, 8, 1, 3, 5, 7, 0, 4, 1, 2, 0, 4, 0, 4, 4, 9, 2, 2, 6, 2, 9]","[0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9]"
|
|
||||||
54,"[8, 7, 8, 9, 6, 6, 8, 2, 5, 2, 2, 1, 9, 5, 6, 1, 2, 5, 3, 4, 7, 8, 2, 9, 3, 0, 6, 9, 5, 0, 4, 1, 0, 2, 5, 5, 9, 0, 9, 8, 4, 2, 5, 6, 1, 2, 8, 4, 3, 1, 3, 6, 7, 0, 7, 8, 3, 7, 6, 6, 3, 1, 7, 5]","[0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9]"
|
|
||||||
55,"[4, 5, 9, 0, 5, 2, 9, 0, 4, 1, 2, 7, 7, 1, 2, 7, 1, 6, 9, 9, 2, 7, 2, 1, 5, 7, 9, 8, 8, 0, 1, 9, 1, 4, 8, 7, 9, 7, 0, 6, 8, 8, 5, 7, 6, 0, 4, 6, 2, 5, 4, 4, 9, 0, 9, 9, 4, 6, 9, 0, 6, 5, 7, 4]","[0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
56,"[0, 4, 3, 7, 2, 8, 2, 6, 1, 1, 6, 2, 0, 4, 4, 3, 0, 5, 0, 8, 1, 6, 4, 2, 0, 6, 4, 4, 3, 8, 7, 9, 1, 5, 8, 9, 3, 8, 3, 8, 6, 1, 5, 7, 1, 3, 6, 8, 2, 1, 2, 6, 4, 3, 7, 8, 3, 0, 9, 0, 1, 8, 5, 0]","[0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9]"
|
|
||||||
57,"[1, 5, 8, 0, 6, 2, 0, 3, 4, 1, 4, 1, 7, 9, 5, 6, 3, 2, 7, 5, 0, 2, 4, 8, 0, 3, 1, 8, 8, 1, 7, 1, 8, 0, 0, 1, 5, 1, 5, 6, 9, 2, 1, 4, 5, 4, 8, 7, 3, 3, 8, 6, 0, 3, 1, 4, 3, 3, 2, 3, 2, 9, 7, 1]","[0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9]"
|
|
||||||
58,"[6, 5, 9, 8, 9, 9, 1, 5, 8, 3, 9, 4, 0, 7, 2, 6, 5, 4, 9, 2, 7, 0, 4, 8, 1, 8, 7, 0, 9, 5, 7, 2, 5, 4, 6, 9, 7, 8, 9, 4, 6, 6, 7, 6, 5, 6, 4, 8, 7, 8, 6, 4, 7, 4, 9, 4, 6, 4, 7, 4, 4, 7, 9, 4]","[0, 0, 0, 1, 1, 2, 2, 2, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
59,"[3, 1, 0, 7, 6, 4, 0, 3, 0, 9, 7, 8, 9, 2, 6, 2, 3, 1, 1, 6, 9, 1, 1, 8, 0, 7, 7, 3, 5, 6, 6, 3, 3, 5, 1, 2, 3, 7, 3, 8, 1, 6, 7, 8, 8, 7, 7, 1, 3, 2, 4, 6, 4, 5, 3, 4, 0, 4, 1, 3, 1, 3, 6, 3]","[0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 9, 9, 9]"
|
|
||||||
60,"[6, 3, 3, 5, 2, 7, 1, 9, 5, 7, 3, 4, 0, 8, 1, 0, 7, 4, 0, 3, 5, 7, 2, 9, 7, 3, 8, 1, 6, 5, 8, 4, 2, 1, 5, 3, 0, 2, 2, 3, 9, 5, 6, 9, 0, 5, 1, 4, 3, 1, 8, 4, 5, 3, 8, 8, 9, 4, 6, 5, 7, 6, 1, 6]","[0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9]"
|
|
||||||
61,"[1, 8, 5, 3, 9, 7, 4, 0, 9, 2, 6, 6, 8, 1, 0, 5, 2, 2, 0, 5, 2, 9, 6, 6, 2, 9, 1, 8, 7, 7, 0, 2, 8, 2, 8, 1, 5, 5, 9, 0, 1, 5, 3, 8, 3, 0, 3, 9, 7, 4, 1, 4, 8, 7, 7, 0, 6, 8, 8, 3, 8, 4, 1, 9]","[0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
62,"[6, 0, 1, 8, 9, 2, 9, 8, 5, 1, 2, 1, 9, 5, 7, 0, 2, 3, 4, 9, 9, 5, 5, 3, 0, 4, 9, 8, 3, 5, 5, 3, 6, 9, 6, 8, 9, 3, 0, 6, 4, 3, 0, 3, 4, 6, 7, 8, 5, 6, 2, 8, 7, 3, 6, 1, 4, 5, 5, 2, 2, 4, 6, 3]","[0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
63,"[9, 2, 7, 0, 4, 3, 4, 7, 5, 7, 3, 4, 9, 0, 0, 4, 9, 7, 3, 5, 3, 8, 4, 3, 0, 1, 3, 8, 7, 8, 6, 0, 7, 9, 8, 8, 2, 4, 6, 4, 9, 1, 8, 5, 8, 3, 3, 6, 3, 1, 4, 3, 3, 5, 3, 1, 5, 9, 6, 8, 3, 5, 9, 1]","[0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
64,"[4, 7, 1, 9, 9, 6, 9, 2, 5, 1, 1, 4, 9, 2, 1, 5, 2, 5, 8, 2, 6, 0, 4, 4, 2, 3, 3, 3, 6, 7, 7, 9, 1, 8, 8, 7, 8, 9, 7, 1, 5, 4, 8, 3, 6, 4, 7, 7, 0, 7, 1, 4, 1, 2, 2, 9, 9, 6, 6, 5, 9, 1, 0, 1]","[0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
65,"[3, 3, 2, 9, 6, 1, 3, 3, 1, 1, 1, 6, 6, 0, 6, 3, 0, 0, 6, 6, 4, 7, 7, 1, 4, 2, 8, 4, 3, 2, 0, 6, 9, 4, 0, 5, 8, 6, 0, 6, 6, 4, 7, 8, 3, 3, 3, 8, 5, 9, 6, 7, 5, 0, 8, 9, 6, 0, 3, 3, 4, 7, 7, 1]","[0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 9, 9, 9, 9]"
|
|
||||||
66,"[4, 8, 5, 9, 2, 7, 0, 1, 5, 7, 4, 4, 2, 6, 0, 4, 6, 1, 3, 7, 0, 1, 5, 4, 7, 1, 0, 3, 0, 5, 8, 4, 7, 6, 0, 9, 6, 0, 9, 4, 0, 1, 1, 5, 2, 4, 6, 7, 0, 6, 5, 6, 0, 0, 5, 9, 3, 5, 9, 1, 4, 2, 7, 9]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 8, 8, 9, 9, 9, 9, 9, 9]"
|
|
||||||
67,"[6, 0, 3, 8, 4, 6, 5, 1, 8, 8, 3, 2, 8, 8, 8, 9, 2, 2, 3, 8, 3, 5, 7, 1, 8, 9, 5, 6, 7, 8, 4, 2, 7, 4, 2, 7, 8, 1, 4, 4, 0, 1, 2, 6, 9, 1, 7, 1, 0, 9, 6, 2, 3, 0, 3, 4, 1, 0, 8, 6, 5, 2, 3, 2]","[0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9]"
|
|
||||||
68,"[5, 6, 8, 0, 1, 2, 1, 8, 4, 8, 2, 4, 0, 0, 5, 0, 2, 4, 6, 3, 3, 1, 2, 0, 1, 7, 6, 0, 1, 2, 8, 0, 7, 1, 8, 9, 1, 8, 2, 0, 8, 4, 1, 2, 3, 2, 9, 6, 4, 8, 7, 0, 9, 0, 8, 8, 8, 7, 9, 6, 3, 3, 7, 4]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 5, 5, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9]"
|
|
||||||
69,"[1, 3, 4, 2, 7, 2, 8, 2, 9, 5, 2, 5, 7, 0, 4, 8, 1, 6, 7, 1, 7, 1, 6, 4, 5, 7, 9, 7, 5, 3, 9, 2, 4, 2, 0, 8, 2, 3, 2, 4, 5, 7, 6, 2, 5, 2, 6, 7, 7, 0, 0, 3, 6, 7, 9, 3, 3, 6, 0, 8, 4, 9, 1, 4]","[0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 9, 9, 9, 9, 9]"
|
|
||||||
70,"[1, 8, 4, 6, 5, 4, 2, 8, 4, 2, 2, 7, 3, 0, 5, 9, 4, 9, 4, 5, 9, 8, 3, 8, 0, 8, 9, 5, 2, 4, 0, 2, 7, 0, 7, 6, 3, 7, 1, 8, 2, 3, 7, 4, 1, 3, 1, 4, 6, 6, 0, 4, 4, 8, 5, 6, 5, 3, 7, 5, 7, 6, 9, 1]","[0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9]"
|
|
||||||
71,"[9, 7, 2, 0, 0, 4, 1, 6, 9, 7, 1, 0, 5, 2, 9, 0, 5, 7, 1, 2, 0, 8, 8, 1, 3, 1, 9, 4, 8, 8, 3, 6, 5, 5, 4, 4, 3, 1, 0, 0, 2, 5, 2, 9, 9, 7, 3, 0, 4, 3, 1, 1, 1, 6, 4, 2, 0, 7, 4, 6, 3, 7, 9, 4]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
72,"[7, 1, 0, 2, 8, 6, 6, 8, 5, 9, 9, 5, 8, 2, 7, 2, 4, 1, 8, 7, 7, 0, 8, 4, 7, 6, 6, 0, 0, 8, 5, 3, 8, 6, 4, 0, 4, 3, 0, 6, 2, 0, 4, 4, 8, 1, 7, 1, 5, 7, 9, 5, 0, 8, 6, 1, 3, 4, 9, 8, 5, 4, 5, 2]","[0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9]"
|
|
||||||
73,"[1, 3, 6, 7, 4, 8, 7, 7, 5, 9, 0, 8, 4, 4, 6, 1, 2, 7, 2, 8, 1, 9, 7, 3, 7, 5, 0, 2, 3, 8, 1, 4, 1, 8, 2, 7, 6, 9, 9, 6, 3, 6, 3, 1, 0, 1, 0, 1, 1, 5, 8, 0, 3, 0, 4, 1, 1, 6, 2, 0, 7, 7, 3, 5]","[0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9]"
|
|
||||||
74,"[2, 7, 3, 7, 0, 1, 8, 4, 7, 1, 8, 4, 0, 1, 5, 6, 3, 6, 7, 6, 2, 7, 6, 6, 2, 7, 0, 5, 2, 8, 7, 0, 8, 9, 2, 4, 4, 3, 9, 9, 4, 0, 5, 2, 8, 1, 5, 4, 0, 6, 2, 4, 2, 7, 5, 5, 6, 5, 0, 8, 8, 0, 3, 9]","[0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9]"
|
|
||||||
75,"[9, 9, 1, 3, 6, 8, 9, 9, 8, 2, 8, 2, 0, 7, 7, 2, 6, 9, 4, 6, 9, 0, 5, 5, 5, 3, 7, 4, 3, 8, 1, 6, 9, 9, 6, 7, 0, 5, 3, 2, 0, 5, 6, 8, 8, 3, 8, 6, 5, 3, 4, 3, 7, 5, 2, 5, 6, 9, 8, 4, 6, 7, 2, 0]","[0, 0, 0, 0, 0, 1, 1, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
76,"[8, 5, 7, 6, 7, 0, 5, 9, 2, 8, 3, 8, 7, 5, 5, 6, 8, 6, 7, 9, 3, 2, 9, 2, 0, 8, 0, 3, 1, 4, 2, 4, 0, 2, 1, 1, 8, 9, 1, 3, 3, 6, 2, 8, 1, 5, 7, 3, 9, 2, 2, 8, 3, 8, 7, 8, 4, 9, 9, 6, 1, 5, 7, 3]","[0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
77,"[6, 0, 7, 6, 5, 5, 8, 7, 3, 1, 8, 6, 4, 6, 3, 5, 9, 3, 0, 5, 2, 6, 7, 0, 7, 0, 3, 8, 9, 7, 9, 3, 3, 1, 2, 9, 0, 2, 8, 5, 0, 7, 2, 7, 1, 9, 2, 0, 5, 6, 3, 7, 0, 6, 8, 2, 8, 4, 5, 4, 3, 8, 1, 8]","[0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9]"
|
|
||||||
78,"[1, 8, 1, 1, 1, 4, 5, 8, 1, 0, 3, 3, 4, 0, 6, 5, 5, 1, 6, 1, 6, 0, 6, 6, 8, 9, 3, 8, 7, 3, 5, 3, 4, 8, 4, 1, 0, 4, 2, 3, 0, 1, 2, 0, 7, 9, 3, 6, 8, 8, 3, 2, 5, 5, 1, 9, 4, 7, 8, 2, 1, 9, 6, 9]","[0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9]"
|
|
||||||
79,"[1, 4, 4, 0, 8, 2, 9, 3, 9, 4, 2, 0, 5, 7, 9, 3, 1, 8, 0, 1, 5, 4, 2, 8, 2, 3, 8, 5, 7, 9, 0, 6, 5, 4, 5, 1, 6, 1, 4, 0, 0, 3, 2, 8, 4, 7, 8, 6, 0, 5, 4, 8, 7, 7, 9, 2, 8, 9, 2, 8, 9, 3, 3, 9]","[0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 6, 6, 6, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
80,"[4, 5, 0, 0, 5, 2, 4, 6, 1, 6, 1, 0, 6, 1, 6, 3, 5, 4, 3, 5, 9, 6, 0, 8, 3, 8, 3, 8, 1, 5, 7, 4, 1, 8, 5, 0, 1, 9, 9, 0, 9, 3, 8, 6, 7, 7, 8, 0, 9, 4, 6, 7, 1, 3, 1, 3, 7, 3, 1, 6, 8, 4, 1, 3]","[0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9]"
|
|
||||||
81,"[7, 9, 4, 5, 4, 3, 6, 4, 6, 2, 5, 1, 4, 1, 8, 9, 4, 0, 5, 4, 8, 3, 4, 1, 3, 6, 1, 3, 7, 3, 0, 2, 7, 6, 3, 5, 1, 2, 1, 9, 9, 1, 0, 0, 5, 0, 6, 4, 5, 2, 4, 8, 3, 5, 0, 2, 6, 7, 1, 3, 5, 4, 4, 8]","[0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 8, 8, 8, 8, 9, 9, 9, 9]"
|
|
||||||
82,"[5, 5, 6, 7, 3, 4, 4, 1, 6, 2, 7, 6, 1, 0, 2, 7, 7, 6, 8, 6, 3, 3, 5, 4, 1, 2, 9, 9, 0, 9, 0, 3, 3, 8, 1, 4, 5, 3, 9, 0, 7, 0, 5, 6, 9, 7, 9, 3, 2, 5, 3, 7, 1, 3, 0, 0, 6, 1, 4, 8, 0, 5, 6, 6]","[0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 9, 9, 9, 9, 9, 9]"
|
|
||||||
83,"[6, 4, 9, 2, 4, 8, 7, 3, 2, 2, 1, 0, 0, 2, 7, 6, 0, 6, 1, 4, 0, 1, 4, 4, 6, 1, 1, 6, 9, 8, 1, 7, 3, 5, 6, 7, 8, 1, 2, 0, 9, 5, 0, 7, 6, 8, 7, 0, 1, 5, 9, 6, 4, 3, 1, 9, 7, 4, 4, 4, 6, 9, 9, 2]","[0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
84,"[6, 6, 3, 3, 6, 2, 8, 0, 2, 8, 4, 2, 1, 7, 0, 1, 8, 0, 2, 5, 7, 8, 0, 4, 0, 5, 0, 5, 2, 3, 9, 0, 8, 6, 1, 0, 8, 2, 5, 9, 6, 6, 6, 3, 7, 5, 7, 8, 5, 5, 2, 8, 0, 6, 0, 7, 7, 8, 2, 0, 5, 7, 0, 1]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9]"
|
|
||||||
85,"[0, 0, 3, 4, 5, 3, 6, 8, 9, 9, 2, 0, 7, 8, 1, 0, 2, 0, 6, 4, 3, 7, 4, 0, 8, 5, 4, 7, 9, 2, 7, 7, 6, 1, 8, 0, 1, 7, 3, 0, 8, 4, 8, 0, 0, 1, 7, 9, 6, 9, 9, 9, 3, 9, 0, 7, 0, 7, 4, 6, 9, 3, 8, 5]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 5, 5, 5, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
86,"[7, 0, 8, 1, 8, 3, 8, 3, 7, 2, 3, 9, 6, 1, 8, 4, 3, 3, 6, 2, 6, 8, 6, 3, 3, 6, 4, 6, 7, 1, 6, 1, 1, 7, 4, 7, 1, 1, 8, 5, 7, 9, 8, 3, 2, 4, 7, 6, 4, 5, 9, 4, 2, 5, 7, 7, 4, 6, 2, 2, 0, 5, 8, 6]","[0, 0, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9]"
|
|
||||||
87,"[3, 7, 2, 5, 8, 9, 5, 9, 5, 4, 2, 1, 4, 3, 1, 7, 4, 6, 6, 6, 2, 9, 2, 7, 9, 6, 4, 1, 1, 0, 9, 5, 1, 1, 9, 2, 1, 0, 2, 2, 3, 7, 7, 1, 1, 9, 7, 6, 0, 5, 5, 7, 3, 4, 8, 5, 8, 5, 9, 3, 7, 3, 1, 6]","[0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
88,"[1, 7, 0, 5, 1, 9, 3, 8, 5, 5, 5, 0, 8, 4, 0, 4, 5, 4, 0, 4, 3, 8, 8, 7, 1, 8, 2, 5, 1, 9, 7, 8, 9, 9, 1, 2, 9, 4, 6, 2, 9, 5, 3, 7, 0, 6, 3, 2, 2, 0, 0, 5, 8, 5, 2, 1, 5, 0, 9, 7, 7, 6, 3, 7]","[0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
89,"[4, 9, 2, 8, 8, 4, 7, 7, 9, 4, 8, 0, 2, 8, 2, 0, 2, 1, 8, 3, 1, 8, 7, 1, 3, 8, 9, 9, 7, 8, 9, 4, 4, 1, 7, 9, 9, 0, 4, 2, 8, 1, 4, 4, 5, 6, 0, 7, 5, 9, 1, 8, 6, 7, 0, 7, 3, 1, 0, 3, 1, 7, 8, 4]","[0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
90,"[5, 4, 9, 0, 2, 8, 3, 8, 2, 5, 7, 6, 3, 6, 2, 4, 0, 9, 0, 1, 7, 6, 1, 6, 6, 9, 2, 8, 9, 5, 9, 1, 0, 9, 7, 1, 6, 8, 2, 4, 6, 7, 1, 6, 8, 5, 5, 7, 5, 9, 9, 1, 5, 5, 0, 4, 3, 9, 6, 4, 0, 5, 4, 7]","[0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 3, 3, 3, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
91,"[5, 2, 3, 3, 2, 4, 8, 2, 8, 5, 4, 3, 4, 2, 1, 4, 5, 2, 1, 7, 4, 1, 2, 8, 9, 3, 2, 9, 9, 7, 9, 4, 7, 8, 4, 7, 4, 7, 3, 8, 2, 1, 6, 9, 9, 7, 3, 3, 2, 7, 8, 6, 5, 5, 5, 5, 5, 2, 9, 5, 5, 0, 7, 8]","[0, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
92,"[9, 1, 8, 8, 4, 1, 2, 2, 8, 9, 4, 3, 2, 4, 3, 8, 2, 4, 3, 6, 6, 1, 8, 0, 2, 0, 0, 9, 6, 5, 8, 5, 1, 2, 7, 2, 2, 7, 5, 8, 1, 0, 9, 5, 1, 6, 1, 5, 3, 9, 0, 7, 4, 9, 2, 8, 2, 6, 2, 4, 2, 7, 5, 2]","[0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9]"
|
|
||||||
93,"[1, 6, 4, 9, 3, 8, 7, 4, 4, 8, 5, 8, 6, 4, 0, 6, 9, 2, 6, 6, 3, 5, 2, 5, 4, 1, 5, 6, 2, 5, 8, 0, 8, 2, 9, 0, 9, 2, 5, 2, 6, 8, 9, 2, 2, 4, 4, 7, 6, 9, 5, 9, 5, 6, 0, 8, 4, 8, 5, 2, 4, 2, 5, 9]","[0, 0, 0, 0, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
94,"[1, 8, 1, 7, 3, 2, 5, 2, 1, 7, 2, 7, 0, 5, 4, 1, 8, 5, 7, 2, 5, 5, 2, 3, 9, 9, 0, 6, 6, 3, 6, 7, 8, 9, 3, 7, 0, 8, 9, 3, 4, 5, 3, 4, 7, 6, 6, 3, 3, 4, 1, 8, 5, 4, 0, 0, 9, 9, 7, 0, 8, 8, 0, 4]","[0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9]"
|
|
||||||
95,"[1, 0, 0, 5, 2, 5, 6, 2, 7, 6, 3, 8, 4, 2, 2, 9, 2, 4, 3, 2, 3, 3, 6, 6, 2, 3, 7, 4, 0, 2, 5, 6, 7, 3, 6, 7, 4, 9, 6, 6, 2, 9, 9, 5, 2, 5, 2, 2, 1, 9, 5, 0, 7, 5, 4, 7, 9, 1, 3, 3, 9, 1, 2, 1]","[0, 0, 0, 0, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 8, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
96,"[9, 0, 9, 7, 7, 6, 4, 6, 0, 4, 4, 3, 9, 2, 1, 8, 8, 3, 7, 1, 6, 3, 6, 6, 4, 4, 3, 4, 8, 9, 6, 6, 6, 2, 2, 0, 1, 6, 0, 1, 3, 0, 6, 7, 6, 5, 5, 9, 3, 5, 0, 8, 6, 1, 7, 0, 1, 7, 4, 0, 5, 7, 4, 3]","[0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 9, 9, 9, 9, 9]"
|
|
||||||
97,"[3, 3, 7, 0, 0, 8, 4, 0, 5, 8, 0, 9, 3, 5, 9, 7, 8, 3, 3, 2, 1, 8, 7, 0, 3, 3, 3, 8, 4, 4, 0, 1, 0, 0, 2, 3, 5, 8, 7, 0, 0, 3, 9, 5, 7, 3, 7, 7, 6, 4, 7, 5, 5, 1, 3, 8, 8, 1, 2, 0, 5, 6, 3, 5]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9]"
|
|
||||||
98,"[4, 7, 8, 4, 8, 0, 7, 7, 8, 4, 5, 2, 1, 7, 4, 3, 6, 1, 4, 5, 6, 0, 3, 6, 5, 7, 6, 3, 6, 6, 1, 8, 4, 3, 4, 0, 1, 3, 0, 3, 4, 5, 4, 2, 2, 9, 0, 9, 1, 3, 9, 6, 5, 4, 0, 5, 3, 2, 1, 6, 2, 8, 5, 7]","[0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 9, 9, 9]"
|
|
||||||
99,"[0, 5, 6, 5, 0, 0, 2, 6, 3, 2, 0, 6, 7, 4, 7, 9, 2, 7, 8, 0, 5, 5, 8, 8, 0, 9, 0, 8, 5, 3, 7, 9, 9, 2, 0, 6, 9, 2, 4, 1, 0, 5, 6, 4, 6, 4, 4, 0, 2, 2, 9, 1, 3, 0, 0, 0, 3, 4, 6, 3, 8, 1, 5, 5]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9]"
|
|
||||||
|
@ -1,790 +0,0 @@
|
|||||||
# Copyright (c) 2023 ETH Zurich.
|
|
||||||
# All rights reserved.
|
|
||||||
#
|
|
||||||
# Use of this source code is governed by a BSD-style license that can be
|
|
||||||
# found in the LICENSE file.
|
|
||||||
#
|
|
||||||
# main author: Nils Blach
|
|
||||||
|
|
||||||
import os
|
|
||||||
import logging
|
|
||||||
import datetime
|
|
||||||
import json
|
|
||||||
import csv
|
|
||||||
from typing import Dict, List, Callable, Union
|
|
||||||
from graph_of_thoughts import controller, language_models, operations, prompter, parser
|
|
||||||
|
|
||||||
# This is a hack to also allow execution of this file from the examples directory
|
|
||||||
try:
|
|
||||||
from . import utils
|
|
||||||
except ImportError:
|
|
||||||
import utils
|
|
||||||
|
|
||||||
|
|
||||||
class SortingPrompter(prompter.Prompter):
|
|
||||||
"""
|
|
||||||
SortingPrompter provides the generation of prompts specific to the sorting
|
|
||||||
example for the language models.
|
|
||||||
|
|
||||||
Inherits from the Prompter class and implements its abstract methods.
|
|
||||||
"""
|
|
||||||
|
|
||||||
sort_prompt = """<Instruction> Sort the following list of numbers in ascending order. Output only the sorted list of numbers, no additional text. </Instruction>
|
|
||||||
|
|
||||||
<Examples>
|
|
||||||
Input: [5, 1, 0, 1, 2, 0, 4, 8, 1, 9, 5, 1, 3, 3, 9, 7]
|
|
||||||
Output: [0, 0, 1, 1, 1, 1, 2, 3, 3, 4, 5, 5, 7, 8, 9, 9]
|
|
||||||
|
|
||||||
Input: [3, 7, 0, 2, 8, 1, 2, 2, 2, 4, 7, 8, 5, 5, 3, 9, 4, 3, 5, 6, 6, 4, 4, 5, 2, 0, 9, 3, 3, 9, 2, 1]
|
|
||||||
Output: [0, 0, 1, 1, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 9]
|
|
||||||
|
|
||||||
Input: [4, 4, 9, 7, 9, 7, 0, 0, 4, 9, 1, 7, 9, 5, 8, 7, 5, 6, 3, 8, 6, 7, 5, 8, 5, 0, 6, 3, 7, 0, 5, 3, 7, 5, 2, 4, 4, 9, 0, 7, 8, 2, 7, 7, 7, 2, 1, 3, 9, 9, 7, 9, 6, 6, 4, 5, 4, 2, 0, 8, 9, 0, 2, 2]
|
|
||||||
Output: [0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9]
|
|
||||||
</Examples>
|
|
||||||
|
|
||||||
Input: {input}"""
|
|
||||||
|
|
||||||
sort_prompt_cot = """<Instruction> Sort the following list of numbers in ascending order. You can generate any intermediate lists, but the final output should be the sorted list of numbers, prefixed with "Output: ". </Instruction>
|
|
||||||
|
|
||||||
<Approach>
|
|
||||||
To sort the list of numbers follow these steps:
|
|
||||||
1. Split the list of numbers into two to four unsorted sublists, each containing an equal number of elements from the original list (make sure they don't overlap).
|
|
||||||
2. Sort each of the unsorted sublists.
|
|
||||||
3. Merge the sorted sublists into a single sorted list using the merging algorithm from merge sort.
|
|
||||||
</Approach>
|
|
||||||
|
|
||||||
<Examples>
|
|
||||||
Input: [4, 5, 3, 3, 7, 3, 0, 5, 0, 2, 8, 0, 2, 1, 6, 9]
|
|
||||||
Unsorted Subarrays:
|
|
||||||
[4, 5, 3, 3, 7, 3, 0, 5]
|
|
||||||
[0, 2, 8, 0, 2, 1, 6, 9]
|
|
||||||
Sorted Subarrays:
|
|
||||||
[0, 3, 3, 3, 4, 5, 5, 7]
|
|
||||||
[0, 0, 1, 2, 2, 6, 8, 9]
|
|
||||||
Output: [0, 0, 0, 1, 2, 2, 3, 3, 3, 4, 5, 5, 6, 7, 8, 9]
|
|
||||||
|
|
||||||
Input: [6, 4, 5, 7, 5, 6, 9, 7, 6, 9, 4, 6, 9, 8, 1, 9, 2, 4, 9, 0, 7, 6, 5, 6, 6, 2, 8, 3, 9, 5, 6, 1]
|
|
||||||
Unsorted Subarrays:
|
|
||||||
[6, 4, 5, 7, 5, 6, 9, 7, 6, 9, 4, 6, 9, 8, 1, 9]
|
|
||||||
[2, 4, 9, 0, 7, 6, 5, 6, 6, 2, 8, 3, 9, 5, 6, 1]
|
|
||||||
Sorted Subarrays:
|
|
||||||
[1, 4, 4, 5, 5, 6, 6, 6, 6, 7, 7, 8, 9, 9, 9, 9]
|
|
||||||
[0, 1, 2, 2, 3, 4, 5, 5, 6, 6, 6, 6, 7, 8, 9, 9]
|
|
||||||
Output: [0, 1, 1, 2, 2, 3, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 8, 8, 9, 9, 9, 9, 9, 9]
|
|
||||||
|
|
||||||
Input: [3, 7, 0, 2, 8, 1, 2, 2, 2, 4, 7, 8, 5, 5, 3, 9, 4, 3, 5, 6, 6, 4, 4, 5, 2, 0, 9, 3, 3, 9, 2, 1, 9, 3, 1, 8, 1, 8, 6, 0, 1, 6, 1, 7, 4, 4, 6, 3, 3, 7, 9, 3, 6, 0, 3, 4, 5, 6, 6, 9, 9, 9, 7, 3]
|
|
||||||
Unsorted Subarrays:
|
|
||||||
[3, 7, 0, 2, 8, 1, 2, 2, 2, 4, 7, 8, 5, 5, 3, 9]
|
|
||||||
[4, 3, 5, 6, 6, 4, 4, 5, 2, 0, 9, 3, 3, 9, 2, 1]
|
|
||||||
[9, 3, 1, 8, 1, 8, 6, 0, 1, 6, 1, 7, 4, 4, 6, 3]
|
|
||||||
[3, 7, 9, 3, 6, 0, 3, 4, 5, 6, 6, 9, 9, 9, 7, 3]
|
|
||||||
Sorted Subarrays:
|
|
||||||
[0, 1, 2, 2, 2, 2, 3, 3, 4, 5, 5, 7, 7, 8, 8, 9]
|
|
||||||
[0, 1, 2, 2, 3, 3, 3, 4, 4, 4, 5, 5, 6, 6, 9, 9]
|
|
||||||
[0, 1, 1, 1, 1, 3, 3, 4, 4, 6, 6, 6, 7, 8, 8, 9]
|
|
||||||
[0, 3, 3, 3, 3, 4, 5, 6, 6, 6, 7, 7, 9, 9, 9, 9]
|
|
||||||
Output: [0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9]
|
|
||||||
</Examples>
|
|
||||||
|
|
||||||
Input: {input}"""
|
|
||||||
|
|
||||||
tot_improve_prompt = """<Instruction> The following two lists represent an unsorted list of numbers and a sorted variant of that list. The sorted variant is not correct. Fix the sorted variant so that it is correct.
|
|
||||||
Make sure that the output list is sorted in ascending order, has the same number of elements as the input list ({length}), and contains the same elements as the input list. </Instruction>
|
|
||||||
|
|
||||||
<Approach>
|
|
||||||
To fix the incorrectly sorted list follow these steps:
|
|
||||||
1. For each number from 0 to 9, compare the frequency of that number in the incorrectly sorted list to the frequency of that number in the input list.
|
|
||||||
2. Iterate through the incorrectly sorted list and add or remove numbers as needed to make the frequency of each number in the incorrectly sorted list match the frequency of that number in the input list.
|
|
||||||
</Approach>
|
|
||||||
|
|
||||||
<Examples>
|
|
||||||
Input: [3, 7, 0, 2, 8, 1, 2, 2, 2, 4, 7, 8, 5, 5, 3, 9]
|
|
||||||
Incorrectly Sorted: [0, 0, 0, 0, 0, 1, 2, 2, 3, 3, 4, 4, 4, 5, 5, 7, 7, 8, 8, 9, 9, 9, 9]
|
|
||||||
Reason: The incorrectly sorted list contains four extra 0s, two extra 4s and three extra 9s and is missing two 2s.
|
|
||||||
Output: [0, 1, 2, 2, 2, 2, 3, 3, 4, 5, 5, 7, 7, 8, 8, 9]
|
|
||||||
|
|
||||||
Input: [6, 4, 5, 7, 5, 6, 9, 7, 6, 9, 4, 6, 9, 8, 1, 9, 2, 4, 9, 0, 7, 6, 5, 6, 6, 2, 8, 3, 9, 5, 6, 1]
|
|
||||||
Incorrectly Sorted: [0, 1, 1, 2, 2, 3, 4, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 7, 7, 7, 8, 8, 9, 9, 9, 9, 9]
|
|
||||||
Reason: The incorrectly sorted list contains two extra 4s and is missing two 6s and one 9.
|
|
||||||
Output: [0, 1, 1, 2, 2, 3, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 8, 8, 9, 9, 9, 9, 9, 9]
|
|
||||||
|
|
||||||
Input: [4, 4, 9, 7, 9, 7, 0, 0, 4, 9, 1, 7, 9, 5, 8, 7, 5, 6, 3, 8, 6, 7, 5, 8, 5, 0, 6, 3, 7, 0, 5, 3, 7, 5, 2, 4, 4, 9, 0, 7, 8, 2, 7, 7, 7, 2, 1, 3, 9, 9, 7, 9, 6, 6, 4, 5, 4, 2, 0, 8, 9, 0, 2, 2]
|
|
||||||
Incorrectly Sorted: [0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 2, 2, 2, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 5, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9]
|
|
||||||
Reason: The incorrectly sorted list contains one extra 8 and is missing two 2s, one 3, three 4s, two 5s, one 6, six 7s and one 9.
|
|
||||||
Output: [0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9]
|
|
||||||
</Examples>
|
|
||||||
|
|
||||||
Input: {input}
|
|
||||||
Incorrectly Sorted: {incorrectly_sorted}
|
|
||||||
"""
|
|
||||||
|
|
||||||
got_split_prompt = """<Instruction> Split the following list of 64 numbers into 4 lists of 16 numbers each, the first list should contain the first 16 numbers, the second list the second 16 numbers, the third list the third 16 numbers and the fourth list the fourth 16 numbers.
|
|
||||||
Only output the final 4 lists in the following format without any additional text or thoughts!:
|
|
||||||
{{
|
|
||||||
"List 1": [3, 4, 3, 5, 7, 8, 1, ...],
|
|
||||||
"List 2": [2, 9, 2, 4, 7, 1, 5, ...],
|
|
||||||
"List 3": [6, 9, 8, 1, 9, 2, 4, ...],
|
|
||||||
"List 4": [9, 0, 7, 6, 5, 6, 6, ...]
|
|
||||||
}} </Instruction>
|
|
||||||
|
|
||||||
<Example>
|
|
||||||
Input: [3, 1, 9, 3, 7, 5, 5, 4, 8, 1, 5, 3, 3, 2, 3, 0, 9, 7, 2, 2, 4, 4, 8, 5, 0, 8, 7, 3, 3, 8, 7, 0, 9, 5, 1, 6, 7, 6, 8, 9, 0, 3, 0, 6, 3, 4, 8, 0, 6, 9, 8, 4, 1, 2, 9, 0, 4, 8, 8, 9, 9, 8, 5, 9]
|
|
||||||
Output:
|
|
||||||
{{
|
|
||||||
"List 1": [3, 1, 9, 3, 7, 5, 5, 4, 8, 1, 5, 3, 3, 2, 3, 0],
|
|
||||||
"List 2": [9, 7, 2, 2, 4, 4, 8, 5, 0, 8, 7, 3, 3, 8, 7, 0],
|
|
||||||
"List 3": [9, 5, 1, 6, 7, 6, 8, 9, 0, 3, 0, 6, 3, 4, 8, 0],
|
|
||||||
"List 4": [6, 9, 8, 4, 1, 2, 9, 0, 4, 8, 8, 9, 9, 8, 5, 9]
|
|
||||||
}}
|
|
||||||
</Example>
|
|
||||||
|
|
||||||
Input: {input}"""
|
|
||||||
|
|
||||||
got_merge_prompt = """<Instruction> Merge the following 2 sorted lists of length {length1} each, into one sorted list of length {length2} using a merge sort style approach.
|
|
||||||
Only output the final merged list without any additional text or thoughts!:</Instruction>
|
|
||||||
|
|
||||||
<Approach>
|
|
||||||
To merge the two lists in a merge-sort style approach, follow these steps:
|
|
||||||
1. Compare the first element of both lists.
|
|
||||||
2. Append the smaller element to the merged list and move to the next element in the list from which the smaller element came.
|
|
||||||
3. Repeat steps 1 and 2 until one of the lists is empty.
|
|
||||||
4. Append the remaining elements of the non-empty list to the merged list.
|
|
||||||
</Approach>
|
|
||||||
|
|
||||||
Merge the following two lists into one sorted list:
|
|
||||||
1: {input1}
|
|
||||||
2: {input2}
|
|
||||||
|
|
||||||
Merged list:
|
|
||||||
"""
|
|
||||||
|
|
||||||
def aggregation_prompt(self, state_dicts: List[Dict], **kwargs) -> str:
|
|
||||||
"""
|
|
||||||
Generate an aggregation prompt for the language model.
|
|
||||||
|
|
||||||
:param state_dicts: The thought states that should be aggregated.
|
|
||||||
:type state_dicts: List[Dict]
|
|
||||||
:param kwargs: Additional keyword arguments.
|
|
||||||
:return: The aggregation prompt.
|
|
||||||
:rtype: str
|
|
||||||
:raise AssertionError: If not exactly two thought states are provided.
|
|
||||||
"""
|
|
||||||
assert len(state_dicts) == 2, "Expected two states for aggregation prompt."
|
|
||||||
len_input1 = len(utils.string_to_list(state_dicts[0]["current"]))
|
|
||||||
len_input2 = len(utils.string_to_list(state_dicts[1]["current"]))
|
|
||||||
if len_input1 == len_input2:
|
|
||||||
length = len_input1
|
|
||||||
elif len_input1 + len_input2 - 32 <= 16:
|
|
||||||
length = 16
|
|
||||||
else:
|
|
||||||
length = 32
|
|
||||||
|
|
||||||
return self.got_merge_prompt.format(
|
|
||||||
input1=state_dicts[0]["current"],
|
|
||||||
input2=state_dicts[1]["current"],
|
|
||||||
length1=length,
|
|
||||||
length2=length * 2,
|
|
||||||
)
|
|
||||||
|
|
||||||
def generate_prompt(
|
|
||||||
self, num_branches: int, original: str, current: str, method: str, **kwargs
|
|
||||||
) -> str:
|
|
||||||
"""
|
|
||||||
Generate a generate prompt for the language model.
|
|
||||||
|
|
||||||
:param num_branches: The number of responses the prompt should ask the LM to generate.
|
|
||||||
:type num_branches: int
|
|
||||||
:param original: Input list of numbers.
|
|
||||||
:type original: str
|
|
||||||
:param current: Intermediate solution.
|
|
||||||
:type current: str
|
|
||||||
:param method: Method for which the generate prompt is generated.
|
|
||||||
:type method: str
|
|
||||||
:param kwargs: Additional keyword arguments.
|
|
||||||
:return: The generate prompt.
|
|
||||||
:rtype: str
|
|
||||||
:raise AssertionError: If the requested number of branches is not one.
|
|
||||||
"""
|
|
||||||
|
|
||||||
assert num_branches == 1, "Branching should be done via multiple requests."
|
|
||||||
if current is None or current == "":
|
|
||||||
input = original
|
|
||||||
else:
|
|
||||||
input = current
|
|
||||||
if method.startswith("io"):
|
|
||||||
return self.sort_prompt.format(input=input)
|
|
||||||
elif method.startswith("cot"):
|
|
||||||
return self.sort_prompt_cot.format(input=input)
|
|
||||||
elif method.startswith("tot"):
|
|
||||||
if current is None or current == "":
|
|
||||||
return self.sort_prompt.format(input=input)
|
|
||||||
return self.tot_improve_prompt.format(
|
|
||||||
input=original,
|
|
||||||
incorrectly_sorted=current,
|
|
||||||
length=len(utils.string_to_list(original)),
|
|
||||||
)
|
|
||||||
elif method.startswith("got"):
|
|
||||||
if current is None or current == "":
|
|
||||||
return self.got_split_prompt.format(input=input)
|
|
||||||
# if current is just a sublist of the original input, return the split prompt
|
|
||||||
if kwargs["phase"] == 1:
|
|
||||||
return self.sort_prompt.format(input=current)
|
|
||||||
|
|
||||||
if (
|
|
||||||
"unsorted_sublist" in kwargs
|
|
||||||
and kwargs["unsorted_sublist"] != ""
|
|
||||||
and len(kwargs["unsorted_sublist"]) < len(original) - 5
|
|
||||||
):
|
|
||||||
original = kwargs["unsorted_sublist"]
|
|
||||||
|
|
||||||
return self.tot_improve_prompt.format(
|
|
||||||
input=original,
|
|
||||||
incorrectly_sorted=current,
|
|
||||||
length=len(utils.string_to_list(original)),
|
|
||||||
)
|
|
||||||
|
|
||||||
def improve_prompt(self, **kwargs) -> str:
|
|
||||||
"""
|
|
||||||
Generate an improve prompt for the language model.
|
|
||||||
|
|
||||||
:param kwargs: Additional keyword arguments.
|
|
||||||
:return: The improve prompt.
|
|
||||||
:rtype: str
|
|
||||||
"""
|
|
||||||
pass
|
|
||||||
|
|
||||||
def validation_prompt(self, **kwargs) -> str:
|
|
||||||
"""
|
|
||||||
Generate a validation prompt for the language model.
|
|
||||||
|
|
||||||
:param kwargs: Additional keyword arguments.
|
|
||||||
:return: The validation prompt.
|
|
||||||
:rtype: str
|
|
||||||
"""
|
|
||||||
pass
|
|
||||||
|
|
||||||
def score_prompt(self, state_dicts: List[Dict], **kwargs) -> str:
|
|
||||||
"""
|
|
||||||
Generate a score prompt for the language model.
|
|
||||||
|
|
||||||
:param state_dicts: The thought states that should be scored,
|
|
||||||
if more than one, they should be scored together.
|
|
||||||
:type state_dicts: List[Dict]
|
|
||||||
:param kwargs: Additional keyword arguments.
|
|
||||||
:return: The score prompt.
|
|
||||||
:rtype: str
|
|
||||||
"""
|
|
||||||
pass
|
|
||||||
|
|
||||||
|
|
||||||
class SortingParser(parser.Parser):
|
|
||||||
"""
|
|
||||||
SortingParser provides the parsing of language model reponses specific to
|
|
||||||
the sorting example.
|
|
||||||
|
|
||||||
Inherits from the Parser class and implements its abstract methods.
|
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(self) -> None:
|
|
||||||
"""
|
|
||||||
Inits the response cache.
|
|
||||||
"""
|
|
||||||
self.cache = {}
|
|
||||||
|
|
||||||
def parse_aggregation_answer(
|
|
||||||
self, states: List[Dict], texts: List[str]
|
|
||||||
) -> Union[Dict, List[Dict]]:
|
|
||||||
"""
|
|
||||||
Parse the response from the language model for an aggregation prompt.
|
|
||||||
|
|
||||||
:param states: The thought states used to generate the prompt.
|
|
||||||
:type states: List[Dict]
|
|
||||||
:param texts: The responses to the prompt from the language model.
|
|
||||||
:type texts: List[str]
|
|
||||||
:return: The new thought states after parsing the respones from the language model.
|
|
||||||
:rtype: Union[Dict, List[Dict]]
|
|
||||||
:raise AssertionError: If not exactly two thought states are provided.
|
|
||||||
"""
|
|
||||||
|
|
||||||
assert len(states) == 2, "Expected two states for aggregation answer."
|
|
||||||
new_states = []
|
|
||||||
for text in texts:
|
|
||||||
answers = text.strip().split("\n")
|
|
||||||
if any(["Output" in answer for answer in answers]):
|
|
||||||
# cut elements until last output is found
|
|
||||||
for answer in reversed(answers):
|
|
||||||
if "Output" in answer:
|
|
||||||
answers = answers[answers.index(answer) :]
|
|
||||||
break
|
|
||||||
|
|
||||||
answers_stripped = [
|
|
||||||
answer for answer in answers if "[" in answer and "]" in answer
|
|
||||||
]
|
|
||||||
if len(answers_stripped) == 0:
|
|
||||||
for answer in answers:
|
|
||||||
answer = "[" + answer + "]"
|
|
||||||
try:
|
|
||||||
answer_converted = utils.string_to_list(answer)
|
|
||||||
if len(answer_converted) > 0:
|
|
||||||
answers_stripped.append(answer)
|
|
||||||
except:
|
|
||||||
pass
|
|
||||||
if len(answers_stripped) == 0:
|
|
||||||
logging.warning(
|
|
||||||
f"Could not parse aggregation answer: {text}. Returning empty list."
|
|
||||||
)
|
|
||||||
answer = "[]"
|
|
||||||
else:
|
|
||||||
answer = [
|
|
||||||
answer[answer.index("[") : answer.index("]") + 1]
|
|
||||||
for answer in answers_stripped
|
|
||||||
][0]
|
|
||||||
states = sorted(states, key=lambda x: x["part"])
|
|
||||||
merged_unsorted_sublists = (
|
|
||||||
states[0]["unsorted_sublist"][:-1]
|
|
||||||
+ ", "
|
|
||||||
+ states[1]["unsorted_sublist"][1:]
|
|
||||||
)
|
|
||||||
new_state = states[0].copy()
|
|
||||||
new_state["current"] = answer
|
|
||||||
new_state["unsorted_sublist"] = merged_unsorted_sublists
|
|
||||||
new_states.append(new_state)
|
|
||||||
return new_states
|
|
||||||
|
|
||||||
def parse_generate_answer(self, state: Dict, texts: List[str]) -> List[Dict]:
|
|
||||||
"""
|
|
||||||
Parse the response from the language model for a generate prompt.
|
|
||||||
|
|
||||||
:param state: The thought state used to generate the prompt.
|
|
||||||
:type state: Dict
|
|
||||||
:param texts: The responses to the prompt from the language model.
|
|
||||||
:type texts: List[str]
|
|
||||||
:return: The new thought states after parsing the respones from the language model.
|
|
||||||
:rtype: List[Dict]
|
|
||||||
"""
|
|
||||||
new_states = []
|
|
||||||
for text in texts:
|
|
||||||
if state["method"] == "got" and state["current"] == "":
|
|
||||||
# We expect a json which contains the four lists named "List 1" to "List 4"
|
|
||||||
# cut everything until the opening bracket and everything after the closing bracket
|
|
||||||
|
|
||||||
try:
|
|
||||||
text = text[text.index("{") : text.index("}") + 1]
|
|
||||||
json_dict = json.loads(text)
|
|
||||||
if len(json_dict.keys()) != 4:
|
|
||||||
logging.warning(
|
|
||||||
f"Expected 4 lists in json, but found {len(json_dict.keys())}."
|
|
||||||
)
|
|
||||||
for key, value in json_dict.items():
|
|
||||||
if "List" not in key:
|
|
||||||
logging.warning(
|
|
||||||
f"Expected key to contain 'List', but found {key}."
|
|
||||||
)
|
|
||||||
continue
|
|
||||||
if not isinstance(value, list):
|
|
||||||
value = utils.string_to_list(value)
|
|
||||||
new_state = state.copy()
|
|
||||||
new_state["current"] = str(value)
|
|
||||||
new_state["unsorted_sublist"] = str(value)
|
|
||||||
new_state["phase"] = 1
|
|
||||||
new_state["part"] = key
|
|
||||||
new_states.append(new_state)
|
|
||||||
except Exception as e:
|
|
||||||
logging.error(
|
|
||||||
f"Could not parse step answer: {text}. Encountered exception: {e}"
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
answers = text.strip().split("\n")
|
|
||||||
answers = [
|
|
||||||
answer for answer in answers if "[" in answer and "]" in answer
|
|
||||||
]
|
|
||||||
if any(["Output" in answer for answer in answers]):
|
|
||||||
# cut elements until last output is found
|
|
||||||
for answer in reversed(answers):
|
|
||||||
if "Output" in answer:
|
|
||||||
answers = answers[answers.index(answer) :]
|
|
||||||
break
|
|
||||||
|
|
||||||
answers = [
|
|
||||||
answer[answer.index("[") : answer.index("]") + 1]
|
|
||||||
for answer in answers
|
|
||||||
]
|
|
||||||
if len(answers) == 0:
|
|
||||||
logging.warning(
|
|
||||||
f"Could not parse step answer: {text}. Returning empty list."
|
|
||||||
)
|
|
||||||
answer = "[]"
|
|
||||||
else:
|
|
||||||
if len(answers) > 1:
|
|
||||||
logging.warning(
|
|
||||||
f"Multiple answers found for step answer: {text}. Using the first one."
|
|
||||||
)
|
|
||||||
answer = answers[0]
|
|
||||||
|
|
||||||
new_state = state.copy()
|
|
||||||
new_state["current"] = answer
|
|
||||||
new_state["phase"] = 2
|
|
||||||
new_states.append(new_state)
|
|
||||||
return new_states
|
|
||||||
|
|
||||||
def parse_improve_answer(self, state: Dict, texts: List[str]) -> Dict:
|
|
||||||
"""
|
|
||||||
Parse the response from the language model for an improve prompt.
|
|
||||||
|
|
||||||
:param state: The thought state used to generate the prompt.
|
|
||||||
:type state: Dict
|
|
||||||
:param texts: The responses to the prompt from the language model.
|
|
||||||
:type texts: List[str]
|
|
||||||
:return: The new thought state after parsing the responses from the language model.
|
|
||||||
:rtype: Dict
|
|
||||||
"""
|
|
||||||
pass
|
|
||||||
|
|
||||||
def parse_validation_answer(self, state: Dict, texts: List[str]) -> bool:
|
|
||||||
"""
|
|
||||||
Parse the response from the language model for a validation prompt.
|
|
||||||
|
|
||||||
:param state: The thought state used to generate the prompt.
|
|
||||||
:type state: Dict
|
|
||||||
:param texts: The responses to the prompt from the language model.
|
|
||||||
:type texts: List[str]
|
|
||||||
:return: Whether the thought state is valid or not.
|
|
||||||
:rtype: bool
|
|
||||||
"""
|
|
||||||
pass
|
|
||||||
|
|
||||||
def parse_score_answer(self, states: List[Dict], texts: List[str]) -> List[float]:
|
|
||||||
"""
|
|
||||||
Parse the response from the language model for a score prompt.
|
|
||||||
|
|
||||||
:param states: The thought states used to generate the prompt.
|
|
||||||
:type states: List[Dict]
|
|
||||||
:param texts: The responses to the prompt from the language model.
|
|
||||||
:type texts: List[str]
|
|
||||||
:return: The scores for the thought states.
|
|
||||||
:rtype: List[float]
|
|
||||||
"""
|
|
||||||
pass
|
|
||||||
|
|
||||||
|
|
||||||
def io() -> operations.GraphOfOperations:
|
|
||||||
"""
|
|
||||||
Generates the Graph of Operations for the IO method.
|
|
||||||
|
|
||||||
:return: Graph of Operations
|
|
||||||
:rtype: GraphOfOperations
|
|
||||||
"""
|
|
||||||
operations_graph = operations.GraphOfOperations()
|
|
||||||
|
|
||||||
operations_graph.append_operation(operations.Generate(1, 1))
|
|
||||||
operations_graph.append_operation(operations.Score(1, False, utils.num_errors))
|
|
||||||
operations_graph.append_operation(operations.GroundTruth(utils.test_sorting))
|
|
||||||
|
|
||||||
return operations_graph
|
|
||||||
|
|
||||||
|
|
||||||
def cot() -> operations.GraphOfOperations:
|
|
||||||
"""
|
|
||||||
Generates the Graph of Operations for the CoT method.
|
|
||||||
|
|
||||||
:return: Graph of Operations
|
|
||||||
:rtype: GraphOfOperations
|
|
||||||
"""
|
|
||||||
operations_graph = operations.GraphOfOperations()
|
|
||||||
|
|
||||||
operations_graph.append_operation(operations.Generate(1, 1))
|
|
||||||
operations_graph.append_operation(operations.Score(1, False, utils.num_errors))
|
|
||||||
operations_graph.append_operation(operations.GroundTruth(utils.test_sorting))
|
|
||||||
|
|
||||||
return operations_graph
|
|
||||||
|
|
||||||
|
|
||||||
def tot() -> operations.GraphOfOperations:
|
|
||||||
"""
|
|
||||||
Generates the Graph of Operations for the ToT method.
|
|
||||||
ToT uses a wider tree, where on each level there are more branches.
|
|
||||||
|
|
||||||
:return: Graph of Operations
|
|
||||||
:rtype: GraphOfOperations
|
|
||||||
"""
|
|
||||||
operations_graph = operations.GraphOfOperations()
|
|
||||||
|
|
||||||
operations_graph.append_operation(operations.Generate(1, 20))
|
|
||||||
operations_graph.append_operation(operations.Score(1, False, utils.num_errors))
|
|
||||||
keep_best_1 = operations.KeepBestN(1, False)
|
|
||||||
operations_graph.append_operation(keep_best_1)
|
|
||||||
|
|
||||||
for _ in range(3):
|
|
||||||
operations_graph.append_operation(operations.Generate(1, 20))
|
|
||||||
operations_graph.append_operation(operations.Score(1, False, utils.num_errors))
|
|
||||||
keep_best_2 = operations.KeepBestN(1, False)
|
|
||||||
keep_best_2.add_predecessor(keep_best_1)
|
|
||||||
operations_graph.append_operation(keep_best_2)
|
|
||||||
keep_best_1 = keep_best_2
|
|
||||||
|
|
||||||
operations_graph.append_operation(operations.KeepBestN(1, False))
|
|
||||||
operations_graph.append_operation(operations.GroundTruth(utils.test_sorting))
|
|
||||||
|
|
||||||
return operations_graph
|
|
||||||
|
|
||||||
|
|
||||||
def tot2() -> operations.GraphOfOperations:
|
|
||||||
"""
|
|
||||||
Generates the Graph of Operations for the ToT2 method.
|
|
||||||
ToT2 uses a tree with more levels, but with fewer branches per level.
|
|
||||||
|
|
||||||
:return: Graph of Operations
|
|
||||||
:rtype: GraphOfOperations
|
|
||||||
"""
|
|
||||||
operations_graph = operations.GraphOfOperations()
|
|
||||||
|
|
||||||
operations_graph.append_operation(operations.Generate(1, 10))
|
|
||||||
operations_graph.append_operation(operations.Score(1, False, utils.num_errors))
|
|
||||||
keep_best_1 = operations.KeepBestN(1, False)
|
|
||||||
operations_graph.append_operation(keep_best_1)
|
|
||||||
|
|
||||||
for _ in range(6):
|
|
||||||
operations_graph.append_operation(operations.Generate(1, 10))
|
|
||||||
operations_graph.append_operation(operations.Score(1, False, utils.num_errors))
|
|
||||||
keep_best_2 = operations.KeepBestN(1, False)
|
|
||||||
keep_best_2.add_predecessor(keep_best_1)
|
|
||||||
operations_graph.append_operation(keep_best_2)
|
|
||||||
keep_best_1 = keep_best_2
|
|
||||||
|
|
||||||
operations_graph.append_operation(operations.KeepBestN(1, False))
|
|
||||||
operations_graph.append_operation(operations.KeepBestN(1, False))
|
|
||||||
|
|
||||||
operations_graph.append_operation(operations.KeepBestN(1, False))
|
|
||||||
|
|
||||||
operations_graph.append_operation(operations.GroundTruth(utils.test_sorting))
|
|
||||||
|
|
||||||
return operations_graph
|
|
||||||
|
|
||||||
|
|
||||||
def got() -> operations.GraphOfOperations:
|
|
||||||
"""
|
|
||||||
Generates the Graph of Operations for the GoT method.
|
|
||||||
|
|
||||||
:return: Graph of Operations
|
|
||||||
:rtype: GraphOfOperations
|
|
||||||
"""
|
|
||||||
operations_graph = operations.GraphOfOperations()
|
|
||||||
|
|
||||||
plans = operations.Generate(1, 1)
|
|
||||||
operations_graph.append_operation(plans) # generate the sublists
|
|
||||||
sorted_sublists = []
|
|
||||||
for i in range(1, 5):
|
|
||||||
list_id = f"List {i}"
|
|
||||||
sub_list = operations.Selector(
|
|
||||||
lambda thoughts, list_id=list_id: [
|
|
||||||
thought for thought in thoughts if thought.state["part"] == list_id
|
|
||||||
]
|
|
||||||
)
|
|
||||||
sub_list.add_predecessor(plans)
|
|
||||||
operations_graph.add_operation(sub_list)
|
|
||||||
sort_sub_list = operations.Generate(1, 5)
|
|
||||||
sort_sub_list.add_predecessor(sub_list)
|
|
||||||
operations_graph.add_operation(sort_sub_list)
|
|
||||||
score_sub_list = operations.Score(1, False, utils.num_errors)
|
|
||||||
score_sub_list.add_predecessor(sort_sub_list)
|
|
||||||
operations_graph.add_operation(score_sub_list)
|
|
||||||
keep_best_sub_list = operations.KeepBestN(1, False)
|
|
||||||
keep_best_sub_list.add_predecessor(score_sub_list)
|
|
||||||
operations_graph.add_operation(keep_best_sub_list)
|
|
||||||
|
|
||||||
sorted_sublists.append(keep_best_sub_list)
|
|
||||||
|
|
||||||
aggregate_1 = operations.Aggregate(10)
|
|
||||||
aggregate_1.add_predecessor(sorted_sublists[0])
|
|
||||||
aggregate_1.add_predecessor(sorted_sublists[1])
|
|
||||||
operations_graph.add_operation(aggregate_1)
|
|
||||||
score_aggregate_1 = operations.Score(1, False, utils.num_errors)
|
|
||||||
score_aggregate_1.add_predecessor(aggregate_1)
|
|
||||||
operations_graph.add_operation(score_aggregate_1)
|
|
||||||
keep_best_aggregate_1 = operations.KeepBestN(1, False)
|
|
||||||
keep_best_aggregate_1.add_predecessor(score_aggregate_1)
|
|
||||||
operations_graph.add_operation(keep_best_aggregate_1)
|
|
||||||
|
|
||||||
improve_aggregate_1 = operations.Generate(1, 5)
|
|
||||||
improve_aggregate_1.add_predecessor(keep_best_aggregate_1)
|
|
||||||
operations_graph.add_operation(improve_aggregate_1)
|
|
||||||
improve_score_aggregate_1 = operations.Score(1, False, utils.num_errors)
|
|
||||||
improve_score_aggregate_1.add_predecessor(improve_aggregate_1)
|
|
||||||
improve_score_aggregate_1.add_predecessor(keep_best_aggregate_1)
|
|
||||||
operations_graph.add_operation(improve_score_aggregate_1)
|
|
||||||
improve_keep_best_aggregate_1 = operations.KeepBestN(1, False)
|
|
||||||
improve_keep_best_aggregate_1.add_predecessor(improve_score_aggregate_1)
|
|
||||||
operations_graph.add_operation(improve_keep_best_aggregate_1)
|
|
||||||
|
|
||||||
aggregate_2 = operations.Aggregate(10)
|
|
||||||
aggregate_2.add_predecessor(sorted_sublists[2])
|
|
||||||
aggregate_2.add_predecessor(sorted_sublists[3])
|
|
||||||
operations_graph.add_operation(aggregate_2)
|
|
||||||
score_aggregate_2 = operations.Score(1, False, utils.num_errors)
|
|
||||||
score_aggregate_2.add_predecessor(aggregate_2)
|
|
||||||
operations_graph.add_operation(score_aggregate_2)
|
|
||||||
keep_best_aggregate_2 = operations.KeepBestN(1, False)
|
|
||||||
keep_best_aggregate_2.add_predecessor(score_aggregate_2)
|
|
||||||
operations_graph.add_operation(keep_best_aggregate_2)
|
|
||||||
|
|
||||||
improve_aggregate_2 = operations.Generate(1, 5)
|
|
||||||
improve_aggregate_2.add_predecessor(keep_best_aggregate_2)
|
|
||||||
operations_graph.add_operation(improve_aggregate_2)
|
|
||||||
improve_score_aggregate_2 = operations.Score(1, False, utils.num_errors)
|
|
||||||
improve_score_aggregate_2.add_predecessor(improve_aggregate_2)
|
|
||||||
improve_score_aggregate_2.add_predecessor(keep_best_aggregate_2)
|
|
||||||
operations_graph.add_operation(improve_score_aggregate_2)
|
|
||||||
improve_keep_best_aggregate_2 = operations.KeepBestN(1, False)
|
|
||||||
improve_keep_best_aggregate_2.add_predecessor(improve_score_aggregate_2)
|
|
||||||
operations_graph.add_operation(improve_keep_best_aggregate_2)
|
|
||||||
|
|
||||||
final_aggregate = operations.Aggregate(10)
|
|
||||||
operations_graph.append_operation(final_aggregate)
|
|
||||||
operations_graph.append_operation(operations.Score(1, False, utils.num_errors))
|
|
||||||
keep_best_aggregate_final = operations.KeepBestN(1, False)
|
|
||||||
operations_graph.append_operation(keep_best_aggregate_final)
|
|
||||||
|
|
||||||
operations_graph.append_operation(operations.Generate(1, 10))
|
|
||||||
score_aggr_3 = operations.Score(1, False, utils.num_errors)
|
|
||||||
score_aggr_3.add_predecessor(keep_best_aggregate_final)
|
|
||||||
operations_graph.append_operation(score_aggr_3)
|
|
||||||
operations_graph.append_operation(operations.KeepBestN(1, False))
|
|
||||||
|
|
||||||
operations_graph.append_operation(operations.GroundTruth(utils.test_sorting))
|
|
||||||
|
|
||||||
return operations_graph
|
|
||||||
|
|
||||||
|
|
||||||
def run(
|
|
||||||
data_ids: List[int],
|
|
||||||
methods: List[Callable[[], operations.GraphOfOperations]],
|
|
||||||
budget: float,
|
|
||||||
lm_name: str,
|
|
||||||
) -> float:
|
|
||||||
"""
|
|
||||||
Controller function that executes each specified method for each specified
|
|
||||||
sample while the budget is not exhausted.
|
|
||||||
|
|
||||||
:param data_ids: Indices of the sample to be run.
|
|
||||||
:type data_ids: List[int]
|
|
||||||
:param methods: List of functions to generate Graphs of Operations.
|
|
||||||
:type methods: Each function generates a Graph of Operation.
|
|
||||||
:param budget: Language model budget for the execution in dollars.
|
|
||||||
:type budget: float
|
|
||||||
:param lm_name: Name of the language model to be used.
|
|
||||||
:type lm_name: str
|
|
||||||
:return: Spent budget in dollars.
|
|
||||||
:rtype: float
|
|
||||||
"""
|
|
||||||
|
|
||||||
orig_budget = budget
|
|
||||||
data_path = os.path.join(os.path.dirname(__file__), "sorting_064.csv")
|
|
||||||
data = []
|
|
||||||
with open(data_path, "r") as f:
|
|
||||||
reader = csv.reader(f)
|
|
||||||
next(reader)
|
|
||||||
for row in reader:
|
|
||||||
data.append([int(row[0]), row[1], row[2]])
|
|
||||||
|
|
||||||
if data_ids is None or len(data_ids) == 0:
|
|
||||||
data_ids = list(range(len(data)))
|
|
||||||
selected_data = [data[i] for i in data_ids]
|
|
||||||
|
|
||||||
results_dir = os.path.join(os.path.dirname(__file__), "results")
|
|
||||||
|
|
||||||
if not os.path.exists(results_dir):
|
|
||||||
os.makedirs(results_dir)
|
|
||||||
timestamp = datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
|
|
||||||
extra_info = f"{lm_name}_{'-'.join([method.__name__ for method in methods])}"
|
|
||||||
folder_name = f"{extra_info}_{timestamp}"
|
|
||||||
results_folder = os.path.join(results_dir, folder_name)
|
|
||||||
os.makedirs(results_folder)
|
|
||||||
|
|
||||||
config = {
|
|
||||||
"data": selected_data,
|
|
||||||
"methods": [method.__name__ for method in methods],
|
|
||||||
"lm": lm_name,
|
|
||||||
"budget": budget,
|
|
||||||
}
|
|
||||||
with open(os.path.join(results_folder, "config.json"), "w") as f:
|
|
||||||
json.dump(config, f)
|
|
||||||
|
|
||||||
logging.basicConfig(
|
|
||||||
filename=os.path.join(results_folder, "log.log"),
|
|
||||||
filemode="w",
|
|
||||||
format="%(name)s - %(levelname)s - %(message)s",
|
|
||||||
level=logging.DEBUG,
|
|
||||||
)
|
|
||||||
|
|
||||||
for method in methods:
|
|
||||||
# create a results directory for the method
|
|
||||||
os.makedirs(os.path.join(results_folder, method.__name__))
|
|
||||||
|
|
||||||
for data in selected_data:
|
|
||||||
logging.info(f"Running data {data[0]}: {data[1]}")
|
|
||||||
if budget <= 0.0:
|
|
||||||
logging.error(
|
|
||||||
f"Budget has been depleted, stopping. Data {data[0]} has not been run."
|
|
||||||
)
|
|
||||||
break
|
|
||||||
for method in methods:
|
|
||||||
logging.info(f"Running method {method.__name__}")
|
|
||||||
logging.info(f"Budget left: {budget}")
|
|
||||||
if budget <= 0.0:
|
|
||||||
logging.error(
|
|
||||||
f"Budget has been depleted, stopping. Method {method.__name__} has not been run."
|
|
||||||
)
|
|
||||||
break
|
|
||||||
lm = language_models.ChatGPT(
|
|
||||||
os.path.join(
|
|
||||||
os.path.dirname(__file__),
|
|
||||||
"../../graph_of_thoughts/language_models/config.json",
|
|
||||||
),
|
|
||||||
model_name=lm_name,
|
|
||||||
cache=True,
|
|
||||||
)
|
|
||||||
operations_graph = method()
|
|
||||||
executor = controller.Controller(
|
|
||||||
lm,
|
|
||||||
operations_graph,
|
|
||||||
SortingPrompter(),
|
|
||||||
SortingParser(),
|
|
||||||
{
|
|
||||||
"original": data[1],
|
|
||||||
"current": "",
|
|
||||||
"phase": 0,
|
|
||||||
"method": method.__name__,
|
|
||||||
},
|
|
||||||
)
|
|
||||||
try:
|
|
||||||
executor.run()
|
|
||||||
except Exception as e:
|
|
||||||
logging.error(f"Exception: {e}")
|
|
||||||
path = os.path.join(
|
|
||||||
results_folder,
|
|
||||||
method.__name__,
|
|
||||||
f"{data[0]}.json",
|
|
||||||
)
|
|
||||||
executor.output_graph(path)
|
|
||||||
budget -= lm.cost
|
|
||||||
|
|
||||||
return orig_budget - budget
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
"""
|
|
||||||
Input (x) : an unordered list of 64 numbers between 0 and 9 (inclusive)
|
|
||||||
Output (y) : a sorted list of 64 numbers between 0 and 9 (inclusive)
|
|
||||||
Correct : y == sorted(x)
|
|
||||||
Input Example:
|
|
||||||
[0, 1, 9, 4, 2, 2, 0, 5, 1...]
|
|
||||||
Output Example:
|
|
||||||
[0, 0, 0, 0, 1, 1, 1, 1, 2...]
|
|
||||||
"""
|
|
||||||
budget = 30
|
|
||||||
samples = [item for item in range(0, 100)]
|
|
||||||
approaches = [io, cot, tot, tot2, got]
|
|
||||||
|
|
||||||
spent = run(samples, approaches, budget, "chatgpt")
|
|
||||||
|
|
||||||
logging.info(f"Spent {spent} out of {budget} budget.")
|
|
||||||
@ -1,101 +0,0 @@
|
|||||||
ID,Unsorted,Sorted
|
|
||||||
0,"[2, 9, 0, 0, 4, 2, 1, 2, 7, 7, 7, 2, 7, 8, 1, 3, 1, 5, 9, 1, 2, 8, 5, 0, 1, 8, 8, 3, 1, 9, 6, 9, 1, 4, 8, 0, 3, 5, 8, 5, 9, 4, 9, 8, 6, 1, 9, 6, 4, 9, 2, 6, 2, 9, 6, 3, 4, 0, 7, 4, 7, 5, 9, 6, 3, 5, 3, 9, 7, 2, 8, 7, 3, 4, 2, 9, 9, 9, 7, 0, 0, 9, 3, 7, 5, 9, 8, 4, 4, 5, 2, 3, 9, 4, 5, 8, 3, 7, 1, 4, 3, 0, 1, 6, 2, 3, 2, 1, 2, 1, 9, 3, 5, 8, 8, 1, 6, 0, 9, 6, 6, 7, 4, 7, 6, 9, 1, 6]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
1,"[4, 9, 9, 7, 9, 9, 4, 7, 4, 6, 2, 4, 7, 8, 0, 1, 9, 4, 5, 2, 7, 2, 0, 4, 3, 1, 6, 1, 6, 9, 8, 4, 6, 1, 0, 5, 2, 5, 4, 3, 2, 1, 3, 9, 5, 5, 3, 6, 9, 1, 2, 6, 4, 3, 7, 8, 7, 9, 8, 8, 6, 4, 6, 2, 1, 2, 4, 5, 6, 9, 9, 6, 6, 2, 8, 4, 2, 5, 5, 0, 0, 5, 7, 1, 4, 7, 8, 7, 6, 1, 9, 0, 7, 6, 8, 8, 6, 1, 7, 6, 2, 5, 4, 1, 7, 4, 8, 8, 2, 4, 7, 7, 6, 9, 8, 9, 9, 9, 1, 4, 2, 6, 0, 7, 6, 6, 4, 3]","[0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
2,"[1, 9, 2, 1, 8, 2, 1, 0, 8, 7, 6, 7, 4, 2, 0, 9, 0, 2, 3, 3, 7, 9, 7, 2, 1, 5, 3, 8, 7, 1, 1, 9, 0, 8, 6, 0, 9, 1, 6, 0, 4, 7, 9, 9, 4, 3, 8, 1, 5, 2, 3, 4, 3, 5, 8, 7, 5, 8, 9, 9, 4, 3, 5, 3, 6, 5, 6, 4, 9, 4, 2, 7, 4, 1, 0, 5, 3, 2, 6, 1, 7, 5, 3, 6, 0, 5, 6, 9, 9, 2, 0, 2, 9, 0, 7, 1, 4, 7, 6, 0, 5, 3, 6, 9, 4, 4, 7, 5, 4, 5, 1, 6, 6, 4, 4, 8, 6, 6, 6, 7, 0, 8, 5, 3, 6, 9, 2, 4]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
3,"[7, 1, 9, 5, 7, 0, 3, 9, 8, 5, 8, 2, 0, 3, 3, 0, 6, 4, 8, 6, 8, 1, 8, 9, 7, 4, 8, 6, 7, 5, 6, 6, 2, 1, 6, 5, 1, 4, 8, 4, 1, 7, 9, 6, 9, 2, 2, 5, 0, 3, 5, 1, 9, 6, 0, 0, 8, 6, 7, 7, 1, 7, 5, 1, 5, 5, 8, 9, 1, 0, 3, 6, 0, 0, 5, 4, 4, 4, 0, 7, 9, 8, 3, 3, 5, 7, 2, 0, 6, 4, 9, 5, 6, 3, 7, 3, 3, 0, 4, 3, 9, 4, 0, 8, 9, 5, 8, 7, 0, 9, 4, 3, 2, 2, 4, 6, 9, 9, 8, 9, 2, 4, 8, 1, 2, 5, 7, 6]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
4,"[7, 8, 2, 4, 1, 1, 3, 0, 0, 8, 4, 3, 0, 3, 4, 1, 6, 3, 4, 0, 7, 1, 6, 5, 8, 0, 8, 3, 6, 1, 2, 4, 5, 2, 2, 9, 0, 7, 3, 3, 6, 2, 0, 7, 8, 8, 1, 5, 9, 5, 0, 8, 0, 2, 0, 9, 7, 1, 6, 4, 5, 3, 7, 1, 2, 2, 9, 1, 5, 0, 1, 8, 9, 5, 8, 3, 2, 0, 4, 0, 5, 8, 7, 5, 9, 5, 7, 7, 0, 3, 3, 7, 2, 8, 8, 1, 3, 0, 5, 5, 0, 0, 0, 0, 1, 0, 1, 6, 2, 6, 5, 6, 3, 4, 9, 7, 5, 3, 0, 1, 4, 6, 7, 9, 6, 3, 9, 6]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
5,"[9, 8, 1, 2, 7, 7, 6, 0, 6, 1, 8, 2, 8, 9, 4, 8, 2, 3, 0, 5, 4, 1, 0, 3, 0, 7, 0, 1, 0, 2, 0, 9, 4, 2, 2, 7, 7, 7, 6, 1, 3, 8, 7, 2, 7, 9, 8, 1, 0, 3, 7, 2, 0, 4, 8, 5, 1, 6, 3, 4, 9, 4, 3, 3, 2, 2, 7, 7, 6, 1, 1, 7, 8, 6, 4, 6, 7, 6, 8, 3, 4, 8, 2, 5, 4, 8, 1, 3, 0, 8, 4, 3, 1, 3, 1, 2, 9, 8, 4, 2, 2, 8, 1, 5, 5, 3, 7, 4, 6, 3, 1, 0, 0, 4, 1, 0, 4, 7, 4, 9, 2, 1, 9, 0, 8, 0, 1, 7]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
6,"[8, 4, 8, 3, 6, 1, 4, 4, 8, 7, 5, 8, 9, 4, 5, 6, 5, 6, 1, 3, 7, 0, 7, 3, 2, 3, 8, 5, 8, 5, 3, 7, 3, 4, 1, 3, 1, 6, 5, 2, 6, 7, 7, 5, 3, 7, 3, 6, 4, 4, 6, 9, 9, 6, 4, 6, 4, 7, 4, 0, 1, 2, 9, 6, 2, 5, 2, 3, 0, 3, 8, 2, 1, 0, 5, 8, 1, 4, 6, 5, 2, 8, 3, 3, 8, 1, 2, 1, 6, 3, 7, 4, 4, 9, 9, 5, 5, 6, 1, 9, 7, 7, 5, 9, 6, 2, 6, 7, 3, 2, 5, 4, 4, 1, 2, 5, 0, 1, 5, 5, 5, 6, 0, 1, 1, 7, 7, 5]","[0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
7,"[3, 8, 0, 5, 5, 7, 4, 2, 9, 2, 8, 4, 2, 1, 9, 4, 6, 5, 2, 9, 1, 2, 1, 4, 0, 7, 4, 9, 6, 3, 8, 1, 8, 6, 4, 8, 3, 8, 2, 0, 4, 2, 7, 8, 2, 1, 6, 4, 3, 7, 5, 0, 5, 2, 0, 7, 7, 0, 2, 8, 0, 7, 0, 4, 2, 6, 3, 0, 0, 3, 0, 7, 3, 4, 9, 6, 7, 4, 5, 1, 7, 3, 1, 8, 4, 3, 5, 1, 8, 9, 8, 8, 4, 4, 9, 5, 9, 4, 9, 1, 9, 3, 3, 0, 5, 3, 8, 3, 3, 7, 6, 5, 4, 6, 1, 0, 1, 3, 7, 8, 4, 8, 5, 4, 4, 2, 9, 5]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
8,"[6, 2, 6, 9, 5, 4, 5, 2, 7, 3, 4, 0, 7, 8, 5, 2, 5, 0, 7, 7, 5, 5, 9, 9, 1, 8, 9, 1, 0, 3, 4, 6, 2, 6, 0, 6, 0, 2, 2, 3, 5, 9, 5, 7, 4, 2, 4, 5, 2, 2, 5, 0, 9, 7, 2, 6, 1, 6, 2, 0, 9, 7, 4, 3, 6, 5, 9, 0, 5, 6, 2, 9, 7, 7, 7, 7, 2, 9, 7, 0, 5, 5, 2, 3, 1, 8, 9, 1, 0, 1, 3, 6, 8, 1, 8, 1, 1, 9, 7, 7, 7, 4, 9, 7, 9, 4, 8, 0, 7, 3, 6, 8, 6, 6, 8, 5, 5, 5, 8, 2, 7, 1, 1, 5, 0, 7, 2, 3]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
9,"[3, 7, 3, 8, 0, 7, 7, 7, 6, 5, 3, 8, 4, 0, 2, 1, 1, 2, 0, 7, 2, 5, 9, 3, 0, 7, 1, 2, 7, 1, 2, 2, 5, 6, 2, 3, 4, 8, 5, 8, 6, 4, 5, 9, 4, 6, 2, 3, 2, 5, 0, 0, 3, 6, 9, 4, 8, 4, 3, 6, 4, 2, 0, 3, 9, 3, 2, 2, 5, 6, 0, 1, 4, 0, 3, 3, 7, 9, 8, 6, 9, 6, 7, 1, 2, 6, 5, 9, 6, 9, 0, 2, 9, 6, 2, 3, 6, 0, 5, 1, 2, 0, 4, 0, 8, 3, 9, 4, 3, 6, 0, 3, 9, 7, 3, 6, 1, 0, 6, 0, 4, 4, 5, 8, 4, 9, 8, 4]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
10,"[3, 5, 9, 2, 0, 5, 0, 6, 6, 4, 8, 0, 2, 7, 8, 9, 3, 6, 7, 8, 6, 1, 3, 5, 8, 8, 0, 5, 7, 2, 7, 0, 9, 1, 4, 8, 9, 3, 7, 3, 4, 7, 3, 1, 3, 3, 3, 0, 3, 2, 5, 6, 0, 6, 5, 8, 7, 5, 9, 7, 4, 8, 3, 1, 5, 0, 5, 0, 8, 5, 1, 3, 8, 9, 1, 6, 2, 2, 8, 4, 9, 2, 2, 2, 8, 4, 9, 8, 0, 7, 5, 3, 9, 7, 9, 1, 7, 3, 7, 7, 6, 2, 3, 6, 8, 4, 8, 9, 6, 4, 2, 2, 0, 1, 7, 5, 4, 7, 3, 6, 6, 6, 9, 9, 2, 4, 6, 8]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
11,"[6, 6, 7, 8, 6, 9, 0, 9, 4, 2, 9, 0, 0, 3, 8, 3, 2, 3, 1, 7, 9, 8, 8, 5, 4, 7, 2, 6, 4, 3, 4, 0, 2, 6, 4, 5, 6, 6, 1, 3, 1, 0, 9, 6, 1, 5, 1, 5, 3, 5, 6, 9, 5, 1, 8, 8, 0, 9, 1, 5, 5, 7, 2, 3, 9, 1, 2, 5, 4, 4, 8, 2, 1, 2, 4, 6, 4, 5, 3, 3, 7, 4, 8, 4, 2, 2, 3, 7, 2, 9, 4, 8, 2, 3, 3, 0, 1, 5, 3, 8, 1, 9, 9, 0, 9, 0, 5, 8, 3, 2, 5, 8, 4, 9, 0, 9, 4, 8, 6, 0, 4, 3, 2, 2, 1, 0, 2, 2]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
12,"[6, 5, 3, 7, 0, 7, 6, 1, 8, 8, 3, 3, 3, 2, 8, 9, 6, 9, 6, 2, 0, 4, 6, 5, 5, 9, 5, 6, 0, 6, 8, 6, 8, 1, 0, 5, 5, 7, 7, 6, 9, 9, 2, 5, 8, 1, 9, 9, 7, 9, 7, 0, 8, 1, 6, 8, 4, 7, 0, 8, 6, 8, 9, 1, 2, 0, 1, 0, 1, 7, 5, 7, 0, 5, 3, 2, 5, 6, 1, 4, 2, 4, 7, 0, 8, 3, 1, 6, 4, 5, 2, 6, 5, 2, 4, 7, 1, 3, 4, 5, 5, 9, 3, 0, 3, 0, 0, 1, 3, 7, 0, 0, 2, 2, 1, 2, 7, 9, 2, 0, 2, 2, 8, 1, 4, 4, 6, 0]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
13,"[7, 2, 7, 8, 3, 6, 5, 1, 6, 3, 5, 8, 6, 1, 7, 7, 0, 0, 2, 6, 3, 9, 6, 1, 9, 7, 0, 3, 0, 8, 4, 6, 2, 1, 1, 8, 9, 3, 6, 0, 5, 6, 8, 7, 6, 0, 3, 7, 6, 4, 3, 4, 2, 4, 3, 7, 9, 2, 3, 4, 3, 3, 9, 1, 8, 1, 9, 8, 6, 2, 6, 5, 2, 9, 5, 4, 6, 6, 8, 3, 4, 1, 0, 8, 5, 8, 1, 0, 7, 0, 8, 3, 5, 7, 9, 9, 4, 7, 5, 8, 5, 8, 5, 9, 4, 5, 4, 1, 8, 0, 8, 7, 8, 8, 0, 7, 4, 9, 5, 5, 7, 3, 4, 9, 7, 2, 1, 8]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
14,"[2, 2, 1, 1, 6, 3, 1, 7, 8, 2, 8, 9, 4, 0, 6, 9, 7, 0, 8, 4, 4, 6, 4, 6, 4, 3, 4, 8, 7, 7, 5, 2, 4, 4, 7, 0, 1, 7, 5, 2, 7, 6, 6, 0, 0, 8, 2, 2, 5, 0, 1, 6, 6, 1, 1, 4, 6, 7, 5, 2, 1, 4, 8, 4, 6, 0, 7, 2, 5, 9, 6, 3, 9, 8, 3, 2, 1, 1, 4, 5, 7, 4, 1, 8, 9, 2, 3, 6, 8, 4, 5, 3, 4, 0, 3, 1, 4, 5, 5, 5, 2, 1, 8, 3, 7, 3, 9, 8, 3, 5, 5, 9, 6, 3, 5, 0, 6, 6, 3, 0, 9, 0, 8, 6, 8, 7, 8, 4]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
15,"[7, 8, 9, 1, 0, 4, 1, 9, 0, 5, 3, 4, 0, 1, 9, 4, 7, 0, 6, 7, 8, 2, 1, 5, 9, 0, 5, 5, 7, 7, 6, 6, 6, 4, 6, 7, 4, 2, 3, 5, 8, 7, 3, 5, 7, 0, 1, 7, 8, 4, 8, 2, 1, 5, 0, 2, 4, 3, 1, 6, 5, 6, 0, 0, 8, 3, 0, 9, 0, 5, 7, 2, 6, 5, 9, 7, 3, 0, 2, 6, 0, 8, 7, 9, 5, 0, 8, 8, 4, 3, 4, 6, 1, 1, 5, 3, 8, 9, 9, 0, 9, 1, 4, 0, 4, 2, 6, 2, 4, 6, 6, 3, 9, 7, 4, 1, 9, 9, 7, 6, 9, 2, 4, 9, 7, 6, 8, 8]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
16,"[6, 4, 5, 9, 2, 7, 5, 7, 6, 5, 0, 0, 2, 0, 7, 9, 5, 7, 5, 7, 9, 0, 4, 7, 4, 3, 8, 8, 0, 5, 4, 6, 6, 9, 4, 1, 2, 0, 7, 2, 8, 0, 4, 2, 0, 0, 9, 9, 0, 2, 3, 9, 3, 4, 7, 0, 9, 1, 0, 5, 5, 8, 2, 6, 1, 6, 3, 3, 8, 8, 6, 5, 4, 0, 7, 2, 4, 9, 6, 4, 3, 9, 4, 9, 9, 4, 4, 7, 5, 1, 8, 8, 3, 6, 0, 9, 6, 0, 1, 4, 0, 4, 3, 6, 0, 8, 5, 4, 7, 6, 1, 5, 2, 2, 3, 4, 5, 2, 4, 3, 3, 1, 9, 8, 6, 7, 7, 7]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
17,"[8, 3, 0, 2, 5, 3, 6, 1, 9, 9, 6, 6, 7, 7, 6, 4, 1, 9, 9, 9, 2, 5, 1, 4, 8, 0, 7, 6, 4, 2, 0, 0, 4, 1, 1, 2, 3, 2, 6, 7, 1, 1, 2, 1, 0, 9, 1, 0, 5, 2, 0, 2, 7, 7, 8, 1, 3, 8, 5, 0, 1, 4, 5, 5, 4, 4, 6, 1, 0, 7, 5, 4, 6, 0, 2, 9, 1, 7, 5, 5, 3, 2, 4, 2, 4, 5, 7, 1, 8, 4, 1, 2, 0, 5, 8, 7, 3, 2, 3, 6, 7, 0, 2, 9, 1, 8, 5, 5, 3, 5, 1, 0, 7, 0, 4, 3, 5, 4, 5, 5, 5, 7, 9, 0, 9, 0, 4, 0]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
18,"[5, 0, 0, 0, 3, 6, 0, 8, 0, 9, 2, 8, 4, 1, 8, 7, 3, 3, 5, 6, 7, 1, 7, 7, 0, 1, 8, 4, 8, 7, 6, 6, 0, 6, 3, 9, 0, 3, 4, 4, 5, 4, 5, 8, 2, 7, 6, 3, 9, 6, 1, 9, 3, 0, 3, 5, 9, 2, 1, 7, 1, 9, 2, 9, 8, 6, 7, 1, 3, 3, 5, 8, 8, 8, 9, 6, 8, 2, 1, 6, 5, 3, 4, 8, 5, 6, 0, 9, 5, 0, 0, 1, 0, 4, 7, 4, 3, 4, 5, 3, 3, 0, 9, 0, 5, 3, 2, 3, 7, 9, 2, 9, 4, 4, 6, 1, 3, 6, 1, 4, 0, 0, 5, 2, 0, 7, 8, 4]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
19,"[7, 3, 7, 6, 5, 5, 4, 3, 6, 2, 9, 8, 9, 5, 9, 1, 9, 1, 5, 2, 4, 8, 8, 6, 8, 5, 5, 0, 1, 6, 2, 8, 0, 1, 8, 4, 6, 6, 6, 3, 0, 6, 4, 4, 4, 2, 8, 4, 8, 9, 8, 7, 0, 9, 2, 5, 5, 6, 2, 2, 1, 3, 4, 3, 0, 7, 5, 6, 3, 8, 8, 1, 2, 2, 6, 8, 9, 9, 8, 8, 5, 7, 9, 1, 3, 4, 5, 1, 0, 1, 9, 3, 7, 5, 1, 2, 7, 7, 2, 2, 1, 8, 3, 4, 5, 6, 1, 0, 2, 9, 6, 3, 3, 7, 1, 0, 9, 6, 8, 3, 8, 3, 5, 5, 6, 6, 5, 0]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
20,"[0, 1, 1, 4, 0, 1, 8, 4, 1, 3, 2, 7, 9, 8, 2, 9, 7, 2, 4, 5, 4, 7, 9, 4, 6, 9, 1, 5, 9, 6, 7, 6, 1, 5, 0, 5, 4, 3, 6, 9, 0, 9, 3, 8, 9, 2, 7, 3, 1, 5, 0, 5, 6, 7, 5, 2, 2, 2, 8, 1, 3, 8, 0, 2, 3, 0, 0, 9, 7, 4, 2, 8, 3, 5, 3, 0, 6, 8, 2, 0, 1, 9, 9, 7, 7, 3, 2, 7, 9, 5, 9, 7, 6, 4, 1, 6, 8, 5, 8, 8, 3, 1, 9, 2, 3, 5, 4, 5, 6, 5, 2, 6, 6, 6, 6, 5, 2, 9, 1, 7, 3, 8, 8, 1, 1, 8, 5, 9]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
21,"[0, 1, 3, 2, 6, 4, 7, 5, 8, 7, 1, 1, 0, 1, 3, 2, 0, 1, 4, 2, 9, 8, 8, 8, 7, 8, 8, 4, 3, 1, 8, 1, 7, 0, 0, 2, 7, 1, 3, 2, 8, 2, 6, 4, 7, 7, 9, 6, 3, 6, 1, 8, 2, 2, 1, 1, 7, 4, 9, 9, 1, 5, 9, 3, 6, 5, 7, 3, 1, 1, 3, 6, 8, 5, 5, 0, 5, 7, 4, 0, 9, 3, 4, 5, 2, 8, 6, 0, 2, 6, 1, 7, 6, 5, 0, 2, 9, 5, 4, 4, 8, 3, 7, 6, 4, 3, 8, 6, 8, 4, 2, 6, 3, 5, 5, 5, 9, 6, 8, 9, 2, 1, 0, 8, 6, 7, 0, 4]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
22,"[2, 1, 3, 2, 7, 6, 6, 9, 1, 9, 2, 3, 4, 1, 3, 6, 7, 6, 5, 2, 0, 9, 3, 5, 4, 5, 7, 7, 1, 8, 6, 2, 8, 3, 1, 4, 7, 8, 0, 5, 8, 0, 9, 0, 0, 2, 9, 2, 3, 1, 6, 0, 9, 3, 2, 2, 7, 0, 5, 8, 2, 3, 7, 2, 4, 5, 4, 1, 9, 5, 8, 6, 6, 6, 4, 7, 7, 0, 9, 5, 1, 3, 9, 8, 5, 6, 7, 7, 1, 4, 2, 0, 2, 7, 5, 0, 2, 5, 8, 5, 6, 8, 7, 7, 1, 7, 2, 8, 3, 7, 9, 1, 6, 3, 5, 2, 9, 5, 5, 4, 7, 4, 2, 8, 1, 1, 9, 9]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
23,"[2, 6, 6, 7, 3, 1, 6, 0, 1, 8, 8, 9, 8, 6, 7, 0, 3, 0, 6, 8, 3, 5, 0, 0, 8, 7, 6, 0, 6, 6, 1, 9, 4, 2, 3, 3, 1, 2, 5, 9, 1, 0, 1, 5, 0, 6, 3, 1, 3, 7, 5, 2, 1, 8, 0, 5, 3, 4, 8, 3, 5, 4, 9, 5, 1, 2, 7, 7, 3, 6, 2, 5, 7, 4, 5, 4, 2, 6, 6, 9, 4, 9, 1, 1, 0, 6, 4, 4, 4, 4, 8, 3, 6, 1, 0, 2, 5, 1, 5, 3, 0, 8, 2, 2, 7, 0, 9, 6, 1, 5, 7, 1, 6, 7, 5, 6, 0, 8, 9, 2, 7, 2, 9, 2, 3, 2, 9, 6]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
24,"[3, 6, 6, 1, 3, 0, 7, 7, 8, 0, 2, 0, 3, 0, 6, 0, 9, 8, 5, 3, 4, 3, 9, 6, 2, 4, 8, 9, 3, 1, 3, 9, 9, 0, 0, 9, 8, 9, 4, 9, 6, 6, 9, 0, 0, 0, 0, 1, 1, 1, 2, 5, 7, 2, 5, 2, 6, 9, 4, 8, 9, 2, 4, 7, 1, 9, 9, 2, 8, 3, 0, 6, 6, 4, 6, 7, 4, 0, 3, 5, 8, 8, 0, 1, 4, 7, 6, 3, 6, 2, 3, 4, 0, 1, 6, 9, 4, 6, 3, 5, 5, 1, 1, 9, 7, 8, 4, 8, 6, 9, 4, 6, 0, 5, 6, 3, 4, 3, 4, 3, 0, 6, 3, 6, 1, 7, 6, 9]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
25,"[8, 0, 1, 9, 8, 5, 3, 2, 1, 6, 0, 2, 1, 7, 0, 7, 2, 9, 0, 1, 3, 5, 3, 6, 8, 2, 9, 3, 1, 7, 9, 0, 0, 2, 9, 7, 9, 8, 7, 9, 3, 3, 9, 5, 3, 8, 0, 3, 3, 5, 0, 6, 3, 0, 9, 6, 4, 3, 4, 4, 9, 4, 7, 0, 2, 6, 3, 4, 6, 0, 7, 5, 1, 9, 3, 1, 2, 5, 0, 9, 6, 5, 0, 7, 6, 6, 9, 9, 0, 6, 4, 8, 9, 0, 8, 3, 1, 1, 6, 9, 3, 0, 4, 8, 5, 1, 5, 9, 2, 1, 2, 6, 2, 1, 2, 2, 1, 9, 9, 3, 6, 5, 6, 1, 1, 7, 1, 5]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
26,"[1, 8, 6, 7, 7, 5, 0, 5, 5, 8, 8, 3, 8, 4, 4, 7, 8, 3, 0, 0, 8, 4, 8, 5, 9, 6, 3, 4, 1, 5, 1, 1, 7, 2, 6, 6, 0, 4, 0, 4, 6, 6, 1, 8, 0, 4, 5, 3, 6, 7, 1, 7, 5, 5, 3, 9, 8, 3, 7, 6, 3, 1, 0, 6, 0, 6, 9, 9, 3, 6, 0, 1, 4, 8, 0, 8, 4, 8, 8, 0, 1, 0, 6, 6, 8, 6, 1, 8, 2, 8, 4, 0, 6, 0, 0, 4, 7, 4, 3, 8, 7, 1, 4, 6, 7, 6, 2, 7, 5, 3, 8, 7, 3, 0, 5, 0, 9, 2, 6, 3, 4, 9, 3, 0, 2, 3, 2, 9]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
27,"[2, 9, 2, 1, 4, 1, 3, 7, 5, 4, 3, 7, 4, 6, 0, 5, 8, 5, 6, 8, 5, 7, 5, 9, 5, 4, 4, 2, 2, 3, 8, 3, 4, 6, 1, 7, 3, 4, 4, 2, 6, 2, 1, 4, 7, 2, 6, 1, 5, 3, 2, 0, 0, 1, 6, 3, 8, 7, 2, 4, 5, 5, 5, 5, 9, 4, 5, 0, 2, 2, 8, 7, 7, 7, 8, 2, 6, 4, 7, 8, 4, 8, 8, 2, 0, 6, 9, 6, 9, 9, 8, 0, 2, 4, 3, 4, 5, 2, 3, 7, 5, 8, 7, 8, 7, 5, 5, 1, 4, 7, 0, 2, 2, 6, 0, 0, 4, 1, 7, 0, 5, 4, 9, 6, 3, 3, 8, 9]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
28,"[5, 2, 5, 3, 0, 1, 7, 3, 5, 8, 6, 9, 3, 0, 6, 9, 6, 0, 5, 0, 2, 1, 4, 5, 2, 4, 7, 4, 3, 6, 8, 6, 5, 8, 9, 1, 6, 9, 5, 3, 1, 9, 9, 1, 0, 1, 4, 9, 8, 8, 0, 2, 8, 6, 9, 3, 3, 0, 7, 2, 8, 4, 0, 0, 0, 3, 2, 2, 5, 6, 6, 4, 6, 9, 9, 3, 8, 5, 4, 7, 2, 3, 6, 0, 5, 1, 2, 9, 9, 8, 6, 1, 4, 7, 7, 4, 7, 6, 1, 3, 7, 9, 4, 2, 4, 2, 0, 3, 8, 2, 9, 8, 8, 5, 7, 5, 6, 5, 4, 3, 5, 6, 0, 5, 1, 8, 6, 4]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
29,"[1, 9, 2, 4, 6, 8, 4, 3, 0, 9, 5, 5, 2, 7, 4, 8, 4, 2, 7, 7, 0, 5, 4, 8, 4, 9, 5, 9, 8, 1, 2, 7, 0, 3, 7, 6, 2, 5, 4, 6, 8, 5, 2, 3, 3, 6, 5, 4, 6, 6, 1, 7, 0, 8, 2, 9, 5, 5, 2, 8, 1, 9, 7, 6, 5, 1, 9, 3, 8, 7, 4, 9, 0, 4, 0, 1, 8, 1, 0, 8, 9, 8, 2, 7, 1, 9, 4, 9, 4, 4, 1, 2, 7, 6, 1, 2, 5, 3, 1, 0, 7, 6, 2, 5, 8, 4, 6, 4, 4, 7, 2, 5, 0, 6, 3, 5, 9, 2, 0, 4, 7, 1, 3, 5, 3, 9, 9, 7]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
30,"[5, 7, 0, 3, 2, 8, 8, 1, 8, 3, 0, 7, 7, 6, 2, 9, 2, 2, 3, 5, 3, 8, 0, 7, 8, 7, 6, 1, 7, 0, 7, 1, 2, 1, 5, 0, 6, 4, 6, 9, 5, 4, 1, 2, 2, 7, 8, 3, 0, 1, 5, 7, 2, 7, 2, 6, 7, 2, 9, 3, 5, 2, 7, 8, 8, 8, 4, 7, 1, 3, 8, 5, 4, 3, 8, 3, 8, 9, 2, 7, 6, 4, 2, 5, 7, 9, 8, 4, 1, 9, 4, 1, 0, 0, 1, 5, 0, 6, 0, 2, 1, 7, 6, 4, 0, 5, 4, 2, 3, 9, 1, 9, 7, 0, 4, 5, 2, 5, 7, 6, 3, 3, 4, 9, 9, 1, 7, 4]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
31,"[8, 9, 6, 2, 7, 7, 8, 6, 9, 9, 2, 6, 0, 8, 4, 1, 5, 6, 6, 3, 9, 9, 5, 7, 4, 1, 2, 4, 0, 4, 7, 2, 0, 0, 3, 5, 0, 4, 0, 3, 7, 3, 9, 8, 2, 5, 4, 0, 2, 5, 3, 3, 7, 5, 8, 1, 6, 7, 0, 0, 9, 0, 5, 0, 6, 6, 5, 1, 6, 9, 6, 6, 1, 8, 7, 3, 7, 2, 2, 0, 7, 9, 2, 7, 9, 5, 4, 3, 2, 3, 0, 7, 9, 4, 1, 6, 8, 4, 2, 6, 9, 6, 4, 8, 0, 0, 4, 9, 3, 4, 7, 2, 1, 7, 2, 9, 0, 1, 2, 5, 8, 4, 1, 4, 0, 5, 5, 5]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
32,"[5, 6, 8, 1, 2, 0, 6, 1, 5, 7, 5, 4, 2, 9, 6, 6, 7, 0, 5, 3, 5, 7, 2, 5, 8, 2, 4, 5, 7, 5, 1, 0, 9, 9, 6, 8, 2, 0, 8, 4, 9, 3, 4, 2, 1, 1, 2, 7, 6, 1, 0, 2, 8, 9, 9, 4, 7, 5, 3, 6, 4, 6, 1, 4, 9, 2, 3, 0, 9, 2, 1, 5, 6, 9, 1, 2, 3, 0, 3, 4, 4, 4, 8, 9, 2, 3, 5, 5, 4, 8, 8, 6, 7, 9, 3, 1, 1, 8, 0, 4, 9, 9, 3, 1, 4, 6, 2, 1, 7, 1, 8, 9, 5, 3, 2, 1, 7, 8, 7, 1, 0, 5, 4, 6, 1, 1, 9, 7]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
33,"[2, 8, 6, 1, 4, 7, 1, 1, 9, 7, 2, 6, 3, 3, 5, 5, 7, 3, 5, 6, 7, 6, 7, 9, 3, 4, 3, 4, 5, 9, 5, 1, 5, 1, 1, 7, 1, 1, 7, 1, 2, 6, 8, 1, 0, 6, 1, 7, 5, 9, 5, 5, 4, 1, 7, 1, 4, 0, 0, 5, 4, 1, 5, 9, 7, 7, 4, 9, 9, 0, 9, 7, 0, 8, 9, 9, 3, 5, 8, 3, 6, 5, 2, 8, 6, 1, 5, 2, 4, 3, 4, 1, 3, 7, 7, 2, 2, 9, 0, 8, 3, 0, 0, 4, 7, 4, 5, 1, 6, 3, 2, 9, 7, 5, 1, 7, 7, 5, 5, 9, 9, 6, 9, 2, 9, 7, 1, 3]","[0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
34,"[8, 4, 4, 5, 8, 1, 9, 7, 1, 8, 7, 5, 4, 5, 9, 4, 3, 7, 3, 1, 5, 1, 9, 7, 8, 2, 4, 1, 8, 1, 4, 9, 4, 8, 4, 7, 8, 1, 3, 0, 4, 8, 5, 0, 4, 2, 5, 4, 5, 2, 3, 6, 3, 2, 5, 9, 6, 9, 7, 7, 2, 3, 1, 8, 2, 1, 2, 0, 1, 2, 0, 7, 8, 3, 5, 1, 4, 3, 0, 9, 7, 4, 6, 2, 1, 2, 1, 5, 4, 8, 2, 6, 7, 9, 8, 9, 0, 4, 9, 2, 5, 4, 1, 7, 5, 5, 1, 7, 5, 9, 0, 9, 1, 4, 4, 6, 5, 9, 4, 2, 6, 8, 8, 1, 3, 7, 5, 5]","[0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
35,"[2, 5, 8, 2, 7, 2, 1, 3, 3, 6, 9, 2, 3, 1, 4, 9, 0, 5, 3, 2, 7, 2, 9, 9, 9, 0, 4, 8, 0, 9, 3, 5, 4, 7, 9, 2, 0, 0, 3, 3, 5, 1, 6, 5, 7, 2, 8, 1, 6, 5, 5, 0, 7, 4, 8, 0, 8, 2, 8, 4, 8, 3, 8, 9, 8, 7, 2, 5, 8, 4, 0, 7, 0, 1, 6, 4, 7, 0, 6, 9, 5, 3, 8, 9, 4, 8, 6, 5, 7, 7, 4, 0, 4, 3, 8, 1, 5, 7, 2, 5, 3, 5, 5, 4, 8, 1, 9, 2, 8, 7, 6, 0, 8, 4, 8, 1, 1, 1, 0, 0, 8, 0, 2, 7, 9, 7, 6, 5]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
36,"[7, 7, 5, 9, 4, 9, 5, 3, 5, 1, 9, 7, 3, 7, 7, 2, 1, 0, 2, 6, 3, 3, 4, 0, 7, 7, 2, 5, 0, 3, 4, 0, 6, 5, 5, 1, 8, 4, 4, 8, 7, 3, 1, 8, 0, 8, 1, 6, 9, 7, 5, 6, 5, 4, 2, 1, 8, 2, 6, 6, 1, 6, 9, 7, 5, 9, 5, 7, 8, 4, 3, 9, 5, 0, 8, 6, 0, 3, 6, 3, 8, 4, 9, 8, 8, 1, 6, 5, 3, 7, 1, 0, 7, 5, 0, 4, 0, 6, 4, 2, 9, 8, 4, 1, 3, 5, 7, 3, 7, 7, 3, 9, 1, 6, 9, 1, 9, 9, 7, 2, 8, 2, 6, 5, 6, 4, 3, 5]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
37,"[1, 0, 9, 2, 1, 0, 3, 4, 1, 6, 3, 6, 0, 0, 9, 2, 9, 0, 3, 3, 6, 6, 8, 8, 0, 4, 6, 5, 4, 1, 8, 5, 1, 2, 0, 7, 2, 0, 5, 8, 8, 0, 3, 7, 5, 7, 6, 4, 1, 5, 0, 9, 4, 9, 1, 7, 9, 4, 1, 1, 3, 8, 7, 5, 7, 0, 9, 5, 9, 9, 6, 3, 4, 2, 4, 9, 4, 8, 2, 5, 5, 9, 7, 9, 7, 8, 6, 6, 0, 9, 7, 1, 6, 0, 9, 3, 2, 0, 4, 1, 8, 8, 7, 4, 1, 7, 3, 0, 4, 1, 8, 7, 8, 9, 0, 7, 6, 7, 3, 6, 6, 4, 7, 1, 1, 6, 5, 9]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
38,"[1, 7, 0, 4, 9, 9, 0, 8, 1, 0, 3, 8, 1, 3, 5, 5, 6, 5, 1, 7, 3, 0, 0, 9, 5, 3, 9, 9, 6, 6, 2, 5, 1, 1, 9, 3, 0, 2, 8, 4, 9, 2, 5, 9, 4, 0, 6, 2, 6, 6, 1, 9, 6, 3, 0, 7, 8, 5, 3, 4, 1, 3, 3, 9, 7, 3, 5, 7, 3, 9, 7, 7, 2, 5, 1, 9, 0, 9, 3, 0, 3, 1, 3, 1, 7, 4, 1, 0, 7, 1, 4, 7, 1, 8, 7, 8, 6, 5, 2, 7, 3, 8, 2, 7, 3, 4, 3, 1, 5, 0, 1, 4, 9, 2, 9, 3, 3, 6, 6, 5, 8, 5, 1, 9, 1, 3, 2, 6]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
39,"[8, 6, 2, 4, 6, 7, 8, 4, 0, 4, 6, 7, 8, 5, 5, 7, 6, 4, 3, 6, 3, 7, 9, 7, 4, 2, 4, 8, 1, 3, 6, 9, 8, 4, 2, 2, 2, 0, 5, 8, 4, 3, 1, 3, 1, 7, 7, 0, 6, 6, 0, 5, 1, 0, 1, 0, 5, 8, 4, 6, 7, 3, 0, 9, 2, 6, 6, 3, 0, 1, 1, 5, 3, 8, 8, 8, 4, 0, 6, 1, 6, 6, 3, 5, 0, 8, 6, 0, 2, 9, 4, 2, 7, 5, 2, 6, 2, 6, 0, 9, 0, 6, 0, 0, 5, 6, 3, 7, 4, 3, 1, 4, 0, 3, 3, 2, 2, 5, 7, 6, 0, 0, 2, 5, 0, 5, 9, 6]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9]"
|
|
||||||
40,"[8, 6, 0, 6, 5, 8, 9, 7, 9, 3, 8, 8, 0, 8, 0, 7, 7, 4, 5, 1, 7, 7, 4, 9, 0, 7, 5, 8, 3, 5, 8, 7, 2, 9, 9, 0, 1, 8, 0, 6, 5, 3, 3, 8, 0, 6, 7, 2, 6, 4, 9, 5, 0, 4, 1, 5, 2, 0, 5, 5, 7, 6, 7, 5, 0, 8, 9, 0, 1, 9, 3, 1, 8, 6, 2, 5, 7, 1, 1, 0, 3, 7, 3, 1, 3, 5, 6, 4, 5, 8, 7, 4, 0, 1, 9, 2, 8, 5, 7, 7, 7, 6, 7, 0, 0, 0, 7, 0, 4, 6, 2, 6, 5, 9, 8, 4, 8, 8, 1, 0, 7, 8, 7, 9, 9, 2, 0, 8]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
41,"[5, 1, 8, 2, 8, 4, 4, 5, 0, 7, 5, 3, 3, 6, 1, 1, 5, 9, 4, 7, 3, 1, 0, 0, 4, 9, 8, 5, 0, 8, 8, 7, 9, 0, 3, 4, 8, 4, 7, 4, 2, 4, 1, 9, 4, 8, 5, 8, 6, 0, 1, 8, 6, 3, 4, 5, 0, 2, 8, 0, 4, 4, 9, 5, 2, 8, 7, 4, 9, 6, 2, 3, 8, 7, 1, 0, 8, 0, 1, 1, 0, 0, 2, 1, 1, 2, 8, 8, 0, 1, 3, 5, 0, 5, 5, 1, 1, 5, 3, 4, 3, 7, 3, 0, 0, 1, 3, 4, 1, 9, 4, 6, 7, 1, 8, 3, 4, 7, 0, 3, 3, 2, 7, 5, 7, 2, 7, 7]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
42,"[5, 0, 6, 3, 6, 5, 5, 1, 0, 8, 1, 8, 8, 0, 0, 6, 5, 9, 1, 6, 5, 8, 1, 9, 7, 2, 5, 5, 4, 0, 7, 9, 3, 5, 1, 5, 9, 6, 8, 4, 3, 1, 6, 4, 5, 0, 2, 2, 9, 9, 0, 1, 6, 6, 8, 6, 0, 2, 9, 7, 7, 0, 2, 7, 0, 0, 1, 3, 0, 3, 7, 4, 6, 6, 6, 5, 0, 0, 5, 3, 9, 6, 3, 5, 3, 3, 9, 0, 4, 4, 2, 4, 0, 3, 9, 5, 8, 7, 4, 0, 7, 6, 4, 7, 1, 6, 3, 3, 2, 4, 5, 0, 1, 5, 7, 3, 4, 3, 8, 6, 5, 8, 9, 7, 2, 3, 2, 5]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
43,"[4, 1, 0, 8, 1, 0, 4, 2, 0, 8, 8, 7, 8, 0, 2, 6, 6, 8, 3, 3, 6, 0, 5, 5, 3, 0, 9, 9, 6, 7, 8, 4, 2, 8, 5, 0, 9, 4, 4, 5, 8, 6, 4, 1, 1, 6, 5, 4, 6, 9, 6, 4, 3, 2, 5, 4, 3, 8, 1, 7, 5, 9, 4, 2, 0, 5, 9, 0, 3, 6, 1, 3, 6, 8, 5, 4, 4, 6, 6, 1, 7, 1, 5, 1, 1, 0, 5, 5, 3, 8, 7, 8, 8, 5, 3, 2, 1, 1, 7, 7, 4, 0, 8, 7, 6, 2, 9, 2, 6, 8, 8, 1, 6, 8, 3, 2, 2, 1, 3, 2, 9, 5, 1, 8, 8, 0, 4, 2]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
44,"[6, 2, 0, 8, 6, 4, 3, 8, 9, 4, 7, 0, 7, 7, 6, 5, 1, 0, 4, 1, 5, 9, 5, 2, 1, 6, 9, 9, 1, 3, 6, 0, 2, 7, 4, 8, 8, 7, 1, 2, 5, 4, 8, 8, 8, 1, 2, 7, 1, 4, 8, 5, 8, 4, 0, 7, 5, 8, 6, 9, 6, 6, 1, 6, 0, 0, 7, 9, 2, 1, 0, 2, 9, 4, 3, 9, 8, 5, 0, 3, 1, 5, 6, 0, 2, 4, 1, 7, 1, 9, 0, 6, 9, 7, 9, 2, 9, 1, 1, 8, 4, 1, 7, 9, 6, 0, 9, 3, 8, 0, 7, 3, 1, 0, 2, 0, 7, 9, 3, 6, 4, 2, 3, 1, 3, 0, 5, 1]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
45,"[3, 9, 5, 4, 8, 0, 0, 9, 5, 7, 3, 2, 7, 1, 3, 8, 4, 9, 9, 3, 2, 1, 5, 6, 2, 1, 8, 2, 6, 6, 6, 4, 0, 8, 5, 0, 8, 2, 2, 5, 6, 3, 2, 5, 2, 1, 9, 6, 0, 8, 1, 0, 7, 6, 9, 2, 7, 8, 6, 1, 2, 3, 1, 8, 7, 1, 8, 3, 6, 7, 1, 8, 1, 3, 6, 3, 1, 3, 6, 8, 7, 4, 7, 2, 7, 2, 1, 6, 9, 0, 2, 6, 9, 2, 0, 7, 6, 5, 8, 6, 2, 5, 2, 9, 7, 7, 6, 1, 4, 3, 3, 4, 3, 1, 0, 2, 6, 8, 4, 4, 2, 2, 6, 3, 4, 8, 2, 3]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
46,"[1, 6, 0, 4, 0, 8, 3, 5, 4, 9, 5, 9, 2, 0, 9, 5, 5, 0, 1, 3, 6, 7, 6, 6, 5, 0, 3, 0, 4, 3, 1, 1, 3, 6, 4, 2, 0, 2, 5, 1, 2, 5, 0, 9, 9, 3, 0, 5, 8, 3, 2, 0, 3, 3, 2, 9, 9, 0, 5, 5, 5, 7, 0, 5, 0, 9, 8, 8, 9, 5, 7, 1, 5, 8, 4, 0, 8, 2, 9, 0, 7, 9, 6, 6, 1, 6, 4, 1, 6, 8, 2, 3, 0, 2, 6, 9, 3, 2, 5, 3, 9, 0, 9, 4, 5, 7, 8, 4, 5, 4, 6, 6, 2, 8, 2, 3, 8, 1, 5, 1, 3, 4, 8, 2, 0, 6, 7, 6]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
47,"[2, 4, 2, 5, 1, 5, 7, 3, 0, 5, 5, 2, 0, 5, 1, 7, 5, 8, 5, 5, 1, 4, 9, 9, 7, 7, 7, 4, 5, 9, 3, 8, 6, 8, 7, 2, 8, 3, 6, 9, 1, 2, 1, 2, 7, 7, 6, 7, 2, 2, 1, 8, 7, 3, 7, 3, 8, 0, 6, 3, 9, 0, 1, 4, 7, 5, 4, 8, 5, 4, 5, 1, 7, 4, 7, 4, 0, 5, 8, 8, 5, 0, 2, 5, 8, 9, 6, 6, 9, 7, 5, 7, 8, 4, 6, 6, 5, 2, 4, 8, 9, 2, 6, 0, 0, 3, 2, 4, 0, 9, 1, 5, 9, 8, 3, 8, 7, 0, 3, 2, 6, 5, 2, 3, 4, 4, 3, 2]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
48,"[0, 1, 4, 3, 2, 6, 2, 7, 6, 4, 4, 9, 6, 8, 6, 2, 7, 8, 6, 4, 1, 0, 2, 4, 7, 3, 9, 4, 7, 0, 9, 3, 4, 5, 2, 6, 4, 0, 9, 1, 1, 6, 5, 4, 9, 8, 3, 8, 6, 9, 3, 6, 0, 3, 5, 5, 5, 9, 0, 3, 9, 2, 3, 5, 5, 7, 1, 7, 2, 7, 1, 0, 6, 8, 4, 1, 1, 8, 5, 5, 1, 2, 0, 6, 5, 0, 4, 2, 6, 8, 3, 3, 8, 9, 8, 6, 4, 3, 5, 3, 1, 1, 2, 6, 5, 5, 8, 1, 3, 1, 0, 2, 0, 4, 9, 4, 3, 5, 7, 4, 7, 9, 4, 1, 3, 4, 4, 3]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
49,"[2, 2, 0, 6, 0, 1, 9, 3, 8, 4, 7, 2, 6, 1, 5, 7, 5, 9, 7, 1, 3, 3, 3, 1, 1, 7, 4, 0, 7, 9, 1, 4, 9, 6, 3, 1, 1, 6, 3, 7, 1, 1, 9, 4, 0, 7, 1, 1, 1, 2, 3, 8, 4, 9, 7, 1, 8, 2, 7, 1, 2, 1, 6, 0, 7, 5, 8, 1, 8, 5, 2, 5, 3, 8, 0, 1, 0, 9, 5, 7, 1, 8, 7, 1, 9, 5, 3, 7, 0, 4, 0, 5, 9, 6, 7, 1, 2, 2, 0, 8, 2, 3, 7, 1, 9, 4, 4, 6, 9, 0, 6, 4, 3, 9, 3, 8, 6, 5, 3, 3, 6, 7, 2, 8, 4, 2, 8, 7]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
50,"[4, 3, 5, 1, 3, 9, 0, 5, 0, 1, 2, 4, 4, 0, 5, 3, 1, 1, 0, 0, 6, 8, 3, 9, 0, 8, 8, 4, 8, 3, 2, 6, 7, 9, 0, 4, 9, 5, 6, 0, 7, 8, 0, 3, 6, 5, 4, 0, 7, 1, 7, 5, 5, 6, 2, 3, 7, 4, 2, 3, 6, 0, 8, 2, 6, 2, 0, 4, 1, 4, 8, 9, 7, 0, 6, 2, 6, 6, 9, 2, 0, 6, 8, 1, 6, 8, 7, 1, 9, 5, 0, 3, 8, 5, 8, 8, 4, 0, 4, 1, 1, 0, 5, 9, 0, 7, 7, 7, 8, 6, 8, 9, 2, 8, 1, 5, 0, 0, 3, 9, 2, 3, 7, 6, 6, 8, 9, 7]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
51,"[7, 3, 0, 3, 5, 8, 5, 1, 4, 8, 1, 2, 4, 7, 9, 3, 9, 2, 2, 0, 6, 2, 1, 0, 8, 2, 8, 2, 1, 5, 1, 8, 0, 9, 7, 9, 1, 7, 9, 9, 2, 4, 2, 1, 8, 6, 1, 3, 5, 4, 8, 6, 5, 7, 4, 3, 4, 4, 7, 9, 9, 4, 1, 6, 7, 6, 5, 1, 5, 1, 0, 5, 9, 8, 2, 1, 4, 1, 2, 7, 1, 8, 5, 7, 9, 5, 8, 5, 6, 7, 7, 4, 3, 7, 1, 4, 4, 5, 7, 4, 0, 7, 9, 6, 0, 4, 5, 4, 1, 9, 1, 1, 8, 4, 1, 3, 5, 3, 1, 5, 0, 2, 3, 8, 1, 7, 5, 1]","[0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
52,"[6, 0, 1, 9, 9, 9, 6, 1, 2, 1, 1, 8, 2, 3, 2, 8, 8, 1, 7, 5, 3, 5, 0, 7, 8, 5, 3, 5, 1, 8, 0, 9, 0, 3, 4, 3, 4, 0, 6, 5, 0, 0, 1, 0, 4, 6, 3, 3, 5, 3, 5, 6, 0, 4, 7, 1, 1, 5, 0, 9, 7, 8, 2, 6, 8, 5, 6, 6, 3, 5, 1, 3, 8, 3, 3, 4, 7, 6, 4, 4, 2, 6, 2, 3, 0, 3, 4, 7, 4, 2, 7, 6, 9, 4, 5, 7, 8, 7, 5, 6, 6, 3, 3, 5, 0, 8, 3, 1, 1, 9, 7, 7, 8, 5, 4, 3, 0, 3, 1, 6, 1, 3, 1, 1, 5, 5, 2, 4]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
53,"[9, 1, 7, 4, 1, 4, 1, 6, 7, 5, 4, 0, 4, 2, 5, 9, 6, 0, 4, 2, 1, 7, 5, 1, 8, 9, 1, 7, 3, 3, 4, 4, 7, 4, 2, 6, 5, 2, 6, 5, 3, 4, 9, 7, 5, 7, 5, 9, 4, 2, 7, 3, 2, 8, 3, 4, 6, 6, 0, 7, 7, 1, 2, 8, 9, 6, 4, 0, 4, 4, 7, 5, 6, 4, 4, 2, 2, 4, 8, 0, 7, 3, 4, 0, 3, 7, 0, 1, 4, 7, 0, 8, 5, 3, 2, 7, 2, 3, 2, 6, 8, 9, 1, 8, 0, 2, 5, 5, 8, 6, 8, 7, 2, 9, 7, 6, 1, 1, 2, 3, 2, 8, 9, 4, 2, 5, 5, 2]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
54,"[4, 2, 7, 0, 0, 1, 6, 5, 3, 4, 7, 8, 5, 9, 6, 3, 4, 5, 1, 2, 0, 9, 7, 7, 9, 4, 3, 8, 1, 2, 0, 3, 0, 8, 1, 1, 8, 1, 7, 2, 9, 2, 8, 5, 6, 9, 2, 7, 7, 5, 1, 6, 0, 7, 6, 8, 2, 0, 1, 7, 3, 4, 5, 6, 0, 3, 4, 4, 9, 0, 5, 7, 2, 4, 7, 7, 5, 4, 1, 8, 0, 7, 6, 4, 4, 1, 7, 9, 3, 4, 3, 3, 1, 0, 2, 6, 0, 6, 2, 2, 4, 5, 8, 4, 2, 7, 2, 5, 9, 0, 9, 1, 7, 7, 7, 3, 2, 8, 2, 5, 5, 9, 8, 7, 8, 2, 7, 1]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
55,"[5, 9, 3, 4, 4, 4, 2, 8, 1, 5, 0, 4, 9, 3, 7, 0, 6, 9, 8, 6, 9, 6, 5, 2, 3, 8, 9, 3, 0, 3, 8, 7, 2, 0, 8, 9, 1, 9, 6, 2, 7, 9, 9, 7, 8, 4, 3, 4, 3, 9, 9, 2, 9, 4, 7, 1, 7, 1, 4, 9, 0, 8, 9, 2, 9, 1, 6, 8, 7, 9, 5, 1, 3, 3, 6, 4, 2, 9, 4, 6, 1, 6, 1, 5, 6, 8, 8, 7, 4, 7, 3, 8, 2, 7, 7, 2, 2, 3, 8, 5, 4, 7, 2, 8, 6, 0, 4, 0, 5, 3, 2, 9, 2, 2, 9, 7, 2, 6, 0, 0, 5, 3, 1, 1, 5, 3, 4, 7]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
56,"[9, 8, 8, 8, 3, 4, 0, 1, 5, 7, 1, 6, 2, 4, 6, 4, 8, 5, 8, 7, 3, 8, 7, 4, 4, 2, 0, 8, 6, 7, 3, 0, 4, 8, 3, 8, 6, 9, 5, 6, 2, 4, 7, 4, 1, 0, 5, 0, 5, 7, 5, 7, 4, 1, 6, 1, 3, 4, 0, 7, 4, 5, 2, 2, 5, 7, 4, 3, 7, 9, 5, 6, 3, 4, 5, 7, 1, 8, 0, 9, 0, 6, 5, 5, 1, 0, 4, 0, 3, 6, 6, 5, 9, 0, 9, 0, 4, 7, 3, 7, 0, 9, 3, 0, 8, 1, 8, 7, 8, 8, 3, 9, 0, 2, 3, 7, 7, 6, 5, 9, 2, 0, 9, 1, 3, 0, 4, 9]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
57,"[6, 1, 5, 5, 1, 7, 0, 1, 8, 7, 9, 7, 2, 3, 3, 1, 9, 9, 3, 3, 1, 9, 1, 0, 0, 0, 3, 4, 2, 8, 6, 1, 3, 5, 8, 5, 2, 1, 0, 4, 8, 3, 7, 9, 4, 6, 9, 1, 5, 3, 6, 7, 0, 7, 0, 3, 3, 8, 2, 4, 9, 9, 5, 1, 3, 2, 4, 8, 3, 3, 4, 8, 4, 1, 4, 9, 2, 5, 2, 9, 2, 8, 2, 0, 7, 0, 2, 7, 9, 2, 5, 2, 6, 9, 4, 8, 7, 1, 0, 9, 4, 4, 3, 3, 7, 4, 7, 4, 6, 7, 5, 3, 7, 9, 7, 5, 1, 3, 1, 5, 4, 0, 5, 5, 5, 5, 3, 5]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
58,"[9, 3, 1, 5, 4, 9, 5, 7, 1, 7, 3, 8, 0, 2, 3, 1, 1, 9, 4, 9, 2, 6, 2, 2, 4, 2, 5, 1, 7, 1, 2, 5, 1, 6, 8, 4, 8, 2, 6, 0, 9, 8, 0, 7, 9, 5, 1, 7, 5, 7, 4, 4, 1, 9, 6, 9, 6, 3, 3, 0, 7, 2, 9, 0, 7, 6, 7, 4, 2, 7, 4, 0, 7, 9, 1, 2, 5, 9, 0, 8, 5, 5, 6, 7, 2, 1, 0, 6, 8, 4, 5, 7, 7, 2, 4, 3, 0, 4, 4, 0, 6, 0, 4, 5, 6, 4, 3, 7, 7, 9, 2, 4, 4, 1, 8, 6, 7, 4, 6, 4, 6, 5, 7, 8, 5, 0, 0, 3]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
59,"[3, 7, 5, 6, 3, 5, 4, 0, 6, 2, 8, 9, 6, 2, 8, 0, 5, 5, 8, 6, 6, 6, 2, 8, 3, 8, 7, 9, 5, 9, 2, 1, 1, 4, 5, 4, 1, 6, 5, 7, 3, 9, 8, 0, 5, 8, 1, 0, 0, 2, 2, 9, 8, 7, 0, 4, 8, 4, 7, 9, 0, 9, 1, 2, 5, 9, 4, 6, 4, 6, 8, 5, 9, 6, 0, 8, 7, 9, 6, 3, 5, 3, 6, 1, 9, 7, 9, 5, 0, 2, 8, 8, 4, 5, 0, 7, 3, 8, 6, 7, 8, 1, 3, 1, 8, 9, 4, 6, 0, 4, 2, 3, 6, 3, 5, 7, 3, 7, 7, 9, 1, 7, 6, 1, 7, 4, 7, 8]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
60,"[8, 8, 1, 2, 5, 9, 6, 6, 6, 3, 6, 0, 7, 4, 2, 5, 6, 9, 2, 8, 1, 5, 1, 3, 1, 6, 9, 4, 0, 8, 7, 4, 2, 8, 7, 4, 4, 2, 3, 0, 7, 7, 2, 5, 4, 3, 1, 9, 6, 3, 0, 4, 8, 1, 8, 2, 9, 1, 7, 1, 7, 2, 1, 9, 4, 7, 4, 3, 3, 9, 0, 4, 6, 6, 2, 0, 2, 0, 2, 3, 0, 7, 6, 2, 7, 8, 2, 6, 0, 7, 6, 5, 4, 0, 0, 8, 9, 6, 2, 7, 1, 6, 8, 7, 1, 8, 0, 7, 5, 8, 9, 4, 1, 1, 7, 1, 5, 3, 4, 8, 7, 9, 6, 4, 8, 5, 2, 3]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
61,"[6, 4, 6, 9, 7, 7, 5, 2, 7, 9, 7, 7, 0, 8, 4, 9, 2, 4, 4, 7, 0, 3, 4, 3, 2, 0, 3, 3, 6, 4, 3, 0, 3, 6, 5, 1, 6, 2, 4, 0, 2, 0, 8, 4, 7, 7, 9, 1, 3, 0, 3, 3, 9, 7, 1, 5, 7, 1, 9, 3, 9, 2, 2, 7, 8, 6, 9, 6, 9, 3, 3, 0, 5, 5, 3, 7, 8, 8, 3, 7, 6, 3, 9, 4, 6, 5, 2, 6, 9, 5, 4, 3, 9, 6, 7, 0, 9, 6, 3, 0, 6, 5, 1, 9, 8, 7, 7, 6, 0, 0, 4, 5, 0, 7, 2, 8, 2, 5, 3, 0, 2, 1, 2, 4, 4, 2, 4, 7]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
62,"[2, 7, 2, 5, 9, 8, 8, 2, 4, 6, 1, 2, 7, 3, 9, 3, 7, 0, 4, 3, 6, 5, 7, 3, 3, 5, 7, 2, 3, 9, 1, 8, 5, 6, 0, 5, 5, 3, 7, 0, 9, 3, 4, 6, 1, 5, 5, 9, 5, 2, 3, 3, 1, 0, 3, 8, 8, 3, 2, 0, 5, 0, 8, 8, 9, 7, 3, 5, 9, 9, 4, 1, 4, 7, 7, 2, 0, 3, 2, 9, 8, 4, 2, 8, 0, 9, 8, 7, 9, 0, 1, 1, 7, 2, 0, 1, 3, 2, 9, 9, 8, 9, 0, 2, 2, 3, 5, 7, 3, 0, 4, 2, 4, 6, 4, 0, 7, 7, 4, 0, 3, 4, 3, 0, 2, 8, 9, 6]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
63,"[9, 5, 9, 9, 5, 8, 1, 7, 0, 4, 4, 7, 7, 9, 0, 4, 8, 4, 7, 9, 9, 9, 2, 1, 6, 4, 5, 9, 9, 0, 8, 3, 0, 8, 0, 5, 1, 4, 3, 9, 5, 4, 2, 1, 4, 0, 2, 3, 4, 7, 3, 5, 2, 3, 1, 7, 3, 2, 6, 1, 6, 5, 4, 7, 4, 4, 1, 3, 3, 7, 9, 5, 0, 9, 3, 3, 7, 0, 5, 1, 3, 4, 0, 4, 0, 9, 5, 9, 2, 3, 6, 4, 0, 7, 2, 3, 8, 1, 0, 1, 0, 1, 6, 2, 5, 0, 9, 8, 5, 2, 4, 1, 5, 4, 7, 5, 6, 8, 2, 5, 6, 0, 5, 3, 1, 8, 3, 0]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
64,"[4, 7, 5, 0, 4, 3, 6, 0, 7, 2, 7, 1, 2, 5, 1, 3, 3, 2, 9, 4, 9, 3, 9, 5, 4, 6, 3, 3, 3, 8, 2, 2, 3, 4, 0, 4, 2, 5, 1, 3, 5, 7, 7, 0, 6, 7, 2, 3, 4, 8, 9, 5, 0, 6, 3, 7, 6, 2, 3, 5, 2, 4, 2, 6, 8, 8, 1, 6, 5, 8, 7, 3, 3, 8, 6, 1, 4, 1, 5, 2, 7, 5, 5, 7, 1, 5, 5, 7, 6, 8, 7, 3, 1, 7, 2, 4, 2, 3, 8, 0, 4, 9, 3, 7, 4, 0, 0, 3, 2, 8, 8, 8, 0, 6, 4, 0, 3, 7, 2, 1, 5, 7, 1, 1, 5, 1, 5, 9]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9]"
|
|
||||||
65,"[1, 3, 2, 4, 6, 2, 0, 1, 2, 5, 8, 2, 1, 9, 2, 7, 4, 4, 3, 3, 3, 9, 9, 9, 8, 1, 0, 1, 5, 1, 2, 0, 4, 6, 2, 4, 6, 6, 5, 7, 3, 4, 5, 4, 6, 6, 8, 2, 4, 4, 4, 5, 5, 5, 9, 0, 6, 6, 7, 5, 3, 7, 1, 0, 7, 4, 9, 2, 5, 9, 8, 4, 6, 5, 2, 3, 2, 2, 8, 8, 4, 1, 2, 9, 2, 2, 6, 8, 4, 9, 3, 7, 9, 8, 7, 1, 8, 3, 5, 8, 1, 6, 0, 5, 6, 4, 6, 4, 0, 2, 1, 7, 6, 4, 0, 5, 4, 3, 5, 3, 0, 4, 2, 0, 8, 5, 8, 3]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
66,"[7, 0, 7, 4, 0, 7, 7, 4, 6, 2, 4, 8, 0, 3, 9, 3, 8, 1, 4, 1, 7, 6, 5, 7, 9, 7, 2, 5, 4, 1, 8, 5, 4, 2, 7, 0, 8, 0, 7, 2, 7, 9, 1, 8, 5, 4, 8, 0, 6, 9, 4, 7, 4, 7, 9, 4, 7, 9, 3, 5, 1, 6, 7, 6, 0, 8, 4, 3, 7, 0, 9, 6, 2, 2, 6, 4, 4, 1, 1, 8, 9, 5, 7, 9, 8, 1, 5, 0, 4, 0, 3, 8, 7, 5, 0, 7, 2, 6, 1, 6, 2, 9, 8, 2, 1, 5, 8, 4, 3, 0, 8, 7, 2, 5, 5, 7, 1, 8, 0, 1, 2, 6, 7, 0, 5, 5, 9, 2]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
67,"[7, 5, 3, 2, 5, 1, 8, 9, 3, 8, 8, 4, 0, 1, 5, 8, 4, 9, 4, 7, 1, 1, 0, 7, 0, 1, 8, 7, 2, 6, 3, 9, 3, 1, 3, 4, 3, 2, 3, 8, 9, 0, 8, 3, 2, 4, 7, 4, 3, 3, 9, 4, 8, 9, 0, 3, 3, 6, 9, 8, 2, 7, 6, 7, 8, 5, 5, 1, 1, 4, 2, 2, 6, 7, 3, 6, 6, 6, 8, 4, 9, 7, 8, 5, 3, 2, 2, 7, 7, 0, 8, 5, 9, 1, 6, 5, 9, 0, 0, 4, 9, 1, 9, 5, 7, 7, 3, 0, 3, 5, 6, 8, 5, 9, 9, 8, 9, 6, 6, 9, 6, 4, 8, 9, 6, 1, 0, 0]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
68,"[3, 4, 1, 6, 5, 7, 0, 6, 3, 7, 3, 8, 8, 6, 2, 9, 4, 1, 4, 7, 4, 0, 7, 4, 9, 5, 8, 3, 1, 7, 5, 3, 7, 9, 5, 5, 9, 2, 9, 8, 8, 0, 6, 0, 6, 1, 0, 0, 5, 8, 0, 4, 1, 8, 7, 9, 8, 8, 2, 7, 6, 8, 5, 3, 6, 3, 2, 7, 8, 6, 3, 1, 9, 4, 1, 0, 1, 7, 7, 9, 3, 7, 2, 6, 6, 9, 4, 5, 1, 0, 1, 5, 5, 3, 5, 4, 5, 9, 4, 8, 8, 2, 0, 0, 9, 4, 7, 0, 0, 8, 9, 8, 5, 7, 3, 3, 2, 4, 8, 1, 8, 0, 2, 0, 9, 8, 8, 7]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
69,"[3, 2, 9, 2, 2, 2, 3, 5, 1, 2, 3, 6, 3, 7, 0, 8, 8, 1, 4, 1, 2, 1, 7, 9, 0, 9, 3, 2, 6, 5, 8, 7, 5, 3, 1, 6, 8, 2, 5, 4, 0, 8, 8, 1, 5, 4, 6, 1, 4, 6, 2, 6, 7, 4, 0, 2, 1, 9, 6, 4, 9, 2, 9, 1, 1, 3, 9, 0, 2, 7, 8, 3, 5, 0, 3, 1, 8, 2, 0, 5, 3, 5, 4, 6, 5, 2, 9, 8, 4, 7, 3, 0, 6, 5, 3, 2, 5, 1, 6, 3, 3, 4, 9, 1, 5, 2, 3, 8, 4, 7, 9, 5, 0, 6, 8, 7, 5, 5, 0, 9, 3, 7, 7, 4, 1, 4, 0, 7]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
70,"[2, 8, 9, 7, 7, 8, 0, 0, 5, 4, 4, 3, 6, 5, 9, 9, 2, 2, 5, 1, 4, 4, 0, 2, 2, 5, 2, 0, 6, 3, 1, 9, 1, 7, 4, 1, 0, 2, 9, 3, 7, 3, 5, 1, 3, 8, 2, 4, 9, 2, 6, 9, 8, 1, 2, 9, 7, 6, 4, 2, 6, 8, 5, 6, 1, 8, 7, 0, 5, 2, 2, 9, 1, 8, 7, 4, 2, 2, 8, 6, 9, 2, 3, 4, 9, 7, 9, 3, 6, 5, 1, 1, 5, 8, 0, 6, 0, 2, 5, 9, 3, 6, 4, 1, 3, 3, 7, 2, 6, 5, 2, 8, 3, 9, 2, 6, 5, 3, 4, 0, 0, 2, 4, 8, 0, 0, 2, 7]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
71,"[5, 9, 6, 8, 2, 1, 9, 5, 5, 9, 2, 7, 9, 4, 3, 2, 8, 1, 6, 1, 0, 4, 5, 2, 7, 9, 2, 9, 8, 7, 2, 4, 2, 4, 4, 2, 1, 1, 9, 2, 4, 7, 0, 0, 3, 0, 4, 0, 5, 7, 4, 0, 3, 8, 3, 3, 9, 4, 8, 4, 2, 6, 9, 2, 0, 1, 2, 7, 1, 6, 7, 6, 2, 5, 5, 9, 7, 2, 1, 0, 9, 2, 2, 7, 0, 5, 5, 7, 6, 7, 0, 0, 3, 3, 1, 2, 6, 1, 2, 8, 5, 3, 6, 4, 8, 1, 6, 5, 3, 5, 6, 7, 5, 0, 5, 4, 1, 9, 5, 9, 1, 4, 7, 2, 2, 8, 7, 1]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
72,"[1, 5, 4, 6, 6, 1, 0, 0, 9, 9, 8, 7, 5, 9, 3, 8, 4, 7, 0, 1, 5, 0, 5, 4, 5, 0, 6, 9, 3, 6, 3, 8, 1, 8, 2, 9, 7, 7, 3, 7, 4, 4, 8, 4, 9, 2, 8, 6, 5, 7, 2, 6, 1, 8, 3, 9, 0, 6, 9, 0, 6, 2, 4, 9, 9, 8, 5, 6, 7, 3, 5, 8, 4, 3, 0, 7, 1, 3, 5, 6, 8, 4, 8, 0, 9, 2, 9, 6, 9, 9, 6, 7, 0, 9, 5, 9, 8, 9, 0, 1, 5, 6, 9, 8, 0, 6, 7, 5, 9, 5, 4, 9, 1, 2, 4, 6, 1, 4, 7, 0, 5, 4, 9, 2, 9, 7, 0, 7]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
73,"[3, 4, 8, 9, 2, 5, 9, 2, 1, 5, 1, 9, 1, 9, 2, 0, 8, 7, 9, 3, 4, 6, 7, 0, 8, 4, 8, 0, 3, 6, 6, 6, 1, 9, 3, 3, 3, 7, 6, 1, 1, 8, 4, 2, 0, 4, 4, 4, 3, 9, 7, 3, 4, 1, 7, 9, 7, 2, 9, 5, 5, 5, 6, 8, 5, 7, 7, 3, 0, 6, 5, 0, 5, 3, 0, 3, 7, 5, 1, 0, 3, 2, 9, 5, 1, 2, 3, 2, 3, 1, 2, 3, 7, 1, 2, 7, 1, 4, 1, 6, 2, 4, 2, 1, 6, 4, 9, 6, 1, 5, 8, 9, 9, 3, 1, 1, 6, 6, 4, 8, 2, 4, 8, 5, 4, 2, 6, 9]","[0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
74,"[5, 7, 5, 0, 3, 9, 2, 3, 2, 5, 6, 9, 2, 5, 6, 3, 0, 7, 7, 8, 4, 2, 5, 9, 5, 1, 7, 6, 1, 3, 6, 8, 5, 8, 0, 0, 1, 9, 0, 8, 0, 6, 8, 9, 3, 7, 0, 7, 0, 7, 2, 4, 8, 5, 9, 1, 0, 4, 3, 6, 7, 7, 1, 0, 6, 0, 6, 0, 4, 9, 7, 6, 7, 4, 8, 4, 7, 8, 1, 6, 6, 7, 2, 6, 4, 2, 1, 1, 4, 9, 4, 8, 8, 5, 2, 1, 2, 1, 0, 4, 6, 2, 3, 6, 0, 1, 8, 0, 2, 0, 5, 2, 8, 9, 8, 4, 1, 1, 2, 6, 1, 7, 3, 4, 6, 9, 2, 1]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
75,"[7, 5, 4, 5, 0, 9, 1, 4, 5, 3, 1, 1, 5, 6, 1, 1, 8, 6, 9, 8, 2, 7, 6, 7, 5, 7, 4, 3, 9, 6, 5, 5, 0, 2, 5, 0, 8, 8, 9, 0, 2, 5, 9, 5, 0, 4, 2, 7, 0, 8, 1, 8, 0, 3, 9, 3, 2, 3, 8, 8, 3, 9, 4, 8, 2, 1, 6, 6, 6, 1, 1, 9, 0, 3, 3, 7, 7, 9, 0, 9, 8, 4, 3, 1, 4, 0, 3, 5, 8, 0, 3, 6, 5, 4, 5, 4, 4, 3, 8, 3, 3, 8, 8, 7, 5, 3, 9, 4, 8, 9, 9, 9, 6, 5, 2, 0, 1, 9, 7, 5, 2, 9, 1, 0, 5, 1, 7, 9]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
76,"[7, 1, 9, 9, 9, 0, 1, 2, 8, 5, 0, 9, 1, 6, 1, 6, 2, 3, 9, 8, 4, 8, 2, 4, 2, 8, 9, 5, 2, 8, 2, 7, 5, 2, 8, 8, 9, 8, 8, 3, 5, 1, 5, 8, 9, 1, 1, 5, 0, 0, 1, 4, 2, 4, 9, 5, 9, 5, 7, 1, 9, 7, 8, 2, 5, 7, 0, 3, 5, 0, 4, 1, 2, 4, 6, 0, 0, 2, 2, 7, 8, 9, 4, 6, 6, 1, 7, 7, 6, 3, 4, 0, 0, 5, 3, 3, 3, 8, 1, 9, 5, 5, 9, 0, 0, 3, 7, 8, 1, 5, 5, 9, 6, 9, 5, 0, 7, 0, 9, 6, 4, 6, 5, 3, 9, 4, 2, 1]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
77,"[6, 1, 3, 7, 9, 2, 5, 6, 5, 0, 6, 2, 3, 1, 9, 0, 2, 5, 7, 8, 2, 1, 0, 0, 1, 6, 9, 1, 5, 3, 2, 5, 7, 9, 7, 2, 0, 5, 5, 6, 2, 8, 3, 6, 4, 4, 9, 6, 2, 0, 4, 2, 9, 1, 1, 3, 2, 2, 1, 0, 4, 1, 5, 9, 0, 6, 4, 2, 3, 0, 7, 4, 0, 7, 4, 9, 6, 8, 4, 6, 1, 5, 8, 0, 5, 0, 9, 6, 6, 8, 0, 3, 9, 0, 2, 2, 2, 7, 8, 8, 1, 0, 5, 0, 9, 1, 7, 3, 4, 5, 8, 1, 3, 0, 6, 2, 9, 0, 9, 0, 5, 1, 4, 2, 7, 8, 1, 9]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
78,"[3, 9, 0, 9, 2, 1, 6, 7, 0, 8, 7, 2, 0, 1, 7, 7, 2, 0, 8, 7, 7, 4, 1, 3, 4, 2, 6, 8, 6, 8, 3, 9, 6, 8, 2, 1, 7, 7, 7, 4, 6, 4, 0, 6, 0, 4, 0, 4, 2, 7, 6, 3, 0, 3, 4, 1, 1, 5, 9, 2, 3, 4, 9, 4, 4, 7, 4, 0, 4, 4, 1, 0, 1, 5, 7, 0, 8, 9, 7, 0, 0, 7, 3, 5, 3, 2, 2, 2, 1, 1, 0, 8, 4, 2, 2, 8, 5, 2, 8, 0, 7, 8, 4, 3, 1, 6, 7, 4, 5, 0, 9, 5, 9, 1, 4, 5, 0, 7, 6, 7, 6, 2, 7, 0, 6, 9, 6, 7]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
79,"[8, 6, 1, 7, 1, 8, 3, 3, 5, 7, 1, 5, 5, 7, 5, 1, 1, 1, 2, 4, 0, 1, 1, 9, 3, 9, 7, 8, 8, 5, 2, 0, 8, 4, 7, 5, 1, 9, 0, 6, 0, 6, 1, 5, 2, 2, 7, 9, 5, 9, 9, 5, 6, 3, 8, 7, 3, 4, 9, 9, 3, 1, 0, 1, 0, 9, 7, 6, 4, 9, 4, 3, 0, 1, 1, 6, 1, 0, 8, 0, 2, 3, 1, 9, 2, 8, 2, 1, 2, 3, 9, 3, 6, 7, 3, 9, 1, 3, 2, 1, 4, 4, 1, 4, 7, 2, 5, 5, 0, 4, 5, 8, 7, 9, 9, 9, 3, 4, 5, 5, 2, 0, 1, 1, 6, 3, 7, 0]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
80,"[2, 2, 0, 9, 1, 3, 6, 6, 8, 2, 5, 1, 7, 2, 2, 2, 9, 0, 7, 2, 0, 4, 9, 1, 5, 5, 2, 5, 3, 0, 9, 9, 4, 3, 1, 5, 2, 8, 7, 6, 0, 6, 5, 1, 7, 2, 2, 7, 9, 7, 5, 7, 3, 8, 9, 1, 7, 5, 0, 1, 5, 2, 2, 7, 6, 4, 7, 2, 8, 5, 5, 0, 8, 4, 7, 9, 6, 2, 7, 9, 9, 4, 8, 2, 3, 2, 8, 8, 3, 9, 7, 6, 6, 1, 6, 1, 7, 0, 7, 9, 1, 5, 1, 6, 9, 2, 7, 2, 1, 7, 5, 7, 0, 4, 6, 0, 3, 4, 0, 8, 2, 8, 9, 1, 2, 8, 8, 6]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
81,"[4, 1, 2, 5, 6, 5, 7, 2, 2, 0, 6, 8, 2, 6, 2, 7, 1, 4, 6, 3, 3, 7, 4, 3, 9, 7, 2, 4, 2, 0, 6, 8, 9, 0, 2, 0, 9, 1, 1, 2, 2, 8, 0, 9, 0, 2, 0, 9, 6, 8, 6, 4, 0, 5, 3, 6, 4, 3, 3, 7, 4, 8, 8, 2, 8, 2, 6, 3, 6, 6, 3, 9, 1, 0, 5, 4, 0, 7, 3, 2, 7, 3, 2, 2, 3, 6, 0, 7, 5, 1, 4, 2, 4, 6, 8, 2, 5, 5, 3, 2, 3, 2, 7, 0, 3, 6, 9, 6, 7, 4, 9, 3, 7, 3, 5, 2, 0, 1, 8, 0, 9, 1, 9, 2, 4, 5, 9, 6]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
82,"[0, 2, 8, 3, 5, 5, 5, 0, 1, 8, 8, 3, 8, 2, 5, 5, 2, 0, 9, 9, 7, 5, 7, 0, 3, 7, 8, 4, 0, 0, 7, 8, 6, 8, 4, 8, 5, 2, 5, 7, 6, 8, 8, 4, 8, 1, 1, 2, 0, 6, 9, 8, 5, 9, 7, 6, 2, 9, 9, 4, 0, 6, 8, 9, 6, 3, 4, 6, 2, 6, 7, 1, 2, 1, 2, 8, 4, 0, 1, 3, 4, 5, 6, 7, 0, 8, 4, 3, 1, 8, 9, 9, 5, 3, 2, 8, 4, 6, 2, 6, 5, 1, 6, 9, 1, 3, 3, 2, 8, 6, 9, 0, 4, 7, 7, 5, 0, 4, 1, 7, 9, 2, 4, 1, 4, 7, 9, 4]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
83,"[1, 3, 9, 2, 0, 8, 1, 9, 8, 2, 4, 5, 6, 3, 4, 0, 7, 8, 0, 9, 6, 5, 2, 3, 6, 9, 1, 7, 1, 0, 9, 8, 3, 3, 0, 1, 1, 0, 8, 6, 4, 3, 4, 3, 1, 8, 8, 4, 2, 1, 4, 1, 4, 1, 5, 4, 6, 9, 5, 9, 8, 4, 9, 3, 1, 7, 3, 8, 0, 0, 0, 5, 6, 0, 7, 4, 5, 1, 2, 0, 6, 6, 0, 1, 5, 3, 7, 3, 1, 5, 1, 7, 3, 8, 7, 1, 5, 1, 7, 6, 3, 9, 9, 5, 3, 4, 5, 2, 0, 2, 1, 3, 1, 7, 7, 8, 4, 9, 1, 9, 6, 1, 1, 5, 8, 2, 3, 3]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
84,"[2, 3, 3, 5, 1, 0, 5, 4, 9, 9, 5, 1, 2, 1, 1, 7, 4, 0, 9, 4, 3, 4, 8, 5, 7, 4, 0, 2, 3, 8, 6, 7, 1, 7, 2, 0, 6, 6, 9, 5, 4, 6, 6, 9, 0, 5, 3, 1, 3, 9, 5, 6, 6, 7, 5, 3, 1, 0, 7, 9, 1, 2, 3, 3, 1, 6, 9, 5, 3, 4, 1, 3, 5, 1, 6, 8, 6, 4, 4, 9, 4, 2, 3, 5, 1, 5, 0, 6, 7, 9, 0, 6, 4, 7, 0, 1, 3, 1, 0, 4, 3, 0, 6, 2, 1, 6, 1, 0, 5, 0, 4, 2, 8, 1, 3, 9, 6, 9, 3, 5, 7, 5, 5, 7, 4, 2, 0, 3]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
85,"[9, 5, 0, 6, 4, 5, 2, 5, 4, 4, 5, 9, 7, 4, 2, 9, 1, 7, 1, 0, 3, 1, 0, 9, 1, 1, 6, 1, 7, 0, 1, 8, 4, 9, 5, 0, 8, 7, 8, 9, 6, 0, 2, 5, 3, 6, 8, 5, 6, 9, 2, 7, 0, 9, 6, 2, 8, 5, 3, 2, 7, 4, 6, 1, 5, 2, 2, 7, 1, 4, 8, 8, 7, 5, 5, 1, 7, 0, 4, 6, 0, 2, 6, 1, 7, 1, 6, 8, 4, 3, 6, 6, 8, 0, 4, 2, 9, 8, 2, 8, 6, 6, 2, 4, 1, 3, 6, 9, 2, 9, 2, 3, 7, 1, 2, 9, 8, 2, 9, 6, 4, 2, 0, 9, 4, 6, 4, 7]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
86,"[9, 5, 8, 7, 3, 9, 3, 7, 4, 8, 4, 4, 5, 3, 3, 7, 5, 3, 6, 5, 9, 3, 8, 7, 7, 8, 1, 0, 7, 3, 9, 2, 8, 9, 5, 8, 0, 2, 1, 1, 6, 1, 7, 1, 6, 3, 1, 1, 3, 1, 0, 8, 6, 6, 6, 0, 3, 3, 8, 1, 5, 7, 6, 6, 1, 7, 3, 9, 9, 3, 5, 1, 8, 7, 8, 2, 9, 9, 5, 4, 3, 8, 1, 2, 0, 0, 0, 7, 0, 6, 2, 7, 1, 5, 1, 8, 2, 6, 4, 0, 6, 1, 4, 8, 9, 4, 5, 3, 6, 2, 7, 1, 2, 7, 1, 5, 8, 2, 5, 7, 0, 1, 0, 2, 5, 6, 1, 6]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
87,"[7, 6, 8, 7, 6, 9, 9, 4, 1, 0, 1, 8, 5, 0, 2, 3, 0, 3, 9, 1, 4, 4, 6, 1, 6, 4, 5, 4, 7, 7, 9, 0, 7, 3, 1, 9, 1, 6, 1, 1, 4, 6, 0, 2, 1, 4, 6, 7, 9, 7, 2, 2, 0, 7, 9, 9, 2, 0, 1, 4, 0, 0, 7, 9, 2, 7, 7, 4, 5, 3, 9, 0, 2, 6, 8, 6, 7, 8, 6, 8, 5, 5, 9, 1, 8, 5, 9, 1, 4, 6, 1, 8, 4, 6, 1, 0, 3, 0, 3, 4, 5, 9, 5, 6, 2, 8, 9, 8, 1, 3, 7, 4, 0, 4, 0, 7, 1, 8, 5, 2, 0, 8, 7, 6, 3, 7, 3, 2]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
88,"[4, 4, 2, 1, 3, 1, 7, 8, 7, 8, 9, 1, 7, 0, 5, 7, 8, 7, 7, 9, 6, 9, 3, 9, 2, 5, 1, 2, 5, 5, 8, 9, 1, 3, 0, 4, 6, 0, 6, 1, 9, 8, 1, 9, 6, 2, 5, 6, 9, 4, 6, 0, 9, 2, 0, 8, 6, 6, 0, 7, 6, 2, 5, 4, 1, 7, 0, 0, 9, 6, 0, 0, 7, 9, 1, 8, 8, 2, 5, 5, 2, 0, 1, 0, 5, 1, 7, 7, 0, 8, 1, 0, 0, 6, 7, 6, 8, 2, 3, 7, 9, 7, 0, 0, 8, 6, 2, 5, 3, 1, 4, 1, 5, 6, 6, 4, 8, 3, 6, 9, 9, 4, 8, 3, 2, 3, 2, 7]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
89,"[4, 4, 8, 2, 7, 1, 8, 1, 9, 8, 6, 7, 8, 1, 7, 6, 1, 0, 3, 5, 7, 3, 1, 5, 5, 8, 1, 8, 8, 3, 8, 2, 4, 0, 5, 4, 5, 6, 3, 6, 5, 7, 4, 1, 2, 3, 3, 0, 5, 0, 0, 3, 1, 8, 9, 3, 6, 9, 7, 3, 2, 7, 3, 2, 9, 0, 1, 9, 7, 4, 0, 7, 7, 1, 6, 1, 5, 7, 7, 8, 4, 2, 6, 1, 2, 4, 7, 1, 7, 6, 7, 6, 9, 2, 8, 3, 4, 3, 4, 0, 0, 8, 0, 9, 6, 9, 3, 3, 7, 7, 3, 7, 1, 0, 4, 6, 3, 4, 8, 9, 4, 3, 0, 7, 9, 3, 4, 3]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
90,"[5, 0, 7, 6, 4, 2, 9, 1, 1, 8, 1, 0, 2, 3, 9, 1, 3, 8, 1, 2, 1, 4, 0, 9, 9, 5, 0, 7, 6, 7, 8, 1, 4, 9, 2, 9, 8, 4, 0, 5, 5, 8, 1, 4, 1, 1, 4, 5, 1, 3, 6, 7, 0, 4, 1, 5, 8, 1, 6, 4, 0, 8, 8, 8, 3, 5, 9, 5, 3, 3, 5, 4, 2, 3, 6, 5, 3, 8, 4, 8, 6, 4, 7, 7, 4, 2, 3, 9, 5, 5, 8, 3, 6, 1, 1, 7, 0, 1, 1, 6, 9, 4, 2, 4, 3, 1, 0, 2, 0, 6, 2, 1, 0, 3, 8, 0, 1, 3, 0, 5, 7, 0, 1, 2, 2, 2, 2, 0]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
91,"[9, 9, 1, 1, 9, 3, 0, 1, 6, 6, 8, 1, 7, 1, 2, 4, 2, 0, 5, 4, 3, 7, 6, 9, 9, 4, 6, 6, 3, 5, 5, 3, 0, 5, 8, 1, 1, 3, 5, 0, 2, 2, 8, 2, 2, 0, 2, 6, 1, 1, 3, 1, 6, 8, 5, 2, 3, 3, 5, 6, 1, 4, 3, 9, 2, 0, 8, 7, 7, 2, 8, 9, 3, 1, 5, 3, 7, 1, 8, 6, 6, 3, 2, 5, 6, 6, 7, 8, 6, 4, 7, 2, 3, 0, 0, 6, 0, 6, 0, 0, 5, 8, 1, 3, 9, 8, 5, 1, 4, 7, 3, 3, 3, 0, 7, 9, 2, 6, 4, 7, 4, 8, 2, 7, 6, 0, 1, 6]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
92,"[8, 4, 7, 2, 3, 3, 8, 6, 7, 9, 8, 1, 6, 2, 4, 5, 8, 8, 1, 2, 7, 8, 9, 6, 1, 7, 7, 3, 3, 1, 8, 8, 7, 3, 5, 4, 2, 9, 4, 7, 9, 9, 5, 4, 3, 2, 0, 9, 2, 5, 5, 7, 1, 6, 9, 4, 4, 5, 9, 6, 7, 0, 1, 8, 4, 4, 4, 2, 7, 5, 6, 1, 1, 8, 6, 0, 9, 5, 4, 0, 8, 6, 4, 8, 9, 6, 4, 8, 0, 1, 8, 2, 4, 4, 3, 2, 9, 5, 2, 8, 5, 4, 6, 8, 1, 2, 9, 9, 2, 0, 7, 3, 6, 2, 9, 5, 3, 0, 2, 3, 2, 2, 2, 7, 9, 6, 2, 0]","[0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
93,"[4, 1, 2, 4, 5, 2, 7, 1, 7, 5, 6, 1, 8, 2, 8, 6, 1, 4, 9, 2, 2, 2, 8, 0, 7, 8, 0, 8, 9, 0, 2, 8, 2, 9, 7, 7, 7, 8, 3, 7, 3, 1, 5, 8, 8, 5, 3, 7, 1, 3, 6, 0, 0, 7, 6, 4, 3, 5, 1, 1, 0, 0, 6, 9, 0, 2, 0, 2, 7, 6, 4, 0, 7, 2, 7, 4, 4, 2, 2, 5, 4, 7, 6, 3, 1, 6, 0, 4, 6, 1, 0, 1, 3, 8, 9, 6, 4, 2, 4, 4, 9, 2, 8, 8, 2, 0, 7, 9, 6, 6, 5, 6, 1, 9, 0, 5, 7, 3, 1, 1, 3, 0, 3, 7, 9, 3, 3, 0]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
94,"[0, 8, 7, 8, 3, 7, 8, 4, 2, 4, 6, 6, 3, 4, 2, 0, 7, 0, 4, 0, 4, 6, 0, 2, 5, 3, 9, 9, 7, 4, 2, 5, 6, 3, 6, 1, 5, 6, 2, 6, 7, 6, 5, 9, 8, 8, 1, 1, 0, 8, 1, 7, 2, 7, 1, 2, 8, 7, 5, 5, 3, 0, 9, 0, 2, 0, 6, 0, 1, 8, 3, 3, 6, 5, 2, 5, 9, 0, 9, 6, 6, 2, 8, 9, 3, 5, 2, 6, 6, 6, 7, 1, 3, 4, 1, 3, 3, 5, 0, 8, 8, 0, 0, 9, 6, 0, 7, 1, 8, 4, 7, 5, 8, 8, 9, 9, 1, 4, 1, 4, 5, 7, 3, 9, 1, 5, 8, 5]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
95,"[5, 1, 2, 0, 9, 5, 3, 1, 8, 6, 2, 2, 8, 8, 8, 7, 9, 0, 1, 5, 3, 5, 6, 4, 4, 3, 0, 1, 8, 0, 8, 6, 7, 3, 8, 4, 6, 7, 7, 5, 1, 4, 6, 8, 1, 9, 8, 4, 5, 6, 2, 6, 8, 0, 4, 0, 3, 8, 2, 5, 5, 7, 1, 8, 3, 4, 0, 3, 3, 5, 2, 2, 7, 3, 0, 7, 1, 7, 9, 7, 8, 5, 0, 8, 5, 6, 3, 8, 5, 4, 6, 6, 5, 5, 2, 2, 7, 9, 2, 6, 9, 9, 4, 4, 2, 0, 4, 1, 4, 9, 2, 6, 3, 5, 3, 0, 5, 7, 4, 2, 4, 2, 7, 6, 7, 2, 0, 7]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
96,"[8, 4, 4, 2, 4, 8, 3, 7, 7, 3, 9, 6, 6, 4, 5, 6, 1, 4, 6, 7, 3, 1, 3, 3, 7, 4, 0, 6, 5, 1, 7, 5, 6, 8, 4, 0, 4, 8, 7, 9, 8, 6, 8, 3, 2, 7, 1, 2, 6, 6, 1, 1, 9, 3, 2, 1, 0, 1, 2, 5, 5, 1, 1, 9, 3, 6, 3, 1, 8, 4, 9, 9, 4, 0, 0, 3, 0, 7, 1, 5, 9, 4, 4, 8, 3, 9, 3, 4, 3, 9, 7, 3, 6, 0, 2, 2, 5, 7, 6, 2, 5, 5, 2, 4, 9, 5, 3, 2, 1, 3, 8, 2, 6, 9, 0, 1, 2, 6, 9, 3, 3, 4, 5, 6, 3, 9, 4, 1]","[0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
97,"[2, 1, 7, 3, 8, 7, 7, 8, 0, 7, 7, 8, 7, 6, 5, 9, 9, 6, 5, 0, 1, 7, 5, 6, 7, 0, 6, 8, 2, 4, 2, 4, 0, 0, 2, 9, 2, 1, 9, 8, 4, 7, 2, 1, 2, 5, 5, 5, 7, 7, 6, 6, 8, 1, 5, 2, 2, 9, 3, 8, 9, 4, 8, 2, 0, 8, 6, 4, 0, 2, 0, 2, 9, 4, 2, 7, 5, 8, 4, 0, 4, 6, 3, 3, 6, 4, 1, 4, 7, 5, 8, 9, 4, 8, 2, 6, 6, 0, 8, 2, 7, 5, 8, 1, 5, 6, 8, 7, 9, 6, 8, 9, 7, 3, 9, 4, 3, 9, 5, 2, 8, 8, 4, 5, 2, 0, 6, 0]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
98,"[3, 8, 0, 0, 1, 0, 0, 4, 0, 5, 9, 3, 4, 0, 5, 8, 5, 4, 6, 1, 2, 7, 1, 1, 1, 9, 1, 2, 1, 9, 8, 3, 4, 6, 8, 6, 9, 9, 8, 8, 7, 6, 7, 4, 4, 0, 3, 6, 3, 4, 3, 2, 1, 8, 9, 7, 9, 3, 3, 1, 0, 1, 5, 3, 8, 6, 6, 4, 6, 3, 6, 2, 3, 4, 7, 3, 8, 9, 6, 1, 0, 6, 1, 8, 4, 1, 9, 3, 1, 9, 9, 2, 8, 9, 8, 7, 1, 1, 4, 0, 5, 2, 9, 6, 0, 7, 2, 5, 7, 3, 3, 1, 1, 4, 6, 3, 7, 3, 9, 4, 2, 6, 0, 2, 7, 8, 0, 7]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
99,"[1, 4, 7, 9, 7, 4, 1, 4, 7, 4, 6, 7, 7, 8, 2, 1, 3, 7, 2, 2, 1, 6, 2, 5, 5, 3, 6, 8, 1, 6, 1, 8, 0, 1, 0, 7, 6, 8, 9, 4, 9, 8, 3, 1, 2, 9, 8, 3, 5, 3, 7, 5, 9, 0, 0, 2, 7, 0, 5, 7, 5, 3, 3, 2, 0, 7, 8, 6, 1, 7, 8, 8, 9, 4, 2, 1, 3, 1, 3, 8, 0, 5, 5, 1, 5, 0, 6, 4, 6, 0, 6, 5, 2, 7, 1, 5, 3, 2, 6, 9, 5, 7, 1, 8, 8, 3, 5, 9, 4, 1, 7, 7, 8, 0, 9, 8, 4, 5, 8, 2, 9, 6, 0, 2, 9, 8, 8, 5]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
|
|
||||||
|
@ -1,888 +0,0 @@
|
|||||||
# Copyright (c) 2023 ETH Zurich.
|
|
||||||
# All rights reserved.
|
|
||||||
#
|
|
||||||
# Use of this source code is governed by a BSD-style license that can be
|
|
||||||
# found in the LICENSE file.
|
|
||||||
#
|
|
||||||
# main author: Nils Blach
|
|
||||||
|
|
||||||
import os
|
|
||||||
import logging
|
|
||||||
import datetime
|
|
||||||
import json
|
|
||||||
import csv
|
|
||||||
from typing import Dict, List, Callable, Union
|
|
||||||
from graph_of_thoughts import controller, language_models, operations, prompter, parser
|
|
||||||
|
|
||||||
# This is a hack to also allow execution of this file from the examples directory
|
|
||||||
try:
|
|
||||||
from . import utils
|
|
||||||
except ImportError:
|
|
||||||
import utils
|
|
||||||
|
|
||||||
|
|
||||||
class SortingPrompter(prompter.Prompter):
|
|
||||||
"""
|
|
||||||
SortingPrompter provides the generation of prompts specific to the sorting
|
|
||||||
example for the language models.
|
|
||||||
|
|
||||||
Inherits from the Prompter class and implements its abstract methods.
|
|
||||||
"""
|
|
||||||
|
|
||||||
sort_prompt = """<Instruction> Sort the following list of numbers in ascending order. Output only the sorted list of numbers, no additional text. </Instruction>
|
|
||||||
|
|
||||||
<Examples>
|
|
||||||
Input: [5, 1, 0, 1, 2, 0, 4, 8, 1, 9, 5, 1, 3, 3, 9, 7]
|
|
||||||
Output: [0, 0, 1, 1, 1, 1, 2, 3, 3, 4, 5, 5, 7, 8, 9, 9]
|
|
||||||
|
|
||||||
Input: [3, 7, 0, 2, 8, 1, 2, 2, 2, 4, 7, 8, 5, 5, 3, 9, 4, 3, 5, 6, 6, 4, 4, 5, 2, 0, 9, 3, 3, 9, 2, 1]
|
|
||||||
Output: [0, 0, 1, 1, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 9]
|
|
||||||
|
|
||||||
Input: [4, 4, 9, 7, 9, 7, 0, 0, 4, 9, 1, 7, 9, 5, 8, 7, 5, 6, 3, 8, 6, 7, 5, 8, 5, 0, 6, 3, 7, 0, 5, 3, 7, 5, 2, 4, 4, 9, 0, 7, 8, 2, 7, 7, 7, 2, 1, 3, 9, 9, 7, 9, 6, 6, 4, 5, 4, 2, 0, 8, 9, 0, 2, 2]
|
|
||||||
Output: [0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9]
|
|
||||||
</Examples>
|
|
||||||
|
|
||||||
Input: {input}"""
|
|
||||||
|
|
||||||
sort_prompt_cot = """<Instruction> Sort the following list of numbers in ascending order. You can generate any intermediate lists, but the final output should be the sorted list of numbers, prefixed with "Output: ". </Instruction>
|
|
||||||
|
|
||||||
<Approach>
|
|
||||||
To sort the list of numbers follow these steps:
|
|
||||||
1. Split the list of numbers into two to four unsorted sublists, each containing an equal number of elements from the original list (make sure they don't overlap).
|
|
||||||
2. Sort each of the unsorted sublists.
|
|
||||||
3. Merge the sorted sublists into a single sorted list using the merging algorithm from merge sort.
|
|
||||||
</Approach>
|
|
||||||
|
|
||||||
<Examples>
|
|
||||||
Input: [4, 5, 3, 3, 7, 3, 0, 5, 0, 2, 8, 0, 2, 1, 6, 9]
|
|
||||||
Unsorted Subarrays:
|
|
||||||
[4, 5, 3, 3, 7, 3, 0, 5]
|
|
||||||
[0, 2, 8, 0, 2, 1, 6, 9]
|
|
||||||
Sorted Subarrays:
|
|
||||||
[0, 3, 3, 3, 4, 5, 5, 7]
|
|
||||||
[0, 0, 1, 2, 2, 6, 8, 9]
|
|
||||||
Output: [0, 0, 0, 1, 2, 2, 3, 3, 3, 4, 5, 5, 6, 7, 8, 9]
|
|
||||||
|
|
||||||
Input: [6, 4, 5, 7, 5, 6, 9, 7, 6, 9, 4, 6, 9, 8, 1, 9, 2, 4, 9, 0, 7, 6, 5, 6, 6, 2, 8, 3, 9, 5, 6, 1]
|
|
||||||
Unsorted Subarrays:
|
|
||||||
[6, 4, 5, 7, 5, 6, 9, 7, 6, 9, 4, 6, 9, 8, 1, 9]
|
|
||||||
[2, 4, 9, 0, 7, 6, 5, 6, 6, 2, 8, 3, 9, 5, 6, 1]
|
|
||||||
Sorted Subarrays:
|
|
||||||
[1, 4, 4, 5, 5, 6, 6, 6, 6, 7, 7, 8, 9, 9, 9, 9]
|
|
||||||
[0, 1, 2, 2, 3, 4, 5, 5, 6, 6, 6, 6, 7, 8, 9, 9]
|
|
||||||
Output: [0, 1, 1, 2, 2, 3, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 8, 8, 9, 9, 9, 9, 9, 9]
|
|
||||||
|
|
||||||
Input: [3, 7, 0, 2, 8, 1, 2, 2, 2, 4, 7, 8, 5, 5, 3, 9, 4, 3, 5, 6, 6, 4, 4, 5, 2, 0, 9, 3, 3, 9, 2, 1, 9, 3, 1, 8, 1, 8, 6, 0, 1, 6, 1, 7, 4, 4, 6, 3, 3, 7, 9, 3, 6, 0, 3, 4, 5, 6, 6, 9, 9, 9, 7, 3]
|
|
||||||
Unsorted Subarrays:
|
|
||||||
[3, 7, 0, 2, 8, 1, 2, 2, 2, 4, 7, 8, 5, 5, 3, 9]
|
|
||||||
[4, 3, 5, 6, 6, 4, 4, 5, 2, 0, 9, 3, 3, 9, 2, 1]
|
|
||||||
[9, 3, 1, 8, 1, 8, 6, 0, 1, 6, 1, 7, 4, 4, 6, 3]
|
|
||||||
[3, 7, 9, 3, 6, 0, 3, 4, 5, 6, 6, 9, 9, 9, 7, 3]
|
|
||||||
Sorted Subarrays:
|
|
||||||
[0, 1, 2, 2, 2, 2, 3, 3, 4, 5, 5, 7, 7, 8, 8, 9]
|
|
||||||
[0, 1, 2, 2, 3, 3, 3, 4, 4, 4, 5, 5, 6, 6, 9, 9]
|
|
||||||
[0, 1, 1, 1, 1, 3, 3, 4, 4, 6, 6, 6, 7, 8, 8, 9]
|
|
||||||
[0, 3, 3, 3, 3, 4, 5, 6, 6, 6, 7, 7, 9, 9, 9, 9]
|
|
||||||
Output: [0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9]
|
|
||||||
</Examples>
|
|
||||||
|
|
||||||
Input: {input}"""
|
|
||||||
|
|
||||||
tot_improve_prompt = """<Instruction> The following two lists represent an unsorted list of numbers and a sorted variant of that list. The sorted variant is not correct. Fix the sorted variant so that it is correct.
|
|
||||||
Make sure that the output list is sorted in ascending order, has the same number of elements as the input list ({length}), and contains the same elements as the input list. </Instruction>
|
|
||||||
|
|
||||||
<Approach>
|
|
||||||
To fix the incorrectly sorted list follow these steps:
|
|
||||||
1. For each number from 0 to 9, compare the frequency of that number in the incorrectly sorted list to the frequency of that number in the input list.
|
|
||||||
2. Iterate through the incorrectly sorted list and add or remove numbers as needed to make the frequency of each number in the incorrectly sorted list match the frequency of that number in the input list.
|
|
||||||
</Approach>
|
|
||||||
|
|
||||||
<Examples>
|
|
||||||
Input: [3, 7, 0, 2, 8, 1, 2, 2, 2, 4, 7, 8, 5, 5, 3, 9]
|
|
||||||
Incorrectly Sorted: [0, 0, 0, 0, 0, 1, 2, 2, 3, 3, 4, 4, 4, 5, 5, 7, 7, 8, 8, 9, 9, 9, 9]
|
|
||||||
Reason: The incorrectly sorted list contains four extra 0s, two extra 4s and three extra 9s and is missing two 2s.
|
|
||||||
Output: [0, 1, 2, 2, 2, 2, 3, 3, 4, 5, 5, 7, 7, 8, 8, 9]
|
|
||||||
|
|
||||||
Input: [6, 4, 5, 7, 5, 6, 9, 7, 6, 9, 4, 6, 9, 8, 1, 9, 2, 4, 9, 0, 7, 6, 5, 6, 6, 2, 8, 3, 9, 5, 6, 1]
|
|
||||||
Incorrectly Sorted: [0, 1, 1, 2, 2, 3, 4, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 7, 7, 7, 8, 8, 9, 9, 9, 9, 9]
|
|
||||||
Reason: The incorrectly sorted list contains two extra 4s and is missing two 6s and one 9.
|
|
||||||
Output: [0, 1, 1, 2, 2, 3, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 8, 8, 9, 9, 9, 9, 9, 9]
|
|
||||||
|
|
||||||
Input: [4, 4, 9, 7, 9, 7, 0, 0, 4, 9, 1, 7, 9, 5, 8, 7, 5, 6, 3, 8, 6, 7, 5, 8, 5, 0, 6, 3, 7, 0, 5, 3, 7, 5, 2, 4, 4, 9, 0, 7, 8, 2, 7, 7, 7, 2, 1, 3, 9, 9, 7, 9, 6, 6, 4, 5, 4, 2, 0, 8, 9, 0, 2, 2]
|
|
||||||
Incorrectly Sorted: [0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 2, 2, 2, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 5, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9]
|
|
||||||
Reason: The incorrectly sorted list contains one extra 8 and is missing two 2s, one 3, three 4s, two 5s, one 6, six 7s and one 9.
|
|
||||||
Output: [0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9]
|
|
||||||
</Examples>
|
|
||||||
|
|
||||||
Input: {input}
|
|
||||||
Incorrectly Sorted: {incorrectly_sorted}
|
|
||||||
"""
|
|
||||||
|
|
||||||
got_split_prompt = """<Instruction> Split the following list of 128 numbers into 8 lists of 16 numbers each, the first list should contain the first 16 numbers, the second list the second 16 numbers, the third list the third 16 numbers, the fourth list the fourth 16 numbers, the fifth list the fifth 16 numbers and so on.
|
|
||||||
Only output the final 8 lists in the following format without any additional text or thoughts!:
|
|
||||||
{{
|
|
||||||
"List 1": [3, 4, 3, 5, 7, 8, 1, ...],
|
|
||||||
"List 2": [2, 9, 2, 4, 7, 1, 5, ...],
|
|
||||||
"List 3": [6, 9, 8, 1, 9, 2, 4, ...],
|
|
||||||
"List 4": [9, 0, 7, 6, 5, 6, 6, ...],
|
|
||||||
"List 5": [7, 9, 4, 1, 1, 8, 1, ...],
|
|
||||||
"List 6": [1, 9, 0, 4, 3, 3, 5, ...],
|
|
||||||
"List 7": [2, 4, 3, 5, 8, 2, 2, ...],
|
|
||||||
"List 8": [4, 2, 1, 2, 7, 6, 8, ...]
|
|
||||||
}} </Instruction>
|
|
||||||
|
|
||||||
<Example>
|
|
||||||
Input: [6, 0, 2, 3, 8, 3, 0, 2, 4, 5, 4, 1, 3, 6, 9, 8, 3, 1, 2, 6, 5, 3, 9, 8, 9, 1, 6, 1, 0, 2, 8, 9, 5, 3, 1, 2, 7, 9, 4, 8, 8, 9, 3, 2, 8, 4, 7, 4, 3, 8, 7, 3, 6, 4, 0, 0, 6, 8, 1, 5, 8, 7, 5, 1, 4, 0, 8, 6, 1, 3, 6, 1, 7, 6, 8, 7, 3, 7, 8, 2, 0, 8, 2, 6, 0, 0, 9, 9, 8, 6, 9, 4, 8, 5, 5, 0, 0, 9, 3, 9, 4, 0, 5, 6, 2, 4, 6, 7, 7, 7, 8, 0, 4, 9, 1, 4, 8, 5, 1, 4, 4, 7, 4, 9, 3, 9, 6, 7]
|
|
||||||
Output:
|
|
||||||
{{
|
|
||||||
"List 1": [6, 0, 2, 3, 8, 3, 0, 2, 4, 5, 4, 1, 3, 6, 9, 8],
|
|
||||||
"List 2": [3, 1, 2, 6, 5, 3, 9, 8, 9, 1, 6, 1, 0, 2, 8, 9],
|
|
||||||
"List 3": [5, 3, 1, 2, 7, 9, 4, 8, 8, 9, 3, 2, 8, 4, 7, 4],
|
|
||||||
"List 4": [3, 8, 7, 3, 6, 4, 0, 0, 6, 8, 1, 5, 8, 7, 5, 1],
|
|
||||||
"List 5": [4, 0, 8, 6, 1, 3, 6, 1, 7, 6, 8, 7, 3, 7, 8, 2],
|
|
||||||
"List 6": [0, 8, 2, 6, 0, 0, 9, 9, 8, 6, 9, 4, 8, 5, 5, 0],
|
|
||||||
"List 7": [0, 9, 3, 9, 4, 0, 5, 6, 2, 4, 6, 7, 7, 7, 8, 0],
|
|
||||||
"List 8": [4, 9, 1, 4, 8, 5, 1, 4, 4, 7, 4, 9, 3, 9, 6, 7]
|
|
||||||
}}
|
|
||||||
</Example>
|
|
||||||
|
|
||||||
Input: {input}"""
|
|
||||||
|
|
||||||
got_merge_prompt = """<Instruction> Merge the following 2 sorted lists of length {length1} each, into one sorted list of length {length2} using a merge sort style approach.
|
|
||||||
Only output the final merged list without any additional text or thoughts!:</Instruction>
|
|
||||||
|
|
||||||
<Approach>
|
|
||||||
To merge the two lists in a merge-sort style approach, follow these steps:
|
|
||||||
1. Compare the first element of both lists.
|
|
||||||
2. Append the smaller element to the merged list and move to the next element in the list from which the smaller element came.
|
|
||||||
3. Repeat steps 1 and 2 until one of the lists is empty.
|
|
||||||
4. Append the remaining elements of the non-empty list to the merged list.
|
|
||||||
</Approach>
|
|
||||||
|
|
||||||
Merge the following two lists into one sorted list:
|
|
||||||
1: {input1}
|
|
||||||
2: {input2}
|
|
||||||
|
|
||||||
Merged list:
|
|
||||||
"""
|
|
||||||
|
|
||||||
def aggregation_prompt(self, state_dicts: List[Dict], **kwargs) -> str:
|
|
||||||
"""
|
|
||||||
Generate an aggregation prompt for the language model.
|
|
||||||
|
|
||||||
:param state_dicts: The thought states that should be aggregated.
|
|
||||||
:type state_dicts: List[Dict]
|
|
||||||
:param kwargs: Additional keyword arguments.
|
|
||||||
:return: The aggregation prompt.
|
|
||||||
:rtype: str
|
|
||||||
:raise AssertionError: If not exactly two thought states are provided.
|
|
||||||
"""
|
|
||||||
assert len(state_dicts) == 2, "Expected two states for aggregation prompt."
|
|
||||||
len_input1 = len(utils.string_to_list(state_dicts[0]["current"]))
|
|
||||||
len_input2 = len(utils.string_to_list(state_dicts[1]["current"]))
|
|
||||||
if len_input1 == len_input2:
|
|
||||||
length = len_input1
|
|
||||||
elif len_input1 + len_input2 - 32 <= 16:
|
|
||||||
length = 16
|
|
||||||
elif len_input1 + len_input2 - 64 <= 32:
|
|
||||||
length = 32
|
|
||||||
else:
|
|
||||||
length = 64
|
|
||||||
|
|
||||||
return self.got_merge_prompt.format(
|
|
||||||
input1=state_dicts[0]["current"],
|
|
||||||
input2=state_dicts[1]["current"],
|
|
||||||
length1=length,
|
|
||||||
length2=length * 2,
|
|
||||||
)
|
|
||||||
|
|
||||||
def generate_prompt(
|
|
||||||
self, num_branches: int, original: str, current: str, method: str, **kwargs
|
|
||||||
) -> str:
|
|
||||||
"""
|
|
||||||
Generate a generate prompt for the language model.
|
|
||||||
|
|
||||||
:param num_branches: The number of responses the prompt should ask the LM to generate.
|
|
||||||
:type num_branches: int
|
|
||||||
:param original: Input list of numbers.
|
|
||||||
:type original: str
|
|
||||||
:param current: Intermediate solution.
|
|
||||||
:type current: str
|
|
||||||
:param method: Method for which the generate prompt is generated.
|
|
||||||
:type method: str
|
|
||||||
:param kwargs: Additional keyword arguments.
|
|
||||||
:return: The generate prompt.
|
|
||||||
:rtype: str
|
|
||||||
:raise AssertionError: If the requested number of branches is not one.
|
|
||||||
"""
|
|
||||||
|
|
||||||
assert num_branches == 1, "Branching should be done via multiple requests."
|
|
||||||
if current is None or current == "":
|
|
||||||
input = original
|
|
||||||
else:
|
|
||||||
input = current
|
|
||||||
if method.startswith("io"):
|
|
||||||
return self.sort_prompt.format(input=input)
|
|
||||||
elif method.startswith("cot"):
|
|
||||||
return self.sort_prompt_cot.format(input=input)
|
|
||||||
elif method.startswith("tot"):
|
|
||||||
if current is None or current == "":
|
|
||||||
return self.sort_prompt.format(input=input)
|
|
||||||
return self.tot_improve_prompt.format(
|
|
||||||
input=original,
|
|
||||||
incorrectly_sorted=current,
|
|
||||||
length=len(utils.string_to_list(original)),
|
|
||||||
)
|
|
||||||
elif method.startswith("got"):
|
|
||||||
if current is None or current == "":
|
|
||||||
return self.got_split_prompt.format(input=input)
|
|
||||||
# if current is just a sublist of the original input, return the split prompt
|
|
||||||
if kwargs["phase"] == 1:
|
|
||||||
return self.sort_prompt.format(input=current)
|
|
||||||
|
|
||||||
if (
|
|
||||||
"unsorted_sublist" in kwargs
|
|
||||||
and kwargs["unsorted_sublist"] != ""
|
|
||||||
and len(kwargs["unsorted_sublist"]) < len(original) - 5
|
|
||||||
):
|
|
||||||
original = kwargs["unsorted_sublist"]
|
|
||||||
|
|
||||||
return self.tot_improve_prompt.format(
|
|
||||||
input=original,
|
|
||||||
incorrectly_sorted=current,
|
|
||||||
length=len(utils.string_to_list(original)),
|
|
||||||
)
|
|
||||||
|
|
||||||
def improve_prompt(self, **kwargs) -> str:
|
|
||||||
"""
|
|
||||||
Generate an improve prompt for the language model.
|
|
||||||
|
|
||||||
:param kwargs: Additional keyword arguments.
|
|
||||||
:return: The improve prompt.
|
|
||||||
:rtype: str
|
|
||||||
"""
|
|
||||||
pass
|
|
||||||
|
|
||||||
def validation_prompt(self, **kwargs) -> str:
|
|
||||||
"""
|
|
||||||
Generate a validation prompt for the language model.
|
|
||||||
|
|
||||||
:param kwargs: Additional keyword arguments.
|
|
||||||
:return: The validation prompt.
|
|
||||||
:rtype: str
|
|
||||||
"""
|
|
||||||
pass
|
|
||||||
|
|
||||||
def score_prompt(self, state_dicts: List[Dict], **kwargs) -> str:
|
|
||||||
"""
|
|
||||||
Generate a score prompt for the language model.
|
|
||||||
|
|
||||||
:param state_dicts: The thought states that should be scored,
|
|
||||||
if more than one, they should be scored together.
|
|
||||||
:type state_dicts: List[Dict]
|
|
||||||
:param kwargs: Additional keyword arguments.
|
|
||||||
:return: The score prompt.
|
|
||||||
:rtype: str
|
|
||||||
"""
|
|
||||||
pass
|
|
||||||
|
|
||||||
|
|
||||||
class SortingParser(parser.Parser):
|
|
||||||
"""
|
|
||||||
SortingParser provides the parsing of language model reponses specific to
|
|
||||||
the sorting example.
|
|
||||||
|
|
||||||
Inherits from the Parser class and implements its abstract methods.
|
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(self) -> None:
|
|
||||||
"""
|
|
||||||
Inits the response cache.
|
|
||||||
"""
|
|
||||||
self.cache = {}
|
|
||||||
|
|
||||||
def parse_aggregation_answer(
|
|
||||||
self, states: List[Dict], texts: List[str]
|
|
||||||
) -> Union[Dict, List[Dict]]:
|
|
||||||
"""
|
|
||||||
Parse the response from the language model for an aggregation prompt.
|
|
||||||
|
|
||||||
:param states: The thought states used to generate the prompt.
|
|
||||||
:type states: List[Dict]
|
|
||||||
:param texts: The responses to the prompt from the language model.
|
|
||||||
:type texts: List[str]
|
|
||||||
:return: The new thought states after parsing the respones from the language model.
|
|
||||||
:rtype: Union[Dict, List[Dict]]
|
|
||||||
:raise AssertionError: If not exactly two thought states are provided.
|
|
||||||
"""
|
|
||||||
|
|
||||||
assert len(states) == 2, "Expected two states for aggregation answer."
|
|
||||||
new_states = []
|
|
||||||
for text in texts:
|
|
||||||
answers = text.strip().split("\n")
|
|
||||||
if any(["Output" in answer for answer in answers]):
|
|
||||||
# cut elements until last output is found
|
|
||||||
for answer in reversed(answers):
|
|
||||||
if "Output" in answer:
|
|
||||||
answers = answers[answers.index(answer) :]
|
|
||||||
break
|
|
||||||
|
|
||||||
answers_stripped = [
|
|
||||||
answer for answer in answers if "[" in answer and "]" in answer
|
|
||||||
]
|
|
||||||
if len(answers_stripped) == 0:
|
|
||||||
for answer in answers:
|
|
||||||
answer = "[" + answer + "]"
|
|
||||||
try:
|
|
||||||
answer_converted = utils.string_to_list(answer)
|
|
||||||
if len(answer_converted) > 0:
|
|
||||||
answers_stripped.append(answer)
|
|
||||||
except:
|
|
||||||
pass
|
|
||||||
if len(answers_stripped) == 0:
|
|
||||||
logging.warning(
|
|
||||||
f"Could not parse aggregation answer: {text}. Returning empty list."
|
|
||||||
)
|
|
||||||
answer = "[]"
|
|
||||||
else:
|
|
||||||
answer = [
|
|
||||||
answer[answer.index("[") : answer.index("]") + 1]
|
|
||||||
for answer in answers_stripped
|
|
||||||
][0]
|
|
||||||
states = sorted(states, key=lambda x: x["part"])
|
|
||||||
merged_unsorted_sublists = (
|
|
||||||
states[0]["unsorted_sublist"][:-1]
|
|
||||||
+ ", "
|
|
||||||
+ states[1]["unsorted_sublist"][1:]
|
|
||||||
)
|
|
||||||
new_state = states[0].copy()
|
|
||||||
new_state["current"] = answer
|
|
||||||
new_state["unsorted_sublist"] = merged_unsorted_sublists
|
|
||||||
new_states.append(new_state)
|
|
||||||
return new_states
|
|
||||||
|
|
||||||
def parse_generate_answer(self, state: Dict, texts: List[str]) -> List[Dict]:
|
|
||||||
"""
|
|
||||||
Parse the response from the language model for a generate prompt.
|
|
||||||
|
|
||||||
:param state: The thought state used to generate the prompt.
|
|
||||||
:type state: Dict
|
|
||||||
:param texts: The responses to the prompt from the language model.
|
|
||||||
:type texts: List[str]
|
|
||||||
:return: The new thought states after parsing the respones from the language model.
|
|
||||||
:rtype: List[Dict]
|
|
||||||
"""
|
|
||||||
new_states = []
|
|
||||||
for text in texts:
|
|
||||||
if state["method"] == "got" and state["current"] == "":
|
|
||||||
# We expect a json which contains the four lists named "List 1" to "List 4"
|
|
||||||
# cut everything until the opening bracket and everything after the closing bracket
|
|
||||||
try:
|
|
||||||
text = text[text.index("{") : text.index("}") + 1]
|
|
||||||
json_dict = json.loads(text)
|
|
||||||
if len(json_dict.keys()) != 4:
|
|
||||||
logging.warning(
|
|
||||||
f"Expected 4 lists in json, but found {len(json_dict.keys())}."
|
|
||||||
)
|
|
||||||
for key, value in json_dict.items():
|
|
||||||
if "List" not in key:
|
|
||||||
logging.warning(
|
|
||||||
f"Expected key to contain 'List', but found {key}."
|
|
||||||
)
|
|
||||||
continue
|
|
||||||
if not isinstance(value, list):
|
|
||||||
value = utils.string_to_list(value)
|
|
||||||
new_state = state.copy()
|
|
||||||
new_state["current"] = str(value)
|
|
||||||
new_state["unsorted_sublist"] = str(value)
|
|
||||||
new_state["phase"] = 1
|
|
||||||
new_state["part"] = key
|
|
||||||
new_states.append(new_state)
|
|
||||||
except Exception as e:
|
|
||||||
logging.error(
|
|
||||||
f"Could not parse step answer: {text}. Encountered exception: {e}"
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
answers = text.strip().split("\n")
|
|
||||||
answers = [
|
|
||||||
answer for answer in answers if "[" in answer and "]" in answer
|
|
||||||
]
|
|
||||||
if any(["Output" in answer for answer in answers]):
|
|
||||||
# cut elements until last output is found
|
|
||||||
for answer in reversed(answers):
|
|
||||||
if "Output" in answer:
|
|
||||||
answers = answers[answers.index(answer) :]
|
|
||||||
break
|
|
||||||
|
|
||||||
answers = [
|
|
||||||
answer[answer.index("[") : answer.index("]") + 1]
|
|
||||||
for answer in answers
|
|
||||||
]
|
|
||||||
if len(answers) == 0:
|
|
||||||
logging.warning(
|
|
||||||
f"Could not parse step answer: {text}. Returning empty list."
|
|
||||||
)
|
|
||||||
answer = "[]"
|
|
||||||
else:
|
|
||||||
if len(answers) > 1:
|
|
||||||
logging.warning(
|
|
||||||
f"Multiple answers found for step answer: {text}. Using the first one."
|
|
||||||
)
|
|
||||||
answer = answers[0]
|
|
||||||
|
|
||||||
new_state = state.copy()
|
|
||||||
new_state["current"] = answer
|
|
||||||
new_state["phase"] = 2
|
|
||||||
new_states.append(new_state)
|
|
||||||
return new_states
|
|
||||||
|
|
||||||
def parse_improve_answer(self, state: Dict, texts: List[str]) -> Dict:
|
|
||||||
"""
|
|
||||||
Parse the response from the language model for an improve prompt.
|
|
||||||
|
|
||||||
:param state: The thought state used to generate the prompt.
|
|
||||||
:type state: Dict
|
|
||||||
:param texts: The responses to the prompt from the language model.
|
|
||||||
:type texts: List[str]
|
|
||||||
:return: The new thought state after parsing the responses from the language model.
|
|
||||||
:rtype: Dict
|
|
||||||
"""
|
|
||||||
pass
|
|
||||||
|
|
||||||
def parse_validation_answer(self, state: Dict, texts: List[str]) -> bool:
|
|
||||||
"""
|
|
||||||
Parse the response from the language model for a validation prompt.
|
|
||||||
|
|
||||||
:param state: The thought state used to generate the prompt.
|
|
||||||
:type state: Dict
|
|
||||||
:param texts: The responses to the prompt from the language model.
|
|
||||||
:type texts: List[str]
|
|
||||||
:return: Whether the thought state is valid or not.
|
|
||||||
:rtype: bool
|
|
||||||
"""
|
|
||||||
pass
|
|
||||||
|
|
||||||
def parse_score_answer(self, states: List[Dict], texts: List[str]) -> List[float]:
|
|
||||||
"""
|
|
||||||
Parse the response from the language model for a score prompt.
|
|
||||||
|
|
||||||
:param states: The thought states used to generate the prompt.
|
|
||||||
:type states: List[Dict]
|
|
||||||
:param texts: The responses to the prompt from the language model.
|
|
||||||
:type texts: List[str]
|
|
||||||
:return: The scores for the thought states.
|
|
||||||
:rtype: List[float]
|
|
||||||
"""
|
|
||||||
pass
|
|
||||||
|
|
||||||
|
|
||||||
def io() -> operations.GraphOfOperations:
|
|
||||||
"""
|
|
||||||
Generates the Graph of Operations for the IO method.
|
|
||||||
|
|
||||||
:return: Graph of Operations
|
|
||||||
:rtype: GraphOfOperations
|
|
||||||
"""
|
|
||||||
operations_graph = operations.GraphOfOperations()
|
|
||||||
|
|
||||||
operations_graph.append_operation(operations.Generate(1, 1))
|
|
||||||
operations_graph.append_operation(operations.Score(1, False, utils.num_errors))
|
|
||||||
operations_graph.append_operation(operations.GroundTruth(utils.test_sorting))
|
|
||||||
|
|
||||||
return operations_graph
|
|
||||||
|
|
||||||
|
|
||||||
def cot() -> operations.GraphOfOperations:
|
|
||||||
"""
|
|
||||||
Generates the Graph of Operations for the CoT method.
|
|
||||||
|
|
||||||
:return: Graph of Operations
|
|
||||||
:rtype: GraphOfOperations
|
|
||||||
"""
|
|
||||||
operations_graph = operations.GraphOfOperations()
|
|
||||||
|
|
||||||
operations_graph.append_operation(operations.Generate(1, 1))
|
|
||||||
operations_graph.append_operation(operations.Score(1, False, utils.num_errors))
|
|
||||||
operations_graph.append_operation(operations.GroundTruth(utils.test_sorting))
|
|
||||||
|
|
||||||
return operations_graph
|
|
||||||
|
|
||||||
|
|
||||||
def tot() -> operations.GraphOfOperations:
|
|
||||||
"""
|
|
||||||
Generates the Graph of Operations for the ToT method.
|
|
||||||
ToT uses a wider tree, where on each level there are more branches.
|
|
||||||
|
|
||||||
:return: Graph of Operations
|
|
||||||
:rtype: GraphOfOperations
|
|
||||||
"""
|
|
||||||
operations_graph = operations.GraphOfOperations()
|
|
||||||
|
|
||||||
operations_graph.append_operation(operations.Generate(1, 20))
|
|
||||||
operations_graph.append_operation(operations.Score(1, False, utils.num_errors))
|
|
||||||
keep_best_1 = operations.KeepBestN(1, False)
|
|
||||||
operations_graph.append_operation(keep_best_1)
|
|
||||||
|
|
||||||
for _ in range(3):
|
|
||||||
operations_graph.append_operation(operations.Generate(1, 20))
|
|
||||||
operations_graph.append_operation(operations.Score(1, False, utils.num_errors))
|
|
||||||
keep_best_2 = operations.KeepBestN(1, False)
|
|
||||||
keep_best_2.add_predecessor(keep_best_1)
|
|
||||||
operations_graph.append_operation(keep_best_2)
|
|
||||||
keep_best_1 = keep_best_2
|
|
||||||
|
|
||||||
operations_graph.append_operation(operations.KeepBestN(1, False))
|
|
||||||
operations_graph.append_operation(operations.GroundTruth(utils.test_sorting))
|
|
||||||
|
|
||||||
return operations_graph
|
|
||||||
|
|
||||||
|
|
||||||
def tot2() -> operations.GraphOfOperations:
|
|
||||||
"""
|
|
||||||
Generates the Graph of Operations for the ToT2 method.
|
|
||||||
ToT2 uses a tree with more levels, but with fewer branches per level.
|
|
||||||
|
|
||||||
:return: Graph of Operations
|
|
||||||
:rtype: GraphOfOperations
|
|
||||||
"""
|
|
||||||
operations_graph = operations.GraphOfOperations()
|
|
||||||
|
|
||||||
operations_graph.append_operation(operations.Generate(1, 10))
|
|
||||||
operations_graph.append_operation(operations.Score(1, False, utils.num_errors))
|
|
||||||
keep_best_1 = operations.KeepBestN(1, False)
|
|
||||||
operations_graph.append_operation(keep_best_1)
|
|
||||||
|
|
||||||
for _ in range(9):
|
|
||||||
operations_graph.append_operation(operations.Generate(1, 10))
|
|
||||||
operations_graph.append_operation(operations.Score(1, False, utils.num_errors))
|
|
||||||
keep_best_2 = operations.KeepBestN(1, False)
|
|
||||||
keep_best_2.add_predecessor(keep_best_1)
|
|
||||||
operations_graph.append_operation(keep_best_2)
|
|
||||||
keep_best_1 = keep_best_2
|
|
||||||
|
|
||||||
operations_graph.append_operation(operations.KeepBestN(1, False))
|
|
||||||
operations_graph.append_operation(operations.KeepBestN(1, False))
|
|
||||||
|
|
||||||
operations_graph.append_operation(operations.KeepBestN(1, False))
|
|
||||||
|
|
||||||
operations_graph.append_operation(operations.GroundTruth(utils.test_sorting))
|
|
||||||
|
|
||||||
return operations_graph
|
|
||||||
|
|
||||||
|
|
||||||
def got() -> operations.GraphOfOperations:
|
|
||||||
"""
|
|
||||||
Generates the Graph of Operations for the GoT method.
|
|
||||||
|
|
||||||
:return: Graph of Operations
|
|
||||||
:rtype: GraphOfOperations
|
|
||||||
"""
|
|
||||||
operations_graph = operations.GraphOfOperations()
|
|
||||||
|
|
||||||
plans = operations.Generate(1, 1)
|
|
||||||
operations_graph.append_operation(plans) # generate the sublists
|
|
||||||
sorted_sublists = []
|
|
||||||
for i in range(1, 9):
|
|
||||||
list_id = f"List {i}"
|
|
||||||
sub_list = operations.Selector(
|
|
||||||
lambda thoughts, list_id=list_id: [
|
|
||||||
thought for thought in thoughts if thought.state["part"] == list_id
|
|
||||||
]
|
|
||||||
)
|
|
||||||
sub_list.add_predecessor(plans)
|
|
||||||
operations_graph.add_operation(sub_list)
|
|
||||||
sort_sub_list = operations.Generate(1, 5)
|
|
||||||
sort_sub_list.add_predecessor(sub_list)
|
|
||||||
operations_graph.add_operation(sort_sub_list)
|
|
||||||
score_sub_list = operations.Score(1, False, utils.num_errors)
|
|
||||||
score_sub_list.add_predecessor(sort_sub_list)
|
|
||||||
operations_graph.add_operation(score_sub_list)
|
|
||||||
keep_best_sub_list = operations.KeepBestN(1, False)
|
|
||||||
keep_best_sub_list.add_predecessor(score_sub_list)
|
|
||||||
operations_graph.add_operation(keep_best_sub_list)
|
|
||||||
|
|
||||||
sorted_sublists.append(keep_best_sub_list)
|
|
||||||
|
|
||||||
aggregate_1 = operations.Aggregate(10)
|
|
||||||
aggregate_1.add_predecessor(sorted_sublists[0])
|
|
||||||
aggregate_1.add_predecessor(sorted_sublists[1])
|
|
||||||
operations_graph.add_operation(aggregate_1)
|
|
||||||
score_aggregate_1 = operations.Score(1, False, utils.num_errors)
|
|
||||||
score_aggregate_1.add_predecessor(aggregate_1)
|
|
||||||
operations_graph.add_operation(score_aggregate_1)
|
|
||||||
keep_best_aggregate_1 = operations.KeepBestN(1, False)
|
|
||||||
keep_best_aggregate_1.add_predecessor(score_aggregate_1)
|
|
||||||
operations_graph.add_operation(keep_best_aggregate_1)
|
|
||||||
|
|
||||||
improve_aggregate_1 = operations.Generate(1, 5)
|
|
||||||
improve_aggregate_1.add_predecessor(keep_best_aggregate_1)
|
|
||||||
operations_graph.add_operation(improve_aggregate_1)
|
|
||||||
improve_score_aggregate_1 = operations.Score(1, False, utils.num_errors)
|
|
||||||
improve_score_aggregate_1.add_predecessor(improve_aggregate_1)
|
|
||||||
improve_score_aggregate_1.add_predecessor(keep_best_aggregate_1)
|
|
||||||
operations_graph.add_operation(improve_score_aggregate_1)
|
|
||||||
improve_keep_best_aggregate_1 = operations.KeepBestN(1, False)
|
|
||||||
improve_keep_best_aggregate_1.add_predecessor(improve_score_aggregate_1)
|
|
||||||
operations_graph.add_operation(improve_keep_best_aggregate_1)
|
|
||||||
|
|
||||||
aggregate_2 = operations.Aggregate(10)
|
|
||||||
aggregate_2.add_predecessor(sorted_sublists[2])
|
|
||||||
aggregate_2.add_predecessor(sorted_sublists[3])
|
|
||||||
operations_graph.add_operation(aggregate_2)
|
|
||||||
score_aggregate_2 = operations.Score(1, False, utils.num_errors)
|
|
||||||
score_aggregate_2.add_predecessor(aggregate_2)
|
|
||||||
operations_graph.add_operation(score_aggregate_2)
|
|
||||||
keep_best_aggregate_2 = operations.KeepBestN(1, False)
|
|
||||||
keep_best_aggregate_2.add_predecessor(score_aggregate_2)
|
|
||||||
operations_graph.add_operation(keep_best_aggregate_2)
|
|
||||||
|
|
||||||
improve_aggregate_2 = operations.Generate(1, 5)
|
|
||||||
improve_aggregate_2.add_predecessor(keep_best_aggregate_2)
|
|
||||||
operations_graph.add_operation(improve_aggregate_2)
|
|
||||||
improve_score_aggregate_2 = operations.Score(1, False, utils.num_errors)
|
|
||||||
improve_score_aggregate_2.add_predecessor(improve_aggregate_2)
|
|
||||||
improve_score_aggregate_2.add_predecessor(keep_best_aggregate_2)
|
|
||||||
operations_graph.add_operation(improve_score_aggregate_2)
|
|
||||||
improve_keep_best_aggregate_2 = operations.KeepBestN(1, False)
|
|
||||||
improve_keep_best_aggregate_2.add_predecessor(improve_score_aggregate_2)
|
|
||||||
operations_graph.add_operation(improve_keep_best_aggregate_2)
|
|
||||||
|
|
||||||
aggregate_3 = operations.Aggregate(10)
|
|
||||||
aggregate_3.add_predecessor(sorted_sublists[4])
|
|
||||||
aggregate_3.add_predecessor(sorted_sublists[5])
|
|
||||||
operations_graph.add_operation(aggregate_3)
|
|
||||||
score_aggregate_3 = operations.Score(1, False, utils.num_errors)
|
|
||||||
score_aggregate_3.add_predecessor(aggregate_3)
|
|
||||||
operations_graph.add_operation(score_aggregate_3)
|
|
||||||
keep_best_aggregate_3 = operations.KeepBestN(1, False)
|
|
||||||
keep_best_aggregate_3.add_predecessor(score_aggregate_3)
|
|
||||||
operations_graph.add_operation(keep_best_aggregate_3)
|
|
||||||
|
|
||||||
improve_aggregate_3 = operations.Generate(1, 5)
|
|
||||||
improve_aggregate_3.add_predecessor(keep_best_aggregate_3)
|
|
||||||
operations_graph.add_operation(improve_aggregate_3)
|
|
||||||
improve_score_aggregate_3 = operations.Score(1, False, utils.num_errors)
|
|
||||||
improve_score_aggregate_3.add_predecessor(improve_aggregate_3)
|
|
||||||
improve_score_aggregate_3.add_predecessor(keep_best_aggregate_3)
|
|
||||||
operations_graph.add_operation(improve_score_aggregate_3)
|
|
||||||
improve_keep_best_aggregate_3 = operations.KeepBestN(1, False)
|
|
||||||
improve_keep_best_aggregate_3.add_predecessor(improve_score_aggregate_3)
|
|
||||||
operations_graph.add_operation(improve_keep_best_aggregate_3)
|
|
||||||
|
|
||||||
aggregate_4 = operations.Aggregate(10)
|
|
||||||
aggregate_4.add_predecessor(sorted_sublists[6])
|
|
||||||
aggregate_4.add_predecessor(sorted_sublists[7])
|
|
||||||
operations_graph.add_operation(aggregate_4)
|
|
||||||
score_aggregate_4 = operations.Score(1, False, utils.num_errors)
|
|
||||||
score_aggregate_4.add_predecessor(aggregate_4)
|
|
||||||
operations_graph.add_operation(score_aggregate_4)
|
|
||||||
keep_best_aggregate_4 = operations.KeepBestN(1, False)
|
|
||||||
keep_best_aggregate_4.add_predecessor(score_aggregate_4)
|
|
||||||
operations_graph.add_operation(keep_best_aggregate_4)
|
|
||||||
|
|
||||||
improve_aggregate_4 = operations.Generate(1, 5)
|
|
||||||
improve_aggregate_4.add_predecessor(keep_best_aggregate_4)
|
|
||||||
operations_graph.add_operation(improve_aggregate_4)
|
|
||||||
improve_score_aggregate_4 = operations.Score(1, False, utils.num_errors)
|
|
||||||
improve_score_aggregate_4.add_predecessor(improve_aggregate_4)
|
|
||||||
improve_score_aggregate_4.add_predecessor(keep_best_aggregate_4)
|
|
||||||
operations_graph.add_operation(improve_score_aggregate_4)
|
|
||||||
improve_keep_best_aggregate_4 = operations.KeepBestN(1, False)
|
|
||||||
improve_keep_best_aggregate_4.add_predecessor(improve_score_aggregate_4)
|
|
||||||
operations_graph.add_operation(improve_keep_best_aggregate_4)
|
|
||||||
|
|
||||||
aggregate_1_2 = operations.Aggregate(10)
|
|
||||||
aggregate_1_2.add_predecessor(improve_keep_best_aggregate_1)
|
|
||||||
aggregate_1_2.add_predecessor(improve_keep_best_aggregate_2)
|
|
||||||
operations_graph.add_operation(aggregate_1_2)
|
|
||||||
score_aggregate_1_2 = operations.Score(1, False, utils.num_errors)
|
|
||||||
score_aggregate_1_2.add_predecessor(aggregate_1_2)
|
|
||||||
operations_graph.add_operation(score_aggregate_1_2)
|
|
||||||
keep_best_aggregate_1_2 = operations.KeepBestN(1, False)
|
|
||||||
keep_best_aggregate_1_2.add_predecessor(score_aggregate_1_2)
|
|
||||||
operations_graph.add_operation(keep_best_aggregate_1_2)
|
|
||||||
|
|
||||||
improve_aggregate_1_2 = operations.Generate(1, 5)
|
|
||||||
improve_aggregate_1_2.add_predecessor(keep_best_aggregate_1_2)
|
|
||||||
operations_graph.add_operation(improve_aggregate_1_2)
|
|
||||||
improve_score_aggregate_1_2 = operations.Score(1, False, utils.num_errors)
|
|
||||||
improve_score_aggregate_1_2.add_predecessor(improve_aggregate_1_2)
|
|
||||||
improve_score_aggregate_1_2.add_predecessor(keep_best_aggregate_1_2)
|
|
||||||
operations_graph.add_operation(improve_score_aggregate_1_2)
|
|
||||||
improve_keep_best_aggregate_1_2 = operations.KeepBestN(1, False)
|
|
||||||
improve_keep_best_aggregate_1_2.add_predecessor(improve_score_aggregate_1_2)
|
|
||||||
operations_graph.add_operation(improve_keep_best_aggregate_1_2)
|
|
||||||
|
|
||||||
aggregate_3_4 = operations.Aggregate(10)
|
|
||||||
aggregate_3_4.add_predecessor(improve_keep_best_aggregate_3)
|
|
||||||
aggregate_3_4.add_predecessor(improve_keep_best_aggregate_4)
|
|
||||||
operations_graph.add_operation(aggregate_3_4)
|
|
||||||
score_aggregate_3_4 = operations.Score(1, False, utils.num_errors)
|
|
||||||
score_aggregate_3_4.add_predecessor(aggregate_3_4)
|
|
||||||
operations_graph.add_operation(score_aggregate_3_4)
|
|
||||||
keep_best_aggregate_3_4 = operations.KeepBestN(1, False)
|
|
||||||
keep_best_aggregate_3_4.add_predecessor(score_aggregate_3_4)
|
|
||||||
operations_graph.add_operation(keep_best_aggregate_3_4)
|
|
||||||
|
|
||||||
improve_aggregate_3_4 = operations.Generate(1, 5)
|
|
||||||
improve_aggregate_3_4.add_predecessor(keep_best_aggregate_3_4)
|
|
||||||
operations_graph.add_operation(improve_aggregate_3_4)
|
|
||||||
improve_score_aggregate_3_4 = operations.Score(1, False, utils.num_errors)
|
|
||||||
improve_score_aggregate_3_4.add_predecessor(improve_aggregate_3_4)
|
|
||||||
improve_score_aggregate_3_4.add_predecessor(keep_best_aggregate_3_4)
|
|
||||||
operations_graph.add_operation(improve_score_aggregate_3_4)
|
|
||||||
improve_keep_best_aggregate_3_4 = operations.KeepBestN(1, False)
|
|
||||||
improve_keep_best_aggregate_3_4.add_predecessor(improve_score_aggregate_3_4)
|
|
||||||
operations_graph.add_operation(improve_keep_best_aggregate_3_4)
|
|
||||||
|
|
||||||
final_aggregate = operations.Aggregate(10)
|
|
||||||
operations_graph.append_operation(final_aggregate)
|
|
||||||
operations_graph.append_operation(operations.Score(1, False, utils.num_errors))
|
|
||||||
keep_best_aggregate_final = operations.KeepBestN(1, False)
|
|
||||||
operations_graph.append_operation(keep_best_aggregate_final)
|
|
||||||
|
|
||||||
operations_graph.append_operation(operations.Generate(1, 10))
|
|
||||||
score_aggr_3 = operations.Score(1, False, utils.num_errors)
|
|
||||||
score_aggr_3.add_predecessor(keep_best_aggregate_final)
|
|
||||||
operations_graph.append_operation(score_aggr_3)
|
|
||||||
keep_final_best = operations.KeepBestN(1, False)
|
|
||||||
keep_final_best.add_predecessor(keep_best_aggregate_final)
|
|
||||||
operations_graph.append_operation(keep_final_best)
|
|
||||||
|
|
||||||
operations_graph.append_operation(operations.GroundTruth(utils.test_sorting))
|
|
||||||
|
|
||||||
return operations_graph
|
|
||||||
|
|
||||||
|
|
||||||
def run(
|
|
||||||
data_ids: List[int],
|
|
||||||
methods: List[Callable[[], operations.GraphOfOperations]],
|
|
||||||
budget: float,
|
|
||||||
lm_name: str,
|
|
||||||
) -> float:
|
|
||||||
"""
|
|
||||||
Controller function that executes each specified method for each specified
|
|
||||||
sample while the budget is not exhausted.
|
|
||||||
|
|
||||||
:param data_ids: Indices of the sample to be run.
|
|
||||||
:type data_ids: List[int]
|
|
||||||
:param methods: List of functions to generate Graphs of Operations.
|
|
||||||
:type methods: Each function generates a Graph of Operation.
|
|
||||||
:param budget: Language model budget for the execution in dollars.
|
|
||||||
:type budget: float
|
|
||||||
:param lm_name: Name of the language model to be used.
|
|
||||||
:type lm_name: str
|
|
||||||
:return: Spent budget in dollars.
|
|
||||||
:rtype: float
|
|
||||||
"""
|
|
||||||
|
|
||||||
orig_budget = budget
|
|
||||||
data_path = os.path.join(os.path.dirname(__file__), "sorting_128.csv")
|
|
||||||
data = []
|
|
||||||
with open(data_path, "r") as f:
|
|
||||||
reader = csv.reader(f)
|
|
||||||
next(reader)
|
|
||||||
for row in reader:
|
|
||||||
data.append([int(row[0]), row[1], row[2]])
|
|
||||||
|
|
||||||
if data_ids is None or len(data_ids) == 0:
|
|
||||||
data_ids = list(range(len(data)))
|
|
||||||
selected_data = [data[i] for i in data_ids]
|
|
||||||
|
|
||||||
results_dir = os.path.join(os.path.dirname(__file__), "results")
|
|
||||||
|
|
||||||
if not os.path.exists(results_dir):
|
|
||||||
os.makedirs(results_dir)
|
|
||||||
timestamp = datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
|
|
||||||
extra_info = f"{lm_name}_{'-'.join([method.__name__ for method in methods])}"
|
|
||||||
folder_name = f"{extra_info}_{timestamp}"
|
|
||||||
results_folder = os.path.join(results_dir, folder_name)
|
|
||||||
os.makedirs(results_folder)
|
|
||||||
|
|
||||||
config = {
|
|
||||||
"data": selected_data,
|
|
||||||
"methods": [method.__name__ for method in methods],
|
|
||||||
"lm": lm_name,
|
|
||||||
"budget": budget,
|
|
||||||
}
|
|
||||||
with open(os.path.join(results_folder, "config.json"), "w") as f:
|
|
||||||
json.dump(config, f)
|
|
||||||
|
|
||||||
logging.basicConfig(
|
|
||||||
filename=os.path.join(results_folder, "log.log"),
|
|
||||||
filemode="w",
|
|
||||||
format="%(name)s - %(levelname)s - %(message)s",
|
|
||||||
level=logging.DEBUG,
|
|
||||||
)
|
|
||||||
|
|
||||||
for method in methods:
|
|
||||||
os.makedirs(os.path.join(results_folder, method.__name__))
|
|
||||||
|
|
||||||
for data in selected_data:
|
|
||||||
logging.info(f"Running data {data[0]}: {data[1]}")
|
|
||||||
if budget <= 0.0:
|
|
||||||
logging.error(
|
|
||||||
f"Budget has been depleted, stopping. Data {data[0]} has not been run."
|
|
||||||
)
|
|
||||||
break
|
|
||||||
for method in methods:
|
|
||||||
logging.info(f"Running method {method.__name__}")
|
|
||||||
logging.info(f"Budget left: {budget}")
|
|
||||||
if budget <= 0.0:
|
|
||||||
logging.error(
|
|
||||||
f"Budget has been depleted, stopping. Method {method.__name__} has not been run."
|
|
||||||
)
|
|
||||||
break
|
|
||||||
lm = language_models.ChatGPT(
|
|
||||||
os.path.join(
|
|
||||||
os.path.dirname(__file__),
|
|
||||||
"../../graph_of_thoughts/language_models/config.json",
|
|
||||||
),
|
|
||||||
model_name=lm_name,
|
|
||||||
cache=True,
|
|
||||||
)
|
|
||||||
operations_graph = method()
|
|
||||||
executor = controller.Controller(
|
|
||||||
lm,
|
|
||||||
operations_graph,
|
|
||||||
SortingPrompter(),
|
|
||||||
SortingParser(),
|
|
||||||
{
|
|
||||||
"original": data[1],
|
|
||||||
"current": "",
|
|
||||||
"phase": 0,
|
|
||||||
"method": method.__name__,
|
|
||||||
},
|
|
||||||
)
|
|
||||||
try:
|
|
||||||
executor.run()
|
|
||||||
except Exception as e:
|
|
||||||
logging.error(f"Exception: {e}")
|
|
||||||
path = os.path.join(
|
|
||||||
results_folder,
|
|
||||||
method.__name__,
|
|
||||||
f"{data[0]}.json",
|
|
||||||
)
|
|
||||||
executor.output_graph(path)
|
|
||||||
budget -= lm.cost
|
|
||||||
|
|
||||||
return orig_budget - budget
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
"""
|
|
||||||
Input (x) : an unordered list of 128 numbers between 0 and 9 (inclusive)
|
|
||||||
Output (y) : a sorted list of 128 numbers between 0 and 9 (inclusive)
|
|
||||||
Correct : y == sorted(x)
|
|
||||||
Input Example:
|
|
||||||
[0, 1, 9, 4, 2, 2, 0, 5, 1...]
|
|
||||||
Output Example:
|
|
||||||
[0, 0, 0, 0, 1, 1, 1, 1, 2...]
|
|
||||||
"""
|
|
||||||
budget = 30
|
|
||||||
samples = [item for item in range(0, 100)]
|
|
||||||
approaches = [io, cot, tot, tot2, got]
|
|
||||||
|
|
||||||
spent = run(samples, approaches, budget, "chatgpt")
|
|
||||||
|
|
||||||
logging.info(f"Spent {spent} out of {budget} budget.")
|
|
||||||
@ -1,78 +0,0 @@
|
|||||||
# Copyright (c) 2023 ETH Zurich.
|
|
||||||
# All rights reserved.
|
|
||||||
#
|
|
||||||
# Use of this source code is governed by a BSD-style license that can be
|
|
||||||
# found in the LICENSE file.
|
|
||||||
#
|
|
||||||
# main author: Nils Blach
|
|
||||||
|
|
||||||
from typing import Dict, List
|
|
||||||
|
|
||||||
|
|
||||||
def string_to_list(string: str) -> List[int]:
|
|
||||||
"""
|
|
||||||
Helper function to convert a list encoded inside a string into a Python
|
|
||||||
list object of string elements.
|
|
||||||
|
|
||||||
:param string: Input string containing a list.
|
|
||||||
:type string: str
|
|
||||||
:return: List of string elements.
|
|
||||||
:rtype: List[str]
|
|
||||||
:raise AssertionError: If input string does not contain a list.
|
|
||||||
"""
|
|
||||||
|
|
||||||
assert string[0] == "[" and string[-1] == "]", "String is not a list."
|
|
||||||
return [int(num) for num in string[1:-1].split(",")]
|
|
||||||
|
|
||||||
|
|
||||||
def test_sorting(state: Dict) -> bool:
|
|
||||||
"""
|
|
||||||
Function to test whether the final solution matches ground truth.
|
|
||||||
|
|
||||||
:param state: Thought state that represents the final solution.
|
|
||||||
:type state: Dict
|
|
||||||
:return: Returns whether the solution matches the ground truth.
|
|
||||||
:rtype: bool
|
|
||||||
"""
|
|
||||||
|
|
||||||
try:
|
|
||||||
correct_list = sorted(string_to_list(state["original"]))
|
|
||||||
sorted_list = string_to_list(state["current"])
|
|
||||||
return sorted_list == correct_list
|
|
||||||
except:
|
|
||||||
return False
|
|
||||||
|
|
||||||
|
|
||||||
def num_errors(state: Dict) -> float:
|
|
||||||
"""
|
|
||||||
Function to locally count the number of errors that serves as a score.
|
|
||||||
|
|
||||||
:param state: Thought state to be scored.
|
|
||||||
:type state: Dict
|
|
||||||
:return: Number of errors.
|
|
||||||
:rtype: float
|
|
||||||
"""
|
|
||||||
|
|
||||||
try:
|
|
||||||
unsorted_list = state["original"]
|
|
||||||
if (
|
|
||||||
"unsorted_sublist" in state
|
|
||||||
and state["unsorted_sublist"] != ""
|
|
||||||
and state["unsorted_sublist"] is not None
|
|
||||||
and len(state["unsorted_sublist"]) < len(unsorted_list) - 5
|
|
||||||
):
|
|
||||||
unsorted_list = state["unsorted_sublist"]
|
|
||||||
correct_list = sorted(string_to_list(unsorted_list))
|
|
||||||
current_list = string_to_list(state["current"])
|
|
||||||
num_errors = 0
|
|
||||||
for i in range(10):
|
|
||||||
num_errors += abs(
|
|
||||||
sum([1 for num in current_list if num == i])
|
|
||||||
- sum([1 for num in correct_list if num == i])
|
|
||||||
)
|
|
||||||
num_errors += sum(
|
|
||||||
[1 for num1, num2 in zip(current_list, current_list[1:]) if num1 > num2]
|
|
||||||
)
|
|
||||||
return num_errors
|
|
||||||
except:
|
|
||||||
return 300
|
|
||||||
@ -1 +0,0 @@
|
|||||||
"""HTTP API helpers (FastAPI) for running Graph of Thoughts with OpenRouter."""
|
|
||||||
@ -1,4 +0,0 @@
|
|||||||
from graph_of_thoughts.api.app import run
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
run()
|
|
||||||
@ -1,192 +0,0 @@
|
|||||||
# Copyright (c) 2023 ETH Zurich.
|
|
||||||
# All rights reserved.
|
|
||||||
#
|
|
||||||
# Use of this source code is governed by a BSD-style license that can be
|
|
||||||
# found in the LICENSE file.
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import logging
|
|
||||||
import os
|
|
||||||
import time
|
|
||||||
import uuid
|
|
||||||
from typing import Any, Dict, List, Optional
|
|
||||||
|
|
||||||
from graph_of_thoughts.api.got_openai_pipeline import (
|
|
||||||
ChatCompletionParser,
|
|
||||||
ChatCompletionPrompter,
|
|
||||||
build_default_chat_graph,
|
|
||||||
extract_assistant_text,
|
|
||||||
format_chat_messages,
|
|
||||||
)
|
|
||||||
from graph_of_thoughts.controller import Controller
|
|
||||||
from graph_of_thoughts.language_models.openrouter import (
|
|
||||||
OpenRouter,
|
|
||||||
OpenRouterBadRequestError,
|
|
||||||
OpenRouterRateLimitError,
|
|
||||||
)
|
|
||||||
|
|
||||||
try:
|
|
||||||
from fastapi import FastAPI, HTTPException
|
|
||||||
from fastapi.responses import JSONResponse
|
|
||||||
from pydantic import BaseModel, Field
|
|
||||||
except ImportError as e:
|
|
||||||
raise ImportError(
|
|
||||||
"FastAPI and Pydantic are required for the HTTP API. "
|
|
||||||
'Install with: pip install "graph_of_thoughts[api]"'
|
|
||||||
) from e
|
|
||||||
|
|
||||||
|
|
||||||
class ChatMessage(BaseModel):
|
|
||||||
role: str
|
|
||||||
content: str
|
|
||||||
|
|
||||||
|
|
||||||
class ChatCompletionRequest(BaseModel):
|
|
||||||
model: Optional[str] = None
|
|
||||||
messages: List[ChatMessage]
|
|
||||||
temperature: Optional[float] = None
|
|
||||||
max_tokens: Optional[int] = None
|
|
||||||
stream: Optional[bool] = False
|
|
||||||
n: Optional[int] = Field(default=1, ge=1, le=1)
|
|
||||||
|
|
||||||
|
|
||||||
def _get_config_path() -> str:
|
|
||||||
return os.environ.get(
|
|
||||||
"OPENROUTER_CONFIG",
|
|
||||||
os.path.join(
|
|
||||||
os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
|
|
||||||
"language_models",
|
|
||||||
"config.openrouter.yaml",
|
|
||||||
),
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def _run_controller(lm: OpenRouter, user_text: str) -> str:
|
|
||||||
graph = build_default_chat_graph(num_candidates=3)
|
|
||||||
ctrl = Controller(
|
|
||||||
lm,
|
|
||||||
graph,
|
|
||||||
ChatCompletionPrompter(),
|
|
||||||
ChatCompletionParser(),
|
|
||||||
{"input": user_text},
|
|
||||||
)
|
|
||||||
ctrl.run()
|
|
||||||
return extract_assistant_text(ctrl.get_final_thoughts())
|
|
||||||
|
|
||||||
|
|
||||||
app = FastAPI(
|
|
||||||
title="Graph of Thoughts (OpenRouter)",
|
|
||||||
version="0.1.0",
|
|
||||||
description="OpenAI-compatible chat completions backed by Graph of Operations + OpenRouter.",
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
@app.on_event("startup")
|
|
||||||
def _startup() -> None:
|
|
||||||
logging.basicConfig(level=os.environ.get("LOG_LEVEL", "INFO"))
|
|
||||||
|
|
||||||
|
|
||||||
@app.get("/v1/models")
|
|
||||||
def list_models() -> Dict[str, Any]:
|
|
||||||
path = _get_config_path()
|
|
||||||
if not os.path.isfile(path):
|
|
||||||
return {"object": "list", "data": []}
|
|
||||||
from graph_of_thoughts.language_models.openrouter import load_openrouter_config
|
|
||||||
|
|
||||||
cfg = load_openrouter_config(path)
|
|
||||||
models = cfg.get("models") or []
|
|
||||||
if isinstance(models, str):
|
|
||||||
models = [models]
|
|
||||||
data = [
|
|
||||||
{
|
|
||||||
"id": m,
|
|
||||||
"object": "model",
|
|
||||||
"created": int(time.time()),
|
|
||||||
"owned_by": "openrouter",
|
|
||||||
}
|
|
||||||
for m in models
|
|
||||||
]
|
|
||||||
return {"object": "list", "data": data}
|
|
||||||
|
|
||||||
|
|
||||||
@app.post("/v1/chat/completions")
|
|
||||||
def chat_completions(body: ChatCompletionRequest) -> JSONResponse:
|
|
||||||
if body.stream:
|
|
||||||
raise HTTPException(
|
|
||||||
status_code=400,
|
|
||||||
detail="stream=true is not supported; use stream=false.",
|
|
||||||
)
|
|
||||||
if body.n != 1:
|
|
||||||
raise HTTPException(status_code=400, detail="Only n=1 is supported.")
|
|
||||||
|
|
||||||
path = _get_config_path()
|
|
||||||
if not os.path.isfile(path):
|
|
||||||
raise HTTPException(
|
|
||||||
status_code=500,
|
|
||||||
detail=f"OpenRouter config not found at {path}. Set OPENROUTER_CONFIG.",
|
|
||||||
)
|
|
||||||
|
|
||||||
lm = OpenRouter(config_path=path)
|
|
||||||
user_text = format_chat_messages(
|
|
||||||
[{"role": m.role, "content": m.content} for m in body.messages]
|
|
||||||
)
|
|
||||||
try:
|
|
||||||
lm.set_request_overrides(
|
|
||||||
model=body.model,
|
|
||||||
temperature=body.temperature,
|
|
||||||
max_tokens=body.max_tokens,
|
|
||||||
)
|
|
||||||
try:
|
|
||||||
answer = _run_controller(lm, user_text)
|
|
||||||
finally:
|
|
||||||
lm.clear_request_overrides()
|
|
||||||
except OpenRouterRateLimitError as e:
|
|
||||||
raise HTTPException(status_code=429, detail=str(e)) from e
|
|
||||||
except OpenRouterBadRequestError as e:
|
|
||||||
raise HTTPException(status_code=400, detail=str(e)) from e
|
|
||||||
|
|
||||||
model_id = (
|
|
||||||
body.model
|
|
||||||
or lm.generation_model_id
|
|
||||||
or lm.last_model_id
|
|
||||||
or (lm.models[0] if lm.models else "openrouter")
|
|
||||||
)
|
|
||||||
resp_id = f"chatcmpl-{uuid.uuid4().hex}"
|
|
||||||
now = int(time.time())
|
|
||||||
payload = {
|
|
||||||
"id": resp_id,
|
|
||||||
"object": "chat.completion",
|
|
||||||
"created": now,
|
|
||||||
"model": model_id,
|
|
||||||
"choices": [
|
|
||||||
{
|
|
||||||
"index": 0,
|
|
||||||
"message": {"role": "assistant", "content": answer},
|
|
||||||
"finish_reason": "stop",
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"usage": {
|
|
||||||
"prompt_tokens": lm.prompt_tokens,
|
|
||||||
"completion_tokens": lm.completion_tokens,
|
|
||||||
"total_tokens": lm.prompt_tokens + lm.completion_tokens,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
return JSONResponse(content=payload)
|
|
||||||
|
|
||||||
|
|
||||||
def run() -> None:
|
|
||||||
import uvicorn
|
|
||||||
|
|
||||||
host = os.environ.get("HOST", "0.0.0.0")
|
|
||||||
port = int(os.environ.get("PORT", "8000"))
|
|
||||||
uvicorn.run(
|
|
||||||
"graph_of_thoughts.api.app:app",
|
|
||||||
host=host,
|
|
||||||
port=port,
|
|
||||||
reload=os.environ.get("RELOAD", "").lower() in ("1", "true", "yes"),
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
run()
|
|
||||||
@ -1,123 +0,0 @@
|
|||||||
# Copyright (c) 2023 ETH Zurich.
|
|
||||||
# All rights reserved.
|
|
||||||
#
|
|
||||||
# Use of this source code is governed by a BSD-style license that can be
|
|
||||||
# found in the LICENSE file.
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import json
|
|
||||||
import re
|
|
||||||
from typing import Any, Dict, List, Union
|
|
||||||
|
|
||||||
from graph_of_thoughts.operations import GraphOfOperations, operations
|
|
||||||
from graph_of_thoughts.operations.thought import Thought
|
|
||||||
from graph_of_thoughts.parser import Parser
|
|
||||||
from graph_of_thoughts.prompter import Prompter
|
|
||||||
|
|
||||||
|
|
||||||
def format_chat_messages(messages: List[Dict[str, str]]) -> str:
|
|
||||||
parts: List[str] = []
|
|
||||||
for m in messages:
|
|
||||||
role = m.get("role", "user")
|
|
||||||
content = m.get("content", "")
|
|
||||||
if not isinstance(content, str):
|
|
||||||
content = str(content)
|
|
||||||
parts.append(f"{role.upper()}:\n{content}")
|
|
||||||
return "\n\n".join(parts)
|
|
||||||
|
|
||||||
|
|
||||||
class ChatCompletionPrompter(Prompter):
|
|
||||||
"""Prompter for a small generate → score → keep-best Graph of Operations."""
|
|
||||||
|
|
||||||
def generate_prompt(self, num_branches: int, **kwargs: Any) -> str:
|
|
||||||
problem = kwargs.get("input", "")
|
|
||||||
return (
|
|
||||||
"You are a careful assistant. Read the conversation below and produce "
|
|
||||||
"one candidate answer for the USER's latest needs.\n\n"
|
|
||||||
f"{problem}\n\n"
|
|
||||||
"Reply with your answer only, no preamble."
|
|
||||||
)
|
|
||||||
|
|
||||||
def score_prompt(self, state_dicts: List[Dict], **kwargs: Any) -> str:
|
|
||||||
lines = [
|
|
||||||
"You evaluate candidate answers for the same problem. "
|
|
||||||
"Score each candidate from 0 (worst) to 10 (best) on correctness, "
|
|
||||||
"completeness, and relevance.",
|
|
||||||
"",
|
|
||||||
"Return ONLY a JSON array of numbers, one score per candidate in order, e.g. [7, 5, 9].",
|
|
||||||
"",
|
|
||||||
]
|
|
||||||
for i, st in enumerate(state_dicts):
|
|
||||||
cand = st.get("candidate", "")
|
|
||||||
lines.append(f"Candidate {i}:\n{cand}\n")
|
|
||||||
return "\n".join(lines)
|
|
||||||
|
|
||||||
def aggregation_prompt(self, state_dicts: List[Dict], **kwargs: Any) -> str:
|
|
||||||
raise RuntimeError("aggregation_prompt is not used by the chat completion pipeline")
|
|
||||||
|
|
||||||
def improve_prompt(self, **kwargs: Any) -> str:
|
|
||||||
raise RuntimeError("improve_prompt is not used by the chat completion pipeline")
|
|
||||||
|
|
||||||
def validation_prompt(self, **kwargs: Any) -> str:
|
|
||||||
raise RuntimeError("validation_prompt is not used by the chat completion pipeline")
|
|
||||||
|
|
||||||
|
|
||||||
class ChatCompletionParser(Parser):
|
|
||||||
def parse_generate_answer(self, state: Dict, texts: List[str]) -> List[Dict]:
|
|
||||||
out: List[Dict] = []
|
|
||||||
for i, t in enumerate(texts):
|
|
||||||
out.append({"candidate": (t or "").strip(), "branch_index": i})
|
|
||||||
return out
|
|
||||||
|
|
||||||
def parse_score_answer(self, states: List[Dict], texts: List[str]) -> List[float]:
|
|
||||||
raw = texts[0] if texts else ""
|
|
||||||
scores = self._scores_from_text(raw, len(states))
|
|
||||||
if len(scores) < len(states):
|
|
||||||
scores.extend([0.0] * (len(states) - len(scores)))
|
|
||||||
return scores[: len(states)]
|
|
||||||
|
|
||||||
def _scores_from_text(self, raw: str, n: int) -> List[float]:
|
|
||||||
raw = raw.strip()
|
|
||||||
try:
|
|
||||||
data = json.loads(raw)
|
|
||||||
if isinstance(data, list):
|
|
||||||
return [float(x) for x in data]
|
|
||||||
except (json.JSONDecodeError, ValueError, TypeError):
|
|
||||||
pass
|
|
||||||
nums = re.findall(r"-?\d+(?:\.\d+)?", raw)
|
|
||||||
return [float(x) for x in nums[:n]]
|
|
||||||
|
|
||||||
def parse_aggregation_answer(
|
|
||||||
self, states: List[Dict], texts: List[str]
|
|
||||||
) -> Union[Dict, List[Dict]]:
|
|
||||||
raise RuntimeError("parse_aggregation_answer is not used")
|
|
||||||
|
|
||||||
def parse_improve_answer(self, state: Dict, texts: List[str]) -> Dict:
|
|
||||||
raise RuntimeError("parse_improve_answer is not used")
|
|
||||||
|
|
||||||
def parse_validation_answer(self, state: Dict, texts: List[str]) -> bool:
|
|
||||||
raise RuntimeError("parse_validation_answer is not used")
|
|
||||||
|
|
||||||
|
|
||||||
def build_default_chat_graph(num_candidates: int = 3) -> GraphOfOperations:
|
|
||||||
g = GraphOfOperations()
|
|
||||||
g.append_operation(
|
|
||||||
operations.Generate(
|
|
||||||
num_branches_prompt=1, num_branches_response=num_candidates
|
|
||||||
)
|
|
||||||
)
|
|
||||||
g.append_operation(operations.Score(combined_scoring=True))
|
|
||||||
g.append_operation(operations.KeepBestN(1))
|
|
||||||
return g
|
|
||||||
|
|
||||||
|
|
||||||
def extract_assistant_text(final_thoughts_list: List[List[Thought]]) -> str:
|
|
||||||
"""``get_final_thoughts`` returns one list per leaf operation; we take the first leaf's first thought."""
|
|
||||||
if not final_thoughts_list:
|
|
||||||
return ""
|
|
||||||
thoughts = final_thoughts_list[0]
|
|
||||||
if not thoughts:
|
|
||||||
return ""
|
|
||||||
state = thoughts[0].state or {}
|
|
||||||
return str(state.get("candidate", ""))
|
|
||||||
@ -1,28 +0,0 @@
|
|||||||
# Controller
|
|
||||||
|
|
||||||
The Controller class is responsible for traversing the Graph of Operations (GoO), which is a static structure that is constructed once, before the execution starts.
|
|
||||||
GoO prescribes the execution plan of thought operations and the Controller invokes their execution, generating the Graph Reasoning State (GRS).
|
|
||||||
|
|
||||||
In order for a GoO to be executed, an instance of Large Language Model (LLM) must be supplied to the controller (along with other required objects).
|
|
||||||
Please refer to the [Language Models](../language_models/README.md) section for more information about LLMs.
|
|
||||||
|
|
||||||
The following section describes how to instantiate the Controller to run a defined GoO.
|
|
||||||
|
|
||||||
## Controller Instantiation
|
|
||||||
- Requires custom `Prompter`, `Parser`, as well as instantiated `GraphOfOperations` and `AbstractLanguageModel` - creation of these is described separately.
|
|
||||||
- Prepare initial state (thought) as dictionary - this can be used in the initial prompts by the operations.
|
|
||||||
```
|
|
||||||
lm = ...create
|
|
||||||
graph_of_operations = ...create
|
|
||||||
|
|
||||||
executor = controller.Controller(
|
|
||||||
lm,
|
|
||||||
graph_of_operations,
|
|
||||||
<CustomPrompter()>,
|
|
||||||
<CustomParser()>,
|
|
||||||
<initial state>,
|
|
||||||
)
|
|
||||||
executor.run()
|
|
||||||
executor.output_graph("path/to/output.json")
|
|
||||||
```
|
|
||||||
- After the run the graph is written to an output file, which contains individual operations, their thoughts, information about scores and validity and total amount of used tokens / cost.
|
|
||||||
@ -1 +0,0 @@
|
|||||||
from .controller import Controller
|
|
||||||
@ -1,152 +0,0 @@
|
|||||||
# Copyright (c) 2023 ETH Zurich.
|
|
||||||
# All rights reserved.
|
|
||||||
#
|
|
||||||
# Use of this source code is governed by a BSD-style license that can be
|
|
||||||
# found in the LICENSE file.
|
|
||||||
#
|
|
||||||
# main author: Nils Blach
|
|
||||||
|
|
||||||
import json
|
|
||||||
import logging
|
|
||||||
from typing import List
|
|
||||||
from graph_of_thoughts.language_models import AbstractLanguageModel
|
|
||||||
from graph_of_thoughts.operations import GraphOfOperations, Thought
|
|
||||||
from graph_of_thoughts.prompter import Prompter
|
|
||||||
from graph_of_thoughts.parser import Parser
|
|
||||||
|
|
||||||
|
|
||||||
class Controller:
|
|
||||||
"""
|
|
||||||
Controller class to manage the execution flow of the Graph of Operations,
|
|
||||||
generating the Graph Reasoning State.
|
|
||||||
This involves language models, graph operations, prompting, and parsing.
|
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(
|
|
||||||
self,
|
|
||||||
lm: AbstractLanguageModel,
|
|
||||||
graph: GraphOfOperations,
|
|
||||||
prompter: Prompter,
|
|
||||||
parser: Parser,
|
|
||||||
problem_parameters: dict,
|
|
||||||
) -> None:
|
|
||||||
"""
|
|
||||||
Initialize the Controller instance with the language model,
|
|
||||||
operations graph, prompter, parser, and problem parameters.
|
|
||||||
|
|
||||||
:param lm: An instance of the AbstractLanguageModel.
|
|
||||||
:type lm: AbstractLanguageModel
|
|
||||||
:param graph: The Graph of Operations to be executed.
|
|
||||||
:type graph: OperationsGraph
|
|
||||||
:param prompter: An instance of the Prompter class, used to generate prompts.
|
|
||||||
:type prompter: Prompter
|
|
||||||
:param parser: An instance of the Parser class, used to parse responses.
|
|
||||||
:type parser: Parser
|
|
||||||
:param problem_parameters: Initial parameters/state of the problem.
|
|
||||||
:type problem_parameters: dict
|
|
||||||
"""
|
|
||||||
self.logger = logging.getLogger(self.__class__.__module__)
|
|
||||||
self.lm = lm
|
|
||||||
self.graph = graph
|
|
||||||
self.prompter = prompter
|
|
||||||
self.parser = parser
|
|
||||||
self.problem_parameters = problem_parameters
|
|
||||||
self.run_executed = False
|
|
||||||
|
|
||||||
def run(self) -> None:
|
|
||||||
"""
|
|
||||||
Run the controller and execute the operations from the Graph of
|
|
||||||
Operations based on their readiness.
|
|
||||||
Ensures the program is in a valid state before execution.
|
|
||||||
:raises AssertionError: If the Graph of Operation has no roots.
|
|
||||||
:raises AssertionError: If the successor of an operation is not in the Graph of Operations.
|
|
||||||
"""
|
|
||||||
self.logger.debug("Checking that the program is in a valid state")
|
|
||||||
assert self.graph.roots is not None, "The operations graph has no root"
|
|
||||||
self.logger.debug("The program is in a valid state")
|
|
||||||
|
|
||||||
execution_queue = [
|
|
||||||
operation
|
|
||||||
for operation in self.graph.operations
|
|
||||||
if operation.can_be_executed()
|
|
||||||
]
|
|
||||||
|
|
||||||
while len(execution_queue) > 0:
|
|
||||||
current_operation = execution_queue.pop(0)
|
|
||||||
self.logger.info("Executing operation %s", current_operation.operation_type)
|
|
||||||
current_operation.execute(
|
|
||||||
self.lm, self.prompter, self.parser, **self.problem_parameters
|
|
||||||
)
|
|
||||||
self.logger.info("Operation %s executed", current_operation.operation_type)
|
|
||||||
for operation in current_operation.successors:
|
|
||||||
assert (
|
|
||||||
operation in self.graph.operations
|
|
||||||
), "The successor of an operation is not in the operations graph"
|
|
||||||
if operation.can_be_executed():
|
|
||||||
execution_queue.append(operation)
|
|
||||||
self.logger.info("All operations executed")
|
|
||||||
self.run_executed = True
|
|
||||||
|
|
||||||
def get_final_thoughts(self) -> List[List[Thought]]:
|
|
||||||
"""
|
|
||||||
Retrieve the final thoughts after all operations have been executed.
|
|
||||||
|
|
||||||
:return: List of thoughts for each operation in the graph's leaves.
|
|
||||||
:rtype: List[List[Thought]]
|
|
||||||
:raises AssertionError: If the `run` method hasn't been executed yet.
|
|
||||||
"""
|
|
||||||
assert self.run_executed, "The run method has not been executed"
|
|
||||||
return [operation.get_thoughts() for operation in self.graph.leaves]
|
|
||||||
|
|
||||||
def output_graph(self, path: str) -> None:
|
|
||||||
"""
|
|
||||||
Serialize the state and results of the operations graph to a JSON file.
|
|
||||||
|
|
||||||
:param path: The path to the output file.
|
|
||||||
:type path: str
|
|
||||||
"""
|
|
||||||
output = []
|
|
||||||
for operation in self.graph.operations:
|
|
||||||
operation_serialized = {
|
|
||||||
"operation": operation.operation_type.name,
|
|
||||||
"thoughts": [thought.state for thought in operation.get_thoughts()],
|
|
||||||
}
|
|
||||||
if any([thought.scored for thought in operation.get_thoughts()]):
|
|
||||||
operation_serialized["scored"] = [
|
|
||||||
thought.scored for thought in operation.get_thoughts()
|
|
||||||
]
|
|
||||||
operation_serialized["scores"] = [
|
|
||||||
thought.score for thought in operation.get_thoughts()
|
|
||||||
]
|
|
||||||
if any([thought.validated for thought in operation.get_thoughts()]):
|
|
||||||
operation_serialized["validated"] = [
|
|
||||||
thought.validated for thought in operation.get_thoughts()
|
|
||||||
]
|
|
||||||
operation_serialized["validity"] = [
|
|
||||||
thought.valid for thought in operation.get_thoughts()
|
|
||||||
]
|
|
||||||
if any(
|
|
||||||
[
|
|
||||||
thought.compared_to_ground_truth
|
|
||||||
for thought in operation.get_thoughts()
|
|
||||||
]
|
|
||||||
):
|
|
||||||
operation_serialized["compared_to_ground_truth"] = [
|
|
||||||
thought.compared_to_ground_truth
|
|
||||||
for thought in operation.get_thoughts()
|
|
||||||
]
|
|
||||||
operation_serialized["problem_solved"] = [
|
|
||||||
thought.solved for thought in operation.get_thoughts()
|
|
||||||
]
|
|
||||||
output.append(operation_serialized)
|
|
||||||
|
|
||||||
output.append(
|
|
||||||
{
|
|
||||||
"prompt_tokens": self.lm.prompt_tokens,
|
|
||||||
"completion_tokens": self.lm.completion_tokens,
|
|
||||||
"cost": self.lm.cost,
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
with open(path, "w") as file:
|
|
||||||
file.write(json.dumps(output, indent=2))
|
|
||||||
@ -1,112 +0,0 @@
|
|||||||
# Language Models
|
|
||||||
|
|
||||||
The Language Models module is responsible for managing the large language models (LLMs) used by the Controller.
|
|
||||||
|
|
||||||
Currently, the framework supports the following LLMs:
|
|
||||||
- GPT-4 / GPT-3.5 (Remote - OpenAI API)
|
|
||||||
- OpenRouter (Remote - [OpenRouter](https://openrouter.ai/) OpenAI-compatible API, multi-key / multi-model rotation)
|
|
||||||
- LLaMA-2 (Local - HuggingFace Transformers)
|
|
||||||
|
|
||||||
The following sections describe how to instantiate individual LLMs and how to add new LLMs to the framework.
|
|
||||||
|
|
||||||
## LLM Instantiation
|
|
||||||
- Create a copy of `config_template.json` named `config.json`.
|
|
||||||
- Fill configuration details based on the used model (below).
|
|
||||||
|
|
||||||
### GPT-4 / GPT-3.5
|
|
||||||
- Adjust the predefined `chatgpt` or `chatgpt4` configurations or create a new configuration with an unique key.
|
|
||||||
|
|
||||||
| Key | Value |
|
|
||||||
|---------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
|
|
||||||
| model_id | Model name based on [OpenAI model overview](https://platform.openai.com/docs/models/overview). |
|
|
||||||
| prompt_token_cost | Price per 1000 prompt tokens based on [OpenAI pricing](https://openai.com/pricing), used for calculating cumulative price per LLM instance. |
|
|
||||||
| response_token_cost | Price per 1000 response tokens based on [OpenAI pricing](https://openai.com/pricing), used for calculating cumulative price per LLM instance. |
|
|
||||||
| temperature | Parameter of OpenAI models that controls the randomness and the creativity of the responses (higher temperature = more diverse and unexpected responses). Value between 0.0 and 2.0, default is 1.0. More information can be found in the [OpenAI API reference](https://platform.openai.com/docs/api-reference/completions/create#completions/create-temperature). |
|
|
||||||
| max_tokens | The maximum number of tokens to generate in the chat completion. Value depends on the maximum context size of the model specified in the [OpenAI model overview](https://platform.openai.com/docs/models/overview). More information can be found in the [OpenAI API reference](https://platform.openai.com/docs/api-reference/chat/create#chat/create-max_tokens). |
|
|
||||||
| stop | String or array of strings specifying sequences of characters which if detected, stops further generation of tokens. More information can be found in the [OpenAI API reference](https://platform.openai.com/docs/api-reference/chat/create#chat/create-stop). |
|
|
||||||
| organization | Organization to use for the API requests (may be empty). |
|
|
||||||
| api_key | Personal API key that will be used to access OpenAI API. |
|
|
||||||
|
|
||||||
- Instantiate the language model based on the selected configuration key (predefined / custom).
|
|
||||||
```python
|
|
||||||
from graph_of_thoughts.language_models import ChatGPT
|
|
||||||
|
|
||||||
lm = ChatGPT(
|
|
||||||
"path/to/config.json",
|
|
||||||
model_name=<configuration key>
|
|
||||||
)
|
|
||||||
```
|
|
||||||
|
|
||||||
### OpenRouter
|
|
||||||
- Copy `config.openrouter.example.yaml` (or `.json`) to `config.openrouter.yaml` next to this module, or pass an explicit path.
|
|
||||||
- Set `api_keys` (list) and `models` (list). Each request picks a **random** key and a **random** model (uniform over the lists). If the HTTP API passes a `model` field, that model id is used for that request instead of a random one.
|
|
||||||
- Optional: `http_referer` and `x_title` for OpenRouter attribution headers (see [OpenRouter docs](https://openrouter.ai/docs)).
|
|
||||||
- HTTP **429** responses trigger exponential backoff and further rotation; **400** responses are retried a limited number of times with a new key/model pair, then surfaced as an error.
|
|
||||||
|
|
||||||
```python
|
|
||||||
from graph_of_thoughts.language_models import OpenRouter
|
|
||||||
|
|
||||||
lm = OpenRouter("/path/to/config.openrouter.yaml")
|
|
||||||
```
|
|
||||||
|
|
||||||
### LLaMA-2
|
|
||||||
- Requires local hardware to run inference and a HuggingFace account.
|
|
||||||
- Adjust the predefined `llama7b-hf`, `llama13b-hf` or `llama70b-hf` configurations or create a new configuration with an unique key.
|
|
||||||
|
|
||||||
| Key | Value |
|
|
||||||
|---------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
|
|
||||||
| model_id | Specifies HuggingFace LLaMA-2 model identifier (`meta-llama/<model_id>`). |
|
|
||||||
| cache_dir | Local directory where the model will be downloaded and accessed. |
|
|
||||||
| prompt_token_cost | Price per 1000 prompt tokens (currently not used - local model = no cost). |
|
|
||||||
| response_token_cost | Price per 1000 response tokens (currently not used - local model = no cost). |
|
|
||||||
| temperature | Parameter that controls the randomness and the creativity of the responses (higher temperature = more diverse and unexpected responses). Value between 0.0 and 1.0, default is 0.6. |
|
|
||||||
| top_k | Top-K sampling method described in [Transformers tutorial](https://huggingface.co/blog/how-to-generate). Default value is set to 10. |
|
|
||||||
| max_tokens | The maximum number of tokens to generate in the chat completion. More tokens require more memory. |
|
|
||||||
|
|
||||||
- Instantiate the language model based on the selected configuration key (predefined / custom).
|
|
||||||
```python
|
|
||||||
from graph_of_thoughts.language_models import Llama2HF
|
|
||||||
|
|
||||||
lm = Llama2HF(
|
|
||||||
"path/to/config.json",
|
|
||||||
model_name=<configuration key>
|
|
||||||
)
|
|
||||||
```
|
|
||||||
- Request access to LLaMA-2 via the [Meta form](https://ai.meta.com/resources/models-and-libraries/llama-downloads/) using the same email address as for the HuggingFace account.
|
|
||||||
- After the access is granted, go to [HuggingFace LLaMA-2 model card](https://huggingface.co/meta-llama/Llama-2-7b-chat-hf), log in and accept the license (a _"You have been granted access to this model"_ message should appear).
|
|
||||||
- Generate HuggingFace access token.
|
|
||||||
- Log in from CLI with: `huggingface-cli login --token <your token>`.
|
|
||||||
|
|
||||||
Note: 4-bit quantization is used to reduce the model size for inference. During instantiation, the model is downloaded from HuggingFace into the cache directory specified in the `config.json`. Running queries using larger models will require multiple GPUs (splitting across many GPUs is done automatically by the Transformers library).
|
|
||||||
|
|
||||||
## Adding LLMs
|
|
||||||
More LLMs can be added by following these steps:
|
|
||||||
- Create a new class as a subclass of `AbstractLanguageModel`.
|
|
||||||
- Use the constructor for loading the configuration and instantiating the language model (if needed).
|
|
||||||
```python
|
|
||||||
class CustomLanguageModel(AbstractLanguageModel):
|
|
||||||
def __init__(
|
|
||||||
self,
|
|
||||||
config_path: str = "",
|
|
||||||
model_name: str = "llama7b-hf",
|
|
||||||
cache: bool = False
|
|
||||||
) -> None:
|
|
||||||
super().__init__(config_path, model_name, cache)
|
|
||||||
self.config: Dict = self.config[model_name]
|
|
||||||
|
|
||||||
# Load data from configuration into variables if needed
|
|
||||||
|
|
||||||
# Instantiate LLM if needed
|
|
||||||
```
|
|
||||||
- Implement the `query` abstract method that is used to get a list of responses from the LLM (remote API call or local model inference).
|
|
||||||
```python
|
|
||||||
def query(self, query: str, num_responses: int = 1) -> Any:
|
|
||||||
# Support caching
|
|
||||||
# Call LLM and retrieve list of responses - based on num_responses
|
|
||||||
# Return LLM response structure (not only raw strings)
|
|
||||||
```
|
|
||||||
- Implement the `get_response_texts` abstract method that is used to get a list of raw texts from the LLM response structure produced by `query`.
|
|
||||||
```python
|
|
||||||
def get_response_texts(self, query_response: Union[List[Any], Any]) -> List[str]:
|
|
||||||
# Retrieve list of raw strings from the LLM response structure
|
|
||||||
```
|
|
||||||
@ -1,10 +0,0 @@
|
|||||||
from .abstract_language_model import AbstractLanguageModel
|
|
||||||
from .chatgpt import ChatGPT
|
|
||||||
from .llamachat_hf import Llama2HF
|
|
||||||
from .openrouter import (
|
|
||||||
OpenRouter,
|
|
||||||
OpenRouterBadRequestError,
|
|
||||||
OpenRouterError,
|
|
||||||
OpenRouterRateLimitError,
|
|
||||||
load_openrouter_config,
|
|
||||||
)
|
|
||||||
@ -1,92 +0,0 @@
|
|||||||
# Copyright (c) 2023 ETH Zurich.
|
|
||||||
# All rights reserved.
|
|
||||||
#
|
|
||||||
# Use of this source code is governed by a BSD-style license that can be
|
|
||||||
# found in the LICENSE file.
|
|
||||||
#
|
|
||||||
# main author: Nils Blach
|
|
||||||
|
|
||||||
from abc import ABC, abstractmethod
|
|
||||||
from typing import List, Dict, Union, Any
|
|
||||||
import json
|
|
||||||
import os
|
|
||||||
import logging
|
|
||||||
|
|
||||||
|
|
||||||
class AbstractLanguageModel(ABC):
|
|
||||||
"""
|
|
||||||
Abstract base class that defines the interface for all language models.
|
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(
|
|
||||||
self, config_path: str = "", model_name: str = "", cache: bool = False
|
|
||||||
) -> None:
|
|
||||||
"""
|
|
||||||
Initialize the AbstractLanguageModel instance with configuration, model details, and caching options.
|
|
||||||
|
|
||||||
:param config_path: Path to the config file. Defaults to "".
|
|
||||||
:type config_path: str
|
|
||||||
:param model_name: Name of the language model. Defaults to "".
|
|
||||||
:type model_name: str
|
|
||||||
:param cache: Flag to determine whether to cache responses. Defaults to False.
|
|
||||||
:type cache: bool
|
|
||||||
"""
|
|
||||||
self.logger = logging.getLogger(self.__class__.__name__)
|
|
||||||
self.config: Dict = None
|
|
||||||
self.model_name: str = model_name
|
|
||||||
self.cache = cache
|
|
||||||
if self.cache:
|
|
||||||
self.response_cache: Dict[str, List[Any]] = {}
|
|
||||||
self.load_config(config_path)
|
|
||||||
self.prompt_tokens: int = 0
|
|
||||||
self.completion_tokens: int = 0
|
|
||||||
self.cost: float = 0.0
|
|
||||||
|
|
||||||
def load_config(self, path: str) -> None:
|
|
||||||
"""
|
|
||||||
Load configuration from a specified path.
|
|
||||||
|
|
||||||
:param path: Path to the config file. If an empty path provided,
|
|
||||||
default is `config.json` in the current directory.
|
|
||||||
:type path: str
|
|
||||||
"""
|
|
||||||
if path == "":
|
|
||||||
current_dir = os.path.dirname(os.path.abspath(__file__))
|
|
||||||
path = os.path.join(current_dir, "config.json")
|
|
||||||
|
|
||||||
with open(path, "r") as f:
|
|
||||||
self.config = json.load(f)
|
|
||||||
|
|
||||||
self.logger.debug(f"Loaded config from {path} for {self.model_name}")
|
|
||||||
|
|
||||||
def clear_cache(self) -> None:
|
|
||||||
"""
|
|
||||||
Clear the response cache.
|
|
||||||
"""
|
|
||||||
self.response_cache.clear()
|
|
||||||
|
|
||||||
@abstractmethod
|
|
||||||
def query(self, query: str, num_responses: int = 1) -> Any:
|
|
||||||
"""
|
|
||||||
Abstract method to query the language model.
|
|
||||||
|
|
||||||
:param query: The query to be posed to the language model.
|
|
||||||
:type query: str
|
|
||||||
:param num_responses: The number of desired responses.
|
|
||||||
:type num_responses: int
|
|
||||||
:return: The language model's response(s).
|
|
||||||
:rtype: Any
|
|
||||||
"""
|
|
||||||
pass
|
|
||||||
|
|
||||||
@abstractmethod
|
|
||||||
def get_response_texts(self, query_responses: Union[List[Any], Any]) -> List[str]:
|
|
||||||
"""
|
|
||||||
Abstract method to extract response texts from the language model's response(s).
|
|
||||||
|
|
||||||
:param query_responses: The responses returned from the language model.
|
|
||||||
:type query_responses: Union[List[Any], Any]
|
|
||||||
:return: List of textual responses.
|
|
||||||
:rtype: List[str]
|
|
||||||
"""
|
|
||||||
pass
|
|
||||||
@ -1,157 +0,0 @@
|
|||||||
# Copyright (c) 2023 ETH Zurich.
|
|
||||||
# All rights reserved.
|
|
||||||
#
|
|
||||||
# Use of this source code is governed by a BSD-style license that can be
|
|
||||||
# found in the LICENSE file.
|
|
||||||
#
|
|
||||||
# main author: Nils Blach
|
|
||||||
|
|
||||||
import backoff
|
|
||||||
import os
|
|
||||||
import random
|
|
||||||
import time
|
|
||||||
from typing import List, Dict, Union
|
|
||||||
from openai import OpenAI, OpenAIError
|
|
||||||
from openai.types.chat.chat_completion import ChatCompletion
|
|
||||||
|
|
||||||
from .abstract_language_model import AbstractLanguageModel
|
|
||||||
|
|
||||||
|
|
||||||
class ChatGPT(AbstractLanguageModel):
|
|
||||||
"""
|
|
||||||
The ChatGPT class handles interactions with the OpenAI models using the provided configuration.
|
|
||||||
|
|
||||||
Inherits from the AbstractLanguageModel and implements its abstract methods.
|
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(
|
|
||||||
self, config_path: str = "", model_name: str = "chatgpt", cache: bool = False
|
|
||||||
) -> None:
|
|
||||||
"""
|
|
||||||
Initialize the ChatGPT instance with configuration, model details, and caching options.
|
|
||||||
|
|
||||||
:param config_path: Path to the configuration file. Defaults to "".
|
|
||||||
:type config_path: str
|
|
||||||
:param model_name: Name of the model, default is 'chatgpt'. Used to select the correct configuration.
|
|
||||||
:type model_name: str
|
|
||||||
:param cache: Flag to determine whether to cache responses. Defaults to False.
|
|
||||||
:type cache: bool
|
|
||||||
"""
|
|
||||||
super().__init__(config_path, model_name, cache)
|
|
||||||
self.config: Dict = self.config[model_name]
|
|
||||||
# The model_id is the id of the model that is used for chatgpt, i.e. gpt-4, gpt-3.5-turbo, etc.
|
|
||||||
self.model_id: str = self.config["model_id"]
|
|
||||||
# The prompt_token_cost and response_token_cost are the costs for 1000 prompt tokens and 1000 response tokens respectively.
|
|
||||||
self.prompt_token_cost: float = self.config["prompt_token_cost"]
|
|
||||||
self.response_token_cost: float = self.config["response_token_cost"]
|
|
||||||
# The temperature of a model is defined as the randomness of the model's output.
|
|
||||||
self.temperature: float = self.config["temperature"]
|
|
||||||
# The maximum number of tokens to generate in the chat completion.
|
|
||||||
self.max_tokens: int = self.config["max_tokens"]
|
|
||||||
# The stop sequence is a sequence of tokens that the model will stop generating at (it will not generate the stop sequence).
|
|
||||||
self.stop: Union[str, List[str]] = self.config["stop"]
|
|
||||||
# The account organization is the organization that is used for chatgpt.
|
|
||||||
self.organization: str = self.config["organization"]
|
|
||||||
if self.organization == "":
|
|
||||||
self.logger.warning("OPENAI_ORGANIZATION is not set")
|
|
||||||
self.api_key: str = os.getenv("OPENAI_API_KEY", self.config["api_key"])
|
|
||||||
if self.api_key == "":
|
|
||||||
raise ValueError("OPENAI_API_KEY is not set")
|
|
||||||
# Initialize the OpenAI Client
|
|
||||||
self.client = OpenAI(api_key=self.api_key, organization=self.organization)
|
|
||||||
|
|
||||||
def query(
|
|
||||||
self, query: str, num_responses: int = 1
|
|
||||||
) -> Union[List[ChatCompletion], ChatCompletion]:
|
|
||||||
"""
|
|
||||||
Query the OpenAI model for responses.
|
|
||||||
|
|
||||||
:param query: The query to be posed to the language model.
|
|
||||||
:type query: str
|
|
||||||
:param num_responses: Number of desired responses, default is 1.
|
|
||||||
:type num_responses: int
|
|
||||||
:return: Response(s) from the OpenAI model.
|
|
||||||
:rtype: Dict
|
|
||||||
"""
|
|
||||||
if self.cache and query in self.response_cache:
|
|
||||||
return self.response_cache[query]
|
|
||||||
|
|
||||||
if num_responses == 1:
|
|
||||||
response = self.chat([{"role": "user", "content": query}], num_responses)
|
|
||||||
else:
|
|
||||||
response = []
|
|
||||||
next_try = num_responses
|
|
||||||
total_num_attempts = num_responses
|
|
||||||
while num_responses > 0 and total_num_attempts > 0:
|
|
||||||
try:
|
|
||||||
assert next_try > 0
|
|
||||||
res = self.chat([{"role": "user", "content": query}], next_try)
|
|
||||||
response.append(res)
|
|
||||||
num_responses -= next_try
|
|
||||||
next_try = min(num_responses, next_try)
|
|
||||||
except Exception as e:
|
|
||||||
next_try = (next_try + 1) // 2
|
|
||||||
self.logger.warning(
|
|
||||||
f"Error in chatgpt: {e}, trying again with {next_try} samples"
|
|
||||||
)
|
|
||||||
time.sleep(random.randint(1, 3))
|
|
||||||
total_num_attempts -= 1
|
|
||||||
|
|
||||||
if self.cache:
|
|
||||||
self.response_cache[query] = response
|
|
||||||
return response
|
|
||||||
|
|
||||||
@backoff.on_exception(backoff.expo, OpenAIError, max_time=10, max_tries=6)
|
|
||||||
def chat(self, messages: List[Dict], num_responses: int = 1) -> ChatCompletion:
|
|
||||||
"""
|
|
||||||
Send chat messages to the OpenAI model and retrieves the model's response.
|
|
||||||
Implements backoff on OpenAI error.
|
|
||||||
|
|
||||||
:param messages: A list of message dictionaries for the chat.
|
|
||||||
:type messages: List[Dict]
|
|
||||||
:param num_responses: Number of desired responses, default is 1.
|
|
||||||
:type num_responses: int
|
|
||||||
:return: The OpenAI model's response.
|
|
||||||
:rtype: ChatCompletion
|
|
||||||
"""
|
|
||||||
response = self.client.chat.completions.create(
|
|
||||||
model=self.model_id,
|
|
||||||
messages=messages,
|
|
||||||
temperature=self.temperature,
|
|
||||||
max_tokens=self.max_tokens,
|
|
||||||
n=num_responses,
|
|
||||||
stop=self.stop,
|
|
||||||
)
|
|
||||||
|
|
||||||
self.prompt_tokens += response.usage.prompt_tokens
|
|
||||||
self.completion_tokens += response.usage.completion_tokens
|
|
||||||
prompt_tokens_k = float(self.prompt_tokens) / 1000.0
|
|
||||||
completion_tokens_k = float(self.completion_tokens) / 1000.0
|
|
||||||
self.cost = (
|
|
||||||
self.prompt_token_cost * prompt_tokens_k
|
|
||||||
+ self.response_token_cost * completion_tokens_k
|
|
||||||
)
|
|
||||||
self.logger.info(
|
|
||||||
f"This is the response from chatgpt: {response}"
|
|
||||||
f"\nThis is the cost of the response: {self.cost}"
|
|
||||||
)
|
|
||||||
return response
|
|
||||||
|
|
||||||
def get_response_texts(
|
|
||||||
self, query_response: Union[List[ChatCompletion], ChatCompletion]
|
|
||||||
) -> List[str]:
|
|
||||||
"""
|
|
||||||
Extract the response texts from the query response.
|
|
||||||
|
|
||||||
:param query_response: The response dictionary (or list of dictionaries) from the OpenAI model.
|
|
||||||
:type query_response: Union[List[ChatCompletion], ChatCompletion]
|
|
||||||
:return: List of response strings.
|
|
||||||
:rtype: List[str]
|
|
||||||
"""
|
|
||||||
if not isinstance(query_response, List):
|
|
||||||
query_response = [query_response]
|
|
||||||
return [
|
|
||||||
choice.message.content
|
|
||||||
for response in query_response
|
|
||||||
for choice in response.choices
|
|
||||||
]
|
|
||||||
@ -1,21 +0,0 @@
|
|||||||
{
|
|
||||||
"base_url": "https://openrouter.ai/api/v1",
|
|
||||||
"api_keys": [
|
|
||||||
"sk-or-v1-replace-me-1",
|
|
||||||
"sk-or-v1-replace-me-2"
|
|
||||||
],
|
|
||||||
"models": [
|
|
||||||
"openai/gpt-4o-mini",
|
|
||||||
"anthropic/claude-3.5-haiku"
|
|
||||||
],
|
|
||||||
"temperature": 0.7,
|
|
||||||
"max_tokens": 4096,
|
|
||||||
"stop": null,
|
|
||||||
"prompt_token_cost": 0.0,
|
|
||||||
"response_token_cost": 0.0,
|
|
||||||
"max_retries_429": 8,
|
|
||||||
"max_retries_400": 3,
|
|
||||||
"base_backoff_seconds": 1.0,
|
|
||||||
"http_referer": "",
|
|
||||||
"x_title": ""
|
|
||||||
}
|
|
||||||
@ -1,29 +0,0 @@
|
|||||||
# Copy to config.openrouter.yaml (or set path explicitly) and fill in keys.
|
|
||||||
# Per chat request, an API key and model are chosen at random (uniform) from the lists.
|
|
||||||
|
|
||||||
base_url: https://openrouter.ai/api/v1
|
|
||||||
|
|
||||||
api_keys:
|
|
||||||
- sk-or-v1-replace-me-1
|
|
||||||
- sk-or-v1-replace-me-2
|
|
||||||
|
|
||||||
models:
|
|
||||||
- openai/gpt-4o-mini
|
|
||||||
- anthropic/claude-3.5-haiku
|
|
||||||
|
|
||||||
temperature: 0.7
|
|
||||||
max_tokens: 4096
|
|
||||||
stop: null
|
|
||||||
|
|
||||||
# Optional cost accounting (set to 0 if unknown)
|
|
||||||
prompt_token_cost: 0.0
|
|
||||||
response_token_cost: 0.0
|
|
||||||
|
|
||||||
# Retries after HTTP 429 / 400 (each retry uses a fresh random key + model)
|
|
||||||
max_retries_429: 8
|
|
||||||
max_retries_400: 3
|
|
||||||
base_backoff_seconds: 1.0
|
|
||||||
|
|
||||||
# Optional OpenRouter attribution headers (recommended by OpenRouter)
|
|
||||||
http_referer: ""
|
|
||||||
x_title: ""
|
|
||||||
@ -1,49 +0,0 @@
|
|||||||
{
|
|
||||||
"chatgpt" : {
|
|
||||||
"model_id": "gpt-3.5-turbo",
|
|
||||||
"prompt_token_cost": 0.0015,
|
|
||||||
"response_token_cost": 0.002,
|
|
||||||
"temperature": 1.0,
|
|
||||||
"max_tokens": 1536,
|
|
||||||
"stop": null,
|
|
||||||
"organization": "",
|
|
||||||
"api_key": ""
|
|
||||||
},
|
|
||||||
"chatgpt4" : {
|
|
||||||
"model_id": "gpt-4",
|
|
||||||
"prompt_token_cost": 0.03,
|
|
||||||
"response_token_cost": 0.06,
|
|
||||||
"temperature": 1.0,
|
|
||||||
"max_tokens": 4096,
|
|
||||||
"stop": null,
|
|
||||||
"organization": "",
|
|
||||||
"api_key": ""
|
|
||||||
},
|
|
||||||
"llama7b-hf" : {
|
|
||||||
"model_id": "Llama-2-7b-chat-hf",
|
|
||||||
"cache_dir": "/llama",
|
|
||||||
"prompt_token_cost": 0.0,
|
|
||||||
"response_token_cost": 0.0,
|
|
||||||
"temperature": 0.6,
|
|
||||||
"top_k": 10,
|
|
||||||
"max_tokens": 4096
|
|
||||||
},
|
|
||||||
"llama13b-hf" : {
|
|
||||||
"model_id": "Llama-2-13b-chat-hf",
|
|
||||||
"cache_dir": "/llama",
|
|
||||||
"prompt_token_cost": 0.0,
|
|
||||||
"response_token_cost": 0.0,
|
|
||||||
"temperature": 0.6,
|
|
||||||
"top_k": 10,
|
|
||||||
"max_tokens": 4096
|
|
||||||
},
|
|
||||||
"llama70b-hf" : {
|
|
||||||
"model_id": "Llama-2-70b-chat-hf",
|
|
||||||
"cache_dir": "/llama",
|
|
||||||
"prompt_token_cost": 0.0,
|
|
||||||
"response_token_cost": 0.0,
|
|
||||||
"temperature": 0.6,
|
|
||||||
"top_k": 10,
|
|
||||||
"max_tokens": 4096
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -1,119 +0,0 @@
|
|||||||
# Copyright (c) 2023 ETH Zurich.
|
|
||||||
# All rights reserved.
|
|
||||||
#
|
|
||||||
# Use of this source code is governed by a BSD-style license that can be
|
|
||||||
# found in the LICENSE file.
|
|
||||||
#
|
|
||||||
# main author: Ales Kubicek
|
|
||||||
|
|
||||||
import os
|
|
||||||
import torch
|
|
||||||
from typing import List, Dict, Union
|
|
||||||
from .abstract_language_model import AbstractLanguageModel
|
|
||||||
|
|
||||||
|
|
||||||
class Llama2HF(AbstractLanguageModel):
|
|
||||||
"""
|
|
||||||
An interface to use LLaMA 2 models through the HuggingFace library.
|
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(
|
|
||||||
self, config_path: str = "", model_name: str = "llama7b-hf", cache: bool = False
|
|
||||||
) -> None:
|
|
||||||
"""
|
|
||||||
Initialize an instance of the Llama2HF class with configuration, model details, and caching options.
|
|
||||||
|
|
||||||
:param config_path: Path to the configuration file. Defaults to an empty string.
|
|
||||||
:type config_path: str
|
|
||||||
:param model_name: Specifies the name of the LLaMA model variant. Defaults to "llama7b-hf".
|
|
||||||
Used to select the correct configuration.
|
|
||||||
:type model_name: str
|
|
||||||
:param cache: Flag to determine whether to cache responses. Defaults to False.
|
|
||||||
:type cache: bool
|
|
||||||
"""
|
|
||||||
super().__init__(config_path, model_name, cache)
|
|
||||||
self.config: Dict = self.config[model_name]
|
|
||||||
# Detailed id of the used model.
|
|
||||||
self.model_id: str = self.config["model_id"]
|
|
||||||
# Costs for 1000 tokens.
|
|
||||||
self.prompt_token_cost: float = self.config["prompt_token_cost"]
|
|
||||||
self.response_token_cost: float = self.config["response_token_cost"]
|
|
||||||
# The temperature is defined as the randomness of the model's output.
|
|
||||||
self.temperature: float = self.config["temperature"]
|
|
||||||
# Top K sampling.
|
|
||||||
self.top_k: int = self.config["top_k"]
|
|
||||||
# The maximum number of tokens to generate in the chat completion.
|
|
||||||
self.max_tokens: int = self.config["max_tokens"]
|
|
||||||
|
|
||||||
# Important: must be done before importing transformers
|
|
||||||
os.environ["TRANSFORMERS_CACHE"] = self.config["cache_dir"]
|
|
||||||
import transformers
|
|
||||||
|
|
||||||
hf_model_id = f"meta-llama/{self.model_id}"
|
|
||||||
model_config = transformers.AutoConfig.from_pretrained(hf_model_id)
|
|
||||||
bnb_config = transformers.BitsAndBytesConfig(
|
|
||||||
load_in_4bit=True,
|
|
||||||
bnb_4bit_quant_type="nf4",
|
|
||||||
bnb_4bit_use_double_quant=True,
|
|
||||||
bnb_4bit_compute_dtype=torch.bfloat16,
|
|
||||||
)
|
|
||||||
|
|
||||||
self.tokenizer = transformers.AutoTokenizer.from_pretrained(hf_model_id)
|
|
||||||
self.model = transformers.AutoModelForCausalLM.from_pretrained(
|
|
||||||
hf_model_id,
|
|
||||||
trust_remote_code=True,
|
|
||||||
config=model_config,
|
|
||||||
quantization_config=bnb_config,
|
|
||||||
device_map="auto",
|
|
||||||
)
|
|
||||||
self.model.eval()
|
|
||||||
torch.no_grad()
|
|
||||||
|
|
||||||
self.generate_text = transformers.pipeline(
|
|
||||||
model=self.model, tokenizer=self.tokenizer, task="text-generation"
|
|
||||||
)
|
|
||||||
|
|
||||||
def query(self, query: str, num_responses: int = 1) -> List[Dict]:
|
|
||||||
"""
|
|
||||||
Query the LLaMA 2 model for responses.
|
|
||||||
|
|
||||||
:param query: The query to be posed to the language model.
|
|
||||||
:type query: str
|
|
||||||
:param num_responses: Number of desired responses, default is 1.
|
|
||||||
:type num_responses: int
|
|
||||||
:return: Response(s) from the LLaMA 2 model.
|
|
||||||
:rtype: List[Dict]
|
|
||||||
"""
|
|
||||||
if self.cache and query in self.response_cache:
|
|
||||||
return self.response_cache[query]
|
|
||||||
sequences = []
|
|
||||||
query = f"<s><<SYS>>You are a helpful assistant. Always follow the intstructions precisely and output the response exactly in the requested format.<</SYS>>\n\n[INST] {query} [/INST]"
|
|
||||||
for _ in range(num_responses):
|
|
||||||
sequences.extend(
|
|
||||||
self.generate_text(
|
|
||||||
query,
|
|
||||||
do_sample=True,
|
|
||||||
top_k=self.top_k,
|
|
||||||
num_return_sequences=1,
|
|
||||||
eos_token_id=self.tokenizer.eos_token_id,
|
|
||||||
max_length=self.max_tokens,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
response = [
|
|
||||||
{"generated_text": sequence["generated_text"][len(query) :].strip()}
|
|
||||||
for sequence in sequences
|
|
||||||
]
|
|
||||||
if self.cache:
|
|
||||||
self.response_cache[query] = response
|
|
||||||
return response
|
|
||||||
|
|
||||||
def get_response_texts(self, query_responses: List[Dict]) -> List[str]:
|
|
||||||
"""
|
|
||||||
Extract the response texts from the query response.
|
|
||||||
|
|
||||||
:param query_responses: The response list of dictionaries generated from the `query` method.
|
|
||||||
:type query_responses: List[Dict]
|
|
||||||
:return: List of response strings.
|
|
||||||
:rtype: List[str]
|
|
||||||
"""
|
|
||||||
return [query_response["generated_text"] for query_response in query_responses]
|
|
||||||
@ -1,287 +0,0 @@
|
|||||||
# Copyright (c) 2023 ETH Zurich.
|
|
||||||
# All rights reserved.
|
|
||||||
#
|
|
||||||
# Use of this source code is governed by a BSD-style license that can be
|
|
||||||
# found in the LICENSE file.
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import json
|
|
||||||
import logging
|
|
||||||
import os
|
|
||||||
import random
|
|
||||||
import time
|
|
||||||
from typing import Any, Dict, List, Optional, Union
|
|
||||||
|
|
||||||
import yaml
|
|
||||||
from openai import APIStatusError, OpenAI
|
|
||||||
from openai.types.chat.chat_completion import ChatCompletion
|
|
||||||
|
|
||||||
from .abstract_language_model import AbstractLanguageModel
|
|
||||||
|
|
||||||
|
|
||||||
class OpenRouterError(Exception):
|
|
||||||
"""Base error for OpenRouter integration."""
|
|
||||||
|
|
||||||
|
|
||||||
class OpenRouterBadRequestError(OpenRouterError):
|
|
||||||
"""Raised when OpenRouter returns HTTP 400 after retries."""
|
|
||||||
|
|
||||||
|
|
||||||
class OpenRouterRateLimitError(OpenRouterError):
|
|
||||||
"""Raised when OpenRouter returns HTTP 429 after retries."""
|
|
||||||
|
|
||||||
|
|
||||||
def load_openrouter_config(path: str) -> Dict[str, Any]:
|
|
||||||
"""Load a YAML or JSON OpenRouter configuration file."""
|
|
||||||
return _load_config_file(path)
|
|
||||||
|
|
||||||
|
|
||||||
def _load_config_file(path: str) -> Dict[str, Any]:
|
|
||||||
ext = os.path.splitext(path)[1].lower()
|
|
||||||
with open(path, "r", encoding="utf-8") as f:
|
|
||||||
if ext in (".yaml", ".yml"):
|
|
||||||
data = yaml.safe_load(f)
|
|
||||||
else:
|
|
||||||
data = json.load(f)
|
|
||||||
if not isinstance(data, dict):
|
|
||||||
raise ValueError(f"Config at {path} must be a JSON/YAML object")
|
|
||||||
return data
|
|
||||||
|
|
||||||
|
|
||||||
class OpenRouter(AbstractLanguageModel):
|
|
||||||
"""
|
|
||||||
OpenRouter-backed language model with per-request rotation of API keys and models.
|
|
||||||
|
|
||||||
Configuration is loaded from YAML or JSON (see ``config.openrouter.example.yaml``).
|
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(
|
|
||||||
self,
|
|
||||||
config_path: str = "",
|
|
||||||
model_name: str = "openrouter",
|
|
||||||
cache: bool = False,
|
|
||||||
) -> None:
|
|
||||||
self._rotation_model_name = model_name
|
|
||||||
self._request_overrides: Dict[str, Any] = {}
|
|
||||||
super().__init__(config_path, model_name, cache)
|
|
||||||
self._apply_openrouter_config()
|
|
||||||
|
|
||||||
def load_config(self, path: str) -> None:
|
|
||||||
if path == "":
|
|
||||||
path = os.path.join(
|
|
||||||
os.path.dirname(os.path.abspath(__file__)),
|
|
||||||
"config.openrouter.yaml",
|
|
||||||
)
|
|
||||||
self.config_path = path
|
|
||||||
self.config = _load_config_file(path)
|
|
||||||
self.logger.debug("Loaded OpenRouter config from %s", path)
|
|
||||||
|
|
||||||
def _apply_openrouter_config(self) -> None:
|
|
||||||
cfg = self.config
|
|
||||||
self.base_url: str = cfg.get("base_url", "https://openrouter.ai/api/v1")
|
|
||||||
keys = cfg.get("api_keys") or []
|
|
||||||
if isinstance(keys, str):
|
|
||||||
keys = [keys]
|
|
||||||
self.api_keys: List[str] = [k for k in keys if k]
|
|
||||||
if not self.api_keys:
|
|
||||||
raise ValueError("OpenRouter config must define non-empty 'api_keys'")
|
|
||||||
|
|
||||||
models = cfg.get("models") or []
|
|
||||||
if isinstance(models, str):
|
|
||||||
models = [models]
|
|
||||||
self.models: List[str] = [m for m in models if m]
|
|
||||||
if not self.models:
|
|
||||||
raise ValueError("OpenRouter config must define non-empty 'models'")
|
|
||||||
|
|
||||||
self.temperature: float = float(cfg.get("temperature", 1.0))
|
|
||||||
self.max_tokens: int = int(cfg.get("max_tokens", 4096))
|
|
||||||
self.stop: Union[str, List[str], None] = cfg.get("stop")
|
|
||||||
self.prompt_token_cost: float = float(cfg.get("prompt_token_cost", 0.0))
|
|
||||||
self.response_token_cost: float = float(cfg.get("response_token_cost", 0.0))
|
|
||||||
|
|
||||||
self.max_retries_429: int = int(cfg.get("max_retries_429", 8))
|
|
||||||
self.max_retries_400: int = int(cfg.get("max_retries_400", 3))
|
|
||||||
self.base_backoff_seconds: float = float(cfg.get("base_backoff_seconds", 1.0))
|
|
||||||
|
|
||||||
self.http_referer: str = cfg.get("http_referer", "") or os.getenv(
|
|
||||||
"OPENROUTER_HTTP_REFERER", ""
|
|
||||||
)
|
|
||||||
self.x_title: str = cfg.get("x_title", "") or os.getenv("OPENROUTER_X_TITLE", "")
|
|
||||||
|
|
||||||
self.model_name = self._rotation_model_name
|
|
||||||
self.last_model_id: Optional[str] = None
|
|
||||||
self.generation_model_id: Optional[str] = None
|
|
||||||
|
|
||||||
def set_request_overrides(self, **kwargs: Any) -> None:
|
|
||||||
"""Optional per-request parameters (used by the HTTP API). Cleared with :meth:`clear_request_overrides`."""
|
|
||||||
self._request_overrides = {k: v for k, v in kwargs.items() if v is not None}
|
|
||||||
|
|
||||||
def clear_request_overrides(self) -> None:
|
|
||||||
self._request_overrides = {}
|
|
||||||
|
|
||||||
def _pick_key(self) -> str:
|
|
||||||
return random.choice(self.api_keys)
|
|
||||||
|
|
||||||
def _pick_model(self, override: Optional[str]) -> str:
|
|
||||||
if override:
|
|
||||||
return override
|
|
||||||
o = self._request_overrides.get("model")
|
|
||||||
if o:
|
|
||||||
return str(o)
|
|
||||||
return random.choice(self.models)
|
|
||||||
|
|
||||||
def _effective_temperature(self) -> float:
|
|
||||||
t = self._request_overrides.get("temperature")
|
|
||||||
return float(t) if t is not None else self.temperature
|
|
||||||
|
|
||||||
def _effective_max_tokens(self) -> int:
|
|
||||||
m = self._request_overrides.get("max_tokens")
|
|
||||||
return int(m) if m is not None else self.max_tokens
|
|
||||||
|
|
||||||
def _client_for_key(self, api_key: str) -> OpenAI:
|
|
||||||
headers: Dict[str, str] = {}
|
|
||||||
if self.http_referer:
|
|
||||||
headers["HTTP-Referer"] = self.http_referer
|
|
||||||
if self.x_title:
|
|
||||||
headers["X-Title"] = self.x_title
|
|
||||||
return OpenAI(
|
|
||||||
base_url=self.base_url,
|
|
||||||
api_key=api_key,
|
|
||||||
default_headers=headers or None,
|
|
||||||
)
|
|
||||||
|
|
||||||
def _sleep_backoff(self, attempt: int) -> None:
|
|
||||||
cap = 60.0
|
|
||||||
delay = min(
|
|
||||||
self.base_backoff_seconds * (2**attempt) + random.random(),
|
|
||||||
cap,
|
|
||||||
)
|
|
||||||
self.logger.warning("Backing off %.2fs (attempt %d)", delay, attempt + 1)
|
|
||||||
time.sleep(delay)
|
|
||||||
|
|
||||||
def chat(
|
|
||||||
self,
|
|
||||||
messages: List[Dict[str, str]],
|
|
||||||
num_responses: int = 1,
|
|
||||||
model_override: Optional[str] = None,
|
|
||||||
) -> ChatCompletion:
|
|
||||||
"""
|
|
||||||
Call OpenRouter chat completions with rotation and retries for 429/400.
|
|
||||||
"""
|
|
||||||
attempts_429 = 0
|
|
||||||
attempts_400 = 0
|
|
||||||
attempt = 0
|
|
||||||
last_exc: Optional[Exception] = None
|
|
||||||
|
|
||||||
while True:
|
|
||||||
api_key = self._pick_key()
|
|
||||||
model_id = self._pick_model(model_override)
|
|
||||||
client = self._client_for_key(api_key)
|
|
||||||
try:
|
|
||||||
response = client.chat.completions.create(
|
|
||||||
model=model_id,
|
|
||||||
messages=messages,
|
|
||||||
temperature=self._effective_temperature(),
|
|
||||||
max_tokens=self._effective_max_tokens(),
|
|
||||||
n=num_responses,
|
|
||||||
stop=self.stop,
|
|
||||||
)
|
|
||||||
if response.usage is not None:
|
|
||||||
self.prompt_tokens += response.usage.prompt_tokens or 0
|
|
||||||
self.completion_tokens += response.usage.completion_tokens or 0
|
|
||||||
pt_k = float(self.prompt_tokens) / 1000.0
|
|
||||||
ct_k = float(self.completion_tokens) / 1000.0
|
|
||||||
self.cost = (
|
|
||||||
self.prompt_token_cost * pt_k
|
|
||||||
+ self.response_token_cost * ct_k
|
|
||||||
)
|
|
||||||
self.last_model_id = model_id
|
|
||||||
if self.generation_model_id is None:
|
|
||||||
self.generation_model_id = model_id
|
|
||||||
self.logger.info(
|
|
||||||
"OpenRouter response model=%s id=%s", model_id, response.id
|
|
||||||
)
|
|
||||||
return response
|
|
||||||
except APIStatusError as e:
|
|
||||||
last_exc = e
|
|
||||||
code = e.status_code
|
|
||||||
if code == 429:
|
|
||||||
if attempts_429 >= self.max_retries_429:
|
|
||||||
raise OpenRouterRateLimitError(
|
|
||||||
f"OpenRouter rate limited after {attempts_429} retries: {e.message}"
|
|
||||||
) from e
|
|
||||||
attempts_429 += 1
|
|
||||||
self._sleep_backoff(attempt)
|
|
||||||
attempt += 1
|
|
||||||
continue
|
|
||||||
if code == 400:
|
|
||||||
self.logger.warning(
|
|
||||||
"OpenRouter HTTP 400 (will retry with rotated key/model if allowed): %s body=%s",
|
|
||||||
e.message,
|
|
||||||
e.body,
|
|
||||||
)
|
|
||||||
if attempts_400 >= self.max_retries_400:
|
|
||||||
raise OpenRouterBadRequestError(
|
|
||||||
f"OpenRouter bad request after {attempts_400} retries: {e.message}"
|
|
||||||
) from e
|
|
||||||
attempts_400 += 1
|
|
||||||
attempt += 1
|
|
||||||
time.sleep(random.uniform(0.2, 0.8))
|
|
||||||
continue
|
|
||||||
raise
|
|
||||||
except Exception:
|
|
||||||
self.logger.exception("Unexpected error calling OpenRouter")
|
|
||||||
raise
|
|
||||||
|
|
||||||
def query(
|
|
||||||
self, query: str, num_responses: int = 1
|
|
||||||
) -> Union[List[ChatCompletion], ChatCompletion]:
|
|
||||||
if self.cache and query in self.response_cache:
|
|
||||||
return self.response_cache[query]
|
|
||||||
|
|
||||||
messages = [{"role": "user", "content": query}]
|
|
||||||
model_ov = self._request_overrides.get("model")
|
|
||||||
model_override = str(model_ov) if model_ov else None
|
|
||||||
|
|
||||||
if num_responses == 1:
|
|
||||||
response = self.chat(messages, 1, model_override=model_override)
|
|
||||||
else:
|
|
||||||
response = []
|
|
||||||
next_try = num_responses
|
|
||||||
total_num_attempts = num_responses
|
|
||||||
remaining = num_responses
|
|
||||||
while remaining > 0 and total_num_attempts > 0:
|
|
||||||
try:
|
|
||||||
assert next_try > 0
|
|
||||||
res = self.chat(
|
|
||||||
messages, next_try, model_override=model_override
|
|
||||||
)
|
|
||||||
response.append(res)
|
|
||||||
remaining -= next_try
|
|
||||||
next_try = min(remaining, next_try)
|
|
||||||
except Exception as e:
|
|
||||||
next_try = max(1, (next_try + 1) // 2)
|
|
||||||
self.logger.warning(
|
|
||||||
"Error in OpenRouter query: %s, retrying with n=%s",
|
|
||||||
e,
|
|
||||||
next_try,
|
|
||||||
)
|
|
||||||
time.sleep(random.uniform(0.5, 2.0))
|
|
||||||
total_num_attempts -= 1
|
|
||||||
|
|
||||||
if self.cache:
|
|
||||||
self.response_cache[query] = response
|
|
||||||
return response
|
|
||||||
|
|
||||||
def get_response_texts(
|
|
||||||
self, query_response: Union[List[ChatCompletion], ChatCompletion]
|
|
||||||
) -> List[str]:
|
|
||||||
if not isinstance(query_response, list):
|
|
||||||
query_response = [query_response]
|
|
||||||
texts: List[str] = []
|
|
||||||
for response in query_response:
|
|
||||||
for choice in response.choices:
|
|
||||||
c = choice.message.content
|
|
||||||
texts.append(c if c is not None else "")
|
|
||||||
return texts
|
|
||||||
@ -1,70 +0,0 @@
|
|||||||
# Operations
|
|
||||||
|
|
||||||
The Operations module contains operations to manipulate and process thoughts represented by the [Thought](thought.py) class.
|
|
||||||
Operations interface with a language model and use other helper classes like [Prompter](../prompter/prompter.py) and [Parser](../parser/parser.py) for effective communication and extraction of results from the language model.
|
|
||||||
The [Graph of Operations](graph_of_operations.py) class is the main class of the module and is responsible for orchestrating the operations, defining their relationships and maintaining the state of the thought graph, also known as Graph Reasoning State.
|
|
||||||
|
|
||||||
## Graph of Operations
|
|
||||||
The [GraphOfOperations](graph_of_operations.py) class facilitates the creation and management of a directed graph representing the sequence and interrelationships of operations on thoughts. Here’s how you can construct and work with the Graph of Operations:
|
|
||||||
|
|
||||||
### Initialization
|
|
||||||
Creating a new instance of GraphOfOperations:
|
|
||||||
|
|
||||||
```python
|
|
||||||
from graph_of_thoughts.operations import GraphOfOperations
|
|
||||||
|
|
||||||
graph = GraphOfOperations()
|
|
||||||
```
|
|
||||||
|
|
||||||
Upon initialization, the graph will be empty with no operations, roots, or leaves.
|
|
||||||
|
|
||||||
### Adding Operations
|
|
||||||
**Append Operation:** You can append operations to the end of the graph using the append_operation method. This ensures that the operation becomes a successor to all current leaf operations in the graph.
|
|
||||||
```python
|
|
||||||
from graph_of_thoughts.operations import Generate
|
|
||||||
|
|
||||||
operationA = Generate()
|
|
||||||
graph.append_operation(operationA)
|
|
||||||
```
|
|
||||||
**Add Operation with Relationships:** If you want to define specific relationships for an operation, use the add_operation method.
|
|
||||||
```python
|
|
||||||
operationB = Generate()
|
|
||||||
operationB.predecessors.append(operationA)
|
|
||||||
graph.add_operation(operationB)
|
|
||||||
```
|
|
||||||
Remember to set up the predecessors (and optionally successors) for your operation before adding it to the graph.
|
|
||||||
|
|
||||||
## Available Operations
|
|
||||||
The following operations are available in the module:
|
|
||||||
|
|
||||||
**Score:** Collect all thoughts from preceding operations and score them either using the LLM or a custom scoring function.
|
|
||||||
- num_samples (Optional): The number of samples to use for scoring, defaults to 1.
|
|
||||||
- combined_scoring (Optional): Whether to score all thoughts together in a single prompt or separately, defaults to False.
|
|
||||||
- scoring_function (Optional): A function that takes in a list of thought states and returns a list of scores for each thought.
|
|
||||||
|
|
||||||
**ValidateAndImprove:** For each thought, validate it and if it is invalid, improve it.
|
|
||||||
- num_samples (Optional): The number of samples to use for validation, defaults to 1.
|
|
||||||
- improve (Optional): Whether to improve the thought if it is invalid, defaults to True.
|
|
||||||
- num_tries (Optional): The number of times to try improving the thought, before giving up, defaults to 3.
|
|
||||||
- validate_function (Optional): A function that takes in a thought state and returns a boolean indicating whether the thought is valid.
|
|
||||||
|
|
||||||
**Generate:** Generate new thoughts from the current thoughts. If no previous thoughts are available, the thoughts are initialized with the input to the [Controller](../controller/controller.py).
|
|
||||||
- num_branches_prompt (Optional): Number of responses that each prompt should generate (passed to prompter). Defaults to 1.
|
|
||||||
- num_branches_response (Optional): Number of responses the LLM should generate for each prompt. Defaults to 1.
|
|
||||||
|
|
||||||
**Improve:** Improve the current thoughts. This operation is similar to the ValidateAndImprove operation, but it does not validate the thoughts and always tries to improve them.
|
|
||||||
|
|
||||||
**Aggregate:** Aggregate the current thoughts into a single thought. This operation is useful when you want to combine multiple thoughts into a single thought.
|
|
||||||
- num_responses (Optional): Number of responses to request from the LLM (generates multiple new thoughts). Defaults to 1.
|
|
||||||
|
|
||||||
**KeepBestN:** Keep the best N thoughts from the preceding thoughts. Assumes that the thoughts are already scored and throws an error if they are not.
|
|
||||||
- n: The number of thoughts to keep in order of score.
|
|
||||||
- higher_is_better (Optional): Whether higher scores are better (True) or lower scores are better (False). Defaults to True.
|
|
||||||
|
|
||||||
**KeepValid:** Keep only the valid thoughts from the preceding thoughts. Assumes that each thought has already been validated, if not, it will be considered valid.
|
|
||||||
|
|
||||||
**Selector:** Select a number of thoughts from the preceding thoughts using a selection function. This is useful if subsequent operations should only be applied to a subset of the preceding thoughts.
|
|
||||||
- selector: A function that takes in a list of thoughts and returns a list of thoughts to select.
|
|
||||||
|
|
||||||
**GroundTruth**: Evaluates if the preceding/current thoughts solve the problem and equal the ground truth. This operation is useful for terminating the graph and checking if the final thoughts solve the problem, but is only useful if the ground truth is known.
|
|
||||||
- ground_truth_evaluator: A function that takes in a thought state and returns a boolean indicating whether the thought solves the problem.
|
|
||||||
@ -1,14 +0,0 @@
|
|||||||
from .thought import Thought
|
|
||||||
from .graph_of_operations import GraphOfOperations
|
|
||||||
from .operations import (
|
|
||||||
Operation,
|
|
||||||
Score,
|
|
||||||
ValidateAndImprove,
|
|
||||||
Generate,
|
|
||||||
Aggregate,
|
|
||||||
KeepBestN,
|
|
||||||
KeepValid,
|
|
||||||
Selector,
|
|
||||||
GroundTruth,
|
|
||||||
Improve,
|
|
||||||
)
|
|
||||||
@ -1,69 +0,0 @@
|
|||||||
# Copyright (c) 2023 ETH Zurich.
|
|
||||||
# All rights reserved.
|
|
||||||
#
|
|
||||||
# Use of this source code is governed by a BSD-style license that can be
|
|
||||||
# found in the LICENSE file.
|
|
||||||
#
|
|
||||||
# main author: Nils Blach
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
from typing import List
|
|
||||||
|
|
||||||
from graph_of_thoughts.operations.operations import Operation
|
|
||||||
|
|
||||||
|
|
||||||
class GraphOfOperations:
|
|
||||||
"""
|
|
||||||
Represents the Graph of Operations, which prescribes the execution plan of thought operations.
|
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(self) -> None:
|
|
||||||
"""
|
|
||||||
Initializes a new Graph of Operations instance with empty operations, roots, and leaves.
|
|
||||||
The roots are the entry points in the graph with no predecessors.
|
|
||||||
The leaves are the exit points in the graph with no successors.
|
|
||||||
"""
|
|
||||||
self.operations: List[Operation] = []
|
|
||||||
self.roots: List[Operation] = []
|
|
||||||
self.leaves: List[Operation] = []
|
|
||||||
|
|
||||||
def append_operation(self, operation: Operation) -> None:
|
|
||||||
"""
|
|
||||||
Appends an operation to all leaves in the graph and updates the relationships.
|
|
||||||
|
|
||||||
:param operation: The operation to append.
|
|
||||||
:type operation: Operation
|
|
||||||
"""
|
|
||||||
self.operations.append(operation)
|
|
||||||
|
|
||||||
if len(self.roots) == 0:
|
|
||||||
self.roots = [operation]
|
|
||||||
else:
|
|
||||||
for leave in self.leaves:
|
|
||||||
leave.add_successor(operation)
|
|
||||||
|
|
||||||
self.leaves = [operation]
|
|
||||||
|
|
||||||
def add_operation(self, operation: Operation) -> None:
|
|
||||||
"""
|
|
||||||
Add an operation to the graph considering its predecessors and successors.
|
|
||||||
Adjust roots and leaves based on the added operation's position within the graph.
|
|
||||||
|
|
||||||
:param operation: The operation to add.
|
|
||||||
:type operation: Operation
|
|
||||||
"""
|
|
||||||
self.operations.append(operation)
|
|
||||||
if len(self.roots) == 0:
|
|
||||||
self.roots = [operation]
|
|
||||||
self.leaves = [operation]
|
|
||||||
assert (
|
|
||||||
len(operation.predecessors) == 0
|
|
||||||
), "First operation should have no predecessors"
|
|
||||||
else:
|
|
||||||
if len(operation.predecessors) == 0:
|
|
||||||
self.roots.append(operation)
|
|
||||||
for predecessor in operation.predecessors:
|
|
||||||
if predecessor in self.leaves:
|
|
||||||
self.leaves.remove(predecessor)
|
|
||||||
if len(operation.successors) == 0:
|
|
||||||
self.leaves.append(operation)
|
|
||||||
@ -1,900 +0,0 @@
|
|||||||
# Copyright (c) 2023 ETH Zurich.
|
|
||||||
# All rights reserved.
|
|
||||||
#
|
|
||||||
# Use of this source code is governed by a BSD-style license that can be
|
|
||||||
# found in the LICENSE file.
|
|
||||||
#
|
|
||||||
# main author: Nils Blach
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
import logging
|
|
||||||
from enum import Enum
|
|
||||||
from typing import List, Iterator, Dict, Callable, Union
|
|
||||||
from abc import ABC, abstractmethod
|
|
||||||
import itertools
|
|
||||||
|
|
||||||
from graph_of_thoughts.operations.thought import Thought
|
|
||||||
from graph_of_thoughts.language_models import AbstractLanguageModel
|
|
||||||
from graph_of_thoughts.prompter import Prompter
|
|
||||||
from graph_of_thoughts.parser import Parser
|
|
||||||
|
|
||||||
|
|
||||||
class OperationType(Enum):
|
|
||||||
"""
|
|
||||||
Enum to represent different operation types that can be used as unique identifiers.
|
|
||||||
"""
|
|
||||||
|
|
||||||
score: int = 0
|
|
||||||
validate_and_improve: int = 1
|
|
||||||
generate: int = 2
|
|
||||||
improve: int = 3
|
|
||||||
aggregate: int = 4
|
|
||||||
keep_best_n: int = 5
|
|
||||||
keep_valid: int = 6
|
|
||||||
ground_truth_evaluator: int = 7
|
|
||||||
selector: int = 8
|
|
||||||
|
|
||||||
|
|
||||||
class Operation(ABC):
|
|
||||||
"""
|
|
||||||
Abstract base class that defines the interface for all operations.
|
|
||||||
"""
|
|
||||||
|
|
||||||
_ids: Iterator[int] = itertools.count(0)
|
|
||||||
|
|
||||||
operation_type: OperationType = None
|
|
||||||
|
|
||||||
def __init__(self) -> None:
|
|
||||||
"""
|
|
||||||
Initializes a new Operation instance with a unique id, and empty predecessors and successors.
|
|
||||||
"""
|
|
||||||
self.logger: logging.Logger = logging.getLogger(self.__class__.__name__)
|
|
||||||
self.id: int = next(Operation._ids)
|
|
||||||
self.predecessors: List[Operation] = []
|
|
||||||
self.successors: List[Operation] = []
|
|
||||||
self.executed: bool = False
|
|
||||||
|
|
||||||
def can_be_executed(self) -> bool:
|
|
||||||
"""
|
|
||||||
Checks if the operation can be executed based on its predecessors.
|
|
||||||
|
|
||||||
:return: True if all predecessors have been executed, False otherwise.
|
|
||||||
:rtype: bool
|
|
||||||
"""
|
|
||||||
return all(predecessor.executed for predecessor in self.predecessors)
|
|
||||||
|
|
||||||
def get_previous_thoughts(self) -> List[Thought]:
|
|
||||||
"""
|
|
||||||
Iterates over all predecessors and aggregates their thoughts.
|
|
||||||
|
|
||||||
:return: A list of all thoughts from the predecessors.
|
|
||||||
:rtype: List[Thought]
|
|
||||||
"""
|
|
||||||
previous_thoughts: List[Thought] = [
|
|
||||||
thought
|
|
||||||
for predecessor in self.predecessors
|
|
||||||
for thought in predecessor.get_thoughts()
|
|
||||||
]
|
|
||||||
|
|
||||||
return previous_thoughts
|
|
||||||
|
|
||||||
def add_predecessor(self, operation: Operation) -> None:
|
|
||||||
"""
|
|
||||||
Add a preceding operation and update the relationships.
|
|
||||||
|
|
||||||
:param operation: The operation to be set as a predecessor.
|
|
||||||
:type operation: Operation
|
|
||||||
"""
|
|
||||||
self.predecessors.append(operation)
|
|
||||||
operation.successors.append(self)
|
|
||||||
|
|
||||||
def add_successor(self, operation: Operation) -> None:
|
|
||||||
"""
|
|
||||||
Add a succeeding operation and update the relationships.
|
|
||||||
|
|
||||||
:param operation: The operation to be set as a successor.
|
|
||||||
:type operation: Operation
|
|
||||||
"""
|
|
||||||
self.successors.append(operation)
|
|
||||||
operation.predecessors.append(self)
|
|
||||||
|
|
||||||
def execute(
|
|
||||||
self, lm: AbstractLanguageModel, prompter: Prompter, parser: Parser, **kwargs
|
|
||||||
) -> None:
|
|
||||||
"""
|
|
||||||
Execute the operation, assuring that all predecessors have been executed.
|
|
||||||
|
|
||||||
:param lm: The language model to be used.
|
|
||||||
:type lm: AbstractLanguageModel
|
|
||||||
:param prompter: The prompter for crafting prompts.
|
|
||||||
:type prompter: Prompter
|
|
||||||
:param parser: The parser for parsing responses.
|
|
||||||
:type parser: Parser
|
|
||||||
:param kwargs: Additional parameters for execution.
|
|
||||||
:raises AssertionError: If not all predecessors have been executed.
|
|
||||||
"""
|
|
||||||
assert self.can_be_executed(), "Not all predecessors have been executed"
|
|
||||||
self.logger.info(
|
|
||||||
"Executing operation %d of type %s", self.id, self.operation_type
|
|
||||||
)
|
|
||||||
self._execute(lm, prompter, parser, **kwargs)
|
|
||||||
self.logger.debug("Operation %d executed", self.id)
|
|
||||||
self.executed = True
|
|
||||||
|
|
||||||
@abstractmethod
|
|
||||||
def _execute(
|
|
||||||
self, lm: AbstractLanguageModel, prompter: Prompter, parser: Parser, **kwargs
|
|
||||||
) -> None:
|
|
||||||
"""
|
|
||||||
Abstract method for the actual execution of the operation.
|
|
||||||
This should be implemented in derived classes.
|
|
||||||
|
|
||||||
:param lm: The language model to be used.
|
|
||||||
:type lm: AbstractLanguageModel
|
|
||||||
:param prompter: The prompter for crafting prompts.
|
|
||||||
:type prompter: Prompter
|
|
||||||
:param parser: The parser for parsing responses.
|
|
||||||
:type parser: Parser
|
|
||||||
:param kwargs: Additional parameters for execution.
|
|
||||||
"""
|
|
||||||
pass
|
|
||||||
|
|
||||||
@abstractmethod
|
|
||||||
def get_thoughts(self) -> List[Thought]:
|
|
||||||
"""
|
|
||||||
Abstract method to retrieve the thoughts associated with the operation.
|
|
||||||
This should be implemented in derived classes.
|
|
||||||
|
|
||||||
:return: List of associated thoughts.
|
|
||||||
:rtype: List[Thought]
|
|
||||||
"""
|
|
||||||
pass
|
|
||||||
|
|
||||||
|
|
||||||
class Score(Operation):
|
|
||||||
"""
|
|
||||||
Operation to score thoughts.
|
|
||||||
"""
|
|
||||||
|
|
||||||
operation_type: OperationType = OperationType.score
|
|
||||||
|
|
||||||
def __init__(
|
|
||||||
self,
|
|
||||||
num_samples: int = 1,
|
|
||||||
combined_scoring: bool = False,
|
|
||||||
scoring_function: Callable[
|
|
||||||
[Union[List[Dict], Dict]], Union[List[float], float]
|
|
||||||
] = None,
|
|
||||||
) -> None:
|
|
||||||
"""
|
|
||||||
Initializes a new Score operation.
|
|
||||||
|
|
||||||
:param num_samples: Number of samples to use for scoring. Defaults to 1.
|
|
||||||
:type num_samples: int
|
|
||||||
:param combined_scoring: Whether to score all thoughts together or individually. Defaults to False.
|
|
||||||
:type combined_scoring: bool
|
|
||||||
:param scoring_function: A function to score thoughts (if not using LM). Defaults to None.
|
|
||||||
:type scoring_function: Takes a list of thought states or a single thought state and
|
|
||||||
returns a list of scores or a single score.
|
|
||||||
"""
|
|
||||||
super().__init__()
|
|
||||||
self.num_samples: int = num_samples
|
|
||||||
self.combined_scoring: bool = combined_scoring
|
|
||||||
self.thoughts: List[Thought] = []
|
|
||||||
self.scoring_function: Callable[
|
|
||||||
[Union[List[Dict], Dict]], Union[List[float], float]
|
|
||||||
] = scoring_function
|
|
||||||
|
|
||||||
def get_thoughts(self) -> List[Thought]:
|
|
||||||
"""
|
|
||||||
Returns the thoughts associated with the operation.
|
|
||||||
|
|
||||||
:return: List of scored thoughts.
|
|
||||||
:rtype: List[Thought]
|
|
||||||
"""
|
|
||||||
return self.thoughts
|
|
||||||
|
|
||||||
def _execute(
|
|
||||||
self, lm: AbstractLanguageModel, prompter: Prompter, parser: Parser, **kwargs
|
|
||||||
) -> None:
|
|
||||||
"""
|
|
||||||
Executes the scoring operation by scoring the thoughts from the predecessors.
|
|
||||||
If combined scoring is used, the thoughts are scored together, otherwise individually.
|
|
||||||
If a scoring function is provided, it is used, otherwise the LM is prompted.
|
|
||||||
|
|
||||||
:param lm: The language model to be used.
|
|
||||||
:type lm: AbstractLanguageModel
|
|
||||||
:param prompter: The prompter for crafting prompts.
|
|
||||||
:type prompter: Prompter
|
|
||||||
:param parser: The parser for parsing responses.
|
|
||||||
:type parser: Parser
|
|
||||||
:param kwargs: Additional parameters for execution.
|
|
||||||
:raises AssertionError: If operation has no predecessors.
|
|
||||||
"""
|
|
||||||
previous_thoughts: List[Thought] = self.get_previous_thoughts()
|
|
||||||
|
|
||||||
assert (
|
|
||||||
len(self.predecessors) > 0
|
|
||||||
), "Score operation needs at least one predecessor"
|
|
||||||
|
|
||||||
if self.combined_scoring:
|
|
||||||
previous_thoughts_states = [thought.state for thought in previous_thoughts]
|
|
||||||
if self.scoring_function is not None:
|
|
||||||
self.logger.debug(
|
|
||||||
"Using scoring function %s to score states", self.scoring_function
|
|
||||||
)
|
|
||||||
scores = self.scoring_function(previous_thoughts_states)
|
|
||||||
else:
|
|
||||||
prompt = prompter.score_prompt(previous_thoughts_states)
|
|
||||||
self.logger.debug("Prompt for LM: %s", prompt)
|
|
||||||
|
|
||||||
responses = lm.get_response_texts(
|
|
||||||
lm.query(prompt, num_responses=self.num_samples)
|
|
||||||
)
|
|
||||||
self.logger.debug("Responses from LM: %s", responses)
|
|
||||||
scores = parser.parse_score_answer(previous_thoughts_states, responses)
|
|
||||||
for thought, score in zip(previous_thoughts, scores):
|
|
||||||
new_thought = Thought.from_thought(thought)
|
|
||||||
new_thought.score = score
|
|
||||||
self.thoughts.append(new_thought)
|
|
||||||
else:
|
|
||||||
for thought in previous_thoughts:
|
|
||||||
new_thought = Thought.from_thought(thought)
|
|
||||||
if self.scoring_function is not None:
|
|
||||||
self.logger.debug(
|
|
||||||
"Using scoring function %s to score state",
|
|
||||||
self.scoring_function,
|
|
||||||
)
|
|
||||||
score = self.scoring_function(thought.state)
|
|
||||||
else:
|
|
||||||
prompt = prompter.score_prompt([thought.state])
|
|
||||||
self.logger.debug("Prompt for LM: %s", prompt)
|
|
||||||
|
|
||||||
responses = lm.get_response_texts(
|
|
||||||
lm.query(prompt, num_responses=self.num_samples)
|
|
||||||
)
|
|
||||||
self.logger.debug("Responses from LM: %s", responses)
|
|
||||||
score = parser.parse_score_answer([thought.state], responses)[0]
|
|
||||||
|
|
||||||
new_thought.score = score
|
|
||||||
self.thoughts.append(new_thought)
|
|
||||||
|
|
||||||
self.logger.info(
|
|
||||||
"Score operation %d scored %d thoughts",
|
|
||||||
self.id,
|
|
||||||
len(self.thoughts),
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class ValidateAndImprove(Operation):
|
|
||||||
"""
|
|
||||||
Operation to validate and improve thoughts.
|
|
||||||
"""
|
|
||||||
|
|
||||||
operation_type: OperationType = OperationType.validate_and_improve
|
|
||||||
|
|
||||||
def __init__(
|
|
||||||
self,
|
|
||||||
num_samples: int = 1,
|
|
||||||
improve: bool = True,
|
|
||||||
num_tries: int = 3,
|
|
||||||
validate_function: Callable[[Dict], bool] = None,
|
|
||||||
) -> None:
|
|
||||||
"""
|
|
||||||
Initializes a new ValidateAndImprove operation.
|
|
||||||
|
|
||||||
:param num_samples: Number of samples to use for validation. Defaults to 1.
|
|
||||||
:type num_samples: int
|
|
||||||
:param improve: Whether to improve the thought if it is not valid. Defaults to True.
|
|
||||||
:type improve: bool
|
|
||||||
:param num_tries: Number of tries to improve the thought before giving up. Defaults to 3.
|
|
||||||
:type num_tries: int
|
|
||||||
:param validate_function: A function to validate thoughts (if not using LM). Defaults to None.
|
|
||||||
:type validate_function: Takes a thought state and returns a boolean.
|
|
||||||
"""
|
|
||||||
super().__init__()
|
|
||||||
self.num_samples: int = num_samples
|
|
||||||
self.improve: bool = improve
|
|
||||||
self.num_tries: int = num_tries
|
|
||||||
self.validate_function: Callable[[Dict], bool] = validate_function
|
|
||||||
self.thoughts: List[List[Thought]] = []
|
|
||||||
|
|
||||||
def get_thoughts(self) -> List[Thought]:
|
|
||||||
"""
|
|
||||||
Returns the list of final thoughts, after validation and improvement.
|
|
||||||
|
|
||||||
:return: List of final validated and improved thoughts.
|
|
||||||
:rtype: List[Thought]
|
|
||||||
"""
|
|
||||||
return [thought_list[-1] for thought_list in self.thoughts]
|
|
||||||
|
|
||||||
def _execute(
|
|
||||||
self, lm: AbstractLanguageModel, prompter: Prompter, parser: Parser, **kwargs
|
|
||||||
) -> None:
|
|
||||||
"""
|
|
||||||
Executes the ValidateAndImprove operation by validating and improving the predecessors' thoughts.
|
|
||||||
If a validation function is provided, it is used, otherwise the LM is prompted.
|
|
||||||
If improvement is enabled, the LM is prompted to improve the thought, if it is not valid.
|
|
||||||
|
|
||||||
:param lm: The language model to be used.
|
|
||||||
:type lm: AbstractLanguageModel
|
|
||||||
:param prompter: The prompter for crafting prompts.
|
|
||||||
:type prompter: Prompter
|
|
||||||
:param parser: The parser for parsing responses.
|
|
||||||
:type parser: Parser
|
|
||||||
:param kwargs: Additional parameters for execution.
|
|
||||||
:raises AssertionError: If operation has no predecessors.
|
|
||||||
"""
|
|
||||||
previous_thoughts: List[Thought] = self.get_previous_thoughts()
|
|
||||||
|
|
||||||
assert (
|
|
||||||
len(self.predecessors) > 0
|
|
||||||
), "ValidateAndImprove operation needs at least one predecessor"
|
|
||||||
|
|
||||||
for thought in previous_thoughts:
|
|
||||||
thought_list = []
|
|
||||||
current_thought = Thought.from_thought(thought)
|
|
||||||
current_try = 0
|
|
||||||
while True:
|
|
||||||
if self.validate_function is not None:
|
|
||||||
self.logger.debug(
|
|
||||||
"Using validate function %s to score states",
|
|
||||||
self.validate_function,
|
|
||||||
)
|
|
||||||
valid = self.validate_function(current_thought.state)
|
|
||||||
else:
|
|
||||||
prompt = prompter.validation_prompt(**current_thought.state)
|
|
||||||
self.logger.debug("Prompt for LM: %s", prompt)
|
|
||||||
responses = lm.get_response_texts(
|
|
||||||
lm.query(prompt, num_responses=self.num_samples)
|
|
||||||
)
|
|
||||||
self.logger.debug("Responses from LM: %s", responses)
|
|
||||||
|
|
||||||
valid = parser.parse_validation_answer(
|
|
||||||
current_thought.state, responses
|
|
||||||
)
|
|
||||||
current_thought.valid = valid
|
|
||||||
thought_list.append(current_thought)
|
|
||||||
if (
|
|
||||||
not self.improve
|
|
||||||
or current_thought.valid
|
|
||||||
or current_try >= self.num_tries
|
|
||||||
):
|
|
||||||
break
|
|
||||||
improve_prompt = prompter.improve_prompt(**current_thought.state)
|
|
||||||
self.logger.debug("Prompt for LM: %s", improve_prompt)
|
|
||||||
responses = lm.get_response_texts(
|
|
||||||
lm.query(improve_prompt, num_responses=1)
|
|
||||||
)
|
|
||||||
self.logger.debug("Responses from LM: %s", responses)
|
|
||||||
state_update = parser.parse_improve_answer(
|
|
||||||
current_thought.state, responses
|
|
||||||
)
|
|
||||||
current_thought = Thought({**current_thought.state, **state_update})
|
|
||||||
current_try += 1
|
|
||||||
self.thoughts.append(thought_list)
|
|
||||||
|
|
||||||
self.logger.info(
|
|
||||||
"Validate and improve operation %d created %d valid thoughts from %d previous thoughts",
|
|
||||||
self.id,
|
|
||||||
len(
|
|
||||||
[
|
|
||||||
thought_list[-1]
|
|
||||||
for thought_list in self.thoughts
|
|
||||||
if thought_list[-1].valid
|
|
||||||
]
|
|
||||||
),
|
|
||||||
len(previous_thoughts),
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class Generate(Operation):
|
|
||||||
"""
|
|
||||||
Operation to generate thoughts.
|
|
||||||
"""
|
|
||||||
|
|
||||||
operation_type: OperationType = OperationType.generate
|
|
||||||
|
|
||||||
def __init__(
|
|
||||||
self, num_branches_prompt: int = 1, num_branches_response: int = 1
|
|
||||||
) -> None:
|
|
||||||
"""
|
|
||||||
Initializes a new Generate operation.
|
|
||||||
|
|
||||||
:param num_branches_prompt: Number of responses that each prompt should generate (passed to prompter). Defaults to 1.
|
|
||||||
:type num_branches_prompt: int
|
|
||||||
:param num_branches_response: Number of responses the LM should generate for each prompt. Defaults to 1.
|
|
||||||
:type num_branches_response: int
|
|
||||||
"""
|
|
||||||
super().__init__()
|
|
||||||
self.num_branches_prompt: int = num_branches_prompt
|
|
||||||
self.num_branches_response: int = num_branches_response
|
|
||||||
self.thoughts: List[Thought] = []
|
|
||||||
|
|
||||||
def get_thoughts(self) -> List[Thought]:
|
|
||||||
"""
|
|
||||||
Returns the thoughts associated with the operation.
|
|
||||||
|
|
||||||
:return: List of generated thoughts.
|
|
||||||
:rtype: List[Thought]
|
|
||||||
"""
|
|
||||||
return self.thoughts
|
|
||||||
|
|
||||||
def _execute(
|
|
||||||
self, lm: AbstractLanguageModel, prompter: Prompter, parser: Parser, **kwargs
|
|
||||||
) -> None:
|
|
||||||
"""
|
|
||||||
Executes the Generate operation by generating thoughts from the predecessors.
|
|
||||||
The thoughts are generated by prompting the LM with the predecessors' thought states.
|
|
||||||
If there are no predecessors, the kwargs are used as a base state.
|
|
||||||
|
|
||||||
:param lm: The language model to be used.
|
|
||||||
:type lm: AbstractLanguageModel
|
|
||||||
:param prompter: The prompter for crafting prompts.
|
|
||||||
:type prompter: Prompter
|
|
||||||
:param parser: The parser for parsing responses.
|
|
||||||
:type parser: Parser
|
|
||||||
:param kwargs: Additional parameters for execution.
|
|
||||||
"""
|
|
||||||
previous_thoughts: List[Thought] = self.get_previous_thoughts()
|
|
||||||
|
|
||||||
if len(previous_thoughts) == 0 and len(self.predecessors) > 0:
|
|
||||||
return
|
|
||||||
|
|
||||||
if len(previous_thoughts) == 0:
|
|
||||||
# no predecessors, use kwargs as base state
|
|
||||||
previous_thoughts = [Thought(state=kwargs)]
|
|
||||||
|
|
||||||
for thought in previous_thoughts:
|
|
||||||
base_state = thought.state
|
|
||||||
prompt = prompter.generate_prompt(self.num_branches_prompt, **base_state)
|
|
||||||
self.logger.debug("Prompt for LM: %s", prompt)
|
|
||||||
responses = lm.get_response_texts(
|
|
||||||
lm.query(prompt, num_responses=self.num_branches_response)
|
|
||||||
)
|
|
||||||
self.logger.debug("Responses from LM: %s", responses)
|
|
||||||
for new_state in parser.parse_generate_answer(base_state, responses):
|
|
||||||
new_state = {**base_state, **new_state}
|
|
||||||
self.thoughts.append(Thought(new_state))
|
|
||||||
self.logger.debug(
|
|
||||||
"New thought %d created with state %s",
|
|
||||||
self.thoughts[-1].id,
|
|
||||||
self.thoughts[-1].state,
|
|
||||||
)
|
|
||||||
if (
|
|
||||||
len(self.thoughts)
|
|
||||||
> self.num_branches_prompt
|
|
||||||
* self.num_branches_response
|
|
||||||
* len(previous_thoughts)
|
|
||||||
and self.num_branches_prompt > 0
|
|
||||||
):
|
|
||||||
self.logger.warning(
|
|
||||||
"Generate operation %d created more thoughts than expected",
|
|
||||||
self.id,
|
|
||||||
)
|
|
||||||
self.logger.info(
|
|
||||||
"Generate operation %d created %d new thoughts", self.id, len(self.thoughts)
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class Improve(Operation):
|
|
||||||
"""
|
|
||||||
Operation to improve thoughts.
|
|
||||||
"""
|
|
||||||
|
|
||||||
operation_type: OperationType = OperationType.improve
|
|
||||||
|
|
||||||
def __init__(self) -> None:
|
|
||||||
"""
|
|
||||||
Initializes a new Improve operation.
|
|
||||||
"""
|
|
||||||
super().__init__()
|
|
||||||
self.thoughts: List[Thought] = []
|
|
||||||
|
|
||||||
def get_thoughts(self) -> List[Thought]:
|
|
||||||
"""
|
|
||||||
Returns the thoughts associated with the operation after improvement.
|
|
||||||
|
|
||||||
:return: List of improved thoughts.
|
|
||||||
:rtype: List[Thought]
|
|
||||||
"""
|
|
||||||
return self.thoughts
|
|
||||||
|
|
||||||
def _execute(
|
|
||||||
self, lm: AbstractLanguageModel, prompter: Prompter, parser: Parser, **kwargs
|
|
||||||
) -> None:
|
|
||||||
"""
|
|
||||||
Executes the Improve operation by improving the predecessors' thoughts.
|
|
||||||
The thoughts are improved by prompting the LM with the predecessors' thought states.
|
|
||||||
|
|
||||||
:param lm: The language model to be used.
|
|
||||||
:type lm: AbstractLanguageModel
|
|
||||||
:param prompter: The prompter for crafting prompts.
|
|
||||||
:type prompter: Prompter
|
|
||||||
:param parser: The parser for parsing responses.
|
|
||||||
:type parser: Parser
|
|
||||||
:param kwargs: Additional parameters for execution.
|
|
||||||
:raises AssertionError: If operation has no predecessors.
|
|
||||||
"""
|
|
||||||
previous_thoughts: List[Thought] = self.get_previous_thoughts()
|
|
||||||
|
|
||||||
assert len(self.predecessors) > 0, "Needs at least one predecessor"
|
|
||||||
|
|
||||||
for thought in previous_thoughts:
|
|
||||||
improve_prompt = prompter.improve_prompt(**thought.state)
|
|
||||||
self.logger.debug("Prompt for LM: %s", improve_prompt)
|
|
||||||
responses = lm.get_response_texts(lm.query(improve_prompt, num_responses=1))
|
|
||||||
self.logger.debug("Responses from LM: %s", responses)
|
|
||||||
state_update = parser.parse_improve_answer(thought.state, responses)
|
|
||||||
self.thoughts.append(Thought({**thought.state, **state_update}))
|
|
||||||
|
|
||||||
self.logger.info(
|
|
||||||
"Improve operation %d improved %d thoughts", self.id, len(self.thoughts)
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class Aggregate(Operation):
|
|
||||||
"""
|
|
||||||
Operation to aggregate thoughts.
|
|
||||||
"""
|
|
||||||
|
|
||||||
operation_type: OperationType = OperationType.aggregate
|
|
||||||
|
|
||||||
def __init__(self, num_responses: int = 1) -> None:
|
|
||||||
"""
|
|
||||||
Initializes a new Aggregate operation.
|
|
||||||
|
|
||||||
:param num_responses: Number of responses to use for aggregation. Defaults to 1.
|
|
||||||
:type num_responses: int
|
|
||||||
"""
|
|
||||||
super().__init__()
|
|
||||||
self.thoughts: List[Thought] = []
|
|
||||||
self.num_responses: int = num_responses
|
|
||||||
|
|
||||||
def get_thoughts(self) -> List[Thought]:
|
|
||||||
"""
|
|
||||||
Returns the thoughts associated with the operation after aggregation.
|
|
||||||
|
|
||||||
:return: List of aggregated thoughts.
|
|
||||||
:rtype: List[Thought]
|
|
||||||
"""
|
|
||||||
return self.thoughts
|
|
||||||
|
|
||||||
def _execute(
|
|
||||||
self, lm: AbstractLanguageModel, prompter: Prompter, parser: Parser, **kwargs
|
|
||||||
) -> None:
|
|
||||||
"""
|
|
||||||
Executes the Aggregate operation by aggregating the predecessors' thoughts.
|
|
||||||
The thoughts are aggregated by prompting the LM with the predecessors' thought states.
|
|
||||||
|
|
||||||
:param lm: The language model to be used.
|
|
||||||
:type lm: AbstractLanguageModel
|
|
||||||
:param prompter: The prompter for crafting prompts.
|
|
||||||
:type prompter: Prompter
|
|
||||||
:param parser: The parser for parsing responses.
|
|
||||||
:type parser: Parser
|
|
||||||
:param kwargs: Additional parameters for execution.
|
|
||||||
:raises AssertionError: If operation has no predecessors.
|
|
||||||
"""
|
|
||||||
assert (
|
|
||||||
len(self.predecessors) >= 1
|
|
||||||
), "Aggregate operation must have at least one predecessor"
|
|
||||||
|
|
||||||
previous_thoughts: List[Thought] = self.get_previous_thoughts()
|
|
||||||
|
|
||||||
if len(previous_thoughts) == 0:
|
|
||||||
return
|
|
||||||
|
|
||||||
# applied in order of score
|
|
||||||
base_state: Dict = {}
|
|
||||||
for thought in sorted(previous_thoughts, key=lambda thought: thought.score):
|
|
||||||
base_state = {**base_state, **thought.state}
|
|
||||||
|
|
||||||
previous_thought_states = [thought.state for thought in previous_thoughts]
|
|
||||||
prompt = prompter.aggregation_prompt(previous_thought_states)
|
|
||||||
|
|
||||||
self.logger.debug("Prompt for LM: %s", prompt)
|
|
||||||
|
|
||||||
responses = lm.get_response_texts(
|
|
||||||
lm.query(prompt, num_responses=self.num_responses)
|
|
||||||
)
|
|
||||||
|
|
||||||
self.logger.debug("Responses from LM: %s", responses)
|
|
||||||
|
|
||||||
parsed = parser.parse_aggregation_answer(previous_thought_states, responses)
|
|
||||||
|
|
||||||
if isinstance(parsed, dict):
|
|
||||||
parsed = [parsed]
|
|
||||||
for new_state in parsed:
|
|
||||||
self.thoughts.append(Thought({**base_state, **new_state}))
|
|
||||||
|
|
||||||
|
|
||||||
class KeepBestN(Operation):
|
|
||||||
"""
|
|
||||||
Operation to keep the best N thoughts from predecessors based on their score.
|
|
||||||
"""
|
|
||||||
|
|
||||||
operation_type: OperationType = OperationType.keep_best_n
|
|
||||||
|
|
||||||
def __init__(self, n: int, higher_is_better: bool = True) -> None:
|
|
||||||
"""
|
|
||||||
Initializes a new KeepBestN operation.
|
|
||||||
|
|
||||||
:param n: Maximum number of thoughts to keep.
|
|
||||||
:type n: int
|
|
||||||
:param higher_is_better: Whether higher scores are better. Defaults to True.
|
|
||||||
:type higher_is_better: bool
|
|
||||||
:raises AssertionError: If `n` is not greater than zero.
|
|
||||||
"""
|
|
||||||
super().__init__()
|
|
||||||
self.n: int = n
|
|
||||||
assert self.n > 0, "KeepBestN operation must keep at least one thought"
|
|
||||||
self.higher_is_better: bool = higher_is_better
|
|
||||||
self.thoughts: List[Thought] = []
|
|
||||||
|
|
||||||
def get_best_n(self) -> List[Thought]:
|
|
||||||
"""
|
|
||||||
Returns the best N thoughts from the predecessors based on their score.
|
|
||||||
|
|
||||||
:return: List of best N thoughts.
|
|
||||||
:rtype: List[Thought]
|
|
||||||
:raises AssertionError: If not all predecessors have been executed.
|
|
||||||
:raises AssertionError: If not all thoughts have been scored.
|
|
||||||
"""
|
|
||||||
previous_thoughts: List[Thought] = self.get_previous_thoughts()
|
|
||||||
assert all(
|
|
||||||
previous_thought.scored for previous_thought in previous_thoughts
|
|
||||||
), "Not all thoughts have been scored"
|
|
||||||
|
|
||||||
try:
|
|
||||||
return sorted(
|
|
||||||
previous_thoughts,
|
|
||||||
key=lambda thought: thought.score,
|
|
||||||
reverse=self.higher_is_better,
|
|
||||||
)[: self.n]
|
|
||||||
except:
|
|
||||||
self.logger.error("Error in KeepBestN operation")
|
|
||||||
self.logger.error(
|
|
||||||
"Previous operation: %s", [op.id for op in self.predecessors]
|
|
||||||
)
|
|
||||||
self.logger.error("Previous thoughts: %s", previous_thoughts)
|
|
||||||
self.logger.error(
|
|
||||||
"Scores: %s", [thought.score for thought in previous_thoughts]
|
|
||||||
)
|
|
||||||
return sorted(
|
|
||||||
[i for i in previous_thoughts if isinstance(i.score, float)],
|
|
||||||
key=lambda thought: thought.score,
|
|
||||||
reverse=self.higher_is_better,
|
|
||||||
)[: self.n]
|
|
||||||
|
|
||||||
def get_thoughts(self) -> List[Thought]:
|
|
||||||
"""
|
|
||||||
Returns the thoughts kept by the operation.
|
|
||||||
|
|
||||||
:return: List of kept thoughts.
|
|
||||||
:rtype: List[Thought]
|
|
||||||
"""
|
|
||||||
return self.thoughts
|
|
||||||
|
|
||||||
def _execute(
|
|
||||||
self, lm: AbstractLanguageModel, prompter: Prompter, parser: Parser, **kwargs
|
|
||||||
) -> None:
|
|
||||||
"""
|
|
||||||
Executes the KeepBestN operation by keeping the best N thoughts from the predecessors according to their score.
|
|
||||||
|
|
||||||
:param lm: The language model to be used.
|
|
||||||
:type lm: AbstractLanguageModel
|
|
||||||
:param prompter: The prompter for crafting prompts.
|
|
||||||
:type prompter: Prompter
|
|
||||||
:param parser: The parser for parsing responses.
|
|
||||||
:type parser: Parser
|
|
||||||
:param kwargs: Additional parameters for execution.
|
|
||||||
:raises AssertionError: If operation has no predecessors.
|
|
||||||
:raises AssertionError: If not all predecessors have been executed.
|
|
||||||
:raises AssertionError: If not all thoughts have been scored.
|
|
||||||
"""
|
|
||||||
assert (
|
|
||||||
len(self.predecessors) >= 1
|
|
||||||
), "KeepBestN operation must have at least one predecessor"
|
|
||||||
|
|
||||||
self.thoughts = [Thought.from_thought(thought) for thought in self.get_best_n()]
|
|
||||||
|
|
||||||
for thought in self.thoughts:
|
|
||||||
self.logger.debug(
|
|
||||||
"Thought %d with state %s kept", thought.id, thought.state
|
|
||||||
)
|
|
||||||
|
|
||||||
self.logger.info(
|
|
||||||
"KeepBestN operation %d kept %d thoughts", self.id, len(self.thoughts)
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class KeepValid(Operation):
|
|
||||||
"""
|
|
||||||
Operation to keep valid thoughts from predecessors.
|
|
||||||
"""
|
|
||||||
|
|
||||||
operation_type: OperationType = OperationType.keep_valid
|
|
||||||
|
|
||||||
def __init__(self) -> None:
|
|
||||||
"""
|
|
||||||
Initializes a new KeepValid operation.
|
|
||||||
"""
|
|
||||||
super().__init__()
|
|
||||||
self.thoughts: List[Thought] = []
|
|
||||||
|
|
||||||
def get_thoughts(self) -> List[Thought]:
|
|
||||||
"""
|
|
||||||
Returns the thoughts kept by the operation.
|
|
||||||
|
|
||||||
:return: List of kept thoughts.
|
|
||||||
:rtype: List[Thought]
|
|
||||||
"""
|
|
||||||
return self.thoughts
|
|
||||||
|
|
||||||
def _execute(
|
|
||||||
self, lm: AbstractLanguageModel, prompter: Prompter, parser: Parser, **kwargs
|
|
||||||
) -> None:
|
|
||||||
"""
|
|
||||||
Executes the KeepValid operation by keeping the valid thoughts from the predecessors.
|
|
||||||
Keeps unvalidated thoughts as well.
|
|
||||||
|
|
||||||
:param lm: The language model to be used.
|
|
||||||
:type lm: AbstractLanguageModel
|
|
||||||
:param prompter: The prompter for crafting prompts.
|
|
||||||
:type prompter: Prompter
|
|
||||||
:param parser: The parser for parsing responses.
|
|
||||||
:type parser: Parser
|
|
||||||
:param kwargs: Additional parameters for execution.
|
|
||||||
:raises AssertionError: If operation has no predecessors.
|
|
||||||
"""
|
|
||||||
assert (
|
|
||||||
len(self.predecessors) >= 1
|
|
||||||
), "KeepValid operation must have at least one predecessor"
|
|
||||||
|
|
||||||
self.thoughts: List[Thought] = [
|
|
||||||
Thought.from_thought(thought)
|
|
||||||
for thought in self.get_previous_thoughts()
|
|
||||||
if not thought.validated or thought.valid
|
|
||||||
]
|
|
||||||
|
|
||||||
if any(not thought.validated for thought in self.thoughts):
|
|
||||||
self.logger.warning(
|
|
||||||
"KeepValid operation %d has unvalidated thoughts", self.id
|
|
||||||
)
|
|
||||||
|
|
||||||
for thought in self.thoughts:
|
|
||||||
self.logger.debug(
|
|
||||||
"Thought %d with state %s kept", thought.id, thought.state
|
|
||||||
)
|
|
||||||
|
|
||||||
self.logger.info(
|
|
||||||
"KeepValid operation %d kept %d thoughts", self.id, len(self.thoughts)
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class GroundTruth(Operation):
|
|
||||||
"""
|
|
||||||
Operation to evaluate if thoughts correctly solve the problem, using a ground truth evaluator
|
|
||||||
"""
|
|
||||||
|
|
||||||
operation_type: OperationType = OperationType.ground_truth_evaluator
|
|
||||||
|
|
||||||
def __init__(self, ground_truth_evaluator: Callable[[Dict], bool]) -> None:
|
|
||||||
"""
|
|
||||||
Initializes a new GroundTruth operation.
|
|
||||||
|
|
||||||
:param ground_truth_evaluator: A function to evaluate if a thought solves the problem.
|
|
||||||
:type ground_truth_evaluator: A function that takes a thought state and returns a boolean.
|
|
||||||
"""
|
|
||||||
super().__init__()
|
|
||||||
self.ground_truth_evaluator: Callable[[Dict], bool] = ground_truth_evaluator
|
|
||||||
self.thoughts: List[Thought] = []
|
|
||||||
|
|
||||||
def get_thoughts(self) -> List[Thought]:
|
|
||||||
"""
|
|
||||||
Returns the thoughts associated with the operation.
|
|
||||||
|
|
||||||
:return: List of evaluated thoughts.
|
|
||||||
:rtype: List[Thought]
|
|
||||||
"""
|
|
||||||
return self.thoughts
|
|
||||||
|
|
||||||
def _execute(
|
|
||||||
self, lm: AbstractLanguageModel, prompter: Prompter, parser: Parser, **kwargs
|
|
||||||
) -> None:
|
|
||||||
"""
|
|
||||||
Executes the GroundTruth operation by evaluating the predecessors' thoughts using the ground truth evaluator function.
|
|
||||||
|
|
||||||
:param lm: The language model to be used.
|
|
||||||
:type lm: AbstractLanguageModel
|
|
||||||
:param prompter: The prompter for crafting prompts.
|
|
||||||
:type prompter: Prompter
|
|
||||||
:param parser: The parser for parsing responses.
|
|
||||||
:type parser: Parser
|
|
||||||
:param kwargs: Additional parameters for execution.
|
|
||||||
:raises AssertionError: If operation has no predecessor.
|
|
||||||
"""
|
|
||||||
assert (
|
|
||||||
len(self.predecessors) >= 1
|
|
||||||
), "GroundTruth operation must have at least one predecessor"
|
|
||||||
|
|
||||||
previous_thoughts: List[Thought] = self.get_previous_thoughts()
|
|
||||||
|
|
||||||
for thought in previous_thoughts:
|
|
||||||
new_thought = Thought.from_thought(thought)
|
|
||||||
try:
|
|
||||||
new_thought.solved = self.ground_truth_evaluator(new_thought.state)
|
|
||||||
except:
|
|
||||||
new_thought.solved = False
|
|
||||||
self.thoughts.append(new_thought)
|
|
||||||
|
|
||||||
self.logger.info(
|
|
||||||
"GroundTruth operation %d evaluated %d thoughts and %d solved the problem",
|
|
||||||
self.id,
|
|
||||||
len(self.thoughts),
|
|
||||||
len([thought for thought in self.thoughts if thought.solved]),
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class Selector(Operation):
|
|
||||||
"""
|
|
||||||
Operation to select thoughts from predecessors.
|
|
||||||
Useful for separating thoughts to perform different, subsequent operations on them.
|
|
||||||
"""
|
|
||||||
|
|
||||||
operation_type: OperationType = OperationType.selector
|
|
||||||
|
|
||||||
def __init__(self, selector: Callable[[List[Thought]], List[Thought]]) -> None:
|
|
||||||
"""
|
|
||||||
Initializes a new Selector operation.
|
|
||||||
|
|
||||||
:param selector: A function to select thoughts from the predecessors' thoughts.
|
|
||||||
:type selector: A function that takes a list of thoughts and returns a list of thoughts.
|
|
||||||
"""
|
|
||||||
super().__init__()
|
|
||||||
self.selector: Callable[[List[Thought]], List[Thought]] = selector
|
|
||||||
self.thoughts: List[Thought] = []
|
|
||||||
|
|
||||||
def get_thoughts(self) -> List[Thought]:
|
|
||||||
"""
|
|
||||||
Returns the thoughts selected by the operation.
|
|
||||||
|
|
||||||
:return: List of selected thoughts.
|
|
||||||
:rtype: List[Thought]
|
|
||||||
"""
|
|
||||||
return self.thoughts
|
|
||||||
|
|
||||||
def _execute(
|
|
||||||
self, lm: AbstractLanguageModel, prompter: Prompter, parser: Parser, **kwargs
|
|
||||||
) -> None:
|
|
||||||
"""
|
|
||||||
Executes the Selector operation by selecting thoughts from the predecessors using the selector function.
|
|
||||||
If the Selector has no predecessors, the selector function is called with a thought containing the kwargs as state.
|
|
||||||
|
|
||||||
:param lm: The language model to be used.
|
|
||||||
:type lm: AbstractLanguageModel
|
|
||||||
:param prompter: The prompter for crafting prompts.
|
|
||||||
:type prompter: Prompter
|
|
||||||
:param parser: The parser for parsing responses.
|
|
||||||
:type parser: Parser
|
|
||||||
:param kwargs: Additional parameters for execution.
|
|
||||||
"""
|
|
||||||
previous_thoughts: List[Thought] = self.get_previous_thoughts()
|
|
||||||
|
|
||||||
if len(previous_thoughts) == 0:
|
|
||||||
previous_thoughts = [Thought(kwargs)]
|
|
||||||
|
|
||||||
self.thoughts = [
|
|
||||||
Thought.from_thought(thought)
|
|
||||||
for thought in self.selector(previous_thoughts)
|
|
||||||
]
|
|
||||||
|
|
||||||
for thought in self.thoughts:
|
|
||||||
self.logger.debug(
|
|
||||||
"Thought %d with state %s selected", thought.id, thought.state
|
|
||||||
)
|
|
||||||
|
|
||||||
self.logger.info(
|
|
||||||
"Selector operation %d selected %d thoughts", self.id, len(self.thoughts)
|
|
||||||
)
|
|
||||||
@ -1,117 +0,0 @@
|
|||||||
# Copyright (c) 2023 ETH Zurich.
|
|
||||||
# All rights reserved.
|
|
||||||
#
|
|
||||||
# Use of this source code is governed by a BSD-style license that can be
|
|
||||||
# found in the LICENSE file.
|
|
||||||
#
|
|
||||||
# main author: Nils Blach
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
import logging
|
|
||||||
from typing import Iterator, Dict, Optional
|
|
||||||
import itertools
|
|
||||||
|
|
||||||
|
|
||||||
class Thought:
|
|
||||||
"""
|
|
||||||
Represents an LLM thought with its state, constructed by the parser, and various flags.
|
|
||||||
"""
|
|
||||||
|
|
||||||
_ids: Iterator[int] = itertools.count(0)
|
|
||||||
|
|
||||||
def __init__(self, state: Optional[Dict] = None) -> None:
|
|
||||||
"""
|
|
||||||
Initializes a new Thought instance with a state and various default flags.
|
|
||||||
|
|
||||||
:param state: The state of the thought. Defaults to None.
|
|
||||||
:type state: Optional[Dict]
|
|
||||||
"""
|
|
||||||
self.logger: logging.Logger = logging.getLogger(self.__class__.__name__)
|
|
||||||
self.id: int = next(Thought._ids)
|
|
||||||
self.state: Dict = state
|
|
||||||
self._score: float = 0.0
|
|
||||||
self._valid: bool = False
|
|
||||||
self._solved: bool = False
|
|
||||||
self.scored: bool = False
|
|
||||||
self.validated: bool = False
|
|
||||||
self.compared_to_ground_truth: bool = False
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def from_thought(thought: Thought) -> Thought:
|
|
||||||
"""
|
|
||||||
Creates a new thought from an existing one.
|
|
||||||
|
|
||||||
:param thought: An instance of a Thought to clone.
|
|
||||||
:return: A new Thought instance with properties copied from the input thought.
|
|
||||||
"""
|
|
||||||
new_thought = Thought(thought.state)
|
|
||||||
new_thought.score = thought.score
|
|
||||||
new_thought.valid = thought.valid
|
|
||||||
new_thought.solved = thought.solved
|
|
||||||
new_thought.scored = thought.scored
|
|
||||||
new_thought.validated = thought.validated
|
|
||||||
new_thought.compared_to_ground_truth = thought.compared_to_ground_truth
|
|
||||||
return new_thought
|
|
||||||
|
|
||||||
@property
|
|
||||||
def valid(self) -> bool:
|
|
||||||
"""
|
|
||||||
Returns the validity of the thought.
|
|
||||||
|
|
||||||
:return: The validity of the thought.
|
|
||||||
:rtype: bool
|
|
||||||
"""
|
|
||||||
return self._valid
|
|
||||||
|
|
||||||
@valid.setter
|
|
||||||
def valid(self, valid: bool) -> None:
|
|
||||||
"""
|
|
||||||
Sets the validity of the thought and the validated flag.
|
|
||||||
|
|
||||||
:param valid: The validity of the thought.
|
|
||||||
:type valid: bool
|
|
||||||
"""
|
|
||||||
self.validated = True
|
|
||||||
self._valid = valid
|
|
||||||
|
|
||||||
@property
|
|
||||||
def score(self) -> float:
|
|
||||||
"""
|
|
||||||
Returns the score of the thought.
|
|
||||||
|
|
||||||
:return: The score of the thought.
|
|
||||||
:rtype: float
|
|
||||||
"""
|
|
||||||
return self._score
|
|
||||||
|
|
||||||
@score.setter
|
|
||||||
def score(self, new_score: float) -> None:
|
|
||||||
"""
|
|
||||||
Sets the score of the thought and the scored flag.
|
|
||||||
|
|
||||||
:param new_score: The score of the thought.
|
|
||||||
:type new_score: float
|
|
||||||
"""
|
|
||||||
self.scored = True
|
|
||||||
self._score = new_score
|
|
||||||
|
|
||||||
@property
|
|
||||||
def solved(self) -> bool:
|
|
||||||
"""
|
|
||||||
Returns the solved flag of the thought.
|
|
||||||
|
|
||||||
:return: The solved flag of the thought.
|
|
||||||
:rtype: bool
|
|
||||||
"""
|
|
||||||
return self._solved
|
|
||||||
|
|
||||||
@solved.setter
|
|
||||||
def solved(self, solved: bool) -> None:
|
|
||||||
"""
|
|
||||||
Sets the solved flag of the thought and the compared_to_ground_truth flag.
|
|
||||||
|
|
||||||
:param solved: Whether the thought contains a solution to the problem.
|
|
||||||
:type solved: bool
|
|
||||||
"""
|
|
||||||
self.compared_to_ground_truth = True
|
|
||||||
self._solved = solved
|
|
||||||
@ -1 +0,0 @@
|
|||||||
from .parser import Parser
|
|
||||||
@ -1,90 +0,0 @@
|
|||||||
# Copyright (c) 2023 ETH Zurich.
|
|
||||||
# All rights reserved.
|
|
||||||
#
|
|
||||||
# Use of this source code is governed by a BSD-style license that can be
|
|
||||||
# found in the LICENSE file.
|
|
||||||
#
|
|
||||||
# main authors: Robert Gerstenberger, Nils Blach
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
from abc import ABC, abstractmethod
|
|
||||||
from typing import Dict, List, Union
|
|
||||||
|
|
||||||
|
|
||||||
class Parser(ABC):
|
|
||||||
"""
|
|
||||||
Abstract base class that defines the interface for all parsers.
|
|
||||||
Parsers are used to parse the responses from the language models.
|
|
||||||
"""
|
|
||||||
|
|
||||||
@abstractmethod
|
|
||||||
def parse_aggregation_answer(
|
|
||||||
self, states: List[Dict], texts: List[str]
|
|
||||||
) -> Union[Dict, List[Dict]]:
|
|
||||||
"""
|
|
||||||
Parse the response from the language model for a aggregation prompt.
|
|
||||||
|
|
||||||
:param states: The thought states used to generate the prompt.
|
|
||||||
:type states: List[Dict]
|
|
||||||
:param texts: The responses to the prompt from the language model.
|
|
||||||
:type texts: List[str]
|
|
||||||
:return: The new thought states after parsing the response from the language model.
|
|
||||||
:rtype: Union[Dict, List[Dict]]
|
|
||||||
"""
|
|
||||||
pass
|
|
||||||
|
|
||||||
@abstractmethod
|
|
||||||
def parse_improve_answer(self, state: Dict, texts: List[str]) -> Dict:
|
|
||||||
"""
|
|
||||||
Parse the response from the language model for an improve prompt.
|
|
||||||
|
|
||||||
:param state: The thought state used to generate the prompt.
|
|
||||||
:type state: Dict
|
|
||||||
:param texts: The responses to the prompt from the language model.
|
|
||||||
:type texts: List[str]
|
|
||||||
:return: The new thought state after parsing the response from the language model.
|
|
||||||
:rtype: Dict
|
|
||||||
"""
|
|
||||||
pass
|
|
||||||
|
|
||||||
@abstractmethod
|
|
||||||
def parse_generate_answer(self, state: Dict, texts: List[str]) -> List[Dict]:
|
|
||||||
"""
|
|
||||||
Parse the response from the language model for a generate prompt.
|
|
||||||
|
|
||||||
:param state: The thought state used to generate the prompt.
|
|
||||||
:type state: Dict
|
|
||||||
:param texts: The responses to the prompt from the language model.
|
|
||||||
:type texts: List[str]
|
|
||||||
:return: The new thought states after parsing the response from the language model.
|
|
||||||
:rtype: List[Dict]
|
|
||||||
"""
|
|
||||||
pass
|
|
||||||
|
|
||||||
@abstractmethod
|
|
||||||
def parse_validation_answer(self, state: Dict, texts: List[str]) -> bool:
|
|
||||||
"""
|
|
||||||
Parse the response from the language model for a validation prompt.
|
|
||||||
|
|
||||||
:param state: The thought state used to generate the prompt.
|
|
||||||
:type state: Dict
|
|
||||||
:param texts: The responses to the prompt from the language model.
|
|
||||||
:type texts: List[str]
|
|
||||||
:return: Whether the thought state is valid or not.
|
|
||||||
:rtype: bool
|
|
||||||
"""
|
|
||||||
pass
|
|
||||||
|
|
||||||
@abstractmethod
|
|
||||||
def parse_score_answer(self, states: List[Dict], texts: List[str]) -> List[float]:
|
|
||||||
"""
|
|
||||||
Parse the response from the language model for a score prompt.
|
|
||||||
|
|
||||||
:param states: The thought states used to generate the prompt.
|
|
||||||
:type states: List[Dict]
|
|
||||||
:param texts: The responses to the prompt from the language model.
|
|
||||||
:type texts: List[str]
|
|
||||||
:return: The scores for the thought states.
|
|
||||||
:rtype: List[float]
|
|
||||||
"""
|
|
||||||
pass
|
|
||||||
@ -1 +0,0 @@
|
|||||||
from .prompter import Prompter
|
|
||||||
@ -1,86 +0,0 @@
|
|||||||
# Copyright (c) 2023 ETH Zurich.
|
|
||||||
# All rights reserved.
|
|
||||||
#
|
|
||||||
# Use of this source code is governed by a BSD-style license that can be
|
|
||||||
# found in the LICENSE file.
|
|
||||||
#
|
|
||||||
# main authors: Robert Gerstenberger, Nils Blach
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
from abc import ABC, abstractmethod
|
|
||||||
from typing import Dict, List
|
|
||||||
|
|
||||||
|
|
||||||
class Prompter(ABC):
|
|
||||||
"""
|
|
||||||
Abstract base class that defines the interface for all prompters.
|
|
||||||
Prompters are used to generate the prompts for the language models.
|
|
||||||
"""
|
|
||||||
|
|
||||||
@abstractmethod
|
|
||||||
def aggregation_prompt(self, state_dicts: List[Dict], **kwargs) -> str:
|
|
||||||
"""
|
|
||||||
Generate a aggregation prompt for the language model.
|
|
||||||
|
|
||||||
:param state_dicts: The thought states that should be aggregated.
|
|
||||||
:type state_dicts: List[Dict]
|
|
||||||
:param kwargs: Additional keyword arguments.
|
|
||||||
:return: The aggregation prompt.
|
|
||||||
:rtype: str
|
|
||||||
"""
|
|
||||||
pass
|
|
||||||
|
|
||||||
@abstractmethod
|
|
||||||
def improve_prompt(self, **kwargs) -> str:
|
|
||||||
"""
|
|
||||||
Generate an improve prompt for the language model.
|
|
||||||
The thought state is unpacked to allow for additional keyword arguments
|
|
||||||
and concrete implementations to specify required arguments explicitly.
|
|
||||||
|
|
||||||
:param kwargs: Additional keyword arguments.
|
|
||||||
:return: The improve prompt.
|
|
||||||
:rtype: str
|
|
||||||
"""
|
|
||||||
pass
|
|
||||||
|
|
||||||
@abstractmethod
|
|
||||||
def generate_prompt(self, num_branches: int, **kwargs) -> str:
|
|
||||||
"""
|
|
||||||
Generate a generate prompt for the language model.
|
|
||||||
The thought state is unpacked to allow for additional keyword arguments
|
|
||||||
and concrete implementations to specify required arguments explicitly.
|
|
||||||
|
|
||||||
:param num_branches: The number of responses the prompt should ask the LM to generate.
|
|
||||||
:type num_branches: int
|
|
||||||
:param kwargs: Additional keyword arguments.
|
|
||||||
:return: The generate prompt.
|
|
||||||
:rtype: str
|
|
||||||
"""
|
|
||||||
pass
|
|
||||||
|
|
||||||
@abstractmethod
|
|
||||||
def validation_prompt(self, **kwargs) -> str:
|
|
||||||
"""
|
|
||||||
Generate a validation prompt for the language model.
|
|
||||||
The thought state is unpacked to allow for additional keyword arguments
|
|
||||||
and concrete implementations to specify required arguments explicitly.
|
|
||||||
|
|
||||||
:param kwargs: Additional keyword arguments.
|
|
||||||
:return: The validation prompt.
|
|
||||||
:rtype: str
|
|
||||||
"""
|
|
||||||
pass
|
|
||||||
|
|
||||||
@abstractmethod
|
|
||||||
def score_prompt(self, state_dicts: List[Dict], **kwargs) -> str:
|
|
||||||
"""
|
|
||||||
Generate a score prompt for the language model.
|
|
||||||
|
|
||||||
:param state_dicts: The thought states that should be scored,
|
|
||||||
if more than one, they should be scored together.
|
|
||||||
:type state_dicts: List[Dict]
|
|
||||||
:param kwargs: Additional keyword arguments.
|
|
||||||
:return: The score prompt.
|
|
||||||
:rtype: str
|
|
||||||
"""
|
|
||||||
pass
|
|
||||||
@ -1,12 +0,0 @@
|
|||||||
<p align="center">
|
|
||||||
<img src="poster.png" width="40%">
|
|
||||||
</p>
|
|
||||||
|
|
||||||
## Poster
|
|
||||||
The poster presented at the 2024 Association for the Advancement of Artificial Intelligence (AAAI'24) conference is available as [PDF](poster.pdf) and [PNG](poster.png).
|
|
||||||
|
|
||||||
## Plot Data
|
|
||||||
|
|
||||||
The data used to create the figures of the paper can be
|
|
||||||
found in the `final_results_gpt35.tar.bz2` archive. Unpack the archive
|
|
||||||
and run the file `plots.py`.
|
|
||||||
Binary file not shown.
File diff suppressed because one or more lines are too long
|
Before Width: | Height: | Size: 271 KiB |
337
paper/plots.py
337
paper/plots.py
@ -1,337 +0,0 @@
|
|||||||
# Copyright (c) 2023 ETH Zurich.
|
|
||||||
# All rights reserved.
|
|
||||||
#
|
|
||||||
# Use of this source code is governed by a BSD-style license that can be
|
|
||||||
# found in the LICENSE file.
|
|
||||||
#
|
|
||||||
# main author: Nils Blach
|
|
||||||
# contributions: Robert Gerstenberger
|
|
||||||
|
|
||||||
import json
|
|
||||||
import os
|
|
||||||
import matplotlib.pyplot as plt
|
|
||||||
|
|
||||||
|
|
||||||
def get_complete_results(base_directory):
|
|
||||||
results_complete = {}
|
|
||||||
for folder_name in os.listdir(base_directory):
|
|
||||||
folder_path = os.path.join(base_directory, folder_name)
|
|
||||||
if os.path.isdir(folder_path):
|
|
||||||
results_complete[folder_name] = []
|
|
||||||
for file_name in os.listdir(folder_path):
|
|
||||||
if file_name.endswith(".json"):
|
|
||||||
file_path = os.path.join(folder_path, file_name)
|
|
||||||
with open(file_path, "r") as f:
|
|
||||||
data = json.load(f)
|
|
||||||
results_complete[folder_name].append(
|
|
||||||
{"key": int(file_name.split(".")[0]), "data": data}
|
|
||||||
)
|
|
||||||
for key in results_complete.keys():
|
|
||||||
results_complete[key] = sorted(
|
|
||||||
results_complete[key], key=lambda x: x["key"]
|
|
||||||
)
|
|
||||||
return results_complete
|
|
||||||
|
|
||||||
|
|
||||||
def get_final_scores(results_complete):
|
|
||||||
scores = {}
|
|
||||||
for method in results_complete.keys():
|
|
||||||
scores[method] = []
|
|
||||||
for result in results_complete[method]:
|
|
||||||
score = 100
|
|
||||||
solved = False
|
|
||||||
cost = 1
|
|
||||||
prompt_tokens = 0
|
|
||||||
completion_tokens = 0
|
|
||||||
for op in result["data"]:
|
|
||||||
if "operation" in op and op["operation"] == "ground_truth_evaluator":
|
|
||||||
try:
|
|
||||||
score = min(op["scores"])
|
|
||||||
solved = any(op["problem_solved"])
|
|
||||||
except:
|
|
||||||
continue
|
|
||||||
if "cost" in op:
|
|
||||||
cost = op["cost"]
|
|
||||||
prompt_tokens = op["prompt_tokens"]
|
|
||||||
completion_tokens = op["completion_tokens"]
|
|
||||||
scores[method].append(
|
|
||||||
[result["key"], score, solved, prompt_tokens, completion_tokens, cost]
|
|
||||||
)
|
|
||||||
scores[method] = sorted(scores[method], key=lambda x: x[0])
|
|
||||||
return scores
|
|
||||||
|
|
||||||
|
|
||||||
def get_final_scores_doc_merge(results_complete):
|
|
||||||
scores = {}
|
|
||||||
for method in results_complete.keys():
|
|
||||||
scores[method] = []
|
|
||||||
for result in results_complete[method]:
|
|
||||||
score = 0
|
|
||||||
solved = False
|
|
||||||
cost = 1
|
|
||||||
prompt_tokens = 0
|
|
||||||
completion_tokens = 0
|
|
||||||
for op in reversed(result["data"]):
|
|
||||||
if "cost" in op:
|
|
||||||
cost = op["cost"]
|
|
||||||
prompt_tokens = op["prompt_tokens"]
|
|
||||||
completion_tokens = op["completion_tokens"]
|
|
||||||
if "operation" in op and op["operation"] == "score":
|
|
||||||
try:
|
|
||||||
score = max(op["scores"])
|
|
||||||
break
|
|
||||||
except:
|
|
||||||
continue
|
|
||||||
scores[method].append(
|
|
||||||
[result["key"], score, solved, prompt_tokens, completion_tokens, cost]
|
|
||||||
)
|
|
||||||
scores[method] = sorted(scores[method], key=lambda x: x[0])
|
|
||||||
return scores
|
|
||||||
|
|
||||||
|
|
||||||
def get_plotting_data(base_directory, score_method):
|
|
||||||
results_complete = get_complete_results(base_directory)
|
|
||||||
scores = score_method(results_complete)
|
|
||||||
results_plotting = {
|
|
||||||
method: {
|
|
||||||
"scores": [x[1] for x in scores[method]],
|
|
||||||
"solved": sum([1 for x in scores[method] if x[2]]),
|
|
||||||
"costs": [x[5] for x in scores[method]],
|
|
||||||
}
|
|
||||||
for method in scores.keys()
|
|
||||||
}
|
|
||||||
return results_plotting
|
|
||||||
|
|
||||||
|
|
||||||
def plot_results(
|
|
||||||
name,
|
|
||||||
results,
|
|
||||||
methods_order=["io", "cot", "tot", "tot2", "tog"],
|
|
||||||
methods_labels=["IO", "CoT", "ToT", "ToT2", "GoT"],
|
|
||||||
model="GPT-3.5",
|
|
||||||
length=32,
|
|
||||||
y_lower=0,
|
|
||||||
y_upper=16,
|
|
||||||
cost_upper=1.8,
|
|
||||||
display_solved=True,
|
|
||||||
annotation_offset=1,
|
|
||||||
display_left_ylabel=False,
|
|
||||||
display_right_ylabel=False,
|
|
||||||
):
|
|
||||||
methods_order = [method for method in methods_order if method in results]
|
|
||||||
# Extract scores based on the order
|
|
||||||
if name == "set_intersection":
|
|
||||||
scores_ordered = [
|
|
||||||
[min(score, length) for score in results[method]["scores"] if score != 1000]
|
|
||||||
for method in methods_order
|
|
||||||
]
|
|
||||||
elif name == "sorting":
|
|
||||||
scores_ordered = [
|
|
||||||
[
|
|
||||||
min(score, length)
|
|
||||||
for score in results[method]["scores"]
|
|
||||||
if score != 100 and score != 300
|
|
||||||
]
|
|
||||||
for method in methods_order
|
|
||||||
]
|
|
||||||
elif name == "keyword_counting":
|
|
||||||
scores_ordered = [
|
|
||||||
[
|
|
||||||
score
|
|
||||||
for score in results[method]["scores"]
|
|
||||||
if score != 100 and score != 300
|
|
||||||
]
|
|
||||||
for method in methods_order
|
|
||||||
]
|
|
||||||
elif name == "document_merging":
|
|
||||||
scores_ordered = [
|
|
||||||
[score for score in results[method]["scores"]] for method in methods_order
|
|
||||||
]
|
|
||||||
total_costs = [sum(results[method]["costs"]) for method in methods_order]
|
|
||||||
|
|
||||||
# Create figure and axis
|
|
||||||
if name == "keyword_counting" or name == "document_merging":
|
|
||||||
fig, ax = plt.subplots(dpi=150, figsize=(3.75, 5))
|
|
||||||
else:
|
|
||||||
fig, ax = plt.subplots(dpi=150, figsize=(2.5, 5))
|
|
||||||
|
|
||||||
# Create boxplots
|
|
||||||
positions = range(1, len(methods_order) + 1)
|
|
||||||
ax.boxplot(scores_ordered, positions=positions)
|
|
||||||
|
|
||||||
fig_fontsize = 12
|
|
||||||
|
|
||||||
# Set the ticks and labels
|
|
||||||
plt.yticks(fontsize=fig_fontsize)
|
|
||||||
ax.set_xticks(range(1, len(methods_order) + 1))
|
|
||||||
ax.set_xticks(range(1, len(methods_order) + 1))
|
|
||||||
if name == "keyword_counting":
|
|
||||||
ax.set_xticklabels(methods_labels, fontsize=10)
|
|
||||||
else:
|
|
||||||
ax.set_xticklabels(methods_labels, fontsize=fig_fontsize)
|
|
||||||
|
|
||||||
if name == "document_merging":
|
|
||||||
ax.set_ylim(y_lower, 12 if display_solved else 9.75)
|
|
||||||
else:
|
|
||||||
ax.set_ylim(y_lower, (y_upper + 2) if display_solved else y_upper + 1)
|
|
||||||
|
|
||||||
if name == "sorting" or name == "set_intersection":
|
|
||||||
ax1_yticks = range(
|
|
||||||
y_lower, y_upper + 1, 2 if length < 48 else (4 if length < 96 else 8)
|
|
||||||
)
|
|
||||||
ax.set_yticks(ax1_yticks)
|
|
||||||
|
|
||||||
if display_left_ylabel:
|
|
||||||
if name == "keyword_counting":
|
|
||||||
ax.set_ylabel(
|
|
||||||
f"Number of errors; the lower the better", fontsize=fig_fontsize
|
|
||||||
)
|
|
||||||
elif name == "document_merging":
|
|
||||||
ax.set_ylabel(
|
|
||||||
f"Score (out of 10); the higher the better", fontsize=fig_fontsize
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
ax.set_ylabel(
|
|
||||||
f"#incorrect elements; the lower the better", fontsize=fig_fontsize
|
|
||||||
)
|
|
||||||
|
|
||||||
if name == "sorting" or name == "set_intersection":
|
|
||||||
ax.set_title(f"{length} elements")
|
|
||||||
|
|
||||||
ax2 = ax.twinx()
|
|
||||||
ax2.bar(positions, total_costs, alpha=0.5, color="blue", label="Total Cost ($)")
|
|
||||||
ax2.yaxis.set_tick_params(colors="#1919ff", labelsize=fig_fontsize)
|
|
||||||
ax2.set_ylim(0, cost_upper)
|
|
||||||
number_of_ticks = len(ax.get_yticks())
|
|
||||||
tick_interval = cost_upper / (number_of_ticks)
|
|
||||||
ax2_ticks = [tick_interval * i for i in range(number_of_ticks)]
|
|
||||||
|
|
||||||
# Set custom tick positions for ax2
|
|
||||||
ax2.set_yticks(ax2_ticks)
|
|
||||||
|
|
||||||
if display_right_ylabel:
|
|
||||||
ax2.set_ylabel(
|
|
||||||
"Total Cost ($); the lower the better",
|
|
||||||
color="#1919ff",
|
|
||||||
fontsize=fig_fontsize,
|
|
||||||
)
|
|
||||||
|
|
||||||
if display_solved:
|
|
||||||
annotation_height = y_upper + annotation_offset
|
|
||||||
count = 1
|
|
||||||
for method in methods_order:
|
|
||||||
if method not in results:
|
|
||||||
continue
|
|
||||||
solved = results[method]["solved"]
|
|
||||||
ax.text(
|
|
||||||
count,
|
|
||||||
annotation_height,
|
|
||||||
f"{solved}",
|
|
||||||
ha="center",
|
|
||||||
va="bottom",
|
|
||||||
fontsize=fig_fontsize,
|
|
||||||
)
|
|
||||||
count += 1
|
|
||||||
|
|
||||||
model = model.replace(".", "").replace("-", "").lower()
|
|
||||||
if name == "keyword_counting" or name == "document_merging":
|
|
||||||
fig.savefig(f"{name}_{model}.pdf", bbox_inches="tight")
|
|
||||||
else:
|
|
||||||
fig.savefig(f"{name}_{model}_{length}.pdf", bbox_inches="tight")
|
|
||||||
|
|
||||||
|
|
||||||
plot_results(
|
|
||||||
"set_intersection",
|
|
||||||
get_plotting_data("set_intersection_gpt35_032", get_final_scores),
|
|
||||||
methods_order=["io", "cot", "tot", "tot2", "tog2"],
|
|
||||||
length=32,
|
|
||||||
y_upper=19,
|
|
||||||
cost_upper=2,
|
|
||||||
display_solved=True,
|
|
||||||
annotation_offset=0.5,
|
|
||||||
display_left_ylabel=True,
|
|
||||||
display_right_ylabel=True,
|
|
||||||
)
|
|
||||||
|
|
||||||
plot_results(
|
|
||||||
"set_intersection",
|
|
||||||
get_plotting_data("set_intersection_gpt35_064", get_final_scores),
|
|
||||||
methods_order=["io", "cot", "tot", "tot2", "tog2"],
|
|
||||||
length=64,
|
|
||||||
y_upper=32,
|
|
||||||
cost_upper=5.4,
|
|
||||||
display_solved=True,
|
|
||||||
annotation_offset=0.2,
|
|
||||||
display_left_ylabel=True,
|
|
||||||
display_right_ylabel=True,
|
|
||||||
)
|
|
||||||
|
|
||||||
plot_results(
|
|
||||||
"set_intersection",
|
|
||||||
get_plotting_data("set_intersection_gpt35_128", get_final_scores),
|
|
||||||
methods_order=["io", "cot", "tot", "tot2", "tog2"],
|
|
||||||
length=128,
|
|
||||||
y_upper=94,
|
|
||||||
cost_upper=12,
|
|
||||||
display_solved=True,
|
|
||||||
annotation_offset=-3,
|
|
||||||
display_left_ylabel=True,
|
|
||||||
display_right_ylabel=True,
|
|
||||||
)
|
|
||||||
|
|
||||||
plot_results(
|
|
||||||
"sorting",
|
|
||||||
get_plotting_data("sorting_gpt35_032", get_final_scores),
|
|
||||||
length=32,
|
|
||||||
display_solved=False,
|
|
||||||
annotation_offset=0.5,
|
|
||||||
display_left_ylabel=True,
|
|
||||||
display_right_ylabel=True,
|
|
||||||
)
|
|
||||||
|
|
||||||
plot_results(
|
|
||||||
"sorting",
|
|
||||||
get_plotting_data("sorting_gpt35_064", get_final_scores),
|
|
||||||
length=64,
|
|
||||||
y_upper=64,
|
|
||||||
cost_upper=5.1,
|
|
||||||
display_solved=False,
|
|
||||||
display_left_ylabel=True,
|
|
||||||
display_right_ylabel=True,
|
|
||||||
)
|
|
||||||
|
|
||||||
plot_results(
|
|
||||||
"sorting",
|
|
||||||
get_plotting_data("sorting_gpt35_128", get_final_scores),
|
|
||||||
length=128,
|
|
||||||
y_upper=128,
|
|
||||||
cost_upper=17,
|
|
||||||
display_solved=False,
|
|
||||||
display_left_ylabel=True,
|
|
||||||
display_right_ylabel=True,
|
|
||||||
)
|
|
||||||
|
|
||||||
plot_results(
|
|
||||||
"keyword_counting",
|
|
||||||
get_plotting_data("keyword_counting_gpt35", get_final_scores),
|
|
||||||
methods_order=["io", "cot", "tot", "tot2", "gsp4", "gsp8", "gspx"],
|
|
||||||
methods_labels=["IO", "CoT", "ToT", "ToT2", "GoT4", "GoT8", "GoTx"],
|
|
||||||
y_upper=35,
|
|
||||||
cost_upper=9,
|
|
||||||
display_solved=True,
|
|
||||||
annotation_offset=-0.3,
|
|
||||||
display_left_ylabel=True,
|
|
||||||
display_right_ylabel=True,
|
|
||||||
)
|
|
||||||
|
|
||||||
plot_results(
|
|
||||||
"document_merging",
|
|
||||||
get_plotting_data("document_merging_gpt35_16k", get_final_scores_doc_merge),
|
|
||||||
methods_order=["io", "cot", "tot", "gsp", "gsp2"],
|
|
||||||
methods_labels=["IO", "CoT", "ToT", "GoT", "GoT2"],
|
|
||||||
y_upper=10,
|
|
||||||
cost_upper=15,
|
|
||||||
display_solved=False,
|
|
||||||
display_left_ylabel=True,
|
|
||||||
display_right_ylabel=True,
|
|
||||||
)
|
|
||||||
BIN
paper/poster.pdf
BIN
paper/poster.pdf
Binary file not shown.
BIN
paper/poster.png
BIN
paper/poster.png
Binary file not shown.
|
Before Width: | Height: | Size: 1.5 MiB |
@ -1,47 +0,0 @@
|
|||||||
[build-system]
|
|
||||||
requires = ["hatchling"]
|
|
||||||
build-backend = "hatchling.build"
|
|
||||||
|
|
||||||
[project]
|
|
||||||
name = "graph_of_thoughts"
|
|
||||||
version = "0.0.3"
|
|
||||||
authors = [
|
|
||||||
{ name="Maciej Besta", email="maciej.besta@inf.ethz.ch" },
|
|
||||||
{ name="Nils Blach", email="nils.blach@inf.ethz.ch" },
|
|
||||||
{ name="Ales Kubicek", email="akubicek@student.ethz.ch" },
|
|
||||||
{ name="Robert Gerstenberger", email="gerstenberger.robert@gmail.com" },
|
|
||||||
]
|
|
||||||
description = "Python package for Graph of Thoughts that enables solving elaborate problems with Large Language Models"
|
|
||||||
readme = "README.md"
|
|
||||||
license = {file = "LICENSE"}
|
|
||||||
requires-python = ">=3.8"
|
|
||||||
classifiers = [
|
|
||||||
"Programming Language :: Python :: 3",
|
|
||||||
"Operating System :: OS Independent",
|
|
||||||
]
|
|
||||||
dependencies = [
|
|
||||||
"backoff>=2.2.1,<3.0.0",
|
|
||||||
"openai>=1.0.0,<2.0.0",
|
|
||||||
"pyyaml>=6.0.1,<7.0.0",
|
|
||||||
"matplotlib>=3.7.1,<4.0.0",
|
|
||||||
"numpy>=1.24.3,<2.0.0",
|
|
||||||
"pandas>=2.0.3,<3.0.0",
|
|
||||||
"sympy>=1.12,<2.0",
|
|
||||||
"torch>=2.0.1,<3.0.0",
|
|
||||||
"transformers>=4.31.0,<5.0.0",
|
|
||||||
"accelerate>=0.21.0,<1.0.0",
|
|
||||||
"bitsandbytes>=0.41.0,<1.0.0",
|
|
||||||
"scipy>=1.10.1,<2.0.0",
|
|
||||||
]
|
|
||||||
|
|
||||||
[project.optional-dependencies]
|
|
||||||
api = [
|
|
||||||
"fastapi>=0.109.0,<1.0.0",
|
|
||||||
"uvicorn[standard]>=0.27.0,<1.0.0",
|
|
||||||
]
|
|
||||||
|
|
||||||
[project.urls]
|
|
||||||
Homepage = "https://github.com/spcl/graph-of-thoughts"
|
|
||||||
|
|
||||||
[project.scripts]
|
|
||||||
got-openrouter-api = "graph_of_thoughts.api.app:run"
|
|
||||||
Loading…
x
Reference in New Issue
Block a user