|
- import os
- import sys
- import json
- from model_url import get_model_resp, get_url_tokenizer, process_inputstr, json_out_stream
- import pandas as pd
-
- def exists(path):
- """Test whether a path exists. Returns False for broken symbolic links"""
- try:
- os.stat(path)
- except (OSError, ValueError):
- return False
- return True
-
- bbh_multiple_choice_sets = [
- 'temporal_sequences',
- 'disambiguation_qa',
- 'date_understanding',
- 'tracking_shuffled_objects_three_objects',
- 'penguins_in_a_table',
- 'geometric_shapes',
- 'snarks',
- 'ruin_names',
- 'tracking_shuffled_objects_seven_objects',
- 'tracking_shuffled_objects_five_objects',
- 'logical_deduction_three_objects',
- 'hyperbaton',
- 'logical_deduction_five_objects',
- 'logical_deduction_seven_objects',
- 'movie_recommendation',
- 'salient_translation_error_detection',
- 'reasoning_about_colored_objects',
- ]
- bbh_free_form_sets = [
- 'multistep_arithmetic_two',
- 'navigate',
- 'dyck_languages',
- 'word_sorting',
- 'sports_understanding',
- 'boolean_expressions',
- 'object_counting',
- 'formal_fallacies',
- 'causal_judgement',
- 'web_of_lies',
- ]
-
- def run_predict(url, log_path, few_shot = True):
- import numpy as np
- tokenizer = get_url_tokenizer()
-
- id_label = {0: "A", 1: "B", 2: "C", 3: "D"}
- result = []
- File_Dir = "task_dataset/BBH/data"
- for _name in bbh_multiple_choice_sets:
- _hint = None
- # if exists(f"{File_Dir}/lib_prompt/{_name}.txt"):
- # _hint = open(f"{File_Dir}/lib_prompt/{_name}.txt", 'r').read()
- with open(os.path.join(File_Dir, f'{_name}.json'), 'r') as f:
- data = json.load(f)['examples']
- for i in data:
- input, target = i['input'], i['target']
- example = f"Q: {input}\n."
- result.append({"text": example, "src": "bbh"})
-
- for _name in bbh_free_form_sets:
- _hint = None
- with open(os.path.join(File_Dir, f'{_name}.json'), 'r') as f:
- data = json.load(f)['examples']
- for i in data:
- input, target = i['input'], i['target']
- example = f"Follow the given examples and answer the question.\nQ: {input}\nA: Let's think step by step."
- result.append({"text": example, "src": "bbh"})
-
- json_out_stream(log_path + '/bbh.json', result)
|