|
- import os
- import sys
- import json
- from model_url import get_model_resp, get_url_tokenizer
- import pandas as pd
-
- def run_predict(url, log_path, few_shot = True):
- import numpy as np
- tokenizer = get_url_tokenizer()
- MAIN_DIR = os.path.dirname(os.path.abspath(__file__))
- File_Dir = MAIN_DIR + "/task_dataset/LCSTS"
- src_dir = File_Dir+ "/test.src.txt"
- tgt_dir = File_Dir+ "/test.tgt.txt"
-
- src_lines = open(src_dir, 'r', encoding='utf-8').readlines()
- tgt_lines = open(tgt_dir, 'r', encoding='utf-8').readlines()
-
- test_sample = [f"阅读以下文章,并给出简要概述的标题: {content}\n 标题如下:" for content in src_lines]
- #prompt = '阅读以下文章,并给出简短的摘要:{content}\n摘要如下:'
- #test_sample = test_sample[:5]
- #tgt_lines = tgt_lines[:5]
-
- rouge_L_sum = 0
- cnt = 0
- rouge_L = 0
-
- result = {}
- for idx, sample in enumerate(test_sample):
- tmp = {}
- # Tokenize input sentence to ids
- # start_sentence = tokenizer.encode(sample)
- output_samples = get_model_resp(url=url, input_str=sample, tokens_to_generate=100, top_k=1, logprobs=False)
- output_samples = tokenizer.decode(tokenizer.encode(output_samples))
- # Call inference
-
- print(f"----------------------{idx}------------------------")
- # print(f"Input is: {sample}\n")
- # print(f'Output is: {output_samples}\n')
- # print(f'label is: {tgt_lines[idx]}\n')
- tmp["input"] = sample
- tmp["predict"] = output_samples
- tmp['label'] = tgt_lines[idx]
- result[str(idx)] = tmp
- # update eval rouge_L score
- rouge_score = calc_rouge_L(output_samples, tgt_lines[idx])
- if rouge_score > -1:
- rouge_L_sum += rouge_score
- cnt += 1
- rouge_L = rouge_L_sum / cnt
- print(f"rough_L is {rouge_L}")
-
- print("=====================================================================")
- print(f"The Eval LCSTS Rouge is {rouge_L}")
- # Compute rouge score
-
- with open(log_path + '/lcsts.txt', 'w', encoding='utf-8') as file:
- file.write(f"lcsts, zero shot , rouge_L: {rouge_L}, number: {cnt}")
-
- with open(log_path + '/lcsts.json', 'w', encoding='utf-8') as file:
- file.write(json.dumps(result, ensure_ascii=False))
-
- # Rouge_L score
- def my_lcs(string, sub):
- """
- Calculates longest common subsequence for a pair of tokenized strings
- :param string : list of str : tokens from a string split using whitespace
- :param sub : list of str : shorter string, also split using whitespace
- :returns: length (list of int): length of the longest common subsequence between the two strings
- Note: my_lcs only gives length of the longest common subsequence, not the actual LCS
- """
- if len(string) < len(sub):
- sub, string = string, sub
-
- lengths = [[0 for _ in range(0,len(sub)+1)] for _ in range(0,len(string)+1)]
-
- for j in range(1,len(sub)+1):
- for i in range(1, len(string) + 1):
- if string[i - 1] == sub[j - 1]:
- lengths[i][j] = lengths[i-1][j-1] + 1
- else:
- lengths[i][j] = max(lengths[i-1][j] , lengths[i][j-1])
-
- return lengths[len(string)][len(sub)]
-
- def calc_rouge_L(hyp, refs, beta=1.2):
- rouge_L = -1
- try:
- rouge_L_list = []
- for ref in refs:
- lcs = my_lcs(ref, hyp)
- prec = lcs / max(len(hyp), 1e-10)
- rec = lcs / len(ref)
- if prec != 0 and rec !=0:
- rouge_L = ((1 + beta**2)*prec*rec)/float(rec + beta**2*prec)
- else:
- rouge_L = 0.0
- rouge_L_list.append(rouge_L)
- rouge_L = max(rouge_L_list)
- except:
- print('舍弃样本')
- return rouge_L
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
|