microsoft/TypeAgent
Publicmirrored from https://github.com/microsoft/TypeAgentAvailable
python/ta/tools/vizcmp.py
124lines · modecode
| 1 | # Copyright (c) Microsoft Corporation. |
| 2 | # Licensed under the MIT License. |
| 3 | |
| 4 | import argparse |
| 5 | import glob |
| 6 | import os |
| 7 | import re |
| 8 | import statistics |
| 9 | import sys |
| 10 | |
| 11 | from colorama import init as colorama_init, Back, Fore, Style |
| 12 | |
| 13 | |
| 14 | def main(): |
| 15 | parser = argparse.ArgumentParser( |
| 16 | description="Compare evaluation results from multiple files." |
| 17 | ) |
| 18 | parser.add_argument( |
| 19 | "--color", |
| 20 | choices=["auto", "always", "never"], |
| 21 | default="auto", |
| 22 | help="Control color output. Default 'auto' uses colors if stdout is a terminal.", |
| 23 | ) |
| 24 | parser.add_argument( |
| 25 | "files", |
| 26 | nargs="*", |
| 27 | ) |
| 28 | args = parser.parse_args() |
| 29 | |
| 30 | # Initialize colorama according to --color. |
| 31 | match args.color: |
| 32 | case "auto": |
| 33 | colorama_init(strip=not sys.stdout.isatty()) |
| 34 | case "always": |
| 35 | colorama_init(strip=False) |
| 36 | case "never": |
| 37 | colorama_init(strip=True) |
| 38 | case _: |
| 39 | raise ValueError(f"Invalid color option: {args.color}") |
| 40 | |
| 41 | files = args.files or sorted(glob.glob("evals/eval-*.txt")) |
| 42 | table = {} # {file: {counter: score, ...}, ...} |
| 43 | questions = {} # {counter: question, ...} |
| 44 | |
| 45 | # Fill table with scoring data from eval files |
| 46 | for file in files: |
| 47 | with open(file, "r") as f: |
| 48 | lines = f.readlines() |
| 49 | |
| 50 | scores = {} |
| 51 | counter = None |
| 52 | for i, line in enumerate(lines): |
| 53 | if m := re.match(r"^(?:-+|\*+)\s+(\d+)\s+", line): |
| 54 | counter = int(m.group(1)) |
| 55 | elif m := re.match(r"^Score:\s+([\d.]+); Question:\s+(.*)$", line): |
| 56 | score = float(m.group(1)) |
| 57 | scores[counter] = score |
| 58 | question = m.group(2) |
| 59 | if counter not in questions: |
| 60 | questions[counter] = question |
| 61 | elif questions[counter] != question: |
| 62 | print(f"File {file} has a different question for {counter}:") |
| 63 | print(f"< {questions[counter]}") |
| 64 | print(f"> {question}") |
| 65 | |
| 66 | table[file] = scores |
| 67 | |
| 68 | # Print header |
| 69 | all_files = list(table.keys()) |
| 70 | print_header(all_files) |
| 71 | |
| 72 | # Print data |
| 73 | all_counters = sorted( |
| 74 | {counter for data in table.values() for counter in data.keys()}, |
| 75 | key=lambda x: statistics.mean(table[file].get(x, 0.0) for file in all_files), |
| 76 | reverse=True, |
| 77 | ) |
| 78 | for counter in all_counters: |
| 79 | print(f"{counter:>3}:", end="") |
| 80 | for file in all_files: |
| 81 | score = table[file].get(counter, None) |
| 82 | if score is None: |
| 83 | output = Fore.YELLOW + " N/A " + Fore.RESET |
| 84 | output = Style.BRIGHT + output + Style.RESET_ALL |
| 85 | else: |
| 86 | output = f"{score:.3f}" |
| 87 | output = f"{output:>6}" |
| 88 | if score >= 0.97: |
| 89 | output = Fore.GREEN + output + Fore.RESET |
| 90 | if score >= 0.999: |
| 91 | output = Style.BRIGHT + output + Style.RESET_ALL |
| 92 | elif score >= 0.9: |
| 93 | output = Fore.BLUE + output + Fore.RESET |
| 94 | else: |
| 95 | output = Fore.RED + output + Fore.RESET |
| 96 | if score == 0.0: |
| 97 | output = Style.BRIGHT + output + Style.RESET_ALL |
| 98 | print(output, end="") |
| 99 | print(f" {questions.get(counter)}") |
| 100 | |
| 101 | # Print header again |
| 102 | print_footer(all_files) |
| 103 | |
| 104 | |
| 105 | def print_header(all_files): |
| 106 | print(" ", end="") |
| 107 | for i, file in enumerate(all_files): |
| 108 | base = os.path.basename(file) |
| 109 | m = re.match(r"eval-(\d+\w*).*\.txt", base) |
| 110 | if m: |
| 111 | label = m.group(1) |
| 112 | else: |
| 113 | label = "--" |
| 114 | print(f"{label:>6}", end="") |
| 115 | print() |
| 116 | |
| 117 | |
| 118 | def print_footer(all_files): |
| 119 | for i, file in reversed(list(enumerate(all_files))): |
| 120 | print(" |" * i + " " + os.path.basename(file)) |
| 121 | |
| 122 | |
| 123 | if __name__ == "__main__": |
| 124 | main() |
| 125 | |