microsoft/TypeAgent
Publicmirrored from https://github.com/microsoft/TypeAgentAvailable
python/ta/tools/vizcmp.py
97lines · modecode
| 1 | # Copyright (c) Microsoft Corporation. |
| 2 | # Licensed under the MIT License. |
| 3 | |
| 4 | import os |
| 5 | import glob |
| 6 | import re |
| 7 | import statistics |
| 8 | import sys |
| 9 | |
| 10 | from colorama import Back, Fore, Style |
| 11 | |
| 12 | |
| 13 | def main(): |
| 14 | files = sys.argv[1:] or sorted(glob.glob("evals/eval-*.txt")) |
| 15 | table = {} # {file: {counter: score, ...}, ...} |
| 16 | questions = {} # {counter: question, ...} |
| 17 | |
| 18 | # Fill table with scoring data from eval files |
| 19 | for file in files: |
| 20 | with open(file, "r") as f: |
| 21 | lines = f.readlines() |
| 22 | |
| 23 | scores = {} |
| 24 | counter = None |
| 25 | for i, line in enumerate(lines): |
| 26 | if m := re.match(r"^(?:-+|\*+)\s+(\d+)\s+", line): |
| 27 | counter = int(m.group(1)) |
| 28 | elif m := re.match(r"^Score:\s+([\d.]+); Question:\s+(.*)$", line): |
| 29 | score = float(m.group(1)) |
| 30 | scores[counter] = score |
| 31 | question = m.group(2) |
| 32 | if counter not in questions: |
| 33 | questions[counter] = question |
| 34 | elif questions[counter] != question: |
| 35 | print(f"File {file} has a different question for {counter}:") |
| 36 | print(f"< {questions[counter]}") |
| 37 | print(f"> {question}") |
| 38 | |
| 39 | table[file] = scores |
| 40 | |
| 41 | # Print header |
| 42 | all_files = list(table.keys()) |
| 43 | print_header(all_files) |
| 44 | |
| 45 | # Print data |
| 46 | all_counters = sorted( |
| 47 | {counter for data in table.values() for counter in data.keys()}, |
| 48 | key=lambda x: statistics.mean(table[file].get(x, 0.0) for file in all_files), |
| 49 | reverse=True, |
| 50 | ) |
| 51 | for counter in all_counters: |
| 52 | print(f"{counter:>3}:", end="") |
| 53 | for file in all_files: |
| 54 | score = table[file].get(counter, None) |
| 55 | if score is None: |
| 56 | output = Fore.YELLOW + " N/A " + Fore.RESET |
| 57 | output = Style.BRIGHT + output + Style.RESET_ALL |
| 58 | else: |
| 59 | output = f"{score:.3f}" |
| 60 | output = f"{output:>6}" |
| 61 | if score >= 0.97: |
| 62 | output = Fore.GREEN + output + Fore.RESET |
| 63 | if score >= 0.999: |
| 64 | output = Style.BRIGHT + output + Style.RESET_ALL |
| 65 | elif score >= 0.9: |
| 66 | output = Fore.BLUE + output + Fore.RESET |
| 67 | else: |
| 68 | output = Fore.RED + output + Fore.RESET |
| 69 | if score == 0.0: |
| 70 | output = Style.BRIGHT + output + Style.RESET_ALL |
| 71 | print(output, end="") |
| 72 | print(f" {questions.get(counter)}") |
| 73 | |
| 74 | # Print header again |
| 75 | print_footer(all_files) |
| 76 | |
| 77 | |
| 78 | def print_header(all_files): |
| 79 | print(" ", end="") |
| 80 | for i, file in enumerate(all_files): |
| 81 | base = os.path.basename(file) |
| 82 | m = re.match(r"eval-(\d+\w*).*\.txt", base) |
| 83 | if m: |
| 84 | label = m.group(1) |
| 85 | else: |
| 86 | label = "--" |
| 87 | print(f"{label:>6}", end="") |
| 88 | print() |
| 89 | |
| 90 | |
| 91 | def print_footer(all_files): |
| 92 | for i, file in reversed(list(enumerate(all_files))): |
| 93 | print(" |" * i + " " + os.path.basename(file)) |
| 94 | |
| 95 | |
| 96 | if __name__ == "__main__": |
| 97 | main() |
| 98 | |