viz.py

# Copyright (c) Alibaba, Inc. and its affiliates.
# flake8: noqa

import argparse
import os
import re

import json
import pandas as pd
import plotly.graph_objects as go
import seaborn as sns
import streamlit as st
import yaml


def generate_color_palette(n):
    palette = sns.color_palette('hls', n)
    return palette.as_hex()


def read_yaml(yaml_file) -> dict:
    """
    Read yaml file to dict.
    """
    with open(yaml_file, 'r') as f:
        try:
            stream = yaml.safe_load(f)
        except yaml.YAMLError as e:
            print(e)
            raise e

    return stream


def read_jsonl(input_file):
    all_data = []
    with open(input_file, 'r') as input_jsonl:
        for line in input_jsonl:
            data = json.loads(line)
            all_data.append(data)

    return all_data


def cat_view(df, category, models):
    cat_df = df[df['category'] == category] if category != 'all' else df

    model_scores = {}
    for model in models:
        model_a_scores = (
            cat_df[cat_df['model_a'] == model]['scores'].apply(
                lambda x: x[0]).sum())
        model_b_scores = (
            cat_df[cat_df['model_b'] == model]['scores'].apply(
                lambda x: x[1]).sum())

        # calculate count of occurrences for model_a and model_b
        model_a_counts = cat_df[cat_df['model_a'] == model].shape[0]
        model_b_counts = cat_df[cat_df['model_b'] == model].shape[0]

        # calculate average scores for each model
        model_scores[model] = (model_a_scores + model_b_scores) / (
            model_a_counts + model_b_counts)

    return dict(
        category=category,
        count=cat_df['question_id'].nunique(),
        **dict(model_scores),
    )


def get_color(value):
    good_thresholds = [0.2, 0.1, 0.05]
    bad_thresholds = [-0.2, -0.1, -0.05]
    good_colors = ['#32CD32', '#98FF98', '#D0F0C0']
    bad_colors = ['#FF6347', '#FA8072', '#ffcccb']
    color = ''
    for i in range(len(good_thresholds)):
        if value > good_thresholds[i]:
            color = good_colors[i]
            break
    for i in range(len(bad_thresholds)):
        if value < bad_thresholds[i]:
            color = bad_colors[i]
            break
    return 'background-color: %s' % color if color else ''


def get_category_map(category_file):
    if not category_file or not os.path.exists(category_file):
        return dict()

    category_mapping = read_yaml(category_file)
    return category_mapping


def get_category_group(category_map, cat):
    for key, value in category_map.items():
        if cat in value or '*' in value:
            return key
    return cat


def show_table_view(df):
    models = df['model_a'].unique().tolist()
    for model in df['model_b'].unique().tolist():
        if model not in models:
            models.append(model)
    catogories = df['category'].unique().tolist()

    # Calculate the average score for each category
    cat_data = [cat_view(df, category, models) for category in catogories]
    cat_df = pd.DataFrame(cat_data)
    cat_df.sort_values(
        by=models[0], ascending=False, inplace=True, ignore_index=True)

    # Add total data
    total_data = cat_view(df, 'all', models)
    cat_df.loc[cat_df.shape[0]] = [
        '总计',
        total_data['count'],
        *[total_data[model] for model in models],
    ]

    # Render as link for each category
    cat_df['category'] = cat_df['category'].apply(
        lambda x:
        '<a href="?category={}&model_a={}&model_b={}" target="_self">{}</a>'.
        format(x, models[0], models[1], x), )

    # Format the table
    if len(models) == 2:
        cat_df['score diff'] = cat_df.apply(
            lambda x: (x[models[1]] - x[models[0]]) / x[models[0]], axis=1)
        cat_df.rename(
            columns={
                'category': '类别',
                'count': 'Case数量',
                'score diff': '分差百分比',
            },
            inplace=True,
        )
        style_df = cat_df.style.format({
            **dict(zip(models, ['{:.2f}' for model in models])),
            '分差百分比':
            '{:.2%}',
        }).applymap(
            get_color, subset=['分差百分比'])
    else:
        cat_df.rename(
            columns={
                'category': '类别',
                'count': 'Case数量',
            },
            inplace=True,
        )

        def format(row):
            p = '{:.2%}'.format(
                (row[model] - row[baseline_model]) / row[baseline_model])
            p = '+' + p if p[0] != '-' else p
            return f'{row[model]:.2f} ({p})'

        def color(value):
            match = re.search(r'\((.*?)\)', value)
            return get_color(float(match.group(1).strip('%'))
                             / 100) if match else ''

        baseline_model = models[0]
        for model in models[1:]:
            cat_df[model] = cat_df.apply(
                format,
                axis=1,
            )
        cat_df[baseline_model] = cat_df[baseline_model].apply(
            lambda x: '{:.2f}'.format(x))
        style_df = cat_df.style.applymap(color, subset=models[1:])

    # df_html = style_df.to_html(escape=False, index=False) # TODO
    df_html = style_df.to_html()
    st.markdown(df_html, unsafe_allow_html=True)


def show_radar_chart(df):
    score_list = []
    for index, row in df.iterrows():
        score_list.append(
            dict(
                model=row['model_a'],
                category=row['category'],
                score=row['scores'][0]))
        score_list.append(
            dict(
                model=row['model_b'],
                category=row['category'],
                score=row['scores'][1]))

    score_df = pd.DataFrame(score_list)
    df_agg = score_df.groupby(['model',
                               'category'])['score'].mean().reset_index()

    pivot_df = df_agg.pivot(
        index='model', columns='category', values='score').fillna(0)

    categories = pivot_df.columns.tolist()

    fig = go.Figure()

    num_models = len(pivot_df.index.tolist())
    color_palette = generate_color_palette(num_models)
    color_dict = dict(zip(pivot_df.index.tolist(), color_palette))

    for model in pivot_df.index.tolist():
        model_values = pivot_df[pivot_df.index == model].values.tolist()[0]
        model_values.append(model_values[0])  # Make the data cyclic
        fig.add_trace(
            go.Scatterpolar(
                r=model_values,
                theta=categories
                + [categories[0]],  # Make the categories cyclic
                fill='none',
                name=model,
                line=dict(color=color_dict[model]),
            ))

    fig.update_layout(
        polar=dict(radialaxis=dict(visible=True, range=[0, 10])),
        showlegend=True)
    st.plotly_chart(fig)


def show_single_result(df, category, model_a, model_b):
    categories = df['category'].unique().tolist()
    model_names = df['model_a'].unique().tolist()

    col1, col2 = st.columns([1, 3])
    with col1:
        category = st.selectbox(
            '选择类别',
            categories,
            index=categories.index(category) if category in categories else 0,
        )

    df = df[df['category'] == category]
    with col2:
        ques = st.selectbox('选择问题', df['question'].unique().tolist())

    col1, col2 = st.columns(2)
    with col1:
        model_a_options = [model_a]
        model_a = st.selectbox(
            '选择模型A',
            model_a_options,
            index=model_a_options.index(model_a),
        )

    with col2:
        model_b = st.selectbox('选择模型B',
                               [m for m in model_names if m != model_a])

    with st.container():
        st.markdown(
            """
<div style="background-color:#DEEBF7;padding:10px;margin:10px 0;border-radius:8px">
<b>问题：</b>
{ques}
</div>""".format(ques=ques),
            unsafe_allow_html=True,
        )

        col1, col2 = st.columns(2)
        with col1:
            st.write(
                """
<div style="background-color:#E2F0D9;padding:10px;border-radius:8px">
<p><b>{model_a} 回答</b></p>
{output_a}
</div>""".format(
                    model_a=model_a,
                    output_a=df[(df['question'] == ques) & (
                        df['model_a'] == model_a)].iloc[0]['output_a'],
                ),
                unsafe_allow_html=True,
            )
        with col2:
            st.write(
                """
<div style="background-color:#E2F0D9;padding:10px;border-radius:8px">
<p><b>{model_b} 回答</b></p>
{output_b}
</div>""".format(
                    model_b=model_b,
                    output_b=df[(df['question'] == ques) & (
                        df['model_b'] == model_b)].iloc[0]['output_b'],
                ),
                unsafe_allow_html=True,
            )

    score_1 = df[(df['question'] == ques)
                 & (df['model_a'] == model_a)].iloc[0]['scores']
    score_2 = df[(df['question'] == ques)
                 & (df['model_b'] == model_a)].iloc[0]['scores']

    scores = [
        {
            'round': '第一轮',
            model_a: score_1[0],
            model_b: score_1[1]
        },
        {
            'round': '第二轮',
            model_a: score_2[1],
            model_b: score_2[0]
        },
    ]

    score_df = pd.DataFrame(scores)
    styled_df = score_df.style.highlight_max(
        axis=1,
        subset=[model_a, model_b],
        color='lightgreen',
    )
    styled_df.format({
        model_a: '{:.1f}',
        model_b: '{:.1f}',
    })
    # score_html = styled_df.to_html(index=False)       # TODO
    score_html = styled_df.to_html()
    st.markdown(
        """
<div style="background-color:#FBE5D6;padding:10px;margin:10px 0;border-radius:8px">
<b>GPT-4 评分：</b>
{score_html}
</div>""".format(score_html=score_html),
        unsafe_allow_html=True,
    )


def run_app(review_file, category_file):
    category_map = get_category_map(category_file)

    review_file = os.path.abspath(review_file)
    data = read_jsonl(review_file)
    df = pd.DataFrame(data)
    df = df[[
        'model_a',
        'model_b',
        'scores',
        'category',
        'question_id',
        'question',
        'output_a',
        'output_b',
    ]]
    df['category'] = df['category'].apply(
        lambda x: get_category_group(category_map, x))

    query_params = st.experimental_get_query_params()
    if 'category' in query_params:
        st.set_page_config(layout='wide')
        st.write("<a href='/' target='_self'>返回</a>", unsafe_allow_html=True)
        show_single_result(
            df,
            query_params['category'][0],
            query_params['model_a'][0],
            query_params['model_b'][0],
        )
        st.write("<a href='/' target='_self'>返回</a>", unsafe_allow_html=True)
    else:
        st.set_page_config(layout='centered')
        st.write('### 评测结果展示（Arena 模式）')
        st.write('#### 模型分类别得分')
        show_table_view(df)
        st.write('#### 模型得分雷达图')
        show_radar_chart(df)


def parse_args():
    parser = argparse.ArgumentParser(description='Run visualization on a evaluation.')

    parser.add_argument('--review-file', type=str, default='evalscope/registry/data/qa_browser/battle.jsonl', required=True)
    parser.add_argument('--category-file', type=str, default='evalscope/registry/data/qa_browser/category_mapping.yaml', required=True)

    args = parser.parse_args()

    return args


def main():
    args = parse_args()
    print(args)
    run_app(args.review_file, args.category_file)


if __name__ == '__main__':

    print('**Usage:\n streamlit run viz.py -- --review-file evalscope/registry/data/qa_browser/battle.jsonl --category-file evalscope/registry/data/qa_browser/category_mapping.yaml')
    main()