Skip to content

Commit

Permalink
format
Browse files Browse the repository at this point in the history
  • Loading branch information
@infwinston
infwinston committed Jan 18, 2025
1 parent 02943c7 commit 2669ec3
Show file tree
Hide file tree
Showing 12 changed files with 267 additions and 128 deletions.
3 changes: 1 addition & 2 deletions fastchat/serve/monitor/basic_stats.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -142,7 +142,7 @@ def report_basic_stats(log_files):
height=300,
width=1200,
)

# calculate conversion rate for each day (vote / chat)
conversion_rate = {}
for date in chat_dates_counts.index:
Expand All @@ -163,7 +163,6 @@ def report_basic_stats(log_files):
width=1200,
)


# Model call counts
model_hist_all = df_all[df_all["type"] == "chat"]["model"].value_counts()
model_hist_1_day = df_1_day[df_1_day["type"] == "chat"]["model"].value_counts()
Expand Down
92 changes: 66 additions & 26 deletions fastchat/serve/monitor/clean_battle_data.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -146,7 +146,7 @@ def replace_model_name(old_name, tstamp):
"dumbledore-v3": "llama-3.2-vision-90b-instruct",
"potter-v1": "llama-3.2-vision-11b-instruct",
"sharp-game-player-v1": "llama-3.2-3b-instruct",
"zeus-flare-thunder-v1": "llama-3.2-1b-instruct",
"zeus-flare-thunder-v1": "llama-3.2-1b-instruct",
"qwen-vl-max-0809": "qwen2-vl-72b",
"gemini-1.5-pro-002-test-sp": "gemini-1.5-pro-002",
"gemini-1.5-flash-test-5": "gemini-1.5-flash-002",
Expand Down Expand Up @@ -185,7 +185,7 @@ def replace_model_name(old_name, tstamp):
return old_name
else:
return old_name + "-old"
if old_name == "eureka-chatbot":
if old_name == "eureka-chatbot":
if tstamp > 1721651521:
return "eureka-chatbot-v2"
else:
Expand Down Expand Up @@ -293,12 +293,17 @@ def process_data_txt2img(
if row["models"][0] is None or row["models"][1] is None:
count_dict["invalid"] += 1
continue
if not isinstance(row["models"][0], str) or not isinstance(row["models"][1], str):
if not isinstance(row["models"][0], str) or not isinstance(
row["models"][1], str
):
count_dict["invalid"] += 1
continue

# Resolve model names
models_public = [remove_html(row["models"][0]), remove_html(row["models"][1])]
models_public = [
remove_html(row["models"][0]),
remove_html(row["models"][1]),
]
if "model_name" in row["states"][0]:
models_hidden = [
row["states"][0]["model_name"],
Expand Down Expand Up @@ -356,12 +361,12 @@ def process_data_txt2img(
"anony_bothbad_vote": 0,
"anony_leftvote": 0,
"anony_rightvote": 0,
"sanitized_id": shortuuid.uuid()
"sanitized_id": shortuuid.uuid(),
}
all_ips[ip]["count"] += 1
if flag_anony:
all_ips[ip]["anony_count"] += 1
all_ips[ip]["anony_"+row["type"]] += 1
all_ips[ip]["anony_" + row["type"]] += 1

if sanitize_ip:
user_id = f"{all_ips[ip]['sanitized_id']}"
Expand Down Expand Up @@ -389,6 +394,7 @@ def process_data_txt2img(
)
return battles, count_dict, count_leak, all_ips


def process_data(
data,
exclude_model_names,
Expand Down Expand Up @@ -433,12 +439,17 @@ def process_data(
if row["models"][0] is None or row["models"][1] is None:
count_dict["invalid"] += 1
continue
if not isinstance(row["models"][0], str) or not isinstance(row["models"][1], str):
if not isinstance(row["models"][0], str) or not isinstance(
row["models"][1], str
):
count_dict["invalid"] += 1
continue

# Resolve model names
models_public = [remove_html(row["models"][0]), remove_html(row["models"][1])]
models_public = [
remove_html(row["models"][0]),
remove_html(row["models"][1]),
]
if "model_name" in row["states"][0]:
models_hidden = [
row["states"][0]["model_name"],
Expand Down Expand Up @@ -484,7 +495,6 @@ def process_data(
print(state["messages"][state["offset"]][1])
raise ValueError


# Drop conversations if the model names are leaked
messages = ""
for i in range(2):
Expand Down Expand Up @@ -576,12 +586,12 @@ def process_data(
"anony_bothbad_vote": 0,
"anony_leftvote": 0,
"anony_rightvote": 0,
"sanitized_id": shortuuid.uuid()
"sanitized_id": shortuuid.uuid(),
}
all_ips[ip]["count"] += 1
if flag_anony:
all_ips[ip]["anony_count"] += 1
all_ips[ip]["anony_"+row["type"]] += 1
all_ips[ip]["anony_" + row["type"]] += 1

if sanitize_ip:
user_id = f"{all_ips[ip]['sanitized_id']}"
Expand All @@ -607,13 +617,25 @@ def process_data(
)

user_tokens = sum(
[conv["num_tokens"] for conv in conversation_a if conv["role"] == "user"]
[
conv["num_tokens"]
for conv in conversation_a
if conv["role"] == "user"
]
)
assistant_a_tokens = sum(
[conv["num_tokens"] for conv in conversation_a if conv["role"] == "assistant"]
[
conv["num_tokens"]
for conv in conversation_a
if conv["role"] == "assistant"
]
)
assistant_b_tokens = sum(
[conv["num_tokens"] for conv in conversation_b if conv["role"] == "assistant"]
[
conv["num_tokens"]
for conv in conversation_b
if conv["role"] == "assistant"
]
)
context_tokens_a = sum([conv["num_tokens"] for conv in conversation_a[:-1]])
context_tokens_b = sum([conv["num_tokens"] for conv in conversation_b[:-1]])
Expand Down Expand Up @@ -702,30 +724,36 @@ def clean_battle_data(
all_ips[ip]["count"] += sub_all_ips[ip]["count"]
all_ips[ip]["anony_count"] += sub_all_ips[ip]["anony_count"]
all_ips[ip]["anony_tievote"] += sub_all_ips[ip]["anony_tievote"]
all_ips[ip]["anony_bothbad_vote"] += sub_all_ips[ip]["anony_bothbad_vote"]
all_ips[ip]["anony_bothbad_vote"] += sub_all_ips[ip][
"anony_bothbad_vote"
]
all_ips[ip]["anony_leftvote"] += sub_all_ips[ip]["anony_leftvote"]
all_ips[ip]["anony_rightvote"] += sub_all_ips[ip]["anony_rightvote"]

battles.sort(key=lambda x: x["tstamp"])
last_updated_tstamp = battles[-1]["tstamp"]

battles = pd.DataFrame(battles)

# drop rows with same question_id
print(f"before drop dups #battles: {len(battles)}")
battles = battles.drop_duplicates(subset=["question_id"], keep="first")
battles = battles.reset_index(drop=True)
print(f"#battles: {len(battles)}")

battles = battles[battles["anony"]].reset_index(drop=True) if anony_only else battles
battles = (
battles[battles["anony"]].reset_index(drop=True) if anony_only else battles
)
if run_dedup and not (vision or txt2img):
print("Running deduplication...")
battles = utils.dedup_process(battles)
num_dedup_battles = sum(battles["dedup_tag"].apply(lambda x: x["sampled"]))
print(f"#dedup_battles: {num_dedup_battles}")
else:
print("Skip deduplication...")
dedup_tags = np.array([{"high_freq": False, "sampled": True} for _ in range(len(battles))])
dedup_tags = np.array(
[{"high_freq": False, "sampled": True} for _ in range(len(battles))]
)
battles["dedup_tag"] = dedup_tags

last_updated_datetime = datetime.datetime.fromtimestamp(
Expand All @@ -746,7 +774,9 @@ def clean_battle_data(
for votetype in ["tievote", "bothbad_vote", "leftvote", "rightvote"]:
vote_key = "anony_" + votetype
userid_key = "sanitized_id" if sanitize_ip else "ip"
top_30_users = sorted(all_ips.values(), key=lambda x: x[vote_key], reverse=True)[:30]
top_30_users = sorted(
all_ips.values(), key=lambda x: x[vote_key], reverse=True
)[:30]
top_30_ip_id = ["arena_user_" + ip[userid_key] for ip in top_30_users]
battles_top_30_ips = battles[battles["judge"].isin(top_30_ip_id)]
print(f"Top 30 IPs #battles: {len(battles_top_30_ips)}")
Expand All @@ -755,13 +785,15 @@ def clean_battle_data(
for user in top_30_users:
user_ip = user["ip"]
user_id = "arena_user_" + user[userid_key]

ip_battles = battles_top_30_ips[battles_top_30_ips["judge"] == user_id]
win_count = len(ip_battles[ip_battles["winner"] == "model_a"])
tie_count = len(ip_battles[ip_battles["winner"] == "tie"])
loss_count = len(ip_battles[ip_battles["winner"] == "model_b"])
print(f"{user_id}: model_a {win_count}, tie {tie_count}, mobel_b {loss_count}, {user_ip}")

print(
f"{user_id}: model_a {win_count}, tie {tie_count}, mobel_b {loss_count}, {user_ip}"
)

return battles


Expand All @@ -784,8 +816,14 @@ def clean_battle_data(
ban_ip_list = json.load(open(args.ban_ip_file)) if args.ban_ip_file else None

battles = clean_battle_data(
log_files, args.exclude_model_names or [], ban_ip_list, args.sanitize_ip, anony_only=args.anony_only,
run_dedup=args.run_dedup, vision=args.vision, txt2img=args.txt2img
log_files,
args.exclude_model_names or [],
ban_ip_list,
args.sanitize_ip,
anony_only=args.anony_only,
run_dedup=args.run_dedup,
vision=args.vision,
txt2img=args.txt2img,
)
last_updated_tstamp = battles.iloc[-1]["tstamp"]
cutoff_date = datetime.datetime.fromtimestamp(
Expand All @@ -801,7 +839,9 @@ def clean_battle_data(
print(f"Write cleaned data to {output}")

if not args.txt2img:
battles = battles.drop(columns=["conversation_a", "conversation_b", "question_id"])
battles = battles.drop(
columns=["conversation_a", "conversation_b", "question_id"]
)
print("Samples:")
print(battles[:5])

Expand Down
24 changes: 15 additions & 9 deletions fastchat/serve/monitor/clean_chat_data.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,9 @@ def get_log_files(max_num_files=None, is_vision=False):
prefix = ""
if is_vision:
prefix = "vision-tmp-"
name = os.path.expanduser(f"~/fastchat_logs/server{i}/{prefix}{d}-conv.json")
name = os.path.expanduser(
f"~/fastchat_logs/server{i}/{prefix}{d}-conv.json"
)
if os.path.exists(name):
filenames.append(name)
max_num_files = max_num_files or len(filenames)
Expand Down Expand Up @@ -120,9 +122,7 @@ def clean_chat_data(log_files, action_type, remove_prompt=False, exclude_models=
msg = x["content"]
if isinstance(x["content"], list):
msg = x["content"][0]
x["num_tokens"] = len(
encoding.encode(msg, allowed_special="all")
)
x["num_tokens"] = len(encoding.encode(msg, allowed_special="all"))
messages_concat += msg.lower()

if remove_prompt:
Expand Down Expand Up @@ -185,21 +185,27 @@ def clean_chat_data(log_files, action_type, remove_prompt=False, exclude_models=
parser.add_argument("--action-type", type=str, default="chat")
parser.add_argument("--max-num-files", type=int)
parser.add_argument("--vision", action="store_true")
parser.add_argument("--start-time", type=str) # example: 2024-08-01
parser.add_argument("--end-time", type=str) # example: 2024-08-01
parser.add_argument("--start-time", type=str) # example: 2024-08-01
parser.add_argument("--end-time", type=str) # example: 2024-08-01
parser.add_argument("--remove-prompt", action="store_true")
parser.add_argument("--exclude-models", type=str, nargs="+", default=[])
args = parser.parse_args()

log_files = get_log_files(args.max_num_files, args.vision)
# print(log_files)
chats = clean_chat_data(log_files, args.action_type, args.remove_prompt, args.exclude_models)
chats = clean_chat_data(
log_files, args.action_type, args.remove_prompt, args.exclude_models
)
print(len(chats))
# convert to dataframe
chats = pd.DataFrame(chats)
if args.start_time is not None:
chats = chats[pd.to_datetime(chats["tstamp"], unit="s") >= pd.to_datetime(args.start_time)]
chats = chats[pd.to_datetime(chats["tstamp"], unit='s') < pd.to_datetime(args.end_time)]
chats = chats[
pd.to_datetime(chats["tstamp"], unit="s") >= pd.to_datetime(args.start_time)
]
chats = chats[
pd.to_datetime(chats["tstamp"], unit="s") < pd.to_datetime(args.end_time)
]
print(len(chats))

last_updated_tstamp = chats.iloc[-1]["tstamp"]
Expand Down
2 changes: 2 additions & 0 deletions fastchat/serve/monitor/dataset_release_scripts/lmsys_chat_1m/compute_stats.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -98,11 +98,13 @@ def to_remove(x):
# "lmsys/vicuna-7b-v1.5", use_fast=False
# )


def num_tokens_from_string(string: str) -> int:
encoding = tiktoken.encoding_for_model("gpt-4")
num_tokens = len(encoding.encode(string))
return num_tokens


prompts = []
responses = []
for conv in df["conversation"]:
Expand Down
Loading

0 comments on commit 2669ec3

Please sign in to comment.