Computer.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "aa6162d7",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "ce1eb408",
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_json(json_file):\n",
    "    with open(json_file, 'r', encoding='utf-8') as file:\n",
    "        data = json.load(file)\n",
    "    return data\n",
    "\n",
    "def save_jsonl(data, jsonl_file):\n",
    "    with open(jsonl_file, 'w', encoding='utf-8') as file:\n",
    "        for entry in data:\n",
    "            json.dump(entry, file, ensure_ascii=False)\n",
    "            file.write('\\n')\n",
    "\n",
    "def save_json(data, json_file):\n",
    "    with open(json_file, 'w', encoding='utf-8') as file:\n",
    "        json.dump(data, file, ensure_ascii=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "57751d0e",
   "metadata": {},
   "outputs": [],
   "source": [
    "jsonl_file = 'computer_zh_26k.jsonl'\n",
    "#data = load_json(jsonl_file)\n",
    "\n",
    "result = []\n",
    "with open(jsonl_file, 'r', encoding='utf-8') as json_file:\n",
    "\n",
    "    # 遍历JSONL文件的每一行\n",
    "    for line in json_file:\n",
    "        # 解析JSON行\n",
    "        data = json.loads(line)\n",
    "        # 提取\"text\"字段\n",
    "        #text = data[\"text\"]\n",
    "        result.append(data)    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "5aa2fb79",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "20443"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "a55aa498",
   "metadata": {},
   "outputs": [],
   "source": [
    "save_json_file = 'computer.json'\n",
    "save_json(result, save_json_file)\n",
    "#save_jsonl_file = 'computer.jsonl'\n",
    "#save_jsonl(result, save_jsonl_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "7844f7a2",
   "metadata": {},
   "outputs": [],
   "source": [
    "computer = load_json('computer.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "ac40eacc",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_with_category = []\n",
    "data_no_category = []\n",
    "for item in computer:\n",
    "    tmp_no = {\n",
    "        'Question':\"\",\n",
    "        'Answer':\"\"\n",
    "              }\n",
    "    tmp_no['Question'] = item['conversation'][0]['human']\n",
    "    tmp_no['Answer'] = item['conversation'][0]['assistant']\n",
    "    \n",
    "    data_no_category.append(tmp_no)\n",
    "    \n",
    "    tmp_dic = {\n",
    "        'Category':\"\",\n",
    "        'Question':\"\",\n",
    "        'Answer':\"\"\n",
    "              }\n",
    "    tmp_dic['Category'] = item['category']\n",
    "    tmp_dic['Question'] = item['conversation'][0]['human']\n",
    "    tmp_dic['Answer'] = item['conversation'][0]['assistant']\n",
    "    \n",
    "    #json_str = json.dumps(tmp_dic, ensure_ascii=False)\n",
    "    data_with_category.append(tmp_dic)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "94f3fed5",
   "metadata": {},
   "outputs": [],
   "source": [
    "save_json(data_no_category,'computer_no_category_cn.json')\n",
    "save_jsonl(data_no_category,'computer_no_category_cn.jsonl')\n",
    "save_json(data_with_category,'computer_with_category_cn.json')\n",
    "save_jsonl(data_with_category,'computer_with_category_cn.jsonl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "a8e99a86",
   "metadata": {},
   "outputs": [],
   "source": [
    "new = load_json('computer_no_category_cn.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "a0cce768",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "20443"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(new)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f6e774e9",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}