Demo6_tfrecords_file_reading.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "ename": "DuplicateFlagError",
     "evalue": "The flag 'cifar_dir' is defined twice. First from C:\\Users\\William Baker\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py, Second from C:\\Users\\William Baker\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py.  Description from first occurrence: 文件的目录",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mDuplicateFlagError\u001b[0m                        Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-3-fcaf255b2b95>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m     62\u001b[0m \u001b[0mFLAGS\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mapp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mflags\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mFLAGS\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     63\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 64\u001b[1;33m \u001b[0mtf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mapp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mflags\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mDEFINE_string\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"cifar_dir\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m\"./data/cifar10/cifar-10-batches-bin/\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m\"文件的目录\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     65\u001b[0m \u001b[0mtf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mapp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mflags\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mDEFINE_string\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"cifar_tfrecords\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m\"./tmp/cifar.tfrecords\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m\"存进tfrecords的文件\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     66\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\platform\\flags.py\u001b[0m in \u001b[0;36mwrapper\u001b[1;34m(*args, **kwargs)\u001b[0m\n\u001b[0;32m     56\u001b[0m           \u001b[1;34m'Use of the keyword argument names (flag_name, default_value, '\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     57\u001b[0m           'docstring) is deprecated, please use (name, default, help) instead.')\n\u001b[1;32m---> 58\u001b[1;33m     \u001b[1;32mreturn\u001b[0m \u001b[0moriginal_function\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     59\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     60\u001b[0m   \u001b[1;32mreturn\u001b[0m \u001b[0mtf_decorator\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmake_decorator\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0moriginal_function\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mwrapper\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\absl\\flags\\_defines.py\u001b[0m in \u001b[0;36mDEFINE_string\u001b[1;34m(name, default, help, flag_values, **args)\u001b[0m\n\u001b[0;32m    239\u001b[0m   \u001b[0mparser\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0m_argument_parser\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mArgumentParser\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    240\u001b[0m   \u001b[0mserializer\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0m_argument_parser\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mArgumentSerializer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 241\u001b[1;33m   \u001b[0mDEFINE\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mparser\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mname\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdefault\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mhelp\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mflag_values\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mserializer\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    242\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    243\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\absl\\flags\\_defines.py\u001b[0m in \u001b[0;36mDEFINE\u001b[1;34m(parser, name, default, help, flag_values, serializer, module_name, **args)\u001b[0m\n\u001b[0;32m     80\u001b[0m   \"\"\"\n\u001b[0;32m     81\u001b[0m   DEFINE_flag(_flag.Flag(parser, serializer, name, default, help, **args),\n\u001b[1;32m---> 82\u001b[1;33m               flag_values, module_name)\n\u001b[0m\u001b[0;32m     83\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     84\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\absl\\flags\\_defines.py\u001b[0m in \u001b[0;36mDEFINE_flag\u001b[1;34m(flag, flag_values, module_name)\u001b[0m\n\u001b[0;32m    102\u001b[0m   \u001b[1;31m# Copying the reference to flag_values prevents pychecker warnings.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    103\u001b[0m   \u001b[0mfv\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mflag_values\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 104\u001b[1;33m   \u001b[0mfv\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mflag\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mname\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mflag\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    105\u001b[0m   \u001b[1;31m# Tell flag_values who's defining the flag.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    106\u001b[0m   \u001b[1;32mif\u001b[0m \u001b[0mmodule_name\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\absl\\flags\\_flagvalues.py\u001b[0m in \u001b[0;36m__setitem__\u001b[1;34m(self, name, flag)\u001b[0m\n\u001b[0;32m    427\u001b[0m         \u001b[1;31m# module is simply being imported a subsequent time.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    428\u001b[0m         \u001b[1;32mreturn\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 429\u001b[1;33m       \u001b[1;32mraise\u001b[0m \u001b[0m_exceptions\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mDuplicateFlagError\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfrom_flag\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mname\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    430\u001b[0m     \u001b[0mshort_name\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mflag\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mshort_name\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    431\u001b[0m     \u001b[1;31m# If a new flag overrides an old one, we need to cleanup the old flag's\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mDuplicateFlagError\u001b[0m: The flag 'cifar_dir' is defined twice. First from C:\\Users\\William Baker\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py, Second from C:\\Users\\William Baker\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py.  Description from first occurrence: 文件的目录"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import os\n",
    "\n",
    "def csvread(filelist):\n",
    "    \"\"\"\n",
    "    读取CSV文件\n",
    "    :param filelist:文件路径+名字的列表\n",
    "    :return:返回读取的内容\n",
    "    \"\"\"\n",
    "    # 1、构造文件的队列\n",
    "    file_queue = tf.train.string_input_producer(filelist)\n",
    "    # 2、构造CSV阅读器读取队列数据（按照一行读取\n",
    "    reader = tf.TextLineReader()\n",
    "    key, value = reader.read(file_queue)\n",
    "    # 3、对每行内容进行解码\n",
    "    # 其中参数record_defaults：指定每个样本每列的类型及指定默认值，如records = [[\"None\"], [4.0]]\n",
    "    records = [[\"None\"], [\"None\"]]\n",
    "    example, label = tf.decode_csv(value, record_defaults=records)   # 对record解码，并指定每列的类型\n",
    "    # 4、想要读取多个数据，批处理实现\n",
    "    # 批处理的大小对队列、数据的数量没有影响，批处理大小只决定这批次取多少数据\n",
    "    example_batch, label_batch = tf.train.batch([example, label], batch_size=9, num_threads=1, capacity=9)\n",
    "    # print(example, label)\n",
    "    # print(value)\n",
    "    print(example_batch, label_batch)\n",
    "    return example_batch, label_batch\n",
    "\n",
    "def picread(filelist):\n",
    "    \"\"\"\n",
    "    读取狗图片并转换成张量\n",
    "    :param filelist: 文件路径+ 名字的列表\n",
    "    :return: 每张图片的张量\n",
    "    \"\"\"\n",
    "    # 1、构造文件队列\n",
    "    file_queue = tf.train.string_input_producer(filelist)\n",
    "\n",
    "    # 2、构造阅读器去读取图片内容（默认读取一张图片）\n",
    "    reader = tf.WholeFileReader()\n",
    "    key, value = reader.read(file_queue)\n",
    "    print(value)\n",
    "\n",
    "    # 3、对读取的图片数据进行解码\n",
    "    image = tf.image.decode_jpeg(value)\n",
    "    print(image)\n",
    "\n",
    "    # 4、处理图片的大小（统一大小）\n",
    "    image_resize = tf.image.resize_images(image, [200, 200])\n",
    "    print(image_resize)\n",
    "\n",
    "    # 注意：一定要把样本的形状固定 [200, 200, 3],在批处理的时候要求所有数据形状必须定义\n",
    "    image_resize.set_shape([200, 200, 3])\n",
    "\n",
    "    print(image_resize)\n",
    "\n",
    "    # 5、进行批处理\n",
    "    image_batch = tf.train.batch([image_resize], batch_size=20, num_threads=1, capacity=20)\n",
    "\n",
    "    print(image_batch)\n",
    "\n",
    "    return image_batch\n",
    "\n",
    "# 定义cifar的数据等命令行参数\n",
    "FLAGS = tf.app.flags.FLAGS\n",
    "\n",
    "tf.app.flags.DEFINE_string(\"cifar_dir\", \"./data/cifar10/cifar-10-batches-bin/\", \"文件的目录\")\n",
    "tf.app.flags.DEFINE_string(\"cifar_tfrecords\", \"./tmp/cifar.tfrecords\", \"存进tfrecords的文件\")\n",
    "\n",
    "class CifarRead(object):\n",
    "    \"\"\"完成读取二进制文件， 写进tfrecords，读取tfrecords\n",
    "    \"\"\"\n",
    "    def __init__(self, filelist):\n",
    "        # 文件列表\n",
    "        self.file_list = filelist\n",
    "\n",
    "        # 定义读取的图片的一些属性\n",
    "        self.height = 32\n",
    "        self.width = 32\n",
    "        self.channel = 3\n",
    "        # 二进制文件每张图片的字节\n",
    "        self.label_bytes = 1  #每张图片的标签占位1位字节\n",
    "        self.image_bytes = self.height * self.width * self.channel\n",
    "        self.bytes = self.label_bytes + self.image_bytes\n",
    "\n",
    "    def read_and_decode(self):\n",
    "        # 读取cifar文件 转换成张量\n",
    "\n",
    "        # 1、构造文件队列\n",
    "        file_queue = tf.train.string_input_producer(self.file_list)\n",
    "\n",
    "        # 2、构造二进制文件读取器，读取内容, 每个样本的字节数\n",
    "        reader = tf.FixedLengthRecordReader(self.bytes)\n",
    "\n",
    "        key, value = reader.read(file_queue)\n",
    "\n",
    "        # 3、解码内容, 二进制文件内容的解码\n",
    "        label_image = tf.decode_raw(value, tf.uint8)\n",
    "\n",
    "        print(label_image)\n",
    "\n",
    "        # 4、分割出图片和标签数据，切出特征值和目标值\n",
    "        label = tf.cast(tf.slice(label_image, [0], [self.label_bytes]), tf.int32)\n",
    "\n",
    "        image = tf.slice(label_image, [self.label_bytes], [self.image_bytes])\n",
    "\n",
    "        # 5、可以对图片的特征数据进行形状的改变 [3072] --> [32, 32, 3]\n",
    "        image_reshape = tf.reshape(image, [self.height, self.width, self.channel])\n",
    "\n",
    "        print(label, image_reshape)\n",
    "        # 6、批处理数据\n",
    "        image_batch, label_batch = tf.train.batch([image_reshape, label], batch_size=10, num_threads=1, capacity=10)\n",
    "\n",
    "        print(image_batch, label_batch)\n",
    "        return image_batch, label_batch\n",
    "\n",
    "    def write_ro_tfrecords(self, image_batch, label_batch):\n",
    "        \"\"\"\n",
    "        将图片的特征值和目标值存进tfrecords\n",
    "        :param image_batch: 10张图片的特征值\n",
    "        :param label_batch: 10张图片的目标值\n",
    "        :return: None\n",
    "        \"\"\"\n",
    "        # 1、建立TFRecord存储器\n",
    "        writer = tf.python_io.TFRecordWriter(FLAGS.cifar_tfrecords)\n",
    "\n",
    "        # 2、循环地将所有样本写入文件，每张图片样本都要构造example协议\n",
    "        for i in range(10):\n",
    "            # 取出第i个图片数据的特征值和目标值\n",
    "            image = image_batch[i].eval().tostring()\n",
    "\n",
    "            label = int(label_batch[i].eval()[0])\n",
    "\n",
    "            # 构造一个样本的example\n",
    "            example =  tf.train.Example(features=tf.train.Features(feature={\n",
    "                \"image\": tf.train.Feature(bytes_list=tf.train.BytesList(value=[image])),\n",
    "                \"label\": tf.train.Feature(int64_list=tf.train.Int64List(value=[label])),\n",
    "            }))\n",
    "\n",
    "            # 写入单独的样本\n",
    "            writer.write(example.SerializeToString())\n",
    "\n",
    "        # 关闭\n",
    "        writer.close()\n",
    "        return None\n",
    "\n",
    "    def read_from_tfrecords(self):\n",
    "\n",
    "        # 1、构造文件队列\n",
    "        file_queue = tf.train.string_input_producer([FLAGS.cifar_tfrecords])\n",
    "\n",
    "        # 2、构造文件阅读器，读取内容example,value=一个样本的序列化example\n",
    "        reader = tf.TFRecordReader()\n",
    "\n",
    "        key, value = reader.read(file_queue)\n",
    "\n",
    "        # 3、解析example\n",
    "        features = tf.parse_single_example(value, features={\n",
    "            \"image\": tf.FixedLenFeature([], tf.string),\n",
    "            \"label\": tf.FixedLenFeature([], tf.int64),\n",
    "        })\n",
    "\n",
    "        # 4、解码内容, 如果读取的内容格式是string需要解码， 如果是int64,float32不需要解码\n",
    "        image = tf.decode_raw(features[\"image\"], tf.uint8)\n",
    "\n",
    "        # 固定图片的形状，方便与批处理\n",
    "        image_reshape = tf.reshape(image, [self.height, self.width, self.channel])\n",
    "\n",
    "        label = tf.cast(features[\"label\"], tf.int32)\n",
    "\n",
    "        print(image_reshape, label)\n",
    "\n",
    "        # 进行批处理（必须的一步，因为默认都是处理的一个）\n",
    "        image_batch, label_batch = tf.train.batch([image_reshape, label], batch_size=10, num_threads=1, capacity=10)\n",
    "\n",
    "        return image_batch, label_batch\n",
    "\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    # 1、找到文件，放入列表  路径+名字->放到列表当中\n",
    "    file_name = os.listdir(FLAGS.cifar_dir)\n",
    "    filelist = [os.path.join(FLAGS.cifar_dir, file) for file in file_name if file[-3:] == \"bin\"] #切片截取后缀\n",
    "\n",
    "    # print(file_name)\n",
    "    # print(file_list)\n",
    "    cf = CifarRead(filelist)\n",
    "    # image_batch, label_batch = cf.read_and_decode()\n",
    "    image_batch, label_batch = cf.read_from_tfrecords()\n",
    "    # 开启会话，运行结果\n",
    "    with tf.Session() as sess:\n",
    "        # 定义一个线程协调器\n",
    "        coord = tf.train.Coordinator()\n",
    "        # 开启读取文件的线程\n",
    "        threads = tf.train.start_queue_runners(sess, coord=coord)\n",
    "\n",
    "        # 存进tfrecords文件\n",
    "        # print(\"开始存储\")\n",
    "        # cf.write_ro_tfrecords(image_batch, label_batch)\n",
    "        # print(\"结束存储\")\n",
    "        # for i in range()\n",
    "        # 打印读取的内容\n",
    "        print(sess.run([image_batch, label_batch]))\n",
    "\n",
    "        # 回收子线程\n",
    "        coord.request_stop()\n",
    "        coord.join(threads)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}