hokkien_folktw.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: beautifulsoup4 in /Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages (4.7.1)\n",
      "Requirement already satisfied: soupsieve>=1.2 in /Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages (from beautifulsoup4) (1.9.2)\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "pip install beautifulsoup4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: requests in /Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages (2.22.0)\n",
      "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages (from requests) (3.0.4)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages (from requests) (2019.6.16)\n",
      "Requirement already satisfied: idna<2.9,>=2.5 in /Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages (from requests) (2.8)\n",
      "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages (from requests) (1.25.3)\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "pip install requests"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: Pandas in /Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages (0.24.2)\n",
      "Requirement already satisfied: python-dateutil>=2.5.0 in /Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages (from Pandas) (2.8.0)\n",
      "Requirement already satisfied: pytz>=2011k in /Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages (from Pandas) (2019.1)\n",
      "Requirement already satisfied: numpy>=1.12.0 in /Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages (from Pandas) (1.16.4)\n",
      "Requirement already satisfied: six>=1.5 in /Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages (from python-dateutil>=2.5.0->Pandas) (1.12.0)\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "pip install Pandas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: lxml in /Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages (4.3.4)\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "pip install lxml"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from bs4 import BeautifulSoup as soup\n",
    "import requests\n",
    "# grabbing the page\n",
    "\n",
    "def getHTML(url_list):\n",
    "    res = requests.get(url, timeout=30)\n",
    "    return (soup(res.text,'lxml'))\n",
    "\n",
    "def grab(parsed_soup, header, className,): \n",
    "    soup = parsed_soup.findAll(header, {\"class\": className})\n",
    "    return(soup)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "url_list = ['http://www.folktw.com.tw/culture.php','http://www.folktw.com.tw/folksay.php', 'http://www.folktw.com.tw/drama.php']\n",
    "links_list = []\n",
    "for url in url_list:\n",
    "#     print('log', url)\n",
    "    read_html = getHTML(url)\n",
    "    links_list.append(grab(read_html, 'div', 'list'))\n",
    "# print((links_list))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "pact = []\n",
    "count = 1\n",
    "for i in links_list:\n",
    "#     print('new_category_start')\n",
    "    for index in range(len(i)):\n",
    "        \n",
    "        \n",
    "        links = (grab(i[index], 'a', ''))\n",
    "        for li in links:\n",
    "            obj = {}\n",
    "#             print(count)\n",
    "            count += 1\n",
    "            # count individual link nums\n",
    "            link = li.get('href')\n",
    "            title = li.text\n",
    "#             print(title)\n",
    "#             print(link)\n",
    "        \n",
    "            obj['title'] = title\n",
    "            obj['link'] = link\n",
    "\n",
    "            pact.append(obj)\n",
    "    \n",
    "# print(pact)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                          link                      title\n",
      "0      culture_view.php?info=5                 台灣生命禮俗禁忌揭密\n",
      "1     culture_view.php?info=14                   娶鬼妻談「冥婚」\n",
      "2     culture_view.php?info=95                     孝女白瓊探源\n",
      "3      culture_view.php?info=8               從「沉地」傳說談台灣年俗\n",
      "4     culture_view.php?info=15                      新春戒嚴期\n",
      "5     culture_view.php?info=22              不要用北京習俗解釋台灣年俗\n",
      "6     culture_view.php?info=23                    燈火輝煌慶元宵\n",
      "7    culture_view.php?info=109                      乞龜保平安\n",
      "8     culture_view.php?info=25                    飲水思源清明節\n",
      "9     culture_view.php?info=26                   除瘟袪疫的端午節\n",
      "10    culture_view.php?info=27                  最浪漫的節日－七夕\n",
      "11    culture_view.php?info=28                    悲天憫人中元祭\n",
      "12    culture_view.php?info=29                    農曆七月鬼放假\n",
      "13    culture_view.php?info=30                  中元普度　撫慰亡靈\n",
      "14    culture_view.php?info=31                    月到中秋分外明\n",
      "15    culture_view.php?info=32                      冬至大如年\n",
      "16    culture_view.php?info=33                      尾牙與做牙\n",
      "17   culture_view.php?info=112                    多元的民間信仰\n",
      "18   culture_view.php?info=113                 在信仰背後無助的心靈\n",
      "19     culture_view.php?info=9                  神界的大哥大－天公\n",
      "20    culture_view.php?info=17                自然界的守護神－三界公\n",
      "21    culture_view.php?info=48                 農民守護神－神農大帝\n",
      "22    culture_view.php?info=49                 北極星崇拜－玄天上帝\n",
      "23   culture_view.php?info=114                 台灣人的媽媽——媽祖\n",
      "24    culture_view.php?info=50                 台灣的醫神－保生大帝\n",
      "25    culture_view.php?info=53                  情人的剋星－呂洞賓\n",
      "26    culture_view.php?info=52                社區的守護神－二結王公\n",
      "27    culture_view.php?info=51              兒童的守護神－七娘媽、床母\n",
      "28    culture_view.php?info=55                 鄉土的守護神—土地公\n",
      "29    culture_view.php?info=54                  三八擱好湊陣－濟公\n",
      "..                         ...                        ...\n",
      "248    drama_view.php?info=106            放下屠刀證菩提—秀琴《阿育王》\n",
      "249    drama_view.php?info=107          古今多少事都付笑談中—春美《周瑜》\n",
      "250      drama_view.php?info=9                 台灣民俗藝陣中的面具\n",
      "251    drama_view.php?info=108               台灣的行動劇場－民俗藝陣\n",
      "252    drama_view.php?info=109                 台灣的民俗藝陣（一）\n",
      "253    drama_view.php?info=110                 台灣的民俗藝陣（二）\n",
      "254    drama_view.php?info=111                 台灣的民俗藝陣（三）\n",
      "255    drama_view.php?info=112            台灣宋江陣形成之社會背景（一）\n",
      "256    drama_view.php?info=113            台灣宋江陣形成之社會背景（二）\n",
      "257    drama_view.php?info=114                   電子琴花車的魅力\n",
      "258    drama_view.php?info=135            台灣的街頭表演藝術──藝陣傳奇\n",
      "259    drama_view.php?info=136           藝氣風發‧陣勢飛揚─南瀛藝陣風華\n",
      "260     drama_view.php?info=10                   傳統劇團亟需援助\n",
      "261    drama_view.php?info=115                設立國家京劇團的荒謬性\n",
      "262    drama_view.php?info=116                  傳統戲曲的版權問題\n",
      "263    drama_view.php?info=117                    裁撤國家京劇隊\n",
      "264    drama_view.php?info=118                     評審亂象大觀\n",
      "265    drama_view.php?info=119             \\t國家扶植團隊徵選辦法省思\n",
      "266    drama_view.php?info=120                 樂透彩，傳統戲曲摃龜\n",
      "267    drama_view.php?info=121                   傳統曲藝再現紅樓\n",
      "268    drama_view.php?info=122                 針對需求扶植演藝團隊\n",
      "269    drama_view.php?info=123                 文化交流取代金錢外交\n",
      "270    drama_view.php?info=124                  金鐘獎最卑微的獎項\n",
      "271    drama_view.php?info=125  九十三年度教育部文藝創作獎「傳統戲劇劇本」評審感言\n",
      "272    drama_view.php?info=126               有好的劇本才有精采的戲劇\n",
      "273    drama_view.php?info=127                  \\t台灣的勞動歌謠\n",
      "274    drama_view.php?info=128                    遊唱詩人楊秀卿\n",
      "275    drama_view.php?info=129         從失戀歌曲看台灣女性感情自主性的轉變\n",
      "276    drama_view.php?info=131                 媒合駐館不是為了演出\n",
      "277    drama_view.php?info=134                 廖瓊枝的歌仔戲生命史\n",
      "\n",
      "[278 rows x 2 columns]\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "df = pd.DataFrame(pact)\n",
    "print(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}