peach_training_finetuning.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Final-PEACH-training-emnlp2022.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "# Prerequisites"
      ],
      "metadata": {
        "id": "8NlwiGBC4v-J"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "langs = ['EN','DE','FR']\n",
        "vocab_addr = {\n",
        "    'EN' : \"/*address to EN vocab file*/\",\n",
        "    'FR' : \"/*address to FR vocab file*/\",\n",
        "    'DE' : \"/*address to DE vocab file*/\",\n",
        "    'Multi' : \"/*address to multlilingual vocab file*/\",\n",
        "}\n",
        "\n",
        "corpus_addr = {\n",
        "    'EN' : \"/content/en.txt\",\n",
        "    'FR' : \"/*address to FR corpus file*/\",\n",
        "    'DE' : \"/*address to DE corpus file*/\",\n",
        "}\n",
        "\n",
        "word_by_word_output_addr = {\n",
        "    'EN' : \"./en-wbw\",\n",
        "    'FR' : \"/*address to FR word-by-word translation files*/\",\n",
        "    'DE' : \"/*address to DE word-by-word translation files*/\",\n",
        "    'Multi' : \"/*address to  multilingual word-by-word translation improved files*/\"\n",
        "}\n",
        "\n",
        "denosing_output_addr = {\n",
        "    'EN' : \"./en-d.txt\",\n",
        "    'FR' : \"/*address to FR denoising file*/\",\n",
        "    'DE' : \"/*address to DE denoising file*/\",\n",
        "}\n",
        "\n",
        "output_model_addr = {\n",
        "    'EN' : \"./en-m\",\n",
        "    'FR' : \"/*address to FR denosing model output dir*/\",\n",
        "    'DE' : \"/*address to DE denosing model output dir*/\",\n",
        "    'Multi' : \"/*address to multlilingual model output dir*/\",\n",
        "}\n",
        "\n",
        "fine_tune_addr = {\n",
        "    \"EN-FR\" : \"/*address to fine-tuned on EN-FR model output dir*/\"\n",
        "}"
      ],
      "metadata": {
        "id": "YgsgC8f4442M"
      },
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Generating Denoising Pre-Training Data"
      ],
      "metadata": {
        "id": "BNdHlEov4Kqa"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import nltk\n",
        "nltk.download('punkt')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YpNw2k_yb6kG",
        "outputId": "1814952f-083f-4e51-a053-4d65d944033f"
      },
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Unzipping tokenizers/punkt.zip.\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {},
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "input_corpus = corpus_addr['EN']\n",
        "denoising = denosing_output_addr['EN']\n",
        "!python pretrain/peach/denoising/main.py pretrain/peach/denoising/config-en.json $input_corpus $denoising"
      ],
      "metadata": {
        "id": "wcpy3DQU6sC_"
      },
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "input_corpus = corpus_addr['FR']\n",
        "denoising = denosing_output_addr['FR']\n",
        "!python pretrain/peach/denoising/main.py pretrain/peach/denoising/config-en.json $input_corpus $denoising"
      ],
      "metadata": {
        "id": "Yxo2Pz5J70gp"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "input_corpus = corpus_addr['DE']\n",
        "denoising = denosing_output_addr['DE']\n",
        "!python pretrain/peach/denoising/main.py pretrain/peach/denoising/config-en.json $input_corpus $denoising"
      ],
      "metadata": {
        "id": "lwVUvH0s71MO"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Pre-Training Denoising models"
      ],
      "metadata": {
        "id": "YpQ4nXLT3gkw"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install -r PEACH/models/requirements.txt"
      ],
      "metadata": {
        "id": "DqQgXcOegEZ_"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install -e PEACH/models/"
      ],
      "metadata": {
        "id": "IOVME6ShgSU4"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Note:** First, fill the addresses field in the dataset files properly due to the guideline provided, then execute next blocks."
      ],
      "metadata": {
        "id": "sXaEDA0SBCQR"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!cp models/peach/datasets/denoising/*.py /usr/local/lib/python3.7/dist-packages/tensorflow_datasets/translate"
      ],
      "metadata": {
        "id": "epoyVe-qkza0"
      },
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!cat models/peach/datasets/denoising/imports.txt >> /usr/local/lib/python3.7/dist-packages/tensorflow_datasets/translate/__init__.py"
      ],
      "metadata": {
        "id": "JKjTfP5KnmuP"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "vocab = vocab_addr['EN']\n",
        "output_dir = output_model_addr['EN']\n",
        "!python3 models/peach/bin/train.py --params=\"en-denosing\" \\\n",
        "--param_overrides=vocab_filename=$vocab \\\n",
        "--model_dir=$output_dir \\"
      ],
      "metadata": {
        "id": "1Unv_tYYn17g"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "vocab = vocab_addr['FR']\n",
        "output_dir = output_model_addr['FR']\n",
        "!python3 models/peach/bin/train.py --params=\"fr-denosing\" \\\n",
        "--param_overrides=vocab_filename=$vocab \\\n",
        "--model_dir=$output_dir \\"
      ],
      "metadata": {
        "id": "ZZ1nKk_Soff8"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "vocab = vocab_addr['DE']\n",
        "output_dir = output_model_addr['DE']\n",
        "!python3 models/peach/bin/train.py --params=\"de-denosing\" \\\n",
        "--param_overrides=vocab_filename=$vocab \\\n",
        "--model_dir=$output_dir \\"
      ],
      "metadata": {
        "id": "cgVdpmOvoiIh"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Generating Word-By-Word Translation Data"
      ],
      "metadata": {
        "id": "yHjNaxqB4UCO"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "pip install -r PEACH/pretrain/requirements.txt"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "E8QFywIKjTdX",
        "outputId": "c31fc0ad-22e9-4276-b59e-b3142a492f24"
      },
      "execution_count": 40,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: regex in /usr/local/lib/python3.7/dist-packages (from -r PEACH/pretrain/requirements.txt (line 1)) (2022.6.2)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from -r PEACH/pretrain/requirements.txt (line 2)) (1.18.5)\n",
            "Requirement already satisfied: nltk in /usr/local/lib/python3.7/dist-packages (from -r PEACH/pretrain/requirements.txt (line 3)) (3.7)\n",
            "Requirement already satisfied: polyglot in /usr/local/lib/python3.7/dist-packages (from -r PEACH/pretrain/requirements.txt (line 4)) (16.7.4)\n",
            "Requirement already satisfied: PyICU in /usr/local/lib/python3.7/dist-packages (from -r PEACH/pretrain/requirements.txt (line 5)) (2.9)\n",
            "Requirement already satisfied: pycld2 in /usr/local/lib/python3.7/dist-packages (from -r PEACH/pretrain/requirements.txt (line 6)) (0.41)\n",
            "Collecting Morfessor\n",
            "  Downloading Morfessor-2.0.6-py3-none-any.whl (35 kB)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.7/dist-packages (from nltk->-r PEACH/pretrain/requirements.txt (line 3)) (4.64.0)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from nltk->-r PEACH/pretrain/requirements.txt (line 3)) (1.1.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from nltk->-r PEACH/pretrain/requirements.txt (line 3)) (7.1.2)\n",
            "Installing collected packages: Morfessor\n",
            "Successfully installed Morfessor-2.0.6\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!bash PEACH/pretrain/peach/translation/requirements.sh"
      ],
      "metadata": {
        "id": "uH-ur9hMfTf9"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "input_corpus = corpus_addr['EN']\n",
        "output_w_by_w = word_by_word_output_addr['EN']\n",
        "!python pretrain/peach/translation/main.py pretrain/peach/translation/config-en.json $input_corpus $output_w_by_w"
      ],
      "metadata": {
        "id": "WYbly3cZ_kur"
      },
      "execution_count": 50,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "input_corpus = corpus_addr['FR']\n",
        "output_w_by_w = word_by_word_output_addr['FR']\n",
        "!python pretrain/peach/translation/main.py pretrain/peach/translation/config-fr.json $input_corpus $output_w_by_w"
      ],
      "metadata": {
        "id": "P8TyrZ2F_kus"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "input_corpus = corpus_addr['DE']\n",
        "output_w_by_w = word_by_word_output_addr['DE']\n",
        "!python pretrain/peach/translation/main.py pretrain/peach/translation/config-de.json $input_corpus $output_w_by_w"
      ],
      "metadata": {
        "id": "GvoW5UB6_kus"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Improving the Quality of Word-By-Word Translation Data Using Pre-Trained Denosing Models"
      ],
      "metadata": {
        "id": "GDTBU1Ds4cuV"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Note:** First, fill the addresses field in the dataset files properly due to the guideline provided, then execute next blocks."
      ],
      "metadata": {
        "id": "EOIJqlzSBf4O"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!cp models/peach/datasets/improve/*.py /usr/local/lib/python3.7/dist-packages/tensorflow_datasets/translate"
      ],
      "metadata": {
        "id": "jBU0qRMqBf4P"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!cat models/peach/datasets/improve/imports.txt >> /usr/local/lib/python3.7/dist-packages/tensorflow_datasets/translate/__init__.py"
      ],
      "metadata": {
        "id": "xRtxiF0wBf4P"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import os"
      ],
      "metadata": {
        "id": "u-GqcKtmNAXE"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "vocab = vocab_addr['EN']\n",
        "output_dir = os.path.join(output_model_addr['EN'],\"model.ckpt-500000\")\n",
        "number_of_preds = 1000000\n",
        "!python3 models/peach/bin/predict.py --params=\"en-improve\" \\\n",
        "--param_overrides=vocab_filename=$vocab \\\n",
        "--model_dir=$output_dir \\\n",
        "--evaluate_test=\"True\" \\\n",
        "--total_predictions=$number_of_preds \\"
      ],
      "metadata": {
        "id": "pGVT-4ODosQY"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "vocab = vocab_addr['FR']\n",
        "output_dir = os.path.join(output_model_addr['FR'],\"model.ckpt-500000\")\n",
        "number_of_preds = 1000000\n",
        "!python3 models/peach/bin/predict.py --params=\"fr-improve\" \\\n",
        "--param_overrides=vocab_filename=$vocab \\\n",
        "--model_dir=$output_dir/model.ckpt-500000 \\\n",
        "--evaluate_test=\"True\" \\\n",
        "--total_predictions=$number_of_preds \\"
      ],
      "metadata": {
        "id": "AniWnbds62pc"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "vocab = vocab_addr['DE']\n",
        "output_dir = os.path.join(output_model_addr['DE'],\"model.ckpt-500000\")\n",
        "number_of_preds = 1000000\n",
        "!python3 models/peach/bin/predict.py --params=\"de-improve\" \\\n",
        "--param_overrides=vocab_filename=$vocab \\\n",
        "--model_dir=$output_dir/model.ckpt-500000 \\\n",
        "--evaluate_test=\"True\" \\\n",
        "--total_predictions=$number_of_preds \\"
      ],
      "metadata": {
        "id": "YK908jgd64md"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import glob\n",
        "import os\n",
        "en_pred_files = glob.glob(os.path.join(output_model_addr['EN'],\"predictions*.txt\"))\n",
        "en_targ_files = glob.glob(os.path.join(output_model_addr['EN'], \"targets*.txt\"))\n",
        "en_inp_files = glob.glob(os.path.join(output_model_addr['EN'], \"inputs*.txt\"))\n",
        "en_pred_files.sort()\n",
        "en_targ_files.sort()\n",
        "en_inp_files.sort()\n",
        "\n",
        "fr_pred_files = glob.glob(os.path.join(output_model_addr['FR'], \"predictions*.txt\"))\n",
        "fr_targ_files = glob.glob(os.path.join(output_model_addr['FR'], \"targets*.txt\"))\n",
        "fr_inp_files = glob.glob(os.path.join(output_model_addr['FR'], \"inputs*.txt\"))\n",
        "fr_pred_files.sort()\n",
        "fr_targ_files.sort()\n",
        "fr_inp_files.sort()\n",
        "\n",
        "de_pred_files = glob.glob(os.path.join(output_model_addr['DE'], \"predictions*.txt\"))\n",
        "de_targ_files = glob.glob(os.path.join(output_model_addr['DE'], \"targets*.txt\"))\n",
        "de_inp_files = glob.glob(os.path.join(output_model_addr['DE'], \"inputs*.txt\"))\n",
        "de_pred_files.sort()\n",
        "de_targ_files.sort()\n",
        "de_inp_files.sort()\n",
        "\n",
        "pred_files = en_pred_files + fr_pred_files + de_pred_files\n",
        "targ_files = en_targ_files + fr_targ_files + de_targ_files\n",
        "inp_files = en_inp_files + fr_inp_files + de_inp_files\n",
        "\n",
        "preds = []\n",
        "targets = []\n",
        "for pred_file,tar_file,input_file in zip(pred_files,targ_files,inp_files):\n",
        "    with open(pred_file) as inpFilePer, open(tar_file) as inpFileTar, open(input_file) as inpInFile:\n",
        "        i=0\n",
        "        for pred, target in zip(inpFilePer,inpFileTar):\n",
        "            i += 1\n",
        "            if i % 2 == 0:\n",
        "                pred = pred.replace(\"⁇ n ⁇\",\"\").strip().replace(\"⁇\",\"\")\n",
        "                target = target.replace(\"⁇ n ⁇\",\"\").replace(\"⁇ de ⁇ fr ⁇\",\"<de><fr>\").replace(\"⁇ en ⁇ mk ⁇\",\"<en><mk>\").replace(\"⁇ mk ⁇ en ⁇\",\"<mk><en>\").replace(\"⁇ de ⁇ en ⁇\",\"<de><en>\").replace(\"⁇ en ⁇ de ⁇\",\"<en><de>\").replace(\"⁇ en ⁇ fr ⁇\",\"<en><fr>\").replace(\"⁇ fr ⁇ de ⁇\",\"<fr><de>\").replace(\"⁇ fr ⁇ en ⁇\",\"<fr><en>\").strip().replace(\"⁇\",\"\")\n",
        "                preds.append(pred)\n",
        "                targets.append(target)\n",
        "with open(os.path.join(word_by_word_output_addr['Multi'], \"inputs.txt\"),\"w\") as inp, open(os.path.join(word_by_word_output_addr['Multi'], \"outputs.txt\"),\"w\") as out:\n",
        "    for i,o in zip(targets,preds):\n",
        "        inp.write(i + \"\\n\")\n",
        "        out.write(o + \"\\n\")"
      ],
      "metadata": {
        "id": "fM8IPDsx6-Uf"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Pre-Training PEACH"
      ],
      "metadata": {
        "id": "-A0cxjNQ4lmt"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Note:** First, fill the addresses field in the dataset files properly due to the guideline provided, then execute next blocks."
      ],
      "metadata": {
        "id": "0dSZGeAFCc-e"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!cp models/peach/datasets/SPDG/*.py /usr/local/lib/python3.7/dist-packages/tensorflow_datasets/translate"
      ],
      "metadata": {
        "id": "DkW_EIiDCc-f"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!cat models/peach/datasets/SPDG/imports.txt >> /usr/local/lib/python3.7/dist-packages/tensorflow_datasets/translate/__init__.py"
      ],
      "metadata": {
        "id": "dCmoUCS_Cc-f"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "vocab = vocab_addr['Multi']\n",
        "output_dir = output_model_addr['Multi']\n",
        "!python3 models/peach/bin/train.py --params=\"multi-SPDG\" \\\n",
        "--param_overrides=vocab_filename=$vocab \\\n",
        "--model_dir=$output_dir \\"
      ],
      "metadata": {
        "id": "vesh40OeAv5Y"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Fine-Tuning PEACH on Translation"
      ],
      "metadata": {
        "id": "RN4fAWOZ4plH"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Note:** First, fill the addresses field in the dataset files properly due to the guideline provided, then execute next blocks."
      ],
      "metadata": {
        "id": "ID7iMh8cCkTi"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!cp models/peach/datasets/downstream/*.py /usr/local/lib/python3.7/dist-packages/tensorflow_datasets/translate"
      ],
      "metadata": {
        "id": "eZBbOzliCkTj"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!cat models/peach/datasets/downstream/imports.txt >> /usr/local/lib/python3.7/dist-packages/tensorflow_datasets/translate/__init__.py"
      ],
      "metadata": {
        "id": "sX7FUjAMCkTk"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "vocab = vocab_addr['Multi']\n",
        "output_dir = fine_tune_addr['EN-FR']\n",
        "base_model = output_dir = os.path.join(output_model_addr['Multi'],\"model.ckpt-500000\")\n",
        "!python3 models/peach/bin/train.py --params=\"en-fr\" \\\n",
        "--param_overrides=vocab_filename=$vocab \\\n",
        "--model_dir=$output_dir \\\n",
        "--train_init_checkpoint=$base_model"
      ],
      "metadata": {
        "id": "PoZ6lbY-A6Br"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "vocab = vocab_addr['Multi']\n",
        "output_dir = os.path.join(fine_tune_addr['EN-FR'], \"model.ckpt-50000\")\n",
        "!python3 models/peach/bin/evaluate.py --params=\"en-fr\" \\\n",
        "--param_overrides=vocab_filename=$vocab \\\n",
        "--model_dir=$output_dir \\\n",
        "--evaluate_test=\"True\""
      ],
      "metadata": {
        "id": "IjJwZ4u7NWBH"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}