translation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "47205e5e-d28c-43aa-8525-b608721c3ba6",
   "metadata": {},
   "source": [
    "#### Import required modules and load alpaca_data_cleaned file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2b1c1e6e-5984-4dbf-a124-45eb78a1da1d",
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install deepl\n",
    "!pip install openai"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "11ba1146-8564-40db-a1a7-2883458da43c",
   "metadata": {},
   "source": [
    "#### Choose the translator you would like to use"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c778000d-7512-45b8-807e-a694400932a1",
   "metadata": {},
   "outputs": [],
   "source": [
    "TRANSLATOR = \"deepl\" # or openai"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c936db98-c9ac-4846-80e7-3273d54e2ecb",
   "metadata": {},
   "source": [
    "#### Authenticate to deepl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cfadb081-c3ce-4ebf-ba09-5c0894360212",
   "metadata": {},
   "outputs": [],
   "source": [
    "import deepl\n",
    "\n",
    "TARGET_LANG=\"DE\" # e.g. DE, EN,.. \n",
    "FORMALITY=\"less\" # \n",
    "\n",
    "auth_key = \"\"  # replace with your key\n",
    "translator = deepl.Translator(auth_key)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5bfdfc8f-6116-4a55-b1ba-4955b6ad4b83",
   "metadata": {},
   "source": [
    "#### Setup OpenAI information"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "5a586b34-09ac-467e-887e-ea24b28f55eb",
   "metadata": {},
   "outputs": [],
   "source": [
    "import openai\n",
    "\n",
    "openai.api_key = \"\" # replace with your key\n",
    "MODEL = \"gpt-3.5-turbo\"\n",
    "TARGET_LANGUAGE = \"German\" # e.g. \"English\", \"German\", \"Spanish\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "4cf8411b-bf64-4872-9f24-f17f686456a6",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import re\n",
    "import glob\n",
    "\n",
    "input_tasks_path = \"tasks_translated.json\"\n",
    "\n",
    "with open(input_tasks_path, \"rb\") as f:\n",
    "    json_data = json.loads(f.read())\n",
    "    df = pd.DataFrame(json_data)\n",
    "    \n",
    "def write_json_file(blob, file_path):\n",
    "    with open(file_path, 'w') as file:\n",
    "            json.dump(blob, file)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0527cad4-47ca-49b0-be9e-58e54899ce46",
   "metadata": {},
   "source": [
    "### Start translating dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d4407cfe-557a-46e8-8bed-1a467d618a61",
   "metadata": {},
   "source": [
    "#### util functions that help avoid translating content that is not intended for translation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "7cc27072-806d-4b0f-8730-d06b4a95f834",
   "metadata": {},
   "outputs": [],
   "source": [
    "def matches_regex(regex, text):\n",
    "    return bool(re.compile(regex).search(text))\n",
    "\n",
    "\n",
    "def contains_code(text):\n",
    "    # filter based on keywords that indicate code\n",
    "    code_blacklist = ['&&', '||', '<html>', ';\\n', 'SELECT']\n",
    "    \n",
    "    return (\n",
    "            any(code_keyword in text for code_keyword in code_blacklist) |\n",
    "            matches_regex(r'\\w+\\(\\w*\\) \\{', text) | # e.g. myFunc() {\n",
    "            matches_regex(r'def \\w+\\(', text) | # e.g. def parse_list(\n",
    "            matches_regex(r'\\[A-z]+\\.[A-z]+', text) | # e.g. this.language\n",
    "            matches_regex(r': [\\w\\.#]{1,12};', text) | # e.g. font-size: 1.3em;\n",
    "            matches_regex(r'<\\/\\w+>', text) # e.g. </html>\n",
    "           )\n",
    "\n",
    "\n",
    "def contains_words(text):\n",
    "    return matches_regex(r'[A-z]{3,}', text) # words with at least three characters\n",
    "\n",
    "\n",
    "def is_translatable(text):\n",
    "    if text == \"\":\n",
    "        return True # empty string won't be charged by DeepL\n",
    "    return (contains_code(text) is False) & contains_words(text)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3c813ed6-f555-4733-9e41-8300aaf29f12",
   "metadata": {},
   "source": [
    "#### util functions to translate individual columns (instruction, input and output) of each chunck as a list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "c68b81f0-217a-43e0-b607-2c04489690b9",
   "metadata": {},
   "outputs": [],
   "source": [
    "def translate_and_update_series(text_series):\n",
    "    # memorize whether and where the list contains non-translatable content\n",
    "    is_translatable_index = text_series.apply(lambda x: is_translatable(x) is False)\n",
    "    text_list_source_language = text_series.tolist()\n",
    "\n",
    "    # replace non-translatable content with an empty string\n",
    "    text_series[is_translatable_index] = \"\"\n",
    "\n",
    "    # translate list\n",
    "    text_list = text_series.tolist()\n",
    "    if TRANSLATOR == \"deepl\":\n",
    "        translated_list = translate_list_deepl(text_list)\n",
    "    else:\n",
    "        translated_list = translate_list_openai(text_list)\n",
    "\n",
    "    # if list contains non-translatable content, replace accordingly\n",
    "    if is_translatable_index.sum() > 0:\n",
    "        for index, text_is_translatable in enumerate(is_translatable_index.tolist()):\n",
    "            if text_is_translatable:\n",
    "                translated_list[index] = text_list_source_language[index]\n",
    "    return translated_list\n",
    "\n",
    "def create_openai_prompt_string(text):\n",
    "    if ' ' in text:\n",
    "        return f'Please provide the {TARGET_LANGUAGE} translation for these sentences: {text}'\n",
    "    else:\n",
    "        return f'Please provide the {TARGET_LANGUAGE} translation for the following word: {text}'\n",
    "\n",
    "def create_openai_message_list(text_list):\n",
    "    return [None if text == \"\" else {\"role\": \"user\", \"content\": create_openai_prompt_string(text)} for text in text_list]\n",
    "\n",
    "def translate_openai_message(message):\n",
    "    if message is None:\n",
    "        return \"\"\n",
    "    \n",
    "    response = None\n",
    "    while response is None:\n",
    "        try:\n",
    "            response = openai.ChatCompletion.create(\n",
    "                model=MODEL,\n",
    "                messages=[message]\n",
    "            )\n",
    "        except:\n",
    "            pass\n",
    "    return response[\"choices\"][0][\"message\"][\"content\"].strip()\n",
    "\n",
    "def translate_list_openai(text_list):\n",
    "    message_list = create_openai_message_list(text_list)\n",
    "    return [translate_openai_message(message) for message in message_list]\n",
    "\n",
    "def translate_list_deepl(text_list):\n",
    "    # here would be the place to replace the DeepL library with the Google library for example\n",
    "    combined_response = translator.translate_text(text_list, source_lang=\"EN\", target_lang=TARGET_LANG, formality=FORMALITY)\n",
    "    return [response.text for response in combined_response]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3bd08a21-53cd-4847-9f57-c2c86f5176ee",
   "metadata": {},
   "source": [
    "#### Divide dataframe into chunks and translate the chunks sequentially\n",
    "\n",
    "I'm sure this part can be heavily improved (feel free to create a pull request)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "6fe2e739-6efe-4d29-ab73-546df48260ca",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Feel free to increase the chunk size. I was worried that the execution would be interrupted,\n",
    "# so I used a smaller chunk size\n",
    "chunk_size = 5\n",
    "output_dir = './data/output/'\n",
    "\n",
    "def translate_dataframe(df):\n",
    "    os.makedirs(output_dir, exist_ok=True)\n",
    "    number_of_chunks = df.shape[0] // chunk_size\n",
    "    chunked_df_list = np.array_split(df, number_of_chunks)\n",
    "    \n",
    "    start_index = 1\n",
    "    \n",
    "    for index, chunk_df in enumerate(chunked_df_list[start_index:]):\n",
    "        instruction_list_translated = translate_and_update_series(chunk_df.instruction)\n",
    "        input_list_translated = translate_and_update_series(chunk_df.input)\n",
    "        output_list_translated = translate_and_update_series(chunk_df.output)\n",
    "        \n",
    "        translated_df = pd.DataFrame({'instruction': instruction_list_translated, 'input': input_list_translated, 'output': output_list_translated})\n",
    "        translated_dict = translated_df.to_dict('records')\n",
    "        \n",
    "        write_json_file(translated_dict, f'{output_dir}chunk{start_index+index}.json')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4fec387c-4c39-41cd-a168-c63333830a1e",
   "metadata": {},
   "source": [
    "#### Start translating the DataFrame (Warning: Run this cell carefully)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "aa1f9a01-41b7-4551-8211-9cc90a84dabd",
   "metadata": {},
   "outputs": [],
   "source": [
    "translate_dataframe(df)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8e23d3a5-ed8a-462f-8b3d-76c4f17614bc",
   "metadata": {},
   "source": [
    "#### Finally combine all chunked files into one translated task file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "4b5f434e-dacc-4432-97b5-afe6fe7a7b06",
   "metadata": {},
   "outputs": [],
   "source": [
    "def combine_chunks():\n",
    "    translated_tasks_list = []\n",
    "    for index in range(0, len(glob.glob(f'{output_dir}*.json'))):\n",
    "        with open(f'{output_dir}chunk{index}.json', \"rb\") as f:\n",
    "            translated_tasks_list += json.loads(f.read())\n",
    "    write_json_file(translated_tasks_list, f'./translated_tasks_de_{TRANSLATOR}.json')\n",
    "\n",
    "combine_chunks()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "ddit",
   "language": "python",
   "name": "ddit"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}