jupyter/Wordcloud.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "2f0eabc4",
   "metadata": {},
   "source": [
    "# Wordcloud\n",
    "<br>  \n",
    "\n",
    "### References\n",
    "- [jqassistant](https://jqassistant.org)\n",
    "- [py2neo](https://py2neo.org/2021.1/)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 247,
   "id": "4191f259",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plot\n",
    "import typing as typ\n",
    "from neo4j import GraphDatabase\n",
    "from wordcloud import WordCloud, STOPWORDS, ImageColorGenerator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 248,
   "id": "1c5dab37",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Please set the environment variable \"NEO4J_INITIAL_PASSWORD\" in your shell \n",
    "# before starting jupyter notebook to provide the password for the user \"neo4j\". \n",
    "# It is not recommended to hardcode the password into jupyter notebook for security reasons.\n",
    "\n",
    "driver = GraphDatabase.driver(uri=\"bolt://localhost:7687\", auth=(\"neo4j\", os.environ.get(\"NEO4J_INITIAL_PASSWORD\")))\n",
    "driver.verify_connectivity()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 249,
   "id": "c1db254b",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_cypher_query_from_file(filename):\n",
    "    with open(filename) as file:\n",
    "        return ' '.join(file.readlines())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 250,
   "id": "6e8772aa",
   "metadata": {},
   "outputs": [],
   "source": [
    "def query_cypher_to_data_frame(filename: str, parameters_: typ.Optional[typ.Dict[str, typ.Any]] = None):\n",
    "    records, summary, keys = driver.execute_query(get_cypher_query_from_file(filename),parameters_=parameters_)\n",
    "    return pd.DataFrame([r.values() for r in records], columns=keys)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 251,
   "id": "da9e8edb",
   "metadata": {},
   "outputs": [],
   "source": [
    "#The following cell uses the build-in %html \"magic\" to override the CSS style for tables to a much smaller size.\n",
    "#This is especially needed for PDF export of tables with multiple columns."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9deaabce",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%html\n",
    "<style>\n",
    "/* CSS style for smaller dataframe tables. */\n",
    ".dataframe th {\n",
    "    font-size: 8px;\n",
    "}\n",
    ".dataframe td {\n",
    "    font-size: 8px;\n",
    "}\n",
    "</style>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 253,
   "id": "c2496caf",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Main Colormap\n",
    "main_color_map = 'nipy_spectral'"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "97cf6afa",
   "metadata": {},
   "source": [
    "## Word Cloud"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 254,
   "id": "82b46521",
   "metadata": {},
   "outputs": [],
   "source": [
    "customized_stop_words = STOPWORDS.union([\n",
    "    'builder', 'exception', 'abstract', 'helper', 'util', 'callback', 'factory', 'result',\n",
    "    'handler', 'type', 'module', 'name', 'parameter', 'lambda', 'access', 'create', 'message', \n",
    "    'ts', 'js', 'tsx', 'jsx', 'css', 'htm', 'html', 'props', 'use', 'id', 'ref', 'hook', 'event', \n",
    "    'span', 'data', 'context', 'form', 'get', 'set', 'object', 'null', 'new', 'plugin', 'package', \n",
    "    'types', 'dom', 'static', 'view', 'link', 'build', 'element', 'impl', 'function', 'test', \n",
    "    'dev', 'event', 'mock', 'error', 'input', 'sdk', 'api', 'item', 'end', 'value', 'param', 'start'\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 255,
   "id": "d2016fd0",
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_word_cloud_from_text(words: pd.DataFrame, title: str):\n",
    "    \n",
    "    # Join all words into one text separated by spaces\n",
    "    text = \" \".join(word for word in words.word)\n",
    "    number_of_words=len(words.word)\n",
    "    print(\"There are {} words in the dataset for the plot titled '{}'.\".format(number_of_words, title))\n",
    "\n",
    "    if number_of_words <= 0:\n",
    "        return\n",
    "\n",
    "    wordcloud = WordCloud(\n",
    "        width=800, \n",
    "        height=800,\n",
    "        max_words=600, \n",
    "        stopwords=customized_stop_words,\n",
    "        collocations=False,\n",
    "        background_color='white', \n",
    "        colormap='viridis'\n",
    "    ).generate(text)\n",
    "\n",
    "    # Plot the word cloud\n",
    "    plot.figure(figsize=(15,15))\n",
    "    plot.imshow(wordcloud, interpolation='bilinear')\n",
    "    plot.axis(\"off\")\n",
    "    plot.title(title)\n",
    "    plot.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 256,
   "id": "9fbf37ac",
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_word_cloud_with_frequency(words_with_frequency: pd.DataFrame, title: str):\n",
    "    \n",
    "    if words_with_frequency.empty:\n",
    "        return\n",
    "    \n",
    "    # Expects the first column of the DataFrame to contain the words/text and the second column to contain the count/frequency.\n",
    "    words_with_frequency_dict=words_with_frequency.set_index(words_with_frequency.columns[0]).to_dict()[words_with_frequency.columns[1]]\n",
    "    wordcloud = WordCloud(\n",
    "        width=800, \n",
    "        height=800,\n",
    "        max_words=600, \n",
    "        stopwords=customized_stop_words,\n",
    "        collocations=False,\n",
    "        background_color='white', \n",
    "        colormap='viridis'\n",
    "    ).generate_from_frequencies(words_with_frequency_dict)\n",
    "\n",
    "    # Plot the word cloud\n",
    "    plot.figure(figsize=(15,15))\n",
    "    plot.imshow(wordcloud, interpolation='bilinear')\n",
    "    plot.axis(\"off\")\n",
    "    plot.title(title)\n",
    "    plot.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "58dd502f",
   "metadata": {},
   "source": [
    "## WordCloud of names in code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0ed729e3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Query data from graph database\n",
    "words = query_cypher_to_data_frame(\"../cypher/Overview/Words_for_universal_Wordcloud.cypher\")\n",
    "words.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b26bfe71",
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_word_cloud_from_text(words=words, title='Wordcloud of names in code')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9624f5ea",
   "metadata": {},
   "source": [
    "## WordCloud of git authors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b8ae94a2",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Query data from graph database\n",
    "git_author_words_with_frequency = query_cypher_to_data_frame(\"../cypher/Overview/Words_for_git_author_Wordcloud_with_frequency.cypher\")\n",
    "git_author_words_with_frequency.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c1a81f62",
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_word_cloud_with_frequency(git_author_words_with_frequency, 'Wordcloud of git authors (using frequency)')"
   ]
  }
 ],
 "metadata": {
  "authors": [
   {
    "name": "JohT"
   }
  ],
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.9"
  },
  "title": "Object Oriented Design Quality Metrics for Java with Neo4j"
 },
 "nbformat": 4,
 "nbformat_minor": 5
}