llama2_RAG_QA.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "gpuType": "T4"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "# RAG Q&A LLM Chatbot for [Confluence Space](https://templates.atlassian.net/wiki/spaces/SWPRJ) and Github Repositories ([Full-Stack App](https://crowd-wire.github.io/ProjectDocumentation/), [SQL Tables](https://github.com/JannikArndt/sql-auto-doc/tree/master/Examples))"
      ],
      "metadata": {
        "id": "eLznoZ_akil4"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Dependencies"
      ],
      "metadata": {
        "id": "9upuVlI13kH6"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import locale\n",
        "locale.getpreferredencoding = lambda: \"UTF-8\""
      ],
      "metadata": {
        "id": "p4-2IdNKgAsZ",
        "cellView": "code"
      },
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install -qU transformers accelerate einops langchain xformers bitsandbytes faiss-gpu sentence_transformers atlassian-python-api markdownify lark chromadb opensearch-py llama-index"
      ],
      "metadata": {
        "id": "rkD7sdqI_Z6X",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "0a04f16a-4151-4e78-f651-1e65680fd626",
        "cellView": "code"
      },
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[?25l     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m0.0/261.4 kB\u001b[0m \u001b[31m?\u001b[0m eta \u001b[36m-:--:--\u001b[0m\r\u001b[2K     \u001b[91m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[90m╺\u001b[0m\u001b[90m━━━\u001b[0m \u001b[32m235.5/261.4 kB\u001b[0m \u001b[31m6.9 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\r\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m261.4/261.4 kB\u001b[0m \u001b[31m5.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m44.6/44.6 kB\u001b[0m \u001b[31m6.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.9/1.9 MB\u001b[0m \u001b[31m41.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m211.8/211.8 MB\u001b[0m \u001b[31m6.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m92.6/92.6 MB\u001b[0m \u001b[31m8.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m85.5/85.5 MB\u001b[0m \u001b[31m5.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m86.0/86.0 kB\u001b[0m \u001b[31m5.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m168.6/168.6 kB\u001b[0m \u001b[31m18.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m111.6/111.6 kB\u001b[0m \u001b[31m13.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m502.4/502.4 kB\u001b[0m \u001b[31m30.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m258.6/258.6 kB\u001b[0m \u001b[31m23.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m905.1/905.1 kB\u001b[0m \u001b[31m51.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m177.5/177.5 kB\u001b[0m \u001b[31m18.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m47.0/47.0 kB\u001b[0m \u001b[31m5.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m61.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.4/2.4 MB\u001b[0m \u001b[31m74.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m92.9/92.9 kB\u001b[0m \u001b[31m10.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m59.7/59.7 kB\u001b[0m \u001b[31m7.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.4/5.4 MB\u001b[0m \u001b[31m76.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m6.4/6.4 MB\u001b[0m \u001b[31m77.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m57.9/57.9 kB\u001b[0m \u001b[31m7.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m105.3/105.3 kB\u001b[0m \u001b[31m12.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m67.3/67.3 kB\u001b[0m \u001b[31m8.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m699.4/699.4 kB\u001b[0m \u001b[31m62.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.6/1.6 MB\u001b[0m \u001b[31m95.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m72.6/72.6 kB\u001b[0m \u001b[31m10.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m143.0/143.0 kB\u001b[0m \u001b[31m19.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m75.0/75.0 kB\u001b[0m \u001b[31m11.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m220.9/220.9 kB\u001b[0m \u001b[31m29.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.0/2.0 MB\u001b[0m \u001b[31m40.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m49.4/49.4 kB\u001b[0m \u001b[31m7.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m67.0/67.0 kB\u001b[0m \u001b[31m7.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m143.8/143.8 kB\u001b[0m \u001b[31m20.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m46.0/46.0 kB\u001b[0m \u001b[31m6.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m76.9/76.9 kB\u001b[0m \u001b[31m11.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.3/58.3 kB\u001b[0m \u001b[31m8.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m50.8/50.8 kB\u001b[0m \u001b[31m7.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m341.4/341.4 kB\u001b[0m \u001b[31m36.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.4/3.4 MB\u001b[0m \u001b[31m110.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m79.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m130.2/130.2 kB\u001b[0m \u001b[31m16.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m86.8/86.8 kB\u001b[0m \u001b[31m10.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Building wheel for sentence_transformers (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Building wheel for pypika (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "lida 0.0.10 requires kaleido, which is not installed.\n",
            "lida 0.0.10 requires python-multipart, which is not installed.\n",
            "llmx 0.0.15a0 requires cohere, which is not installed.\n",
            "tensorflow-probability 0.22.0 requires typing-extensions<4.6.0, but you have typing-extensions 4.8.0 which is incompatible.\u001b[0m\u001b[31m\n",
            "\u001b[0m"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Model Initialization"
      ],
      "metadata": {
        "id": "1ssKzSN53mj-"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "hf_auth = 'your-hf-key-here'"
      ],
      "metadata": {
        "id": "v672Q_qW4r4C",
        "cellView": "code"
      },
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "id": "fs_mCPKu_NcK",
        "collapsed": true,
        "cellView": "code"
      },
      "outputs": [],
      "source": [
        "from torch import cuda, bfloat16\n",
        "import transformers\n",
        "\n",
        "model_id = 'meta-llama/Llama-2-7b-chat-hf'\n",
        "\n",
        "device = f'cuda:{cuda.current_device()}' if cuda.is_available() else 'cpu'\n",
        "\n",
        "# set quantization configuration to load large model with less GPU memory\n",
        "bnb_config = transformers.BitsAndBytesConfig(\n",
        "    load_in_4bit=True,\n",
        "    bnb_4bit_quant_type='nf4',\n",
        "    bnb_4bit_use_double_quant=True,\n",
        "    bnb_4bit_compute_dtype=bfloat16\n",
        ")\n",
        "\n",
        "# begin initializing HF items with hf auth token\n",
        "model_config = transformers.AutoConfig.from_pretrained(\n",
        "    model_id,\n",
        "    use_auth_token=hf_auth\n",
        ")\n",
        "\n",
        "model = transformers.AutoModelForCausalLM.from_pretrained(\n",
        "    model_id,\n",
        "    trust_remote_code=True,\n",
        "    config=model_config,\n",
        "    quantization_config=bnb_config,\n",
        "    device_map='auto',\n",
        "    use_auth_token=hf_auth\n",
        ")\n",
        "\n",
        "# enable evaluation mode to allow model inference\n",
        "model.eval()\n",
        "\n",
        "print(f\"Model loaded on {device}\")"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Quantization Pros:\n",
        "- Using smaller precision representations require less memory, which simplifies the deployment of models on devices with limited resources.\n",
        "- On hardware with efficient integer operations, such as CPUs, GPUs, and TPUs, quantized models execute quickly.\n",
        "- On mobile devices and in data centers, lower-precision computations are more energy-efficient.\n",
        "\n",
        "Quantization Cons:\n",
        "- Due to precision loss, quantization might diminish model accuracy and negatively affect task performance.\n",
        "- The process of quantization is intricate and lengthy, requiring additional training steps."
      ],
      "metadata": {
        "id": "Ac1og0ipueej"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Tokenizer"
      ],
      "metadata": {
        "id": "cZOAH1zw3qpN"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "tokenizer = transformers.AutoTokenizer.from_pretrained(\n",
        "    model_id,\n",
        "    use_auth_token=hf_auth\n",
        ")"
      ],
      "metadata": {
        "id": "eHJgtlWK_W71",
        "cellView": "code"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Stop On Tokens"
      ],
      "metadata": {
        "id": "H28Ud08W3vdq"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "stop_list = ['\\nHuman:', '\\n```\\n']\n",
        "\n",
        "stop_token_ids = [tokenizer(x)['input_ids'] for x in stop_list]\n",
        "stop_token_ids"
      ],
      "metadata": {
        "id": "UVpqrwro_W-U",
        "collapsed": true,
        "cellView": "code"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import torch\n",
        "\n",
        "stop_token_ids = [torch.LongTensor(x).to(device) for x in stop_token_ids]\n",
        "stop_token_ids"
      ],
      "metadata": {
        "id": "Gv7Fg4wK_XAS",
        "collapsed": true,
        "cellView": "code"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import StoppingCriteria, StoppingCriteriaList\n",
        "\n",
        "# define custom stopping criteria object\n",
        "class StopOnTokens(StoppingCriteria):\n",
        "    def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor, **kwargs) -> bool:\n",
        "        for stop_ids in stop_token_ids:\n",
        "            if torch.eq(input_ids[0][-len(stop_ids):], stop_ids).all():\n",
        "                return True\n",
        "        return False\n",
        "\n",
        "stopping_criteria = StoppingCriteriaList([StopOnTokens()])"
      ],
      "metadata": {
        "id": "rCSVBT1__XB9",
        "cellView": "code"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Text Generation (without RAG)"
      ],
      "metadata": {
        "id": "MyXWGeDx33rz"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "generate_text = transformers.pipeline(\n",
        "    model=model,\n",
        "    tokenizer=tokenizer,\n",
        "    return_full_text=True,  # langchain expects the full text\n",
        "    task='text-generation',\n",
        "    # we pass model parameters here too\n",
        "    stopping_criteria=stopping_criteria,  # without this model rambles during chat\n",
        "    temperature=0.2,  # 'randomness' of outputs, 0.0 is the min and 1.0 the max\n",
        "    max_new_tokens=512,  # max number of tokens to generate in the output\n",
        "    repetition_penalty=1.1  # without this output begins repeating\n",
        ")"
      ],
      "metadata": {
        "id": "eelTj4VY_kP_",
        "cellView": "code"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from langchain.llms import HuggingFacePipeline\n",
        "\n",
        "llm = HuggingFacePipeline(pipeline=generate_text)\n",
        "\n",
        "# checking again that everything is working fine\n",
        "# llm(prompt=\"What is CrowdWire\")"
      ],
      "metadata": {
        "id": "LfbgLlT1_kR-",
        "cellView": "code"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Loading the Documents"
      ],
      "metadata": {
        "id": "TemOkFMp3_Ej"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "import re\n",
        "import uuid\n",
        "from langchain.document_loaders import ConfluenceLoader, DirectoryLoader, TextLoader\n",
        "from langchain.text_splitter import RecursiveCharacterTextSplitter, MarkdownHeaderTextSplitter\n",
        "from llama_index import Document\n",
        "\n",
        "# Confluence Loader (will transform later the web pages to markdown format)\n",
        "confluence_loader = ConfluenceLoader(\n",
        "    url=\"https://templates.atlassian.net/wiki\"\n",
        ")\n",
        "\n",
        "headers_to_split_on = [\n",
        "    (\"#\", \"title\"),\n",
        "    (\"##\", \"header\"),\n",
        "    (\"###\", \"header\"),\n",
        "    (\"####\", \"header\"),\n",
        "    (\"#####\", \"header\"),\n",
        "    (\"######\", \"header\"),\n",
        "]\n",
        "\n",
        "markdown_splitter = MarkdownHeaderTextSplitter(\n",
        "    headers_to_split_on=headers_to_split_on)\n",
        "\n",
        "text_splitter = RecursiveCharacterTextSplitter(\n",
        "    chunk_size=3000, chunk_overlap=200)\n",
        "\n",
        "def clean_text(text):\n",
        "    # remove words with more than 100 characters\n",
        "    text = re.sub(r'\\b\\w{101,}\\b', ' ', text)\n",
        "    # remove extra white spaces\n",
        "    text = re.sub(r' +', ' ', text)\n",
        "    return text\n",
        "\n",
        "# final documents array\n",
        "documents = []"
      ],
      "metadata": {
        "id": "dNF5rjACXHyb",
        "cellView": "code"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Confluence loader"
      ],
      "metadata": {
        "id": "VK8L6zEdbOtU"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Confluence Loader (will transform later the web pages to markdown format)\n",
        "confluence_loader = ConfluenceLoader(\n",
        "    url=\"https://templates.atlassian.net/wiki\"\n",
        ")\n",
        "\n",
        "if callable(confluence_loader.load):\n",
        "  for doc in confluence_loader.load(\n",
        "      space_key=\"SWPRJ\",\n",
        "      include_attachments=False,\n",
        "      keep_markdown_format=True\n",
        "    ):\n",
        "    link = doc.metadata[\"source\"]\n",
        "    source = link.split(\"/\")[-1]\n",
        "    title = doc.metadata[\"title\"]\n",
        "\n",
        "    doc.page_content = clean_text(doc.page_content)\n",
        "    docs = markdown_splitter.split_text(doc.page_content)\n",
        "    docs = text_splitter.split_documents(docs)\n",
        "\n",
        "    for doc in docs:\n",
        "      doc.metadata[\"link\"] = link\n",
        "      doc.metadata[\"source\"] = source\n",
        "      doc.metadata[\"title\"] = title\n",
        "      if \"header\" not in doc.metadata:\n",
        "        doc.metadata[\"header\"] = title\n",
        "      documents.append(Document(\n",
        "          text = doc.page_content,\n",
        "          metadata=doc.metadata,\n",
        "          excluded_llm_metadata_keys = [\"link\"],\n",
        "          excluded_embed_metadata_keys = [\"link\"]\n",
        "        ))"
      ],
      "metadata": {
        "id": "OJcY6yMAaRDR",
        "cellView": "code"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "len(documents)"
      ],
      "metadata": {
        "id": "VFynxSqLJ_9t"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Repository Loader"
      ],
      "metadata": {
        "id": "UlCYYCT0bWcC"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Repository Loader (Markdowns only for now)\n",
        "glob_pattern = \"*.md\"\n",
        "repo_paths = [\"/content/drive/MyDrive/ProjectDocumentation-master\",\n",
        "              \"/content/drive/MyDrive/SQL_Examples\"]\n",
        "repo_links = [\"https://github.com/Crowd-Wire/ProjectDocumentation/tree/master/\",\n",
        "              \"https://github.com/JannikArndt/sql-auto-doc/tree/master/Examples/\"]\n",
        "\n",
        "repo_i = 0\n",
        "for repo_path in repo_paths:\n",
        "  try:\n",
        "    repo_loader = DirectoryLoader(\n",
        "      repo_path,\n",
        "      glob=glob_pattern,\n",
        "      show_progress=True,\n",
        "      recursive=True,\n",
        "      loader_cls=TextLoader,\n",
        "      loader_kwargs={\"encoding\": \"utf8\"})\n",
        "  except Exception as e:\n",
        "    print(f\"Error loading files with pattern '{glob_pattern}': {e}\")\n",
        "    break\n",
        "\n",
        "\n",
        "  if callable(repo_loader.load):\n",
        "    for doc in repo_loader.load():\n",
        "      relative_path = os.path.relpath(doc.metadata['source'], repo_path)\n",
        "      link = repo_links[repo_i] + \"/\" + relative_path\n",
        "\n",
        "      doc.page_content = clean_text(doc.page_content)\n",
        "      docs = markdown_splitter.split_text(doc.page_content)\n",
        "      docs = text_splitter.split_documents(docs)\n",
        "\n",
        "      for doc in docs:\n",
        "        if \"title\" in doc.metadata:\n",
        "          title = doc.metadata[\"title\"]\n",
        "          break\n",
        "\n",
        "      for doc in docs:\n",
        "        doc.metadata['link'] = link\n",
        "        doc.metadata['source'] = relative_path\n",
        "        doc.metadata['title'] = title\n",
        "        if \"header\" not in doc.metadata:\n",
        "          doc.metadata[\"header\"] = title\n",
        "        documents.append(Document(\n",
        "          text = doc.page_content,\n",
        "          metadata=doc.metadata,\n",
        "          excluded_llm_metadata_keys = [\"link\"],\n",
        "          excluded_embed_metadata_keys = [\"link\"]\n",
        "        ))\n",
        "    repo_i += 1\n",
        "\n",
        "del docs"
      ],
      "metadata": {
        "id": "LRUieLTsaRHg",
        "cellView": "code"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Gathered Documents"
      ],
      "metadata": {
        "id": "buAVRuTRUesf"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "len(documents)"
      ],
      "metadata": {
        "id": "FFJvsmO2s25Z",
        "cellView": "code"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def pretty_print_docs(docs):\n",
        "  print(f\"\\n{'-' * 100}\\n\".join([f\"Document {i+1} \\n\\\n",
        "Title: {d.metadata['title']}\\n\\\n",
        "Header: {d.metadata['header']}\\n\\\n",
        "Source: {d.metadata['source']}\\n\\\n",
        "Link: {d.metadata['link']}\\n\\\n",
        "Content:\\n{d.text}\" for i, d in enumerate(docs)]))\n",
        "\n",
        "def pretty_print_answer(answer):\n",
        "  print(f\"Answer: {answer}\\n\\n\")\n",
        "\n",
        "def pretty_print_result(result):\n",
        "  print(f\"Question: {result['query']}\\n\\n\")\n",
        "  print(f\"Answer: {result['result']}\\n\\n\")\n",
        "  print(\"-\" * 100)\n",
        "  print(\"Documents:\\n\")\n",
        "  pretty_print_docs(result[\"source_documents\"])\n",
        "\n",
        "def pretty_print_response(question):\n",
        "  response = query_engine.query(question)\n",
        "  print(f\"Question: {question}\\n\\n\")\n",
        "  print(f\"Answer: {response.response}\\n\\n\")\n",
        "  print(\"-\" * 100)\n",
        "  print(\"Documents:\\n\")\n",
        "  pretty_print_docs(response.source_nodes)"
      ],
      "metadata": {
        "id": "vSvaaVSmoFYh",
        "cellView": "code"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "pretty_print_docs(documents)"
      ],
      "metadata": {
        "id": "W_GkQGhAUU1i",
        "cellView": "code"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Creating Embeddings"
      ],
      "metadata": {
        "id": "bmiXtZ_A4I2j"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from langchain.embeddings import HuggingFaceEmbeddings\n",
        "from llama_index import ServiceContext, LLMPredictor, LangchainEmbedding, set_global_service_context\n",
        "\n",
        "embed_model = LangchainEmbedding(\n",
        "  HuggingFaceEmbeddings(\n",
        "    model_name=\"sentence-transformers/all-mpnet-base-v2\",\n",
        "    model_kwargs={\"device\": \"cuda\"}\n",
        "  )\n",
        ")\n",
        "llm_predictor = LLMPredictor(llm=llm)\n",
        "\n",
        "service_context = ServiceContext.from_defaults(\n",
        "  llm_predictor=llm_predictor,\n",
        "  embed_model=embed_model,\n",
        ")"
      ],
      "metadata": {
        "cellView": "code",
        "id": "t3Nj9DpehXO4"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Preparing the Retrieval QA LLM with RAG"
      ],
      "metadata": {
        "id": "ERYns78X4K9U"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from llama_index import Prompt, VectorStoreIndex\n",
        "from llama_index.response_synthesizers import get_response_synthesizer\n",
        "from llama_index.retrievers import VectorIndexRetriever\n",
        "from llama_index.query_engine import RetrieverQueryEngine\n",
        "from llama_index.vector_stores.types import MetadataInfo, VectorStoreInfo\n",
        "\n",
        "similarity_top_k = 6\n",
        "mmr_threshold = 0.7\n",
        "\n",
        "# define a prompt template\n",
        "text_qa_template = Prompt(\"\"\"You are a QA Bot. Based on the documents answer the question at the end. Don't make up an answer if you don't know the answer.\n",
        "\n",
        "{context_str}\n",
        "\n",
        "Question: {query_str}\n",
        "\n",
        "Answer: \"\"\")\n",
        "\n",
        "\n",
        "# build index\n",
        "index = VectorStoreIndex.from_documents(\n",
        "  documents,\n",
        "  service_context=service_context,\n",
        "  similarity_top_k=similarity_top_k,\n",
        "  vector_store_query_mode=\"mmr\",\n",
        "  vector_store_kwargs={\"mmr_threshold\": mmr_threshold}\n",
        ")\n",
        "\n",
        "# configure retriever\n",
        "retriever = VectorIndexRetriever(\n",
        "    index=index,\n",
        "    similarity_top_k=similarity_top_k,\n",
        "    vector_store_query_mode=\"mmr\",\n",
        "    vector_store_kwargs={\"mmr_threshold\": mmr_threshold},\n",
        ")\n",
        "\n",
        "# configure response synthesizer\n",
        "response_synthesizer = get_response_synthesizer(\n",
        "  text_qa_template=text_qa_template,\n",
        "  service_context=service_context,\n",
        "  response_mode=\"compact\"\n",
        ")\n",
        "\n",
        "# assemble query engine\n",
        "query_engine = RetrieverQueryEngine(\n",
        "    retriever=retriever,\n",
        "    response_synthesizer=response_synthesizer,\n",
        ")"
      ],
      "metadata": {
        "cellView": "code",
        "id": "3oHxxafoqyp3"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Question Answering"
      ],
      "metadata": {
        "id": "wFfnrWff4QRp"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## [Confluence Space](https://templates.atlassian.net/wiki/spaces/SWPRJ)"
      ],
      "metadata": {
        "id": "xUlEDMQDh-6C"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "pretty_print_response(\"What are the confluence user story dashboard requirements?\")"
      ],
      "metadata": {
        "id": "brudhBGNXbx4",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "7820d753-6e86-46f2-ddde-fe6416057196"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/transformers/pipelines/base.py:1101: UserWarning: You seem to be using the pipelines sequentially on GPU. In order to maximize efficiency please use a dataset\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Question: What are the confluence user story dashboard requirements?\n",
            "\n",
            "\n",
            "Answer:  The Confluence user story dashboard requirements are as follows:\n",
            "\n",
            "1. As a user, I want to see the most recent activity related to my work so that I can efficiently triage my work for the day.\n",
            "2. As a user, I want to see comments in the activity feed so that I can see what someone has said without going all the way to a page.\n",
            "3. As a user, I want to be able to save a page for later so that I can quickly go back to it on my desktop.\n",
            "4. As a user, I want to share a page so that I can let my colleagues know to check it out.\n",
            "5. As a user, I want a timestamp on the activity feed items so that I can see how recent they are.\n",
            "\n",
            "\n",
            "----------------------------------------------------------------------------------------------------\n",
            "Documents:\n",
            "\n",
            "Document 1 \n",
            "Title: User story: Dashboard\n",
            "Header: Primary user story\n",
            "Source: User+story%3A+Dashboard\n",
            "Link: https://templates.atlassian.net/wiki/spaces/SWPRJ/pages/33674/User+story%3A+Dashboard\n",
            "Content:\n",
            "grey1dashed**As a** user, **I want** to be see a status of everything related to my work in Confluence, **so that** I can triage my work while on the go.\n",
            "----------------------------------------------------------------------------------------------------\n",
            "Document 2 \n",
            "Title: Sprint planning\n",
            "Header: On this page, you can find our sprint plans, and the incomplete task that will last for the current sprint. New sprint plan will be added each week just after the meeting, if you have any updates to add to it, feel free to add them directly in the meeting notes.\n",
            "Source: Sprint+planning\n",
            "Link: https://templates.atlassian.net/wiki/spaces/SWPRJ/pages/33221/Sprint+planning\n",
            "Content:\n",
            "com.atlassian.confluence.plugins.confluence-business-blueprints:meeting-notes-blueprint56703e11-45eb-4418-a8c4-664d7e0f584c56703e11-45eb-4418-a8c4-664d7e0f584cCreate meeting noteCreate sprint plan  \n",
            "---\n",
            "----------------------------------------------------------------------------------------------------\n",
            "Document 3 \n",
            "Title: User story: Dashboard\n",
            "Header: User story: Dashboard\n",
            "Source: User+story%3A+Dashboard\n",
            "Link: https://templates.atlassian.net/wiki/spaces/SWPRJ/pages/33674/User+story%3A+Dashboard\n",
            "Content:\n",
            "| lEpic link | |\n",
            "| Status | trueGreyDraft |\n",
            "| Developer | |\n",
            "| Designer | |\n",
            "| Product manager | |  \n",
            "---\n",
            "----------------------------------------------------------------------------------------------------\n",
            "Document 4 \n",
            "Title: User story: View pages\n",
            "Header: Primary user story\n",
            "Source: User+story%3A+View+pages\n",
            "Link: https://templates.atlassian.net/wiki/spaces/SWPRJ/pages/33716/User+story%3A+View+pages\n",
            "Content:\n",
            "grey1dashed**As a** user, **I want** to be able to view a Confluence page, **so that** I can read my teams work while on the go.  \n",
            "---\n",
            "----------------------------------------------------------------------------------------------------\n",
            "Document 5 \n",
            "Title: User story: Dashboard\n",
            "Header: Requirements\n",
            "Source: User+story%3A+Dashboard\n",
            "Link: https://templates.atlassian.net/wiki/spaces/SWPRJ/pages/33674/User+story%3A+Dashboard\n",
            "Content:\n",
            "| # | Title | User Story | JIRA Issues | Notes | Importance |\n",
            "| --- | --- | --- | --- | --- | --- |\n",
            "| 1 | Activity feed | **As a**user, **I want** to see the most recent activity related to my work **so that** I can efficiently triage my work for the day. | | * Order pages chronologically with the most recent updates first.\n",
            "* Show a different color link when a page was visited already.\n",
            "| Redmust have |\n",
            "| 2 | Comment excerpt | **As a** user, **I want** to see comments in the activity feed, **so that** I can see what someone has said without going all the way to a page. | | * Maximum length of comment is 280 characters; truncate with an ellipsis.\n",
            "| YellowNice to have |\n",
            "| 3 | Save for later | **As a**user, **I want** be able to save a page for later **so that** I can quickly go back to it on my desktop. | | * Need a saved / not saved status icon.\n",
            "| YellowNice to have |\n",
            "| 4 | Share | **As a**user, **I want** share a page **so that** I can let me colleagues know to check it out. | | * We will support SMS, email, and facsimile.\n",
            "| YellowNice to have |\n",
            "| 5 | Timestamp | **As a**user, **I want** a timestamp on the activity feed items **so that** I can see how recent they are. | | * Timestamp is in relative days/minutes.\n",
            "| YellowNice to have |\n",
            "----------------------------------------------------------------------------------------------------\n",
            "Document 6 \n",
            "Title: Decision log\n",
            "Header: Here you'll be able to find all the answers to any decisions you come across, and the justifications for why any past decisions were made. The decision owner is responsible for following up with stakeholders.\n",
            "Source: Decision+log\n",
            "Link: https://templates.atlassian.net/wiki/spaces/SWPRJ/pages/33219/Decision+log\n",
            "Content:\n",
            "com.atlassian.confluence.plugins.confluence-business-blueprints:file-list-blueprinta2604e03-f7c1-4359-947d-2687c45d215da2604e03-f7c1-4359-947d-2687c45d215dviewCreate decisionCreate decision  \n",
            "---  \n",
            "com.atlassian.confluence.plugins.confluence-business-blueprints:decisions-blueprint 1c90e9ac-e8f9-4608-9388-56d2a87f82d6Record important project decisions and communicate them with your team. DecisionDecisionsSOF3Status,Stakeholders,Outcome,Due date,Ownerdecisionslabel = \"decisions\" and space = \"SWPRJ\"\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "pretty_print_response(\"What is and what is the summary of the confluence 2016-05-23 Retrospective\")"
      ],
      "metadata": {
        "id": "CRIhNPjRJLhS",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "268f652f-c7fe-4398-8e0b-b74eb62b3270"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/transformers/pipelines/base.py:1101: UserWarning: You seem to be using the pipelines sequentially on GPU. In order to maximize efficiency please use a dataset\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Question: What is and what is the summary of the confluence 2016-05-23 Retrospective\n",
            "\n",
            "\n",
            "Answer:  The summary of the confluence 2016-05-23 Retrospective is as follows:\n",
            "\n",
            "* 75% of issues were resolved and deployed.\n",
            "* New QA process helped eliminate last-minute blocking bugs.\n",
            "* Designs were finalized and ready in advance of starting the sprint.\n",
            "* Feedback on pull requests was added within a day.\n",
            "* No critical bugs were introduced.\n",
            "* Build issues prevented us from resolving them and will be moved to a subsequent sprint.\n",
            "* Back-end development took more time than expected and needs to be considered in future estimations.\n",
            "\n",
            "\n",
            "----------------------------------------------------------------------------------------------------\n",
            "Documents:\n",
            "\n",
            "Document 1 \n",
            "Title: 2016-05-23 Retrospective\n",
            "Header: Retrospective\n",
            "Source: 2016-05-23+Retrospective\n",
            "Link: https://templates.atlassian.net/wiki/spaces/SWPRJ/pages/33560/2016-05-23+Retrospective\n",
            "Content:\n",
            "#91c89c#f3f9f4### What did we do well?  \n",
            "* 75% of issues were resolved and deployed.\n",
            "* New QA process helped eliminate last-minute blocking bugs.\n",
            "* Designs were finalised and ready in advance of starting the sprint.\n",
            "* Feedback on pull requests was added within a day.\n",
            "* No critical bugs were introduced.\n",
            "#d04437#fff8f7### What should we have done better?  \n",
            "* Build issues prevented us from resolving  We'll need to move this work to a subsequent sprint.\n",
            "* Back-end development cost more time than expected; we need to consider this in future estimations.\n",
            "----------------------------------------------------------------------------------------------------\n",
            "Document 2 \n",
            "Title: 2016-05-23 Retrospective\n",
            "Header: 2016-05-23 Retrospective\n",
            "Source: 2016-05-23+Retrospective\n",
            "Link: https://templates.atlassian.net/wiki/spaces/SWPRJ/pages/33560/2016-05-23+Retrospective\n",
            "Content:\n",
            "| Date | |\n",
            "| Participants | |  \n",
            "---\n",
            "----------------------------------------------------------------------------------------------------\n",
            "Document 3 \n",
            "Title: What is this site ?\n",
            "Header: What is this site ?\n",
            "Source: 1972475\n",
            "Link: https://templates.atlassian.net/wiki/spaces/SWPRJ/pages/1972475\n",
            "Content:\n",
            "We've created this site to enable you to see how a Confluence site looks like.  \n",
            "You can visit this site just as if it was yours, except that you are in read only, so you're not able to modify the content.  \n",
            "This site is still beta and our teams are working on it in order to display as many features as we can.\n",
            "----------------------------------------------------------------------------------------------------\n",
            "Document 4 \n",
            "Title: 2016-05-23 Retrospective\n",
            "Header: Action items\n",
            "Source: 2016-05-23+Retrospective\n",
            "Link: https://templates.atlassian.net/wiki/spaces/SWPRJ/pages/33560/2016-05-23+Retrospective\n",
            "Content:\n",
            "|  \n",
            "1\n",
            "complete\n",
            "to work on resolving build issues for the next sprint. Due by  \n",
            "|\n",
            "|  \n",
            "5\n",
            "incomplete\n",
            "to do primer on improved estimation process on  \n",
            "|\n",
            "|  \n",
            "0\n",
            "incomplete\n",
            "will buy team birthday cake for  \n",
            "|\n",
            "----------------------------------------------------------------------------------------------------\n",
            "Document 5 \n",
            "Title: Retrospectives\n",
            "Header: Retrospectives\n",
            "Source: Retrospectives\n",
            "Link: https://templates.atlassian.net/wiki/spaces/SWPRJ/pages/33203/Retrospectives\n",
            "Content:\n",
            "com.atlassian.confluence.plugins.confluence-business-blueprints:file-list-blueprinta2604e03-f7c1-4359-947d-2687c45d215da2604e03-f7c1-4359-947d-2687c45d215dviewCreate retrospectiveCreate retrospective  \n",
            "TitleDate,Participantsretrospectivelabel = \"retrospective\" and space = currentSpace()\n",
            "----------------------------------------------------------------------------------------------------\n",
            "Document 6 \n",
            "Title: 2016-05-12 Sprint Planning \"Search\"\n",
            "Header: Project goals\n",
            "Source: 33607\n",
            "Link: https://templates.atlassian.net/wiki/spaces/SWPRJ/pages/33607\n",
            "Content:\n",
            "---\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "pretty_print_response(\"What projects does the confluence space has?\")"
      ],
      "metadata": {
        "id": "F4vcqRyvstwY",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "f8bf34a6-ce9f-4bd6-e4ee-0117eb632b80"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/transformers/pipelines/base.py:1101: UserWarning: You seem to be using the pipelines sequentially on GPU. In order to maximize efficiency please use a dataset\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Question: What projects does the confluence space has?\n",
            "\n",
            "\n",
            "Answer: \n",
            "The confluence space has the following projects:\n",
            "\n",
            "* Demo Project\n",
            "* Search\n",
            "* Retrospectives\n",
            "* Decision Log\n",
            "\n",
            "Note: These are the projects that are available in the sample data provided in the source documents.\n",
            "\n",
            "\n",
            "----------------------------------------------------------------------------------------------------\n",
            "Documents:\n",
            "\n",
            "Document 1 \n",
            "Title: What is this site ?\n",
            "Header: What is this site ?\n",
            "Source: 1972475\n",
            "Link: https://templates.atlassian.net/wiki/spaces/SWPRJ/pages/1972475\n",
            "Content:\n",
            "We've created this site to enable you to see how a Confluence site looks like.  \n",
            "You can visit this site just as if it was yours, except that you are in read only, so you're not able to modify the content.  \n",
            "This site is still beta and our teams are working on it in order to display as many features as we can.\n",
            "----------------------------------------------------------------------------------------------------\n",
            "Document 2 \n",
            "Title: 2016-05-12 Sprint Planning \"Search\"\n",
            "Header: Project goals\n",
            "Source: 33607\n",
            "Link: https://templates.atlassian.net/wiki/spaces/SWPRJ/pages/33607\n",
            "Content:\n",
            "---\n",
            "----------------------------------------------------------------------------------------------------\n",
            "Document 3 \n",
            "Title: Retrospectives\n",
            "Header: Retrospectives\n",
            "Source: Retrospectives\n",
            "Link: https://templates.atlassian.net/wiki/spaces/SWPRJ/pages/33203/Retrospectives\n",
            "Content:\n",
            "com.atlassian.confluence.plugins.confluence-business-blueprints:file-list-blueprinta2604e03-f7c1-4359-947d-2687c45d215da2604e03-f7c1-4359-947d-2687c45d215dviewCreate retrospectiveCreate retrospective  \n",
            "TitleDate,Participantsretrospectivelabel = \"retrospective\" and space = currentSpace()\n",
            "----------------------------------------------------------------------------------------------------\n",
            "Document 4 \n",
            "Title: Demo Project Home\n",
            "Header: Demo Project Home\n",
            "Source: overview\n",
            "Link: https://templates.atlassian.net/wiki/spaces/SWPRJ/overview\n",
            "Content:\n",
            "page, comment, blogpost14trueconcise  \n",
            "---\n",
            "----------------------------------------------------------------------------------------------------\n",
            "Document 5 \n",
            "Title: Decision log\n",
            "Header: Here you'll be able to find all the answers to any decisions you come across, and the justifications for why any past decisions were made. The decision owner is responsible for following up with stakeholders.\n",
            "Source: Decision+log\n",
            "Link: https://templates.atlassian.net/wiki/spaces/SWPRJ/pages/33219/Decision+log\n",
            "Content:\n",
            "com.atlassian.confluence.plugins.confluence-business-blueprints:file-list-blueprinta2604e03-f7c1-4359-947d-2687c45d215da2604e03-f7c1-4359-947d-2687c45d215dviewCreate decisionCreate decision  \n",
            "---  \n",
            "com.atlassian.confluence.plugins.confluence-business-blueprints:decisions-blueprint 1c90e9ac-e8f9-4608-9388-56d2a87f82d6Record important project decisions and communicate them with your team. DecisionDecisionsSOF3Status,Stakeholders,Outcome,Due date,Ownerdecisionslabel = \"decisions\" and space = \"SWPRJ\"\n",
            "----------------------------------------------------------------------------------------------------\n",
            "Document 6 \n",
            "Title: Demo Project Home\n",
            "Header: Demo Project Home\n",
            "Source: overview\n",
            "Link: https://templates.atlassian.net/wiki/spaces/SWPRJ/overview\n",
            "Content:\n",
            "---\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "pretty_print_response(\"How are search results ordered?\")"
      ],
      "metadata": {
        "id": "3bjvnvovtTn4",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "0440be58-e703-44d0-82cf-beb37769f041"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/transformers/pipelines/base.py:1101: UserWarning: You seem to be using the pipelines sequentially on GPU. In order to maximize efficiency please use a dataset\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Question: How are search results ordered?\n",
            "\n",
            "\n",
            "Answer:  According to the document, search results are ordered based on the following options:\n",
            "\n",
            "* By date (surfaces the most recent content)\n",
            "* In alphabetical order (easy to organize and find what you're looking for if you know the title, but not useful if the title is unknown)\n",
            "* By popularity (surfaces the most important current content, but may not be useful if users are searching for something obscure)\n",
            "\n",
            "It is also mentioned that the decision on how to order search results was made based on user research and analytics.\n",
            "\n",
            "\n",
            "----------------------------------------------------------------------------------------------------\n",
            "Documents:\n",
            "\n",
            "Document 1 \n",
            "Title: How should search results be ordered?\n",
            "Header: Background\n",
            "Source: 33510\n",
            "Link: https://templates.atlassian.net/wiki/spaces/SWPRJ/pages/33510\n",
            "Content:\n",
            "The 'Search' function is used to find relevant pages, blogs, files and comments on Confluence. These display the page, blog or file name, the space they're in, and the name of the person who created them. How should these be ordered?\n",
            "----------------------------------------------------------------------------------------------------\n",
            "Document 2 \n",
            "Title: How should search results be ordered?\n",
            "Header: Decision\n",
            "Source: 33510\n",
            "Link: https://templates.atlassian.net/wiki/spaces/SWPRJ/pages/33510\n",
            "Content:\n",
            "| Option | Positives | Negatives | Unknowns |\n",
            "| --- | --- | --- | --- |\n",
            "| By date | Surfaces the most recent content | What about older content that is still relevant? | Will this use date modified or date created? |\n",
            "| In alphabetical order | Easy to organise and find what you're looking for if you know the title Don't have to search through results much as you'll know where to find yours | Not useful if title is unknown  | Do users typically know the title of what they're looking for? |\n",
            "| By popularity | Surfaces the most important current content | Users might be after something obscure, popularity is not necessarily important | Is there any correlation between popularity and what users search for? |\n",
            "----------------------------------------------------------------------------------------------------\n",
            "Document 3 \n",
            "Title: How should search results be ordered?\n",
            "Header: How should search results be ordered?\n",
            "Source: 33510\n",
            "Link: https://templates.atlassian.net/wiki/spaces/SWPRJ/pages/33510\n",
            "Content:\n",
            "| Status | GreenDecided |\n",
            "| Stakeholders | |\n",
            "| Outcome | trueRedOrder by date |\n",
            "| Due date | |\n",
            "| Owner | |\n",
            "----------------------------------------------------------------------------------------------------\n",
            "Document 4 \n",
            "Title: How should search results be ordered?\n",
            "Header: Action items\n",
            "Source: 33510\n",
            "Link: https://templates.atlassian.net/wiki/spaces/SWPRJ/pages/33510\n",
            "Content:\n",
            "|  \n",
            "61\n",
            "complete\n",
            "to conduct user research  \n",
            "|\n",
            "|  \n",
            "66\n",
            "complete\n",
            "to compare user analytics  \n",
            "|\n",
            "|  \n",
            "67\n",
            "incomplete\n",
            "to design user flow  \n",
            "|\n",
            "|  \n",
            "65\n",
            "complete\n",
            "to approve designs  \n",
            "|  \n",
            "grey#FBFBFB1solid### Related to:  \n",
            "---\n",
            "----------------------------------------------------------------------------------------------------\n",
            "Document 5 \n",
            "Title: User story: Search\n",
            "Header: User story: Search\n",
            "Source: User+story%3A+Search\n",
            "Link: https://templates.atlassian.net/wiki/spaces/SWPRJ/pages/33685/User+story%3A+Search\n",
            "Content:\n",
            "| Epic link | |\n",
            "| Status | trueGreenCOMPLETE |\n",
            "| Developer | |\n",
            "| Designer | |\n",
            "| Product manager | |\n",
            "----------------------------------------------------------------------------------------------------\n",
            "Document 6 \n",
            "Title: 2016-05-12 Sprint Planning \"Search\"\n",
            "Header: Goals\n",
            "Source: 33607\n",
            "Link: https://templates.atlassian.net/wiki/spaces/SWPRJ/pages/33607\n",
            "Content:\n",
            "grey1dashed### In this sprint we aim to implement a working prototype for search.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "pretty_print_response(\"What is planned for notifications?\")"
      ],
      "metadata": {
        "id": "iOExj8wms1A4",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "a4f28c51-5730-4631-b31c-523d8bdb46fb"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/transformers/pipelines/base.py:1101: UserWarning: You seem to be using the pipelines sequentially on GPU. In order to maximize efficiency please use a dataset\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Question: What is planned for notifications?\n",
            "\n",
            "\n",
            "Answer:  Based on the provided documents, it appears that the goal for notifications is to implement a working prototype. The team is also considering what the notification feed blank state should look like, and there are several options being considered, including an empty blue background, Confluence pro-tips, and a funny illustration of a puppy looking for content. Additionally, the team is deciding whether to include blog posts in the notification feed.\n",
            "\n",
            "\n",
            "----------------------------------------------------------------------------------------------------\n",
            "Documents:\n",
            "\n",
            "Document 1 \n",
            "Title: 2016-06-19 Sprint Planning \"Notifications\"\n",
            "Header: Goals\n",
            "Source: 33655\n",
            "Link: https://templates.atlassian.net/wiki/spaces/SWPRJ/pages/33655\n",
            "Content:\n",
            "grey1dashed### In this sprint we aim to implement a working prototype for notifications.  \n",
            "---\n",
            "----------------------------------------------------------------------------------------------------\n",
            "Document 2 \n",
            "Title: What does the notification feed blank state look like?\n",
            "Header: Background\n",
            "Source: 33547\n",
            "Link: https://templates.atlassian.net/wiki/spaces/SWPRJ/pages/33547\n",
            "Content:\n",
            "The notification feed displays either the latest activity. What should it look like on new installations, before there's enough activity to fill the screen?\n",
            "----------------------------------------------------------------------------------------------------\n",
            "Document 3 \n",
            "Title: 2016-06-19 Sprint Planning \"Notifications\"\n",
            "Header: Action items\n",
            "Source: 33655\n",
            "Link: https://templates.atlassian.net/wiki/spaces/SWPRJ/pages/33655\n",
            "Content:\n",
            "|  \n",
            "1\n",
            "complete\n",
            "to follow up on process for improving QA notes by  \n",
            "|\n",
            "|  \n",
            "2\n",
            "incomplete\n",
            "will prepare to do training on advanced estimations by  \n",
            "|\n",
            "|  \n",
            "3\n",
            "incomplete\n",
            "to help with setting up new development environment by  \n",
            "|\n",
            "----------------------------------------------------------------------------------------------------\n",
            "Document 4 \n",
            "Title: Should blog posts be included in the notification feed?\n",
            "Header: Background\n",
            "Source: 33526\n",
            "Link: https://templates.atlassian.net/wiki/spaces/SWPRJ/pages/33526\n",
            "Content:\n",
            "Blogs are used for content that is specific mainly to the current time­frame. Blog posts currently display as an infinite scroll, so they surface the latest news and visitors just need to scroll down if they're interested in older content.\n",
            "----------------------------------------------------------------------------------------------------\n",
            "Document 5 \n",
            "Title: User story: Notifications\n",
            "Header: User story: Notifications\n",
            "Source: User+story%3A+Notifications\n",
            "Link: https://templates.atlassian.net/wiki/spaces/SWPRJ/pages/33679/User+story%3A+Notifications\n",
            "Content:\n",
            "| Epic link | |\n",
            "| Status | trueGreenBacklog |\n",
            "| Developer | |\n",
            "| Designer | |\n",
            "| Product manager | |\n",
            "----------------------------------------------------------------------------------------------------\n",
            "Document 6 \n",
            "Title: What does the notification feed blank state look like?\n",
            "Header: Decision\n",
            "Source: 33547\n",
            "Link: https://templates.atlassian.net/wiki/spaces/SWPRJ/pages/33547\n",
            "Content:\n",
            "| Option | Positives | Negatives | Unknowns |\n",
            "| --- | --- | --- | --- |\n",
            "| Empty - just a blue background | No unnecessary noise or distractions Less complexity | Looks a bit ugly | - |\n",
            "| Confluence pro-tips, which change every time you revisit the page | More visually interesting Good way to surface useful features that users might not know about | Distracting Could be overwhelming for new users  | If they're new to Confluence, will they find pro-tips useful? However, they might be previous Confluence users just using a new instance. |\n",
            "| Funny illustration(s) of a puppy looking for content | Everyone loves puppies Visually interesting | Might be distracting | How would this scale as the page fills up with content? |\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "pretty_print_response(\"What are some questions that the the confluence space answers?\")"
      ],
      "metadata": {
        "id": "DtrkjmdfthKD",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "5e32bb5c-c851-4121-d6f1-5ef5295c9527"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/transformers/pipelines/base.py:1101: UserWarning: You seem to be using the pipelines sequentially on GPU. In order to maximize efficiency please use a dataset\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Question: What are some questions that the the confluence space answers?\n",
            "\n",
            "\n",
            "Answer:  Based on the provided sources, the Confluence space answers various questions related to decision-making, retrospectives, sprint planning, and notification feeds. Some of the questions answered by the Confluence space include:\n",
            "\n",
            "* What is this site?\n",
            "* How do I create a decision?\n",
            "* How do I create a retrospective?\n",
            "* Should blog posts be included in the notification feed?\n",
            "* How do I plan my meetings and share notes and actions with my team?\n",
            "\n",
            "The Confluence space also provides answers to questions related to the structure and organization of the platform, such as how to create a new space or how to manage permissions. However, it does not provide answers to questions outside the scope of the Confluence platform, such as general knowledge or industry-specific topics.\n",
            "\n",
            "\n",
            "----------------------------------------------------------------------------------------------------\n",
            "Documents:\n",
            "\n",
            "Document 1 \n",
            "Title: What is this site ?\n",
            "Header: What is this site ?\n",
            "Source: 1972475\n",
            "Link: https://templates.atlassian.net/wiki/spaces/SWPRJ/pages/1972475\n",
            "Content:\n",
            "We've created this site to enable you to see how a Confluence site looks like.  \n",
            "You can visit this site just as if it was yours, except that you are in read only, so you're not able to modify the content.  \n",
            "This site is still beta and our teams are working on it in order to display as many features as we can.\n",
            "----------------------------------------------------------------------------------------------------\n",
            "Document 2 \n",
            "Title: Decision log\n",
            "Header: Here you'll be able to find all the answers to any decisions you come across, and the justifications for why any past decisions were made. The decision owner is responsible for following up with stakeholders.\n",
            "Source: Decision+log\n",
            "Link: https://templates.atlassian.net/wiki/spaces/SWPRJ/pages/33219/Decision+log\n",
            "Content:\n",
            "com.atlassian.confluence.plugins.confluence-business-blueprints:file-list-blueprinta2604e03-f7c1-4359-947d-2687c45d215da2604e03-f7c1-4359-947d-2687c45d215dviewCreate decisionCreate decision  \n",
            "---  \n",
            "com.atlassian.confluence.plugins.confluence-business-blueprints:decisions-blueprint 1c90e9ac-e8f9-4608-9388-56d2a87f82d6Record important project decisions and communicate them with your team. DecisionDecisionsSOF3Status,Stakeholders,Outcome,Due date,Ownerdecisionslabel = \"decisions\" and space = \"SWPRJ\"\n",
            "----------------------------------------------------------------------------------------------------\n",
            "Document 3 \n",
            "Title: CrowdWire - Massive Online Meetings\n",
            "Header: 2nd Milestone\n",
            "Source: README.md\n",
            "Link: https://github.com/Crowd-Wire/ProjectDocumentation/tree/master//README.md\n",
            "Content:\n",
            "- Basic User movement on predefined maps\n",
            "- Video and voice communication\n",
            "- Text Chat feature\n",
            "- Backend Structure and databases\n",
            "- User Interface\n",
            "----------------------------------------------------------------------------------------------------\n",
            "Document 4 \n",
            "Title: Retrospectives\n",
            "Header: Retrospectives\n",
            "Source: Retrospectives\n",
            "Link: https://templates.atlassian.net/wiki/spaces/SWPRJ/pages/33203/Retrospectives\n",
            "Content:\n",
            "com.atlassian.confluence.plugins.confluence-business-blueprints:file-list-blueprinta2604e03-f7c1-4359-947d-2687c45d215da2604e03-f7c1-4359-947d-2687c45d215dviewCreate retrospectiveCreate retrospective  \n",
            "TitleDate,Participantsretrospectivelabel = \"retrospective\" and space = currentSpace()\n",
            "----------------------------------------------------------------------------------------------------\n",
            "Document 5 \n",
            "Title: Should blog posts be included in the notification feed?\n",
            "Header: Decision\n",
            "Source: 33526\n",
            "Link: https://templates.atlassian.net/wiki/spaces/SWPRJ/pages/33526\n",
            "Content:\n",
            "| Option | Positives | Negatives | Unknowns |\n",
            "| --- | --- | --- | --- |\n",
            "| Blog shows in notification feed | Blogs are timely, therefore this surfaces content that is meant to be relevant at that point. News is critical in many organisations. | Too much noise and unnecessary distractions relative to more personal notifications | Do many of our users have blogs? Will this actually affect them? |\n",
            "| Have a separate feed for blogs | Blogs tend to have a specific type of content, such as news and announcements, rather than evergreen information. This lets users select what kind of content they want to consume. | Too many buttons creates unnecessary complexity. | Will our users forget about the other button? Will this result in blog content going unnoticed? |\n",
            "----------------------------------------------------------------------------------------------------\n",
            "Document 6 \n",
            "Title: Sprint planning\n",
            "Header: Sprint plan directory\n",
            "Source: Sprint+planning\n",
            "Link: https://templates.atlassian.net/wiki/spaces/SWPRJ/pages/33221/Sprint+planning\n",
            "Content:\n",
            "com.atlassian.confluence.plugins.confluence-business-blueprints:meeting-notes-blueprint 89caf976-957e-45b7-9ec5-6777a633f09bmeeting-notesPlan your meetings and share notes and actions with your team. Meeting notesmeeting-notes\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## [Full-Stack App GH Repo](https://crowd-wire.github.io/ProjectDocumentation/)"
      ],
      "metadata": {
        "id": "MiqzbWoXiPrS"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "pretty_print_response(\"What is CrowdWire?\")"
      ],
      "metadata": {
        "id": "o-H9mE47iQVf",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "eaf6090c-5cac-49de-d887-2dcb828624fc"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/transformers/pipelines/base.py:1101: UserWarning: You seem to be using the pipelines sequentially on GPU. In order to maximize efficiency please use a dataset\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Question: What is CrowdWire?\n",
            "\n",
            "\n",
            "Answer:  Based on the provided documents, CrowdWire appears to be a web-based platform designed for massive online meetings. It utilizes various technologies such as FastAPI, ReactJS, and Mediasoup to provide features like world-editor, full integration of each micro-service, proximity video and voice chatting with high availability, CI/CD, and file exchange. Additionally, the platform allows users to run it locally by executing certain commands inside the `frontend/` directory.\n",
            "\n",
            "\n",
            "----------------------------------------------------------------------------------------------------\n",
            "Documents:\n",
            "\n",
            "Document 1 \n",
            "Title: CrowdWire - Massive Online Meetings\n",
            "Header: 3rd Milestone\n",
            "Source: README.md\n",
            "Link: https://github.com/Crowd-Wire/ProjectDocumentation/tree/master//README.md\n",
            "Content:\n",
            "- World-Editor\n",
            "- Full Integration of each micro-service\n",
            "- Proximity video and voice chatting with high availability\n",
            "- CI/CD\n",
            "- File Exchange feature\n",
            "----------------------------------------------------------------------------------------------------\n",
            "Document 2 \n",
            "Title: Welcome to CrowdWire\n",
            "Header: Project organization\n",
            "Source: docs/index.md\n",
            "Link: https://github.com/Crowd-Wire/ProjectDocumentation/tree/master//docs/index.md\n",
            "Content:\n",
            ".github/workflows # The CI pipelines with github Actions.\n",
            "api/ # The backend Rest API with FastAPI.\n",
            "frontend/ # The project website with ReactJS.\n",
            "k8s/ # The configuration files for deployment with Kubernets.\n",
            "media_server/ # The media server for communications with Mediasoup.\n",
            "README.md # The project description and team information.\n",
            "----------------------------------------------------------------------------------------------------\n",
            "Document 3 \n",
            "Title: CrowdWire - Massive Online Meetings\n",
            "Header: How to run locally:\n",
            "Source: docs/frontend.md\n",
            "Link: https://github.com/Crowd-Wire/ProjectDocumentation/tree/master//docs/frontend.md\n",
            "Content:\n",
            "To run locally execute the following commands inside the `frontend/` directory:  \n",
            "```\n",
            "$ npm install\n",
            "$ npm start\n",
            "```\n",
            "----------------------------------------------------------------------------------------------------\n",
            "Document 4 \n",
            "Title: CrowdWire - Massive Online Meetings\n",
            "Header: 1st Milestone\n",
            "Source: README.md\n",
            "Link: https://github.com/Crowd-Wire/ProjectDocumentation/tree/master//README.md\n",
            "Content:\n",
            "- Set up docs platforms tools\n",
            "- Develop a prototype\n",
            "- Research technologies and define the system architecture\n",
            "- Start testing and developing shortly each area/framework individually\n",
            "----------------------------------------------------------------------------------------------------\n",
            "Document 5 \n",
            "Title: Welcome to CrowdWire\n",
            "Header: Overview\n",
            "Source: docs/media_server.md\n",
            "Link: https://github.com/Crowd-Wire/ProjectDocumentation/tree/master//docs/media_server.md\n",
            "Content:\n",
            "[Mediasoup](https://mediasoup.org/) has an SFU (Selective Forwarding Unit) topology, where the server routes media around between participants while balancing its limitations with the media inputs it receives. With this approach, participants send their media to the server and receive others' media in separate streams, one each.  \n",
            "For more details of how mediasoup works and how we interacted with it within the browser, check out their documentation:  \n",
            "- [Server Side Library](https://mediasoup.org/documentation/v3/mediasoup/api/)\n",
            "- [Client Side Library](https://mediasoup.org/documentation/v3/mediasoup-client/api/)\n",
            "----------------------------------------------------------------------------------------------------\n",
            "Document 6 \n",
            "Title: Welcome to CrowdWire\n",
            "Header: Deployment Diagram\n",
            "Source: docs/architecture.md\n",
            "Link: https://github.com/Crowd-Wire/ProjectDocumentation/tree/master//docs/architecture.md\n",
            "Content:\n",
            "![Deployment_Diagram](assets/Images/Architecture Diagrams/Deployment_Diagram.png)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "pretty_print_response(\"What is the structure of CrowdWire kubernets charts?\")"
      ],
      "metadata": {
        "id": "6MKajnxziQXs",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "19fe432f-0e03-4f80-f6a6-f1dd9cfd1a80"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/transformers/pipelines/base.py:1101: UserWarning: You seem to be using the pipelines sequentially on GPU. In order to maximize efficiency please use a dataset\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Question: What is the structure of CrowdWire kubernets charts?\n",
            "\n",
            "\n",
            "Answer:  The structure of CrowdWire kubernetes charts is organized into an umbrella chart containing all microservices as subcharts. The subcharts include:\n",
            "\n",
            "* api\n",
            "* frontend\n",
            "* rabbitmq\n",
            "* postgresql\n",
            "* redis\n",
            "\n",
            "Note: The `rabbitmq`, `postgresql`, and `redis` charts belong to Bitnami.\n",
            "\n",
            "\n",
            "----------------------------------------------------------------------------------------------------\n",
            "Documents:\n",
            "\n",
            "Document 1 \n",
            "Title: Welcome to CrowdWire\n",
            "Header: Run our Helm Charts\n",
            "Source: docs/deployment.md\n",
            "Link: https://github.com/Crowd-Wire/ProjectDocumentation/tree/master//docs/deployment.md\n",
            "Content:\n",
            "To manage the packaging of the Kubernetes resources, we are using [Helm]( https://helm.sh/). If you want to make use of our charts locally, do the following steps. Before installing this chart please check the previous deployment notes.  \n",
            "```\n",
            "$cd charts\n",
            "\n",
            "$helm dep build\n",
            "\n",
            "$helm install crowdwire .\n",
            "```  \n",
            "To observe the initialization of all Pods, use the following command:  \n",
            "```\n",
            "kubectl get all -o wide\n",
            "```  \n",
            "This is the structure of our Charts: An Umbrella Chart containing all our microservices, as subcharts.  \n",
            "```\n",
            "├── Chart.lock\n",
            "├── Chart.yaml\n",
            "├── subcharts\n",
            "│ ├── api\n",
            "│ ├── frontend\n",
            "|\t├── mediaserver\n",
            "│ ├── postgresql\n",
            "│ ├── rabbitmq\n",
            "│ └── redis\n",
            "├── templates\n",
            "│ ├── _helpers.tpl\n",
            "│ └── ingress.yaml\n",
            "└── values.yaml\n",
            "```  \n",
            "Important to refer that the `rabbitmq`, `postgresql` and `redis` charts belong to [Bitnami](https://github.com/bitnami) .\n",
            "----------------------------------------------------------------------------------------------------\n",
            "Document 2 \n",
            "Title: Welcome to CrowdWire\n",
            "Header: DB Modelation\n",
            "Source: docs/REST_API.md\n",
            "Link: https://github.com/Crowd-Wire/ProjectDocumentation/tree/master//docs/REST_API.md\n",
            "Content:\n",
            "To model our database we used the following diagram:\n",
            "----------------------------------------------------------------------------------------------------\n",
            "Document 3 \n",
            "Title: Welcome to CrowdWire\n",
            "Header: Operations\n",
            "Source: docs/media_server.md\n",
            "Link: https://github.com/Crowd-Wire/ProjectDocumentation/tree/master//docs/media_server.md\n",
            "Content:\n",
            "In the `media_server/src/utils/startRabbit` directory it is possible to see the operations that are handled by the media server using typescript definitions:  \n",
            "![image](assets/Images/handler_map.png)  \n",
            "And also what's returned at those operations:  \n",
            "![image](assets/Images/outgoing_map.png)\n",
            "----------------------------------------------------------------------------------------------------\n",
            "Document 4 \n",
            "Title: CrowdWire - Massive Online Meetings\n",
            "Header: 2nd Milestone\n",
            "Source: README.md\n",
            "Link: https://github.com/Crowd-Wire/ProjectDocumentation/tree/master//README.md\n",
            "Content:\n",
            "- Basic User movement on predefined maps\n",
            "- Video and voice communication\n",
            "- Text Chat feature\n",
            "- Backend Structure and databases\n",
            "- User Interface\n",
            "----------------------------------------------------------------------------------------------------\n",
            "Document 5 \n",
            "Title: Welcome to CrowdWire\n",
            "Header: Kubernetes Metrics Server\n",
            "Source: docs/deployment.md\n",
            "Link: https://github.com/Crowd-Wire/ProjectDocumentation/tree/master//docs/deployment.md\n",
            "Content:\n",
            "In order to make use of the autoscaling functionality on Kubernetes, it is needed to install the metric server. If you are using using a Cloud Provider for your cluster, you should not need to run any command of the following!  \n",
            "If you are using minikube, you can simply run the following command:  \n",
            "```\n",
            "$ minikube addons enable metrics-server\n",
            "```  \n",
            "Without minikube, you will need to do the following steps:  \n",
            "```\n",
            "$ curl https://github.com/kubernetes-sigs/metrics-server/releases/download/v0.4.4/components.yaml\n",
            "$ vi component.yaml\n",
            "```  \n",
            "Change the Deployment Resource, by adding the following flag `--kubelet-insecure-tls`:  \n",
            "```yaml\n",
            "spec:\n",
            "containers:\n",
            "- args:\n",
            "- --cert-dir=/tmp\n",
            "- --secure-port=4443\n",
            "- --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname\n",
            "- --kubelet-use-node-status-port\n",
            "- --kubelet-insecure-tls\n",
            "\n",
            "```\n",
            "----------------------------------------------------------------------------------------------------\n",
            "Document 6 \n",
            "Title: Welcome to CrowdWire\n",
            "Header: Project organization\n",
            "Source: docs/index.md\n",
            "Link: https://github.com/Crowd-Wire/ProjectDocumentation/tree/master//docs/index.md\n",
            "Content:\n",
            ".github/workflows # The CI pipelines with github Actions.\n",
            "api/ # The backend Rest API with FastAPI.\n",
            "frontend/ # The project website with ReactJS.\n",
            "k8s/ # The configuration files for deployment with Kubernets.\n",
            "media_server/ # The media server for communications with Mediasoup.\n",
            "README.md # The project description and team information.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "pretty_print_response(\"What are the technologies stack used in CrowdWire?\")"
      ],
      "metadata": {
        "id": "n-Ft-CvQiQZ2",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "661ae729-02ec-4208-d5d5-31932aba499e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/transformers/pipelines/base.py:1101: UserWarning: You seem to be using the pipelines sequentially on GPU. In order to maximize efficiency please use a dataset\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Question: What are the technologies stack used in CrowdWire?\n",
            "\n",
            "\n",
            "Answer:  Based on the document provided, the following technologies are used in CrowdWire:\n",
            "\n",
            "* Backend REST API with FastAPI\n",
            "* Frontend website with ReactJS\n",
            "* Configuration files for deployment with Kubernetes (k8s)\n",
            "* Media server for communications with Mediasoup\n",
            "* World-Editor\n",
            "* Proximity video and voice chatting with high availability\n",
            "* CI/CD\n",
            "* File Exchange feature\n",
            "\n",
            "Please note that this list may not be exhaustive as there might be other technologies used in CrowdWire that are not mentioned in the provided documents.\n",
            "\n",
            "\n",
            "----------------------------------------------------------------------------------------------------\n",
            "Documents:\n",
            "\n",
            "Document 1 \n",
            "Title: CrowdWire - Massive Online Meetings\n",
            "Header: 1st Milestone\n",
            "Source: README.md\n",
            "Link: https://github.com/Crowd-Wire/ProjectDocumentation/tree/master//README.md\n",
            "Content:\n",
            "- Set up docs platforms tools\n",
            "- Develop a prototype\n",
            "- Research technologies and define the system architecture\n",
            "- Start testing and developing shortly each area/framework individually\n",
            "----------------------------------------------------------------------------------------------------\n",
            "Document 2 \n",
            "Title: Welcome to CrowdWire\n",
            "Header: Project organization\n",
            "Source: docs/index.md\n",
            "Link: https://github.com/Crowd-Wire/ProjectDocumentation/tree/master//docs/index.md\n",
            "Content:\n",
            ".github/workflows # The CI pipelines with github Actions.\n",
            "api/ # The backend Rest API with FastAPI.\n",
            "frontend/ # The project website with ReactJS.\n",
            "k8s/ # The configuration files for deployment with Kubernets.\n",
            "media_server/ # The media server for communications with Mediasoup.\n",
            "README.md # The project description and team information.\n",
            "----------------------------------------------------------------------------------------------------\n",
            "Document 3 \n",
            "Title: CrowdWire - Massive Online Meetings\n",
            "Header: 3rd Milestone\n",
            "Source: README.md\n",
            "Link: https://github.com/Crowd-Wire/ProjectDocumentation/tree/master//README.md\n",
            "Content:\n",
            "- World-Editor\n",
            "- Full Integration of each micro-service\n",
            "- Proximity video and voice chatting with high availability\n",
            "- CI/CD\n",
            "- File Exchange feature\n",
            "----------------------------------------------------------------------------------------------------\n",
            "Document 4 \n",
            "Title: Welcome to CrowdWire\n",
            "Header: System Technology Architecture\n",
            "Source: docs/architecture.md\n",
            "Link: https://github.com/Crowd-Wire/ProjectDocumentation/tree/master//docs/architecture.md\n",
            "Content:\n",
            "![System_Technology_Architecture](assets/Images/Architecture Diagrams/System_Technology_Architecture.png)\n",
            "----------------------------------------------------------------------------------------------------\n",
            "Document 5 \n",
            "Title: CrowdWire - Massive Online Meetings\n",
            "Header: 4th Milestone\n",
            "Source: README.md\n",
            "Link: https://github.com/Crowd-Wire/ProjectDocumentation/tree/master//README.md\n",
            "Content:\n",
            "- Map and User Permissions well integrated\n",
            "- Stress Testing to all features\n",
            "- Integrate additional features\n",
            "----------------------------------------------------------------------------------------------------\n",
            "Document 6 \n",
            "Title: Welcome to CrowdWire\n",
            "Header: Deployment Diagram\n",
            "Source: docs/architecture.md\n",
            "Link: https://github.com/Crowd-Wire/ProjectDocumentation/tree/master//docs/architecture.md\n",
            "Content:\n",
            "![Deployment_Diagram](assets/Images/Architecture Diagrams/Deployment_Diagram.png)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "pretty_print_response(\"What is the Crowdwire media server framework?\")"
      ],
      "metadata": {
        "id": "ueO_IXsbpeHp",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b3a186d1-121e-4f9f-90ed-f91db00184e6"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/transformers/pipelines/base.py:1101: UserWarning: You seem to be using the pipelines sequentially on GPU. In order to maximize efficiency please use a dataset\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Question: What is the Crowdwire media server framework?\n",
            "\n",
            "\n",
            "Answer:  The Crowdwire media server framework is a Selective Forwarding Unit (SFU) topology that routes media between participants while balancing its limitations with the media inputs it receives. It uses Mediasoup as its server side library and client side library for interaction within the browser.\n",
            "\n",
            "\n",
            "----------------------------------------------------------------------------------------------------\n",
            "Documents:\n",
            "\n",
            "Document 1 \n",
            "Title: Welcome to CrowdWire\n",
            "Header: Overview\n",
            "Source: docs/media_server.md\n",
            "Link: https://github.com/Crowd-Wire/ProjectDocumentation/tree/master//docs/media_server.md\n",
            "Content:\n",
            "[Mediasoup](https://mediasoup.org/) has an SFU (Selective Forwarding Unit) topology, where the server routes media around between participants while balancing its limitations with the media inputs it receives. With this approach, participants send their media to the server and receive others' media in separate streams, one each.  \n",
            "For more details of how mediasoup works and how we interacted with it within the browser, check out their documentation:  \n",
            "- [Server Side Library](https://mediasoup.org/documentation/v3/mediasoup/api/)\n",
            "- [Client Side Library](https://mediasoup.org/documentation/v3/mediasoup-client/api/)\n",
            "----------------------------------------------------------------------------------------------------\n",
            "Document 2 \n",
            "Title: Welcome to CrowdWire\n",
            "Header: System Technology Architecture\n",
            "Source: docs/architecture.md\n",
            "Link: https://github.com/Crowd-Wire/ProjectDocumentation/tree/master//docs/architecture.md\n",
            "Content:\n",
            "![System_Technology_Architecture](assets/Images/Architecture Diagrams/System_Technology_Architecture.png)\n",
            "----------------------------------------------------------------------------------------------------\n",
            "Document 3 \n",
            "Title: Welcome to CrowdWire\n",
            "Header: How to run locally:\n",
            "Source: docs/media_server.md\n",
            "Link: https://github.com/Crowd-Wire/ProjectDocumentation/tree/master//docs/media_server.md\n",
            "Content:\n",
            "Execute inside the `media_server/` directory:  \n",
            "- `npm i`\n",
            "- `npm run build`\n",
            "- `npm start`  \n",
            "Edit the `src/config.js` file according to your needs and create an `.env` file accordingly to the `.env.example`.\n",
            "----------------------------------------------------------------------------------------------------\n",
            "Document 4 \n",
            "Title: CrowdWire - Massive Online Meetings\n",
            "Header: 1st Milestone\n",
            "Source: README.md\n",
            "Link: https://github.com/Crowd-Wire/ProjectDocumentation/tree/master//README.md\n",
            "Content:\n",
            "- Set up docs platforms tools\n",
            "- Develop a prototype\n",
            "- Research technologies and define the system architecture\n",
            "- Start testing and developing shortly each area/framework individually\n",
            "----------------------------------------------------------------------------------------------------\n",
            "Document 5 \n",
            "Title: Welcome to CrowdWire\n",
            "Header: Operations\n",
            "Source: docs/media_server.md\n",
            "Link: https://github.com/Crowd-Wire/ProjectDocumentation/tree/master//docs/media_server.md\n",
            "Content:\n",
            "In the `media_server/src/utils/startRabbit` directory it is possible to see the operations that are handled by the media server using typescript definitions:  \n",
            "![image](assets/Images/handler_map.png)  \n",
            "And also what's returned at those operations:  \n",
            "![image](assets/Images/outgoing_map.png)\n",
            "----------------------------------------------------------------------------------------------------\n",
            "Document 6 \n",
            "Title: CrowdWire - Massive Online Meetings\n",
            "Header: 3rd Milestone\n",
            "Source: README.md\n",
            "Link: https://github.com/Crowd-Wire/ProjectDocumentation/tree/master//README.md\n",
            "Content:\n",
            "- World-Editor\n",
            "- Full Integration of each micro-service\n",
            "- Proximity video and voice chatting with high availability\n",
            "- CI/CD\n",
            "- File Exchange feature\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "pretty_print_response(\"What is the Mediasoup website link?\")"
      ],
      "metadata": {
        "id": "dF9rvVLXiQb1",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "657b0793-a3cc-4528-ea1b-a43160d53b71"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/transformers/pipelines/base.py:1101: UserWarning: You seem to be using the pipelines sequentially on GPU. In order to maximize efficiency please use a dataset\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Question: What is the Mediasoup website link?\n",
            "\n",
            "\n",
            "Answer:  The Mediasoup website link is <https://mediasoup.org/>.\n",
            "\n",
            "\n",
            "----------------------------------------------------------------------------------------------------\n",
            "Documents:\n",
            "\n",
            "Document 1 \n",
            "Title: Welcome to CrowdWire\n",
            "Header: Overview\n",
            "Source: docs/media_server.md\n",
            "Link: https://github.com/Crowd-Wire/ProjectDocumentation/tree/master//docs/media_server.md\n",
            "Content:\n",
            "[Mediasoup](https://mediasoup.org/) has an SFU (Selective Forwarding Unit) topology, where the server routes media around between participants while balancing its limitations with the media inputs it receives. With this approach, participants send their media to the server and receive others' media in separate streams, one each.  \n",
            "For more details of how mediasoup works and how we interacted with it within the browser, check out their documentation:  \n",
            "- [Server Side Library](https://mediasoup.org/documentation/v3/mediasoup/api/)\n",
            "- [Client Side Library](https://mediasoup.org/documentation/v3/mediasoup-client/api/)\n",
            "----------------------------------------------------------------------------------------------------\n",
            "Document 2 \n",
            "Title: What is this site ?\n",
            "Header: What is this site ?\n",
            "Source: 1972475\n",
            "Link: https://templates.atlassian.net/wiki/spaces/SWPRJ/pages/1972475\n",
            "Content:\n",
            "We've created this site to enable you to see how a Confluence site looks like.  \n",
            "You can visit this site just as if it was yours, except that you are in read only, so you're not able to modify the content.  \n",
            "This site is still beta and our teams are working on it in order to display as many features as we can.\n",
            "----------------------------------------------------------------------------------------------------\n",
            "Document 3 \n",
            "Title: Welcome to CrowdWire\n",
            "Header: Data Structures\n",
            "Source: docs/media_server.md\n",
            "Link: https://github.com/Crowd-Wire/ProjectDocumentation/tree/master//docs/media_server.md\n",
            "Content:\n",
            "For the Data Structures, we store the rooms associated with the media server, and each room has a [Worker](https://mediasoup.org/documentation/v3/mediasoup/api/#Worker), a [Router](https://mediasoup.org/documentation/v3/mediasoup/api/#Router) and a state associated.  \n",
            "![image](assets/Images/data_structures.png)  \n",
            "A state has [Transports](https://mediasoup.org/documentation/v3/mediasoup/api/#Transport) associated as well as a map [Producers](https://mediasoup.org/documentation/v3/mediasoup/api/#Producer) for each type (video and audio) and a list of [Consumers](https://mediasoup.org/documentation/v3/mediasoup/api/#Consumer) of those Producers.  \n",
            "![image](assets/Images/my_peer.png)\n",
            "----------------------------------------------------------------------------------------------------\n",
            "Document 4 \n",
            "Title: Files: Screenshots\n",
            "Header: Files: Screenshots\n",
            "Source: Files%3A+Screenshots\n",
            "Link: https://templates.atlassian.net/wiki/spaces/SWPRJ/pages/33299/Files%3A+Screenshots\n",
            "Content:\n",
            "Screenshots for use with promotional materialsPlease only use these in approved ways. Contact the marketing team if you have questions or before you distribute these externally.  \n",
            "true\n",
            "----------------------------------------------------------------------------------------------------\n",
            "Document 5 \n",
            "Title: Welcome to CrowdWire\n",
            "Header: How to run locally:\n",
            "Source: docs/media_server.md\n",
            "Link: https://github.com/Crowd-Wire/ProjectDocumentation/tree/master//docs/media_server.md\n",
            "Content:\n",
            "Execute inside the `media_server/` directory:  \n",
            "- `npm i`\n",
            "- `npm run build`\n",
            "- `npm start`  \n",
            "Edit the `src/config.js` file according to your needs and create an `.env` file accordingly to the `.env.example`.\n",
            "----------------------------------------------------------------------------------------------------\n",
            "Document 6 \n",
            "Title: Should blog posts be included in the notification feed?\n",
            "Header: Action items\n",
            "Source: 33526\n",
            "Link: https://templates.atlassian.net/wiki/spaces/SWPRJ/pages/33526\n",
            "Content:\n",
            "|  \n",
            "61\n",
            "incomplete\n",
            "to conduct user research into when users want to view blogs compared to pages (does this need to be on mobile or not?)  \n",
            "|\n",
            "|  \n",
            "62\n",
            "incomplete\n",
            "to check analytics for how many users have blogs, and how often blog posts get created  \n",
            "|  \n",
            "grey#FBFBFB1solid### Related to:  \n",
            "---\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "pretty_print_response(\"How are messages traded between client, backend and the media server?\")"
      ],
      "metadata": {
        "id": "91JXaEQoiQdj",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "63afefa2-e587-4d79-d643-fcd6824785e5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/transformers/pipelines/base.py:1101: UserWarning: You seem to be using the pipelines sequentially on GPU. In order to maximize efficiency please use a dataset\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Question: How are messages traded between client, backend and the media server?\n",
            "\n",
            "\n",
            "Answer:  Based on the provided sources, messages are traded between client, backend, and the media server through the use of rabbitMQ.\n",
            "\n",
            "\n",
            "----------------------------------------------------------------------------------------------------\n",
            "Documents:\n",
            "\n",
            "Document 1 \n",
            "Title: Welcome to CrowdWire\n",
            "Header: Overview\n",
            "Source: docs/media_server.md\n",
            "Link: https://github.com/Crowd-Wire/ProjectDocumentation/tree/master//docs/media_server.md\n",
            "Content:\n",
            "[Mediasoup](https://mediasoup.org/) has an SFU (Selective Forwarding Unit) topology, where the server routes media around between participants while balancing its limitations with the media inputs it receives. With this approach, participants send their media to the server and receive others' media in separate streams, one each.  \n",
            "For more details of how mediasoup works and how we interacted with it within the browser, check out their documentation:  \n",
            "- [Server Side Library](https://mediasoup.org/documentation/v3/mediasoup/api/)\n",
            "- [Client Side Library](https://mediasoup.org/documentation/v3/mediasoup-client/api/)\n",
            "----------------------------------------------------------------------------------------------------\n",
            "Document 2 \n",
            "Title: Welcome to CrowdWire\n",
            "Header: Operations\n",
            "Source: docs/media_server.md\n",
            "Link: https://github.com/Crowd-Wire/ProjectDocumentation/tree/master//docs/media_server.md\n",
            "Content:\n",
            "In the `media_server/src/utils/startRabbit` directory it is possible to see the operations that are handled by the media server using typescript definitions:  \n",
            "![image](assets/Images/handler_map.png)  \n",
            "And also what's returned at those operations:  \n",
            "![image](assets/Images/outgoing_map.png)\n",
            "----------------------------------------------------------------------------------------------------\n",
            "Document 3 \n",
            "Title: Welcome to CrowdWire\n",
            "Header: Rabbitmq\n",
            "Source: docs/REST_API.md\n",
            "Link: https://github.com/Crowd-Wire/ProjectDocumentation/tree/master//docs/REST_API.md\n",
            "Content:\n",
            "We used [aio-pika](https://aio-pika.readthedocs.io/en/latest/) to integrate rabbitMQ with our API, taking advantage of the fact that FastAPI is asynchronous. Communication between MediaSoup server and FastAPI is made using rabbitMQ.  \n",
            "Also rabbitMQ is used to share information between the API replicas created by kubernetes. Each of the replicas is consumer and producer in the `replicas queue`. Here we are using `FANOUT` exchange in order to broadcast the message to every replica.  \n",
            "The different protocols used in the rabbitMQ channels are present in the `RabbitProtocol` class:  \n",
            "```python\n",
            "class RabbitProtocol:\n",
            "GET_RECV_TRACKS_DONE = '@get-recv-tracks-done'\n",
            "SEND_TRACK_SEND_DONE = '@send-track-send-done'\n",
            "SEND_FILE_SEND_DONE = '@send-file-send-done'\n",
            "CONNECT_TRANSPORT_RECV_DONE = '@connect-transport-recv-done'\n",
            "CONNECT_TRANSPORT_SEND_DONE = '@connect-transport-send-done'\n",
            "YOU_JOINED_AS_PEER = 'you-joined-as-peer'\n",
            "YOU_JOINED_AS_SPEAKER = 'you-joined-as-speaker'\n",
            "YOU_ARE_NOW_A_SPEAKER = 'you-are-now-a-speaker'\n",
            "NEW_PEER_PRODUCER = 'new-peer-producer'\n",
            "NEW_PEER_DATA_PRODUCER = 'new-peer-data-producer'\n",
            "ERROR = 'error'\n",
            "CREATE_NEW_REPLICA = 'CREATE_NEW_REPLICA'\n",
            "```\n",
            "----------------------------------------------------------------------------------------------------\n",
            "Document 4 \n",
            "Title: CrowdWire - Massive Online Meetings\n",
            "Header: Context\n",
            "Source: README.md\n",
            "Link: https://github.com/Crowd-Wire/ProjectDocumentation/tree/master//README.md\n",
            "Content:\n",
            "With the emergence of Covid-19 all around the World, many people, companies and organizations started adopting remote procedures like video-conferences to continue doing the commons tasks they used to do \"in person\". Therefore, the search and use of these types of systems that allow video and voice calls has increased a lot.\n",
            "----------------------------------------------------------------------------------------------------\n",
            "Document 5 \n",
            "Title: Welcome to CrowdWire\n",
            "Header: Data Structures\n",
            "Source: docs/media_server.md\n",
            "Link: https://github.com/Crowd-Wire/ProjectDocumentation/tree/master//docs/media_server.md\n",
            "Content:\n",
            "For the Data Structures, we store the rooms associated with the media server, and each room has a [Worker](https://mediasoup.org/documentation/v3/mediasoup/api/#Worker), a [Router](https://mediasoup.org/documentation/v3/mediasoup/api/#Router) and a state associated.  \n",
            "![image](assets/Images/data_structures.png)  \n",
            "A state has [Transports](https://mediasoup.org/documentation/v3/mediasoup/api/#Transport) associated as well as a map [Producers](https://mediasoup.org/documentation/v3/mediasoup/api/#Producer) for each type (video and audio) and a list of [Consumers](https://mediasoup.org/documentation/v3/mediasoup/api/#Consumer) of those Producers.  \n",
            "![image](assets/Images/my_peer.png)\n",
            "----------------------------------------------------------------------------------------------------\n",
            "Document 6 \n",
            "Title: 2016-06-19 Sprint Planning \"Notifications\"\n",
            "Header: Goals\n",
            "Source: 33655\n",
            "Link: https://templates.atlassian.net/wiki/spaces/SWPRJ/pages/33655\n",
            "Content:\n",
            "grey1dashed### In this sprint we aim to implement a working prototype for notifications.  \n",
            "---\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## [SQL Tables GH Repo](https://github.com/JannikArndt/sql-auto-doc/tree/master/Examples)"
      ],
      "metadata": {
        "id": "d0UMpYAriQk_"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "pretty_print_response(\"What is the column joined?\")"
      ],
      "metadata": {
        "id": "nuyaCUu1iQqJ",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "14a95ed8-9d0d-4e74-8c7b-605de021c895"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/transformers/pipelines/base.py:1101: UserWarning: You seem to be using the pipelines sequentially on GPU. In order to maximize efficiency please use a dataset\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Question: What is the column joined?\n",
            "\n",
            "\n",
            "Answer:  The column joined represents the date when the employee joined our company.\n",
            "\n",
            "\n",
            "----------------------------------------------------------------------------------------------------\n",
            "Documents:\n",
            "\n",
            "Document 1 \n",
            "Title: Welcome to CrowdWire\n",
            "Header: DB Modelation\n",
            "Source: docs/REST_API.md\n",
            "Link: https://github.com/Crowd-Wire/ProjectDocumentation/tree/master//docs/REST_API.md\n",
            "Content:\n",
            "To model our database we used the following diagram:\n",
            "----------------------------------------------------------------------------------------------------\n",
            "Document 2 \n",
            "Title: data.users\n",
            "Header: data.users\n",
            "Source: tables.md\n",
            "Link: https://github.com/JannikArndt/sql-auto-doc/tree/master/Examples//tables.md\n",
            "Content:\n",
            "Employees who have software licenses.  \n",
            "| ID | Name | Type(Length) | Nullable | Default | Example | Comment |\n",
            "| ---- | ------ | -------------- | ---------- | --------- | ---------------------------- | ----------------------------------------- |\n",
            "| 1 | id | bigint(8) | false | 0 | 5 | Primary Key, technical |\n",
            "| 2 | name | nvarchar(800) | false | 0 | “Peter Peterson” | Name of the employee |\n",
            "| 3 | email | nvarchar(800) | false | 0 | “peter.peterson@company.com” | E-mail address of the employee |\n",
            "| 4 | age | int(4) | true | 0 | 42 | The age of the employee |\n",
            "| 5 | joined | datetime2(8) | true | 0 | 2017-01-01 | Date when the employee joined our company |\n",
            "----------------------------------------------------------------------------------------------------\n",
            "Document 3 \n",
            "Title: Welcome to CrowdWire\n",
            "Header: High Level Architecture\n",
            "Source: docs/architecture.md\n",
            "Link: https://github.com/Crowd-Wire/ProjectDocumentation/tree/master//docs/architecture.md\n",
            "Content:\n",
            "![High_Level_Architecture](assets/Images/Architecture Diagrams/High_Level_Architecture.png)\n",
            "----------------------------------------------------------------------------------------------------\n",
            "Document 4 \n",
            "Title: data.users\n",
            "Header: data.users\n",
            "Source: tables.md\n",
            "Link: https://github.com/JannikArndt/sql-auto-doc/tree/master/Examples//tables.md\n",
            "Content:\n",
            "Software bought for or by the company  \n",
            "| ID | Name | Type(Length) | Nullable | Default | Example | Comment |\n",
            "| ---- | --------- | -------------- | ---------- | --------- | ---------------- | ------------------------------------------ |\n",
            "| 1 | id | int(4) | false | 0 | 5 | Primary Key, technical |\n",
            "| 2 | name | nvarchar(800) | true | 0 | “Office 2016” | Name of the software |\n",
            "| 3 | creator | nvarchar(800) | true | 0 | “Microsoft” | Creator behind the software |\n",
            "| 4 | website | nvarchar(800) | true | 0 | “www.office.com” | Website of the software product or creator |\n",
            "| 5 | installed | datetime2(8) | true | 0 | 2016-04-16 | Date the software was first installed |\n",
            "----------------------------------------------------------------------------------------------------\n",
            "Document 5 \n",
            "Title: What is this site ?\n",
            "Header: What is this site ?\n",
            "Source: 1972475\n",
            "Link: https://templates.atlassian.net/wiki/spaces/SWPRJ/pages/1972475\n",
            "Content:\n",
            "We've created this site to enable you to see how a Confluence site looks like.  \n",
            "You can visit this site just as if it was yours, except that you are in read only, so you're not able to modify the content.  \n",
            "This site is still beta and our teams are working on it in order to display as many features as we can.\n",
            "----------------------------------------------------------------------------------------------------\n",
            "Document 6 \n",
            "Title: data.users\n",
            "Header: data.users\n",
            "Source: data.md\n",
            "Link: https://github.com/JannikArndt/sql-auto-doc/tree/master/Examples//data.md\n",
            "Content:\n",
            "Employees who have software licenses.  \n",
            "| ID | Name | Type(Length) | Nullable | Default | Example | Comment |\n",
            "| ---- | ------ | -------------- | ---------- | --------- | ---------------------------- | ----------------------------------------- |\n",
            "| 1 | id | bigint(8) | false | 0 | 5 | Primary Key, technical |\n",
            "| 2 | name | nvarchar(800) | false | 0 | “Peter Peterson” | Name of the employee |\n",
            "| 3 | email | nvarchar(800) | false | 0 | “peter.peterson@company.com” | E-mail address of the employee |\n",
            "| 4 | age | int(4) | true | 0 | 42 | The age of the employee |\n",
            "| 5 | joined | datetime2(8) | true | 0 | 2017-01-01 | Date when the employee joined our company |\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "pretty_print_response(\"What is the table users?\")"
      ],
      "metadata": {
        "id": "Lx8pl5PMiQsW",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "96139fc4-4a69-4b46-df75-e4b8ca950209"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/transformers/pipelines/base.py:1101: UserWarning: You seem to be using the pipelines sequentially on GPU. In order to maximize efficiency please use a dataset\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Question: What is the table users?\n",
            "\n",
            "\n",
            "Answer:  The table \"users\" has the following columns:\n",
            "\n",
            "* ID (bigint(8)) - Primary key and technical column\n",
            "* Name (nvarchar(800)) - Non-technical column\n",
            "* Email (nvarchar(800)) - Non-technical column\n",
            "* Age (int(4)) - Technical column with a default value of 0\n",
            "* Joined (datetime2(8)) - Technical column with a default value of the current date and time\n",
            "\n",
            "Note: The table structure and data types may vary based on the specific implementation and requirements of the application.\n",
            "\n",
            "\n",
            "----------------------------------------------------------------------------------------------------\n",
            "Documents:\n",
            "\n",
            "Document 1 \n",
            "Title: Welcome to CrowdWire\n",
            "Header: DB Modelation\n",
            "Source: docs/REST_API.md\n",
            "Link: https://github.com/Crowd-Wire/ProjectDocumentation/tree/master//docs/REST_API.md\n",
            "Content:\n",
            "To model our database we used the following diagram:\n",
            "----------------------------------------------------------------------------------------------------\n",
            "Document 2 \n",
            "Title: How-to open access rights for our customers\n",
            "Header: How-to open access rights for our customers\n",
            "Source: How-to+open+access+rights+for+our+customers\n",
            "Link: https://templates.atlassian.net/wiki/spaces/SWPRJ/pages/1975532/How-to+open+access+rights+for+our+customers\n",
            "Content:\n",
            "Add the steps involved:  \n",
            "1. Open the customer dashboard\n",
            "2. Click on the blue button\n",
            "3. Give grant access rights\n",
            "4. Click Save  \n",
            "\"Admin\" will have rights to see remove or add new users.\n",
            "----------------------------------------------------------------------------------------------------\n",
            "Document 3 \n",
            "Title: data.users\n",
            "Header: data.users\n",
            "Source: tables.md\n",
            "Link: https://github.com/JannikArndt/sql-auto-doc/tree/master/Examples//tables.md\n",
            "Content:\n",
            "Software bought for or by the company  \n",
            "| ID | Name | Type(Length) | Nullable | Default | Example | Comment |\n",
            "| ---- | --------- | -------------- | ---------- | --------- | ---------------- | ------------------------------------------ |\n",
            "| 1 | id | int(4) | false | 0 | 5 | Primary Key, technical |\n",
            "| 2 | name | nvarchar(800) | true | 0 | “Office 2016” | Name of the software |\n",
            "| 3 | creator | nvarchar(800) | true | 0 | “Microsoft” | Creator behind the software |\n",
            "| 4 | website | nvarchar(800) | true | 0 | “www.office.com” | Website of the software product or creator |\n",
            "| 5 | installed | datetime2(8) | true | 0 | 2016-04-16 | Date the software was first installed |\n",
            "----------------------------------------------------------------------------------------------------\n",
            "Document 4 \n",
            "Title: Welcome to CrowdWire\n",
            "Header: System Technology Architecture\n",
            "Source: docs/architecture.md\n",
            "Link: https://github.com/Crowd-Wire/ProjectDocumentation/tree/master//docs/architecture.md\n",
            "Content:\n",
            "![System_Technology_Architecture](assets/Images/Architecture Diagrams/System_Technology_Architecture.png)\n",
            "----------------------------------------------------------------------------------------------------\n",
            "Document 5 \n",
            "Title: data.users\n",
            "Header: data.users\n",
            "Source: tables.md\n",
            "Link: https://github.com/JannikArndt/sql-auto-doc/tree/master/Examples//tables.md\n",
            "Content:\n",
            "Employees who have software licenses.  \n",
            "| ID | Name | Type(Length) | Nullable | Default | Example | Comment |\n",
            "| ---- | ------ | -------------- | ---------- | --------- | ---------------------------- | ----------------------------------------- |\n",
            "| 1 | id | bigint(8) | false | 0 | 5 | Primary Key, technical |\n",
            "| 2 | name | nvarchar(800) | false | 0 | “Peter Peterson” | Name of the employee |\n",
            "| 3 | email | nvarchar(800) | false | 0 | “peter.peterson@company.com” | E-mail address of the employee |\n",
            "| 4 | age | int(4) | true | 0 | 42 | The age of the employee |\n",
            "| 5 | joined | datetime2(8) | true | 0 | 2017-01-01 | Date when the employee joined our company |\n",
            "----------------------------------------------------------------------------------------------------\n",
            "Document 6 \n",
            "Title: Welcome to CrowdWire\n",
            "Header: High Level Architecture\n",
            "Source: docs/architecture.md\n",
            "Link: https://github.com/Crowd-Wire/ProjectDocumentation/tree/master//docs/architecture.md\n",
            "Content:\n",
            "![High_Level_Architecture](assets/Images/Architecture Diagrams/High_Level_Architecture.png)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "pretty_print_response(\"What is the column for the date the software was installed?\")"
      ],
      "metadata": {
        "id": "tStKpkH0iQuS",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "fe7be443-f3a0-4e9b-90a0-02db8f3cc723"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/transformers/pipelines/base.py:1101: UserWarning: You seem to be using the pipelines sequentially on GPU. In order to maximize efficiency please use a dataset\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Question: What is the column for the date the software was installed?\n",
            "\n",
            "\n",
            "Answer:  The column for the date the software was installed is \"installed\".\n",
            "\n",
            "\n",
            "----------------------------------------------------------------------------------------------------\n",
            "Documents:\n",
            "\n",
            "Document 1 \n",
            "Title: data.software\n",
            "Header: data.software\n",
            "Source: data/software.md\n",
            "Link: https://github.com/JannikArndt/sql-auto-doc/tree/master/Examples//data/software.md\n",
            "Content:\n",
            "Software bought for or by the company  \n",
            "| ID | Name | Type(Length) | Nullable | Default | Example | Comment |\n",
            "| ---- | --------- | -------------- | ---------- | --------- | ---------------- | ------------------------------------------ |\n",
            "| 1 | id | int(4) | false | 0 | 5 | Primary Key, technical |\n",
            "| 2 | name | nvarchar(800) | true | 0 | “Office 2016” | Name of the software |\n",
            "| 3 | creator | nvarchar(800) | true | 0 | “Microsoft” | Creator behind the software |\n",
            "| 4 | website | nvarchar(800) | true | 0 | “www.office.com” | Website of the software product or creator |\n",
            "| 5 | installed | datetime2(8) | true | 0 | 2016-04-16 | Date the software was first installed |\n",
            "----------------------------------------------------------------------------------------------------\n",
            "Document 2 \n",
            "Title: 2016-06-19 Sprint Planning \"Notifications\"\n",
            "Header: 2016-06-19 Sprint Planning \"Notifications\"\n",
            "Source: 33655\n",
            "Link: https://templates.atlassian.net/wiki/spaces/SWPRJ/pages/33655\n",
            "Content:\n",
            "| Start date |   |\n",
            "| End date |   |\n",
            "| Team | |\n",
            "----------------------------------------------------------------------------------------------------\n",
            "Document 3 \n",
            "Title: data.users\n",
            "Header: data.users\n",
            "Source: tables.md\n",
            "Link: https://github.com/JannikArndt/sql-auto-doc/tree/master/Examples//tables.md\n",
            "Content:\n",
            "Software bought for or by the company  \n",
            "| ID | Name | Type(Length) | Nullable | Default | Example | Comment |\n",
            "| ---- | --------- | -------------- | ---------- | --------- | ---------------- | ------------------------------------------ |\n",
            "| 1 | id | int(4) | false | 0 | 5 | Primary Key, technical |\n",
            "| 2 | name | nvarchar(800) | true | 0 | “Office 2016” | Name of the software |\n",
            "| 3 | creator | nvarchar(800) | true | 0 | “Microsoft” | Creator behind the software |\n",
            "| 4 | website | nvarchar(800) | true | 0 | “www.office.com” | Website of the software product or creator |\n",
            "| 5 | installed | datetime2(8) | true | 0 | 2016-04-16 | Date the software was first installed |\n",
            "----------------------------------------------------------------------------------------------------\n",
            "Document 4 \n",
            "Title: Welcome to CrowdWire\n",
            "Header: System Technology Architecture\n",
            "Source: docs/architecture.md\n",
            "Link: https://github.com/Crowd-Wire/ProjectDocumentation/tree/master//docs/architecture.md\n",
            "Content:\n",
            "![System_Technology_Architecture](assets/Images/Architecture Diagrams/System_Technology_Architecture.png)\n",
            "----------------------------------------------------------------------------------------------------\n",
            "Document 5 \n",
            "Title: data.users\n",
            "Header: data.users\n",
            "Source: data.md\n",
            "Link: https://github.com/JannikArndt/sql-auto-doc/tree/master/Examples//data.md\n",
            "Content:\n",
            "Software bought for or by the company  \n",
            "| ID | Name | Type(Length) | Nullable | Default | Example | Comment |\n",
            "| ---- | --------- | -------------- | ---------- | --------- | ---------------- | ------------------------------------------ |\n",
            "| 1 | id | int(4) | false | 0 | 5 | Primary Key, technical |\n",
            "| 2 | name | nvarchar(800) | true | 0 | “Office 2016” | Name of the software |\n",
            "| 3 | creator | nvarchar(800) | true | 0 | “Microsoft” | Creator behind the software |\n",
            "| 4 | website | nvarchar(800) | true | 0 | “www.office.com” | Website of the software product or creator |\n",
            "| 5 | installed | datetime2(8) | true | 0 | 2016-04-16 | Date the software was first installed |\n",
            "----------------------------------------------------------------------------------------------------\n",
            "Document 6 \n",
            "Title: 2016-05-12 Sprint Planning \"Search\"\n",
            "Header: 2016-05-12 Sprint Planning \"Search\"\n",
            "Source: 33607\n",
            "Link: https://templates.atlassian.net/wiki/spaces/SWPRJ/pages/33607\n",
            "Content:\n",
            "| Start date |   |\n",
            "| End date |   |\n",
            "| Team | |\n"
          ]
        }
      ]
    }
  ]
}