data_analysis.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "080c8dcb",
   "metadata": {},
   "source": [
    "# Data Analysis\n",
    "\n",
    "Load and parse classification csvs to review segmentation results, user metrics, and identify difficult images."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4e9c25cb",
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a5345641",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import sys\n",
    "import getpass\n",
    "import re\n",
    "import csv\n",
    "import json\n",
    "import math\n",
    "import datetime\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from glob import glob\n",
    "from skimage import io\n",
    "from tqdm.notebook import tqdm\n",
    "from itertools import combinations\n",
    "\n",
    "import custom_cmaps\n",
    "from metrics import iou, average_precision\n",
    "from aggregation import *\n",
    "from helpers import *\n",
    "from matplotlib import pyplot as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b18432aa",
   "metadata": {},
   "source": [
    "## Load the results csv\n",
    "\n",
    "To access the most recent results you must first \"Request new classification export\" from the Zooniverse project. Wait a minute or two after making the request to try downloading the results (or until you get the confirmation email from zooniverse)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1e6ddd06",
   "metadata": {},
   "outputs": [],
   "source": [
    "# necessary to handle overflow of long csv columns\n",
    "_ = csv.field_size_limit(256<<12)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b8f4b2fd-8d94-4667-a78c-ee848b0603c0",
   "metadata": {},
   "source": [
    "Set and verify the parameters in the cell below before going further."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "322588ce",
   "metadata": {},
   "outputs": [],
   "source": [
    "source_dir = \"\" # directory containing the jpg images uploaded to Zooniverse\n",
    "csv_path = \"\" # path to the downloaded classifications csv\n",
    "\n",
    "IS_FLIPBOOK = True # images in source_dir from flipbooks?\n",
    "SPAN = 5 # number of images per flipbook\n",
    "IM_SIZE = (480, 480) # image size that was used for prep_flipbooks/prep_images\n",
    "\n",
    "RETIRE_LIMIT = 10 # annotations before image is retired\n",
    "\n",
    "# checks\n",
    "assert source_dir\n",
    "assert csv_path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "92c1c183",
   "metadata": {},
   "outputs": [],
   "source": [
    "results_df = pd.read_csv(csv_path)\n",
    "\n",
    "# convert the metadata fields to parsable json strings\n",
    "results_df['metadata_json'] = [json.loads(q) for q in results_df.metadata]\n",
    "results_df['annotations_json'] = [json.loads(q) for q in results_df.annotations]\n",
    "results_df['subject_data_json'] = [json.loads(q) for q in results_df.subject_data]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d097b606-e7bd-4bc9-ab4b-8271049a3b73",
   "metadata": {},
   "source": [
    "## Parsing functions\n",
    "\n",
    "Define some helper functions to parse the results."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1ec48c65-d52b-4ae7-8c73-db2880702788",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_image_name(row):\n",
    "    \"\"\"\n",
    "    Return the name of an image from results dataframe row.\n",
    "    \"\"\"\n",
    "    image_idx = SPAN // 2 if IS_FLIPBOOK else 0\n",
    "    image_name = list(row['subject_data_json'].values())[0][f'Image {image_idx}']\n",
    "    return image_name\n",
    "    \n",
    "def get_image_size(row):\n",
    "    \"\"\"\n",
    "    Return the size of an image from results dataframe row, if available.\n",
    "    Otherwise return the default size defined by the IM_SIZE variable.\n",
    "    \"\"\"\n",
    "    image_dims = row['metadata_json']['subject_dimensions'][image_idx]\n",
    "    if image_dims is not None:\n",
    "        w, h = image_dims['naturalWidth'], image_dims['naturalHeight']\n",
    "    else:\n",
    "        w, h = IM_SIZE\n",
    "        \n",
    "    return (w, h)\n",
    "\n",
    "def calculate_time(row):\n",
    "    \"\"\"\n",
    "    Computes time, in minutes, spent on a given annotation \n",
    "    based on row metadata.\n",
    "    \"\"\"\n",
    "    metadata = row['metadata_json']\n",
    "    start, finish = metadata['started_at'], metadata['finished_at']\n",
    "    start = datetime.datetime.strptime(start, '%Y-%m-%dT%H:%M:%S.%fZ')\n",
    "    finish = datetime.datetime.strptime(finish, '%Y-%m-%dT%H:%M:%S.%fZ')\n",
    "    return (finish - start).total_seconds() / 60\n",
    "\n",
    "def parse_annotations(row):\n",
    "    \"\"\"\n",
    "    Parse out details of the segmentation and confidence\n",
    "    rating tasks from a row and return them in a dictionary.\n",
    "    \"\"\"\n",
    "    annotations = row['annotations_json']\n",
    "    \n",
    "    if len(annotations) != 2:\n",
    "        raise Exception(f'Expected 2 annotations (segmentation and confidence)')\n",
    "    \n",
    "    res = {}\n",
    "    for task in annotations:\n",
    "        # segmentation is task T0\n",
    "        if task['task'] == 'T0':\n",
    "            # if not labeled objects, annotation is blank list\n",
    "            if task['value']:\n",
    "                res['n_objects'] = len(task['value'])\n",
    "                res['segmentation'] = [\n",
    "                    polygon_to_array(value['points']) for value in task['value']\n",
    "                ]\n",
    "            else:\n",
    "                res['n_objects'] = 0\n",
    "                res['segmentation'] = task['value']\n",
    "        else:\n",
    "            res['confidence'] = int(task['value'][:1])\n",
    "            \n",
    "    return res"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d9daf5af",
   "metadata": {},
   "source": [
    "## Subject aggregation\n",
    "\n",
    "This section organizes segmentations by subject image such that they can be aggregated into a consensus."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "23e46b69",
   "metadata": {},
   "outputs": [],
   "source": [
    "# store results in a nested dict, keys are subject names and values are attributes\n",
    "subject_annotations = {}\n",
    "for i, row in results_df.iterrows():\n",
    "    subject_id = list(row['subject_data_json'].keys())[0]\n",
    "    \n",
    "    # load entry for subject, or create a new one\n",
    "    image_name = get_image_name(row)\n",
    "    \n",
    "    if image_name not in subject_annotations:\n",
    "        size = get_image_size(row)\n",
    "        subject_annotations[image_name] = {\n",
    "            'shape': size, 'confidences': [],\n",
    "            'annotations': [], 'users': []\n",
    "        }\n",
    "        \n",
    "    # get annotation and metadata\n",
    "    annotation_dict = parse_annotations(row)\n",
    "    polygons = annotation_dict['segmentation']\n",
    "    confidence = annotation_dict['confidence']\n",
    "    \n",
    "    subject_annotations[image_name]['id'] = subject_id\n",
    "    subject_annotations[image_name]['annotations'].append(polygons)\n",
    "    subject_annotations[image_name]['confidences'].append(confidence)\n",
    "    subject_annotations[image_name]['users'].append(row['user_name'])\n",
    "    \n",
    "# mark the retired subjects\n",
    "for k, v in subject_annotations.items():\n",
    "    if len(v['annotations']) >= RETIRE_LIMIT:\n",
    "        v['retired'] = True\n",
    "    else:\n",
    "        v['retired'] = False\n",
    "        \n",
    "all_subjects = list(subject_annotations.keys())\n",
    "retired_subjects = [k for k,v in subject_annotations.items() if v['retired']]\n",
    "print(f'{len(retired_subjects)} retired subjects out of {len(all_subjects)}')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "24157f28-26a2-4038-8882-ec4de05fbd81",
   "metadata": {},
   "source": [
    "### Review random subject annotations and consensus\n",
    "\n",
    "Randomly pick a subject (retired or otherwise) and plot all the user created segmentations along with a consensus segmentation for the image."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "00070fb7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# randomly pick a retired image to analyze\n",
    "choice = np.random.choice(retired_subjects) # or choose from all_subjects instead\n",
    "\n",
    "subject_dict = subject_annotations[choice]\n",
    "subject_id = subject_dict['id']\n",
    "image_shape = subject_dict['shape']\n",
    "\n",
    "usernames = subject_dict['users']\n",
    "\n",
    "# handle the case that 1 user annotated an image multiple times\n",
    "usernames, indices = np.unique(usernames, return_index=True)\n",
    "usernames = usernames.tolist()\n",
    "annotations = [subject_dict['annotations'][ix] for ix in indices]\n",
    "confidences = [subject_dict['confidences'][ix] for ix in indices]\n",
    "\n",
    "# create masks from the polygons\n",
    "masks = []\n",
    "for i,annotation in enumerate(annotations):\n",
    "    mask = poly2segmentation(annotation, image_shape)\n",
    "    masks.append(mask)\n",
    "    \n",
    "# create the consensus instance segmentation\n",
    "instance_scores = mask_aggregation(masks)\n",
    "instance_seg = aggregated_instance_segmentation(instance_scores, 0.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b6cfe03b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# plot all the user segmentations along with the consensus\n",
    "cols = 6\n",
    "rows = int(math.ceil(RETIRE_LIMIT / 5))\n",
    "\n",
    "f, ax = plt.subplots(rows, cols, figsize=(24, 8))\n",
    "\n",
    "image = io.imread(source_dir + choice)\n",
    "ax[0, 0].imshow(image, cmap='gray')\n",
    "ax[0, 0].set_xticks([])\n",
    "ax[0, 0].set_yticks([])\n",
    "ax[0, 0].set_title('Image')\n",
    "\n",
    "ax[1, 0].imshow(image, cmap='gray')\n",
    "ax[1, 0].imshow(instance_seg, alpha=0.5, cmap='hsv_alpha', interpolation='nearest')\n",
    "ax[1, 0].set_xticks([])\n",
    "ax[1, 0].set_yticks([])\n",
    "ax[1, 0].set_title('Image + Consensus')\n",
    "\n",
    "for _ in range(RETIRE_LIMIT - len(masks)):\n",
    "    masks.append(np.zeros_like(image))\n",
    "    usernames.append('Empty')\n",
    "    confidences.append(0)\n",
    "\n",
    "c = 0\n",
    "for y in range(rows):\n",
    "    for x in range(1, cols):\n",
    "        ax[y, x].imshow(masks[c], cmap='hsv_black', interpolation='nearest')\n",
    "        ax[y, x].set_xticks([])\n",
    "        ax[y, x].set_yticks([])\n",
    "        ax[y, x].set_title(f'{usernames[c]} {confidences[c]}')\n",
    "            \n",
    "        c += 1\n",
    "    \n",
    "plt.tight_layout()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "283b3f97-cea5-42cc-b53c-b029faa463cf",
   "metadata": {},
   "source": [
    "### Review subject difficulty\n",
    "\n",
    "Find subjects were users report low annotation confidence or there was a high level of disagreement between individual segmentations and the consensus."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6306f036-bc0b-4fdb-a14c-041dacec2e9d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# add consensus strength to the subject_annotations\n",
    "for imname, subject_dict in tqdm(subject_annotations.items(), total=len(subject_annotations.keys())):\n",
    "    if 'consensus' in subject_annotations[imname]:\n",
    "        continue\n",
    "    \n",
    "    subject_id = subject_dict['id']\n",
    "    image_shape = subject_dict['shape']\n",
    "\n",
    "    masks = []\n",
    "    for annotation in subject_dict['annotations']:\n",
    "        mask = poly2segmentation(annotation, image_shape)\n",
    "        masks.append(mask)\n",
    "\n",
    "    instance_scores = mask_aggregation(masks)\n",
    "    instance_seg = aggregated_instance_segmentation(instance_scores, 0.75)\n",
    "    \n",
    "    # compute ap of each individual annotation against consensus\n",
    "    scores = [average_precision(instance_seg, mask, 0.50, False)[0] for mask in masks]\n",
    "    \n",
    "    avg_confidence = np.mean(confidences)\n",
    "    subject_annotations[imname]['consensus'] = instance_seg\n",
    "    subject_annotations[imname]['consensus_strength'] = np.mean(scores)\n",
    "    \n",
    "subjects = np.array(list(subject_annotations.keys()))\n",
    "subject_ids = np.array([sd['id'] for sd in subject_annotations.values()])\n",
    "user_confs = np.array([np.median(sd['confidences']) for sd in subject_annotations.values()])\n",
    "user_scores = np.array([sd['consensus_strength'] for sd in subject_annotations.values()])\n",
    "\n",
    "# higher means consensus segmentation should be better\n",
    "most_agreed_indices = np.argsort(user_scores)[::-1]\n",
    "most_confident_indices = np.argsort(user_confs)[::-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f251d4c6-c3b3-4522-abd4-32b558e11639",
   "metadata": {},
   "outputs": [],
   "source": [
    "# plot images and consensus segmentations with worst/best agreement between annotators\n",
    "\n",
    "chosen = most_agreed_indices[-10:] # 10 least agreed\n",
    "#chosen = most_agreed_indices[:10] # 10 most agreed\n",
    "\n",
    "f, ax = plt.subplots(2, 5, figsize=(20, 8))\n",
    "\n",
    "c = 0\n",
    "for y in range(2):\n",
    "    for x in range(5):\n",
    "        index = chosen[c]\n",
    "        subj_id = subject_ids[index]\n",
    "        imname = subjects[index]\n",
    "        sc = user_scores[index]\n",
    "        \n",
    "        image = io.imread(os.path.join(source_dir, imname))\n",
    "        mask = subject_annotations[imname]['consensus']\n",
    "        \n",
    "        ax[y, x].imshow(image, cmap='gray')\n",
    "        ax[y, x].imshow(mask, cmap='hsv_alpha', alpha=0.5, interpolation='nearest')\n",
    "        ax[y, x].set_xticks([])\n",
    "        ax[y, x].set_yticks([])\n",
    "        ax[y, x].set_title(f'Subject {subj_id}; Consensus Str. {sc:.3f}')\n",
    "        c += 1\n",
    "        \n",
    "plt.tight_layout()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "01425e60-4d31-4bce-be03-3f0c8889b43a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# plot images and consensus segmentations with lowest/highest median user annotation confidence\n",
    "\n",
    "chosen = most_confident_indices[-10:] # 10 least confident\n",
    "#chosen = most_confident_indices[:10] # 10 most confident\n",
    "\n",
    "f, ax = plt.subplots(2, 5, figsize=(20, 8))\n",
    "\n",
    "c = 0\n",
    "for y in range(2):\n",
    "    for x in range(5):\n",
    "        index = chosen[c]\n",
    "        subj_id = subject_ids[index]\n",
    "        imname = subjects[index]\n",
    "        sc = user_confs[index]\n",
    "        \n",
    "        image = io.imread(os.path.join(source_dir, imname))\n",
    "        mask = subject_annotations[imname]['consensus']\n",
    "        \n",
    "        ax[y, x].imshow(image, cmap='gray')\n",
    "        ax[y, x].imshow(mask, cmap='hsv_alpha', alpha=0.5, interpolation='nearest')\n",
    "        ax[y, x].set_xticks([])\n",
    "        ax[y, x].set_yticks([])\n",
    "        ax[y, x].set_title(f'Subject {subj_id}; Median conf. {sc:.3f}')\n",
    "        c += 1\n",
    "        \n",
    "plt.tight_layout()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7c9ccc13-a9e7-428e-a96b-37eec3c10ad2",
   "metadata": {
    "tags": []
   },
   "source": [
    "## User evaluation\n",
    "\n",
    "This section organizes annotations by user not by subject. Useful for user level metrics like number of images segmented or accuracy (i.e. average precision and IoU)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bfcd8df3",
   "metadata": {},
   "outputs": [],
   "source": [
    "user_annotations = {}\n",
    "for i, row in results_df.iterrows():\n",
    "    # get annotation and metadata\n",
    "    annotation_dict = {}\n",
    "    annotation_dict['image_size'] = get_image_size(row)\n",
    "    annotation_dict['time'] = calculate_time(row)\n",
    "    annotation_dict |= parse_annotations(row)\n",
    "\n",
    "    # update user's dict\n",
    "    user_id = row['user_name']\n",
    "    image_name = get_image_name(row)\n",
    "    user_annotations[user_id] = user_annotations.get(user_id, {}) | {image_name: annotation_dict}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9315becb",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(f'{len(user_annotations.keys())} users have segmented data in this project.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ae661a07",
   "metadata": {},
   "outputs": [],
   "source": [
    "# plot the histogram of the number of annotations per user\n",
    "plt.figure(figsize=(6, 3))\n",
    "plt.hist([len(v) for v in user_annotations.values()])\n",
    "plt.ylabel('Number of users')\n",
    "plt.xlabel('Number of annotations')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6ed5ae6b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# user ranking by number of images annotated\n",
    "users = list(user_annotations.keys())\n",
    "n_annotated = [len(v) for v in user_annotations.values()]\n",
    "rankings = np.argsort(n_annotated)[::-1][:10] # top 10 only\n",
    "\n",
    "for rank, idx in enumerate(rankings, 1):\n",
    "    print(rank, users[idx], n_annotated[idx])"
   ]
  }
 ],
 "metadata": {
  "celltoolbar": "Raw Cell Format",
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}