coral-reef-kaggle-notebook.ipynb

{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!git clone https://github.com/ultralytics/yolov5  # clone\n!cd yolov5\n!pip install -r ./yolov5/requirements.txt  # install\n!pip install -U albumentations #using image augmentation feature in yolov5","metadata":{"execution":{"iopub.status.busy":"2022-02-19T00:12:04.432286Z","iopub.execute_input":"2022-02-19T00:12:04.43258Z","iopub.status.idle":"2022-02-19T00:12:04.436135Z","shell.execute_reply.started":"2022-02-19T00:12:04.432541Z","shell.execute_reply":"2022-02-19T00:12:04.435167Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import random\nimport numpy as np\nimport pandas as pd\nimport cv2\nimport os\nimport ast\nimport shutil\nfrom IPython.display import display\nimport matplotlib.pyplot as plt","metadata":{"execution":{"iopub.status.busy":"2022-02-19T00:12:04.452017Z","iopub.execute_input":"2022-02-19T00:12:04.452303Z","iopub.status.idle":"2022-02-19T00:12:04.456809Z","shell.execute_reply.started":"2022-02-19T00:12:04.452276Z","shell.execute_reply":"2022-02-19T00:12:04.456039Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv(\"../input/tensorflow-great-barrier-reef/train.csv\", index_col=\"image_id\")\ntrain.head(10)","metadata":{"execution":{"iopub.status.busy":"2022-02-18T23:55:22.790061Z","iopub.execute_input":"2022-02-18T23:55:22.790565Z","iopub.status.idle":"2022-02-18T23:55:22.864283Z","shell.execute_reply.started":"2022-02-18T23:55:22.790535Z","shell.execute_reply":"2022-02-18T23:55:22.863541Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img = cv2.imread(\"../input/tensorflow-great-barrier-reef/train_images/video_0/978.jpg\")\n\nstart_pt = (825, 224)\nend_pt = (875, 259)\ncolor = (0, 0, 255)\nthickness = 2\nimage = cv2.rectangle(img, start_pt, end_pt, color, thickness)\nplt.imshow(image)","metadata":{"execution":{"iopub.status.busy":"2022-02-18T23:55:22.86592Z","iopub.execute_input":"2022-02-18T23:55:22.866101Z","iopub.status.idle":"2022-02-18T23:55:23.281489Z","shell.execute_reply.started":"2022-02-18T23:55:22.866077Z","shell.execute_reply":"2022-02-18T23:55:23.280935Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir -p ./datasets/coral/labels\n!mkdir -p ./datasets/coral/images\n\npath_to_txt_folder = \"./datasets/coral/labels\"\nfinal_path_to_images = \"./datasets/coral/images\"\n\ndef txt_data(path):\n    for folder in os.listdir(path):\n        path_to_folder = os.path.join(path, folder)\n        _, video_id = folder.split(\"_\")\n\n        for image in os.listdir(path_to_folder):\n            path_to_image = os.path.join(path_to_folder, image)\n\n            image_id, img_ext = image.split(\".\")\n            image_match = video_id + \"-\" + image_id\n\n            img_temp = cv2.imread(path_to_image)\n            height, width, channels = img_temp.shape\n\n            image_data = train.loc[image_match]\n            annotation_list = ast.literal_eval(image_data.annotations)\n\n            if bool(annotation_list) == True:\n                path_to_file = os.path.join(path_to_txt_folder, image_match + \".txt\")\n                file_ = open(path_to_file, \"w\")\n                for i in annotation_list:\n                    x_min = i['x']\n                    y_min = i['y']\n                    box_width = i['width']\n                    box_height = i['height']\n                    class_label=str(0)\n                    x_min = str(round((x_min+(box_width/2))/width, 10))\n                    y_min = str(round((y_min+(box_height/2))/height, 10))\n                    box_width = str(round(box_width/width, 10))\n                    box_height = str(round(box_height/height, 10))\n                    file_.write(class_label +\" \"+ x_min +\" \"+ y_min +\" \"+ box_width +\" \"+ box_height + \"\\n\")\n                file_.close()\n    \n                image_new_name = os.path.join(final_path_to_images, image_match + \".\" + img_ext)\n#             new_path_to_image = os.path.join(final_path_to_images, image_new_name)\n                shutil.copyfile(path_to_image, image_new_name)\n\ntxt_data(\"../input/tensorflow-great-barrier-reef/train_images\")","metadata":{"execution":{"iopub.status.busy":"2022-02-18T23:55:23.282623Z","iopub.execute_input":"2022-02-18T23:55:23.282939Z","iopub.status.idle":"2022-02-18T23:55:24.031474Z","shell.execute_reply.started":"2022-02-18T23:55:23.282904Z","shell.execute_reply":"2022-02-18T23:55:24.030549Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img = cv2.imread(\"./datasets/coral/images/0-978.jpg\")\n\nstart_pt = (825, 224)\nend_pt = (875, 259)\ncolor = (0, 0, 255)\nthickness = 2\nimage = cv2.rectangle(img, start_pt, end_pt, color, thickness)\nplt.imshow(image)","metadata":{"execution":{"iopub.status.busy":"2022-02-18T23:55:24.032682Z","iopub.execute_input":"2022-02-18T23:55:24.032889Z","iopub.status.idle":"2022-02-18T23:55:24.036233Z","shell.execute_reply.started":"2022-02-18T23:55:24.032863Z","shell.execute_reply":"2022-02-18T23:55:24.035631Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir -p ./datasets/gbr/train/labels\n!mkdir -p ./datasets/gbr/train/images\n!mkdir -p ./datasets/gbr/val/labels\n!mkdir -p ./datasets/gbr/val/images\n\nfile_list = os.listdir(final_path_to_images)\nnp.random.shuffle(file_list)\ntrain_val_split = 0.9\ntrain_len = round(train_val_split * len(file_list))\ntrain_list = file_list[:train_len]\nval_list = file_list[train_len:]\n\ndef split(new_file_path, typ_list=train_list):\n    for file in typ_list:\n        new_image_path = os.path.join(new_file_path, \"images\")\n        image_path = os.path.join(final_path_to_images, file)\n        shutil.move(image_path, new_image_path)\n        \n        indx, _ = file.split(\".\")\n        new_label_path = os.path.join(new_file_path, \"labels\")\n        label_path = os.path.join(path_to_txt_folder, indx + '.txt')\n        shutil.move(label_path, new_label_path)\n        \nsplit(\"./datasets/gbr/train\", train_list)\nsplit(\"./datasets/gbr/val\", val_list)\n!rm -rf \"./datasets/coral\"","metadata":{"execution":{"iopub.status.busy":"2022-02-18T23:55:24.037427Z","iopub.execute_input":"2022-02-18T23:55:24.037793Z","iopub.status.idle":"2022-02-18T23:55:25.196602Z","shell.execute_reply.started":"2022-02-18T23:55:24.037762Z","shell.execute_reply":"2022-02-18T23:55:25.195632Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"shutil.copyfile(\"../input/coral-yaml/coral.yaml\", \"./yolov5/data/coral.yaml\")\nshutil.copyfile(\"../input/coral-yaml/hyp.yaml\", \"./yolov5/data/hyps/hyp.yaml\")","metadata":{"execution":{"iopub.status.busy":"2022-02-18T23:55:25.197929Z","iopub.execute_input":"2022-02-18T23:55:25.198168Z","iopub.status.idle":"2022-02-18T23:55:25.212315Z","shell.execute_reply.started":"2022-02-18T23:55:25.198138Z","shell.execute_reply":"2022-02-18T23:55:25.211176Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from kaggle_secrets import UserSecretsClient\nuser_secrets = UserSecretsClient()\nsecret_value_0 = user_secrets.get_secret(\"wandb_key\")\n\n!pip install wandb\n!wandb login $secret_value_0","metadata":{"execution":{"iopub.status.busy":"2022-02-18T23:55:25.213551Z","iopub.execute_input":"2022-02-18T23:55:25.214309Z","iopub.status.idle":"2022-02-18T23:55:25.217821Z","shell.execute_reply.started":"2022-02-18T23:55:25.214277Z","shell.execute_reply":"2022-02-18T23:55:25.217133Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Train YOLOv5m on coral_dataset for 3 epochs\n!python ./yolov5/train.py --img 1280 --rect --batch 16 --epochs 25 --data ./yolov5/data/coral.yaml --hyp ./yolov5/data/hyps/hyp.yaml --weights yolov5m6.pt","metadata":{"execution":{"iopub.status.busy":"2022-02-18T23:55:25.218958Z","iopub.execute_input":"2022-02-18T23:55:25.219307Z","iopub.status.idle":"2022-02-18T23:55:25.2288Z","shell.execute_reply.started":"2022-02-18T23:55:25.219267Z","shell.execute_reply":"2022-02-18T23:55:25.228234Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nckpt_path = \"./yolov5/runs/train/exp/weights/best.pt\"\n\nmodel = torch.hub.load('../input/yolov5', 'custom',\n                           path=ckpt_path,\n                           source='local',\n                       force_reload=True)\n\nmodel.conf = 0.20\nmodel.iou = 0.45","metadata":{"execution":{"iopub.status.busy":"2022-02-18T23:55:25.230029Z","iopub.execute_input":"2022-02-18T23:55:25.230315Z","iopub.status.idle":"2022-02-18T23:55:31.064693Z","shell.execute_reply.started":"2022-02-18T23:55:25.230284Z","shell.execute_reply":"2022-02-18T23:55:31.06417Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def result_string_calculator(results_dataframe):\n    if results_dataframe.shape[0] != 0:\n        for index,row in results_dataframe.iterrows():\n            x_center = row['xmin'] + ((row['xmax'] - row['xmin'])/2)\n            y_center = row['ymin'] + ((row['ymax'] - row['ymin'])/2)\n            height= row['ymax'] - row['ymin']\n            width = row['xmax'] - row['xmin']\n            confidence_score = round(row['confidence'], 3)\n        output_string = \"{} {} {} {} {} \".format(confidence_score,round(x_center),round(y_center),round(width),round(height))\n    else:\n        output_string = \"0 0 0 0 0\"\n    return output_string","metadata":{"execution":{"iopub.status.busy":"2022-02-18T23:55:31.066028Z","iopub.execute_input":"2022-02-18T23:55:31.068266Z","iopub.status.idle":"2022-02-18T23:55:31.074772Z","shell.execute_reply.started":"2022-02-18T23:55:31.068234Z","shell.execute_reply":"2022-02-18T23:55:31.073691Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import greatbarrierreef\nenv = greatbarrierreef.make_env()   # initialize the environment\niter_test = env.iter_test()    # an iterator which loops over the test set and sample submission","metadata":{"execution":{"iopub.status.busy":"2022-02-18T23:55:31.077433Z","iopub.execute_input":"2022-02-18T23:55:31.078173Z","iopub.status.idle":"2022-02-18T23:55:31.121931Z","shell.execute_reply.started":"2022-02-18T23:55:31.078097Z","shell.execute_reply":"2022-02-18T23:55:31.120922Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for (pixel_array, sample_prediction_df) in iter_test:\n    results = model(pixel_array)\n    print(results)\n    results_df=results.pandas().xyxy[0]\n    sample_prediction_df['annotations'] =  result_string_calculator(results_df) # make your predictions here\n    env.predict(sample_prediction_df)   # register your predictions","metadata":{"execution":{"iopub.status.busy":"2022-02-18T23:55:31.124001Z","iopub.execute_input":"2022-02-18T23:55:31.127288Z","iopub.status.idle":"2022-02-18T23:55:32.329433Z","shell.execute_reply.started":"2022-02-18T23:55:31.127233Z","shell.execute_reply":"2022-02-18T23:55:32.328567Z"},"trusted":true},"execution_count":null,"outputs":[]}]}