Update code for KuroSiwo v2

README.md

@@ -17,7 +17,10 @@
 - [Repository structure](#kuro-siwo-repo-structure)
 - [Pretrained models](#pretrained-models)
 - [Citation](#citation)
+
+
 ### Download Kuro Siwo
+
   #### GRD Data
 - The Kuro Siwo GRD Dataset can be downloaded either:
   - from the following [link](https://www.dropbox.com/scl/fo/xc69aclh0q4lykd22ynkb/AAaDu8gBtoSdOpmffv7JY50?rlkey=uds2b2aot6oubc9hmnrm7myy7&st=21u41kwx&dl=0),
@@ -29,7 +32,8 @@
 
     1. Make sure to grant the necessary rights by executing `chmod +x scripts/download_kuro_siwo.sh`
     2. Execute `scripts/download_kuro_siwo.sh DESIRED_DATASET_ROOT_PATH` e.g: `./download_kuro_siwo.sh KuroRoot`
-
+
+
 #### SLC Data
   - The SLC Preprocessed products can be downloaded from the following [link](https://www.dropbox.com/scl/fo/kknf6ycz6ywffopjxroys/AOIedl2NgWnOXQBEDUGv4m0?rlkey=rb18w8rzpwitg2w3nlhzklnyy&st=p1vv516h&dl=0).
 
@@ -39,6 +43,8 @@
 ### Data preprocessing
 
 The preprocessing pipelines used to generate the GRD and SLC products can be found at `configs/grd_preprocessing.xml` and `configs/slc_preprocessing.xml` repsectively.
+
+
 ### Kuro Siwo repo structure 
   - Kuro Siwo uses the [black](https://github.com/psf/black) python formatter. To activate it install pre-commit, running `pip install pre-commit`
 and execute `pre-commit install`.
@@ -61,16 +67,19 @@ The weights of the top performing models can be accessed using the following lin
   - [FloodViT](https://www.dropbox.com/scl/fi/srw7u4cw1gtxrf4xzmsh7/floodvit.pt?rlkey=snskpq1qrdav5u2jya8k2bocg&dl=0)
   - [SNUNet](https://www.dropbox.com/scl/fi/3vlsveoobqe1wc71s5z2d/best_segmentation.pt?rlkey=xpy2thmozzxfzymr8b13m7n51&dl=0)
 
+
 ### Citation
 If you use this work please cite:
 ```
-@misc{bountos2024kurosiwo33billion,
-      title={Kuro Siwo: 33 billion $m^2$ under the water. A global multi-temporal satellite dataset for rapid flood mapping}, 
-      author={Nikolaos Ioannis Bountos and Maria Sdraka and Angelos Zavras and Ilektra Karasante and Andreas Karavias and Themistocles Herekakis and Angeliki Thanasou and Dimitrios Michail and Ioannis Papoutsis},
-      year={2024},
-      eprint={2311.12056},
-      archivePrefix={arXiv},
-      primaryClass={cs.CV},
-      url={https://arxiv.org/abs/2311.12056}, 
+@inproceedings{NEURIPS2024_43612b06,
+ author = {Bountos, Nikolaos Ioannis and Sdraka, Maria and Zavras, Angelos and Karavias, Andreas and Karasante, Ilektra and Herekakis, Themistocles and Thanasou, Angeliki and Michail, Dimitrios and Papoutsis, Ioannis},
+ booktitle = {Advances in Neural Information Processing Systems},
+ editor = {A. Globerson and L. Mackey and D. Belgrave and A. Fan and U. Paquet and J. Tomczak and C. Zhang},
+ pages = {38105--38121},
+ publisher = {Curran Associates, Inc.},
+ title = {Kuro Siwo: 33 billion m\^{}2 under the water. A global multi-temporal satellite dataset for rapid flood mapping},
+ url = {https://proceedings.neurips.cc/paper_files/paper/2024/file/43612b0662cb6a4986edf859fd6ebafe-Paper-Datasets_and_Benchmarks_Track.pdf},
+ volume = {37},
+ year = {2024}
 }
 ```

catalogue/catalogue.py

@@ -468,12 +468,13 @@ def oper(args):
             return
 
         grid_dict = {}
-
+        invalid_number = 0
         for i, id in enumerate(fcat.ids):
             print(f"Grid {i+1}/{len(cat)}: {id}")
             gridd = fcat.grid(id)
             if not gridd.valid:
                 print(f"Invalid Grid: {id}")
+                invalid_number += 1
                 continue
             if "999999" in str(gridd.path):
                 continue
@@ -498,7 +499,8 @@ def oper(args):
                 "clz": gridd.clz_id,
                 "clz_name": gridd.clz_name,
             }
-
+        print('Num of invalid grids: ', invalid_number)
+        print('Num of valid grids: ', len(grid_dict))
         pickle_path = CFG["PICKLE_PATH"]
 
         Path(os.path.dirname(pickle_path)).mkdir(parents=True, exist_ok=True)

catalogue/catalogue.yaml

@@ -7,7 +7,7 @@ Version: 0.1
 DATA_PATH: &DATA_PATH your/path/to/data
 CAT_PATH: &CAT_PATH !join [*DATA_PATH, 'catalogue.gpkg']
 REPO_PATH: &REPO_PATH !join [*DATA_PATH, 'data']
-PICKLE_PATH: pickle/grid_dict.pkl
+PICKLE_PATH: pickle/KuroV2_grid_dict_test_0_100.pkl
 
 DEVEL: True
 
@@ -359,4 +359,87 @@ Floods:
       - aoi_id: '15'
         aoi_name: Singleton
         cl_zone: 3
-...
+  - act_id: 1111002
+    act_region: Cameroon
+    ref_date: 20200914T080000
+    aois:
+      - aoi_id: '01'
+        aoi_name: Djagala
+        cl_zone: 2
+  - act_id: 1111003
+    act_region: Djibouti
+    ref_date: 20191126T080000
+    aois:
+      - aoi_id: '01'
+        aoi_name: Djibouti
+        cl_zone: 2
+  - act_id: 1111004
+    act_region: USA
+    ref_date: 20170828T080000
+    aois:
+      - aoi_id: '01'
+        aoi_name: Houston
+        cl_zone: 3
+  - act_id: 1111005
+    act_region: Madagascar
+    ref_date: 20200125T080000
+    aois:
+      - aoi_id: '01'
+        aoi_name: Farihy Alaotra
+        cl_zone: 1
+  - act_id: 1111006
+    act_region: Madagascar
+    ref_date: 20200130T080000
+    aois:
+      - aoi_id: '01'
+        aoi_name: Ambato Boeny
+        cl_zone: 1
+  - act_id: 1111007
+    act_region: Nepal
+    ref_date: 20190917T080000
+    aois:
+      - aoi_id: '01'
+        aoi_name: Patna
+        cl_zone: 1
+  - act_id: 1111008
+    act_region: Nicaragua
+    ref_date: 20200815T080000
+    aois:
+      - aoi_id: '01'
+        aoi_name: Dakban
+        cl_zone: 1
+  - act_id: 1111009
+    act_region: Pakistan
+    ref_date: 20220910T080000
+    aois:
+      - aoi_id: '01'
+        aoi_name: Larkana
+        cl_zone: 3
+  - act_id: 1111010
+    act_region: Peru
+    ref_date: 20170310T080000
+    aois:
+      - aoi_id: '01'
+        aoi_name: Sechura
+        cl_zone: 2
+  - act_id: 1111011
+    act_region: Philippines
+    ref_date: 20201112T080000
+    aois:
+      - aoi_id: '01'
+        aoi_name: Tuguegarao
+        cl_zone: 1
+  - act_id: 1111012
+    act_region: Australia
+    ref_date: 20220306T080000
+    aois:
+      - aoi_id: '01'
+        aoi_name: Singleton
+        cl_zone: 3
+  - act_id: 1111013
+    act_region: USA
+    ref_date: 20170503T080000
+    aois:
+      - aoi_id: '01'
+        aoi_name: St Louis
+        cl_zone: 3

configs/train/data_config.json

@@ -1,22 +1,28 @@
 {
   "track": "RandomEvents",
-  "train_pickle": "pickle/grid_dict.pkl",
-  "test_pickle": "pickle/grid_dict.pkl",
-  "negative_pickle":"pickle/negatives_only.pkl",
-  "inputs": ["pre_event_1","pre_event_2", "post_event"],
+  "train_pickle": "pickle/KuroV2_grid_dict.pkl",  // The pickle containing the GRD training data
+  "test_pickle": "pickle/KuroV2_grid_dict_test_0_100.pkl",  // The pickle containing the GRD testing data
+  "negative_pickle":"pickle/negatives_only.pkl",  // Pickle containing only negatives (optional)
+  "slc": true,  // 'true' in order to use SLC data instead of GRD
+  "train_json": "json/slc_grid_pwater_0.0001.json", // The JSON containing the SLC training data
+  "test_json": "json/slc_grid_pwater_0.json",  // The JSON containing the SLC testing data
+  "slc_root_path": "",  // The path containing the SLC data
+  "inputs": ["pre_event_1", "pre_event_2", "post_event"],
   "channels": [ "vv","vh"],
   "water_percentage": "[0,100]",
-  "data_augmentations":false,
+  "data_augmentations": false,
   "clamp_input": 0.15,
   "scale_input": "normalize",
   "data_mean": [0.0953, 0.0264],
   "data_std": [0.0427, 0.0215],
-  "dem_mean":67.0293,
-  "dem_std":1765.0062,
-  "dem":false,
-  "slope":false,
-  "slope_mean":2.9482,
-  "slope_std":79.2493,
-  "reverse_scaling":false,
-  "uint8":false
+  "dem_mean": 67.0293,
+  "dem_std": 1765.0062,
+  "slc_mean": [2.2367e-02, 3.9242e+01, 8.1130e+01, 4.3526e-02],
+  "slc_std":[ 1.2843, 25.6152, 58.0151,  1.2844],
+  "dem": false,
+  "slope": false,
+  "slope_mean": 2.9482,
+  "slope_std": 79.2493,
+  "reverse_scaling": false,
+  "uint8": false
 }