Update code for KuroSiwo v2

catalogue/catalogue.py

@@ -468,12 +468,13 @@ def oper(args):
             return
 
         grid_dict = {}
-
+        invalid_number = 0
         for i, id in enumerate(fcat.ids):
             print(f"Grid {i+1}/{len(cat)}: {id}")
             gridd = fcat.grid(id)
             if not gridd.valid:
                 print(f"Invalid Grid: {id}")
+                invalid_number += 1
                 continue
             if "999999" in str(gridd.path):
                 continue
@@ -498,7 +499,8 @@ def oper(args):
                 "clz": gridd.clz_id,
                 "clz_name": gridd.clz_name,
             }
-
+        print('Num of invalid grids: ', invalid_number)
+        print('Num of valid grids: ', len(grid_dict))
         pickle_path = CFG["PICKLE_PATH"]
 
         Path(os.path.dirname(pickle_path)).mkdir(parents=True, exist_ok=True)

catalogue/catalogue.yaml

@@ -359,4 +359,87 @@ Floods:
       - aoi_id: '15'
         aoi_name: Singleton
         cl_zone: 3
-...
+  - act_id: 1111002
+    act_region: Cameroon
+    ref_date: 20200914T080000
+    aois:
+      - aoi_id: '01'
+        aoi_name: Djagala
+        cl_zone: 2
+  - act_id: 1111003
+    act_region: Djibouti
+    ref_date: 20191121T080000
+    aois:
+      - aoi_id: '01'
+        aoi_name: Djibouti
+        cl_zone: 2
+  - act_id: 1111004
+    act_region: USA
+    ref_date: 20170825T080000
+    aois:
+      - aoi_id: '01'
+        aoi_name: Houston
+        cl_zone: 3
+  - act_id: 1111005
+    act_region: Madagascar
+    ref_date: 20200117T080000
+    aois:
+      - aoi_id: '01'
+        aoi_name: Farihy Alaotra
+        cl_zone: 1
+  - act_id: 1111006
+    act_region: Madagascar
+    ref_date: 20200117T080000
+    aois:
+      - aoi_id: '01'
+        aoi_name: Ambato Boeny
+        cl_zone: 1
+  - act_id: 1111007
+    act_region: Nepal
+    ref_date: 20190911T080000
+    aois:
+      - aoi_id: '01'
+        aoi_name: Patna
+        cl_zone: 1
+  - act_id: 1111008
+    act_region: Nicaragua
+    ref_date: 20200810T080000
+    aois:
+      - aoi_id: '01'
+        aoi_name: Dakban
+        cl_zone: 1
+  - act_id: 1111009
+    act_region: Pakistan
+    ref_date: 20220620T080000
+    aois:
+      - aoi_id: '01'
+        aoi_name: Larkana
+        cl_zone: 3
+  - act_id: 1111010
+    act_region: Peru
+    ref_date: 20170311T080000
+    aois:
+      - aoi_id: '01'
+        aoi_name: Sechura
+        cl_zone: 2
+  - act_id: 1111011
+    act_region: Philippines
+    ref_date: 20201111T080000
+    aois:
+      - aoi_id: '01'
+        aoi_name: Tuguegarao
+        cl_zone: 1
+  - act_id: 1111012
+    act_region: Australia
+    ref_date: 20220306T080000
+    aois:
+      - aoi_id: '01'
+        aoi_name: Singleton
+        cl_zone: 3
+  - act_id: 1111013
+    act_region: USA
+    ref_date: 20170429T080000
+    aois:
+      - aoi_id: '01'
+        aoi_name: St Louis
+        cl_zone: 3

configs/train/data_config.json

@@ -1,22 +1,28 @@
 {
   "track": "RandomEvents",
-  "train_pickle": "pickle/grid_dict.pkl",
-  "test_pickle": "pickle/grid_dict.pkl",
-  "negative_pickle":"pickle/negatives_only.pkl",
+  "train_pickle": "pickle/grid_dict.pkl",  // The pickle containing the GRD training data
+  "test_pickle": "pickle/grid_dict.pkl",  // The pickle containing the GRD testing data
+  "negative_pickle":"pickle/negatives_only.pkl",  // Pickle containing only negatives (optional)
+  "slc": true,  // 'true' in order to use SLC data instead of GRD
+  "train_json": "slc_grid_pwater_0.0001.json", // The JSON containing the SLC training data
+  "test_json": "slc_grid_pwater_0.json",  // The JSON containing the SLC testing data
+  "slc_root_path": "",  // The path containing the SLC data
   "inputs": ["pre_event_1","pre_event_2", "post_event"],
   "channels": [ "vv","vh"],
   "water_percentage": "[0,100]",
-  "data_augmentations":false,
+  "data_augmentations": false,
   "clamp_input": 0.15,
   "scale_input": "normalize",
   "data_mean": [0.0953, 0.0264],
   "data_std": [0.0427, 0.0215],
-  "dem_mean":67.0293,
-  "dem_std":1765.0062,
-  "dem":false,
-  "slope":false,
-  "slope_mean":2.9482,
-  "slope_std":79.2493,
-  "reverse_scaling":false,
-  "uint8":false
+  "dem_mean": 67.0293,
+  "dem_std": 1765.0062,
+  "slc_mean": [2.2367e-02, 3.9242e+01, 8.1130e+01, 4.3526e-02],
+  "slc_std":[ 1.2843, 25.6152, 58.0151,  1.2844],
+  "dem": false,
+  "slope": false,
+  "slope_mean": 2.9482,
+  "slope_std": 79.2493,
+  "reverse_scaling": false,
+  "uint8": false
 }