add mutated token

BiomedSciAI · Apr 14, 2024 · 407a9d1 · 407a9d1
1 parent d3348e4
commit 407a9d1
Show file tree

Hide file tree

Showing 11 changed files with 111 additions and 10 deletions.
diff --git a/...tokenizer/bpe_tokenizer_trained_on_chembl_zinc_with_aug_4272372_samples_balanced_1_1.json b/...tokenizer/bpe_tokenizer_trained_on_chembl_zinc_with_aug_4272372_samples_balanced_1_1.json
@@ -2693,6 +2693,15 @@
       "rstrip": false,
       "normalized": false,
       "special": true
+    },
+    {
+      "id": 299,
+      "content": "<MUTATED>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
     }
   ],
   "normalizer": null,
@@ -3007,6 +3016,7 @@
       "<CDR3_REGION>": 296,
       "<GENERAL_CHAIN>": 297,
       "<SUBMOLECULAR_ENTITY>": 298,
+      "<MUTATED>": 299,
       "#": 527,
       "%": 528,
       "(": 529,
@@ -8948,4 +8958,4 @@
       "c2n c3n("
     ]
   }
-}
+}
diff --git a/...izer/pretrained_tokenizers/bmfm_extended_modular_tokenizer/cell_attributes_tokenizer.json b/...izer/pretrained_tokenizers/bmfm_extended_modular_tokenizer/cell_attributes_tokenizer.json
@@ -2693,6 +2693,15 @@
       "rstrip": false,
       "normalized": false,
       "special": true
+    },
+    {
+      "id": 299,
+      "content": "<MUTATED>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
     }
   ],
   "normalizer": null,
@@ -3013,6 +3022,7 @@
       "<CDR3_REGION>": 296,
       "<GENERAL_CHAIN>": 297,
       "<SUBMOLECULAR_ENTITY>": 298,
+      "<MUTATED>": 299,
       "[CL:0000499]": 3522,
       "[CL:2000060]": 3523,
       "[CL:0000235]": 3524,
@@ -3805,4 +3815,4 @@
     },
     "unk_token": "<UNK>"
   }
-}
+}
diff --git a/...odulartokenizer/pretrained_tokenizers/bmfm_extended_modular_tokenizer/gene_tokenizer.json b/...odulartokenizer/pretrained_tokenizers/bmfm_extended_modular_tokenizer/gene_tokenizer.json
@@ -2693,6 +2693,15 @@
       "rstrip": false,
       "normalized": false,
       "special": true
+    },
+    {
+      "id": 299,
+      "content": "<MUTATED>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
     }
   ],
   "normalizer": null,
@@ -3013,6 +3022,7 @@
       "<CDR3_REGION>": 296,
       "<GENERAL_CHAIN>": 297,
       "<SUBMOLECULAR_ENTITY>": 298,
+      "<MUTATED>": 299,
       "[100130093]": 5000,
       "[100133445]": 5001,
       "[100286793]": 5002,
@@ -97920,4 +97930,4 @@
     },
     "unk_token": "<UNK>"
   }
-}
+}
diff --git a/...enizer/pretrained_tokenizers/bmfm_extended_modular_tokenizer/t5_tokenizer_AA_special.json b/...enizer/pretrained_tokenizers/bmfm_extended_modular_tokenizer/t5_tokenizer_AA_special.json
@@ -2693,6 +2693,15 @@
       "rstrip": false,
       "normalized": false,
       "special": true
+    },
+    {
+      "id": 299,
+      "content": "<MUTATED>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
     }
   ],
   "normalizer": null,
@@ -3013,6 +3022,7 @@
       "<CDR3_REGION>": 296,
       "<GENERAL_CHAIN>": 297,
       "<SUBMOLECULAR_ENTITY>": 298,
+      "<MUTATED>": 299,
       "A": 501,
       "B": 502,
       "C": 503,
@@ -3042,4 +3052,4 @@
     },
     "unk_token": "<UNK>"
   }
-}
+}
diff --git a/...tokenizer/bpe_tokenizer_trained_on_chembl_zinc_with_aug_4272372_samples_balanced_1_1.json b/...tokenizer/bpe_tokenizer_trained_on_chembl_zinc_with_aug_4272372_samples_balanced_1_1.json
@@ -2693,6 +2693,15 @@
       "rstrip": false,
       "normalized": false,
       "special": true
+    },
+    {
+      "id": 299,
+      "content": "<MUTATED>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
     }
   ],
   "normalizer": null,
@@ -3007,6 +3016,7 @@
       "<CDR3_REGION>": 296,
       "<GENERAL_CHAIN>": 297,
       "<SUBMOLECULAR_ENTITY>": 298,
+      "<MUTATED>": 299,
       "#": 527,
       "%": 528,
       "(": 529,
@@ -8948,4 +8958,4 @@
       "c2n c3n("
     ]
   }
-}
+}
diff --git a/...ulartokenizer/pretrained_tokenizers/bmfm_modular_tokenizer/cell_attributes_tokenizer.json b/...ulartokenizer/pretrained_tokenizers/bmfm_modular_tokenizer/cell_attributes_tokenizer.json
@@ -2693,6 +2693,15 @@
       "rstrip": false,
       "normalized": false,
       "special": true
+    },
+    {
+      "id": 299,
+      "content": "<MUTATED>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
     }
   ],
   "normalizer": null,
@@ -3013,6 +3022,7 @@
       "<CDR3_REGION>": 296,
       "<GENERAL_CHAIN>": 297,
       "<SUBMOLECULAR_ENTITY>": 298,
+      "<MUTATED>": 299,
       "[CL:0000499]": 3522,
       "[CL:2000060]": 3523,
       "[CL:0000235]": 3524,
@@ -3805,4 +3815,4 @@
     },
     "unk_token": "<UNK>"
   }
-}
+}
diff --git a/...odulartokenizer/pretrained_tokenizers/bmfm_modular_tokenizer/t5_tokenizer_AA_special.json b/...odulartokenizer/pretrained_tokenizers/bmfm_modular_tokenizer/t5_tokenizer_AA_special.json
@@ -2693,6 +2693,15 @@
       "rstrip": false,
       "normalized": false,
       "special": true
+    },
+    {
+      "id": 299,
+      "content": "<MUTATED>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
     }
   ],
   "normalizer": null,
@@ -3013,6 +3022,7 @@
       "<CDR3_REGION>": 296,
       "<GENERAL_CHAIN>": 297,
       "<SUBMOLECULAR_ENTITY>": 298,
+      "<MUTATED>": 299,
       "A": 501,
       "B": 502,
       "C": 503,
@@ -3042,4 +3052,4 @@
     },
     "unk_token": "<UNK>"
   }
-}
+}
diff --git a/...ngle_path/bpe_tokenizer_trained_on_chembl_zinc_with_aug_4272372_samples_balanced_1_1.json b/...ngle_path/bpe_tokenizer_trained_on_chembl_zinc_with_aug_4272372_samples_balanced_1_1.json
@@ -2693,6 +2693,15 @@
       "rstrip": false,
       "normalized": false,
       "special": true
+    },
+    {
+      "id": 299,
+      "content": "<MUTATED>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
     }
   ],
   "normalizer": null,
@@ -3007,6 +3016,7 @@
       "<CDR3_REGION>": 296,
       "<GENERAL_CHAIN>": 297,
       "<SUBMOLECULAR_ENTITY>": 298,
+      "<MUTATED>": 299,
       "#": 527,
       "%": 528,
       "(": 529,
@@ -8948,4 +8958,4 @@
       "c2n c3n("
     ]
   }
-}
+}
diff --git a/...enizer/pretrained_tokenizers/modular_AA_SMILES_single_path/cell_attributes_tokenizer.json b/...enizer/pretrained_tokenizers/modular_AA_SMILES_single_path/cell_attributes_tokenizer.json
@@ -2693,6 +2693,15 @@
       "rstrip": false,
       "normalized": false,
       "special": true
+    },
+    {
+      "id": 299,
+      "content": "<MUTATED>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
     }
   ],
   "normalizer": null,
@@ -3013,6 +3022,7 @@
       "<CDR3_REGION>": 296,
       "<GENERAL_CHAIN>": 297,
       "<SUBMOLECULAR_ENTITY>": 298,
+      "<MUTATED>": 299,
       "[CL:0000499]": 3522,
       "[CL:2000060]": 3523,
       "[CL:0000235]": 3524,
@@ -3805,4 +3815,4 @@
     },
     "unk_token": "<UNK>"
   }
-}
+}
diff --git a/...okenizer/pretrained_tokenizers/modular_AA_SMILES_single_path/t5_tokenizer_AA_special.json b/...okenizer/pretrained_tokenizers/modular_AA_SMILES_single_path/t5_tokenizer_AA_special.json
@@ -2693,6 +2693,15 @@
       "rstrip": false,
       "normalized": false,
       "special": true
+    },
+    {
+      "id": 299,
+      "content": "<MUTATED>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
     }
   ],
   "normalizer": null,
@@ -3013,6 +3022,7 @@
       "<CDR3_REGION>": 296,
       "<GENERAL_CHAIN>": 297,
       "<SUBMOLECULAR_ENTITY>": 298,
+      "<MUTATED>": 299,
       "A": 501,
       "B": 502,
       "C": 503,
@@ -3042,4 +3052,4 @@
     },
     "unk_token": "<UNK>"
   }
-}
+}
diff --git a/fusedrug/data/tokenizer/modulartokenizer/special_tokens.py b/fusedrug/data/tokenizer/modulartokenizer/special_tokens.py
@@ -320,6 +320,7 @@
     "ALTERNATIVE",
     "GENERAL_CHAIN",
     "CDR3_REGION",
+    "MUTATED"
 ]
 
 AA_tokens = [