From f7440c474d72e32d768d2c2ef1e060d467fe7731 Mon Sep 17 00:00:00 2001 From: Xiao Feiyu Date: Sun, 16 Aug 2020 17:33:24 +0800 Subject: [PATCH] final --- BertToSimple/BiGRU/.gitignore | 103 + BertToSimple/BiGRU/LICENSE | 201 + BertToSimple/BiGRU/README.md | 77 + BertToSimple/BiGRU/README_GQS.md | 14 + BertToSimple/BiGRU/README_zh.md | 97 + BertToSimple/BiGRU/__init__.py | 1 + BertToSimple/BiGRU/callback/__init__.py | 0 BertToSimple/BiGRU/callback/lr_scheduler.py | 544 + .../BiGRU/callback/modelcheckpoint.py | 97 + .../BiGRU/callback/optimization/__init__.py | 0 .../BiGRU/callback/optimization/adabound.py | 111 + .../BiGRU/callback/optimization/adafactor.py | 177 + .../BiGRU/callback/optimization/adamw.py | 90 + .../BiGRU/callback/optimization/lamb.py | 105 + .../BiGRU/callback/optimization/lars.py | 91 + .../BiGRU/callback/optimization/lookahead.py | 105 + .../BiGRU/callback/optimization/nadam.py | 91 + .../BiGRU/callback/optimization/novograd.py | 75 + .../BiGRU/callback/optimization/planradam.py | 66 + .../BiGRU/callback/optimization/radam.py | 88 + .../BiGRU/callback/optimization/ralamb.py | 104 + .../BiGRU/callback/optimization/ralars.py | 120 + .../BiGRU/callback/optimization/sgdw.py | 82 + BertToSimple/BiGRU/callback/progressbar.py | 58 + .../BiGRU/callback/trainingmonitor.py | 69 + ...convert_albert_tf_checkpoint_to_pytorch.py | 76 + BertToSimple/BiGRU/label_and_pre.csv | 17352 +++++++++++++ BertToSimple/BiGRU/metrics/__init__.py | 0 BertToSimple/BiGRU/metrics/custom_metrics.py | 332 + .../BiGRU/metrics/glue_compute_metrics.py | 83 + BertToSimple/BiGRU/model/__init__.py | 1 + .../BiGRU/model/configuration_albert.py | 80 + .../BiGRU/model/configuration_bert.py | 83 + .../BiGRU/model/configuration_utils.py | 207 + BertToSimple/BiGRU/model/file_utils.py | 294 + BertToSimple/BiGRU/model/modeling_albert.py | 1087 + .../BiGRU/model/modeling_albert_bright.py | 1002 + BertToSimple/BiGRU/model/modeling_bert.py | 1149 + BertToSimple/BiGRU/model/modeling_utils.py | 756 + .../BiGRU/model/tokenization_albert.py | 359 + BertToSimple/BiGRU/model/tokenization_bert.py | 441 + .../BiGRU/model/tokenization_utils.py | 1065 + BertToSimple/BiGRU/my_model.py | 152 + BertToSimple/BiGRU/prepare_lm_data_mask.py | 289 + BertToSimple/BiGRU/prepare_lm_data_ngram.py | 316 + .../albert_21_93/config.json | 28 + .../albert_large_zh/config.json | 21 + .../albert_large_zh/vocab.txt | 21128 ++++++++++++++++ BertToSimple/BiGRU/processors/__init__.py | 5 + BertToSimple/BiGRU/processors/glue.py | 534 + BertToSimple/BiGRU/processors/utils.py | 104 + BertToSimple/BiGRU/run_classifier.py | 533 + BertToSimple/BiGRU/run_pretraining.py | 357 + .../BiGRU/scripts/run_classifier_cola.sh | 25 + .../BiGRU/scripts/run_classifier_lcqmc.sh | 24 + .../BiGRU/scripts/run_classifier_mnli.sh | 24 + .../BiGRU/scripts/run_classifier_qqp.sh | 25 + .../BiGRU/scripts/run_classifier_sst2.sh | 25 + .../BiGRU/scripts/run_classifier_stsb.sh | 24 + BertToSimple/BiGRU/tools/common.py | 347 + BertToSimple/textcnn/predict.py | 83 - BertToSimple/textcnn/readme.md | 10 - BertToSimple/textcnn/test.py | 89 - BertToSimple/textcnn/utils.py | 55 - README.md | 8 +- 65 files changed, 50897 insertions(+), 242 deletions(-) create mode 100644 BertToSimple/BiGRU/.gitignore create mode 100644 BertToSimple/BiGRU/LICENSE create mode 100644 BertToSimple/BiGRU/README.md create mode 100644 BertToSimple/BiGRU/README_GQS.md create mode 100644 BertToSimple/BiGRU/README_zh.md create mode 100644 BertToSimple/BiGRU/__init__.py create mode 100644 BertToSimple/BiGRU/callback/__init__.py create mode 100644 BertToSimple/BiGRU/callback/lr_scheduler.py create mode 100644 BertToSimple/BiGRU/callback/modelcheckpoint.py create mode 100644 BertToSimple/BiGRU/callback/optimization/__init__.py create mode 100644 BertToSimple/BiGRU/callback/optimization/adabound.py create mode 100644 BertToSimple/BiGRU/callback/optimization/adafactor.py create mode 100644 BertToSimple/BiGRU/callback/optimization/adamw.py create mode 100644 BertToSimple/BiGRU/callback/optimization/lamb.py create mode 100644 BertToSimple/BiGRU/callback/optimization/lars.py create mode 100644 BertToSimple/BiGRU/callback/optimization/lookahead.py create mode 100644 BertToSimple/BiGRU/callback/optimization/nadam.py create mode 100644 BertToSimple/BiGRU/callback/optimization/novograd.py create mode 100644 BertToSimple/BiGRU/callback/optimization/planradam.py create mode 100644 BertToSimple/BiGRU/callback/optimization/radam.py create mode 100644 BertToSimple/BiGRU/callback/optimization/ralamb.py create mode 100644 BertToSimple/BiGRU/callback/optimization/ralars.py create mode 100644 BertToSimple/BiGRU/callback/optimization/sgdw.py create mode 100644 BertToSimple/BiGRU/callback/progressbar.py create mode 100644 BertToSimple/BiGRU/callback/trainingmonitor.py create mode 100644 BertToSimple/BiGRU/convert_albert_tf_checkpoint_to_pytorch.py create mode 100644 BertToSimple/BiGRU/label_and_pre.csv create mode 100644 BertToSimple/BiGRU/metrics/__init__.py create mode 100644 BertToSimple/BiGRU/metrics/custom_metrics.py create mode 100644 BertToSimple/BiGRU/metrics/glue_compute_metrics.py create mode 100644 BertToSimple/BiGRU/model/__init__.py create mode 100644 BertToSimple/BiGRU/model/configuration_albert.py create mode 100644 BertToSimple/BiGRU/model/configuration_bert.py create mode 100644 BertToSimple/BiGRU/model/configuration_utils.py create mode 100644 BertToSimple/BiGRU/model/file_utils.py create mode 100644 BertToSimple/BiGRU/model/modeling_albert.py create mode 100644 BertToSimple/BiGRU/model/modeling_albert_bright.py create mode 100644 BertToSimple/BiGRU/model/modeling_bert.py create mode 100644 BertToSimple/BiGRU/model/modeling_utils.py create mode 100644 BertToSimple/BiGRU/model/tokenization_albert.py create mode 100644 BertToSimple/BiGRU/model/tokenization_bert.py create mode 100644 BertToSimple/BiGRU/model/tokenization_utils.py create mode 100644 BertToSimple/BiGRU/my_model.py create mode 100644 BertToSimple/BiGRU/prepare_lm_data_mask.py create mode 100644 BertToSimple/BiGRU/prepare_lm_data_ngram.py create mode 100644 BertToSimple/BiGRU/prev_trained_model/albert_21_93/config.json create mode 100644 BertToSimple/BiGRU/prev_trained_model/albert_large_zh/config.json create mode 100644 BertToSimple/BiGRU/prev_trained_model/albert_large_zh/vocab.txt create mode 100644 BertToSimple/BiGRU/processors/__init__.py create mode 100644 BertToSimple/BiGRU/processors/glue.py create mode 100644 BertToSimple/BiGRU/processors/utils.py create mode 100644 BertToSimple/BiGRU/run_classifier.py create mode 100644 BertToSimple/BiGRU/run_pretraining.py create mode 100644 BertToSimple/BiGRU/scripts/run_classifier_cola.sh create mode 100644 BertToSimple/BiGRU/scripts/run_classifier_lcqmc.sh create mode 100644 BertToSimple/BiGRU/scripts/run_classifier_mnli.sh create mode 100644 BertToSimple/BiGRU/scripts/run_classifier_qqp.sh create mode 100644 BertToSimple/BiGRU/scripts/run_classifier_sst2.sh create mode 100644 BertToSimple/BiGRU/scripts/run_classifier_stsb.sh create mode 100644 BertToSimple/BiGRU/tools/common.py delete mode 100644 BertToSimple/textcnn/predict.py delete mode 100644 BertToSimple/textcnn/readme.md delete mode 100644 BertToSimple/textcnn/test.py delete mode 100644 BertToSimple/textcnn/utils.py diff --git a/BertToSimple/BiGRU/.gitignore b/BertToSimple/BiGRU/.gitignore new file mode 100644 index 0000000..507cc3d --- /dev/null +++ b/BertToSimple/BiGRU/.gitignore @@ -0,0 +1,103 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ diff --git a/BertToSimple/BiGRU/LICENSE b/BertToSimple/BiGRU/LICENSE new file mode 100644 index 0000000..261eeb9 --- /dev/null +++ b/BertToSimple/BiGRU/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/BertToSimple/BiGRU/README.md b/BertToSimple/BiGRU/README.md new file mode 100644 index 0000000..ee9c337 --- /dev/null +++ b/BertToSimple/BiGRU/README.md @@ -0,0 +1,77 @@ +[**English Version**](./README.md) | [**中文版说明**](./README_zh.md) + +## albert_pytorch + +This repository contains a PyTorch implementation of the albert model from the paper + +[A Lite Bert For Self-Supervised Learning Language Representations](https://arxiv.org/pdf/1909.11942.pdf) + +by Zhenzhong Lan. Mingda Chen.... + +## Dependencies + +- pytorch=1.10 +- cuda=9.0 +- cudnn=7.5 +- scikit-learn +- sentencepiece + +## Download Pre-trained Models of English + +Official download links: [google albert](https://github.com/google-research/ALBERT) + +Adapt to this version，download pytorch model (google drive): + +**v1** + +- [albert_base_v1.zip](https://drive.google.com/open?id=1dVsVd6j8rCTpqF4UwnqWuUpmkhxRkEie) +- [albert_large_v1.zip](https://drive.google.com/open?id=18dDXuIHXYWibCLlKX5_rZkFxa3VSc5j1) +- [albert_xlarge_v1.zip](https://drive.google.com/open?id=1jidZkLLFeDuQJsXVtenTvV_LU-AYprJn) +- [albert_xxlarge_v1.zip](https://drive.google.com/open?id=1PV8giuCEAR2Lxaffp0cuCjXh1tVg7Vj_) + +**v2** + +- [albert_base_v2.zip](https://drive.google.com/open?id=1byZQmWDgyhrLpj8oXtxBG6AA52c8IHE-) +- [albert_large_v2.zip](https://drive.google.com/open?id=1KpevOXWzR4OTviFNENm_pbKfYAcokl2V) +- [albert_xlarge_v2.zip](https://drive.google.com/open?id=1W6PxOWnQMxavfiFJsxGic06UVXbq70kq) +- [albert_xxlarge_v2.zip](https://drive.google.com/open?id=1o0EhxPqjd7yRLIwlbH_UAuSAV1dtIXBM) + +## Fine-tuning + +１. Place `config.json` and `30k-clean.model` into the `prev_trained_model/albert_base_v2` directory. +example: +```text +├── prev_trained_model +| └── albert_base_v2 +| | └── pytorch_model.bin +| | └── config.json +| | └── 30k-clean.model +``` +2．convert albert tf checkpoint to pytorch +```python +python convert_albert_tf_checkpoint_to_pytorch.py \ + --tf_checkpoint_path=./prev_trained_model/albert_base_tf_v2 \ + --bert_config_file=./prev_trained_model/albert_base_v2/config.json \ + --pytorch_dump_path=./prev_trained_model/albert_base_v2/pytorch_model.bin +``` +The [General Language Understanding Evaluation (GLUE) benchmark](https://gluebenchmark.com/) is a collection of nine sentence- or sentence-pair language understanding tasks for evaluating and analyzing natural language understanding systems. + +Before running anyone of these GLUE tasks you should download the [GLUE data](https://gluebenchmark.com/tasks) by running [this script](https://gist.github.com/W4ngatang/60c2bdb54d156a41194446737ce03e2e) and unpack it to some directory $DATA_DIR. + +3．run `sh scripts/run_classifier_sst2.sh`to fine tuning albert model + +## Result + +Performance of ALBERT on GLUE benchmark results using a single-model setup on **dev**: + +| | Cola| Sst-2| Mnli| Sts-b| +| :------- | :---------: | :---------: |:---------: | :---------: | +| metric | matthews_corrcoef |accuracy |accuracy | pearson | + +| model | Cola| Sst-2| Mnli| Sts-b| +| :------- | :---------: | :---------: |:---------: | :---------: | +| albert_base_v2 | 0.5756 | 0.926 | 0.8418 | 0.9091 | +| albert_large_v2 | 0.5851 |0.9507 | |0.9151 | +| albert_xlarge_v2 | 0.6023 | | |0.9221 | + + diff --git a/BertToSimple/BiGRU/README_GQS.md b/BertToSimple/BiGRU/README_GQS.md new file mode 100644 index 0000000..87eb52b --- /dev/null +++ b/BertToSimple/BiGRU/README_GQS.md @@ -0,0 +1,14 @@ +进行分类任务直接运行run_classifier.py +********************************************************************* +对main函数开始设置的参数进行说明： +model_name_or_path——teahcer model的位置，是分类正确率为93%的ALBERT +continue_train_distill——是否加载训练好（训练一半）的蒸馏模型的参数 +distill_model_dir——训练好的蒸馏模型的参数位置，分类正确率为90% +do_train和do_eval——是否进行训练/推理 +logging_steps——在训练时，训练多少批数据做一次在测试集上预测 +save_steps——训练多少批数据保存一次模型 +********************************************************************* +my_model.py保存了蒸馏模型网络结构 + + + diff --git a/BertToSimple/BiGRU/README_zh.md b/BertToSimple/BiGRU/README_zh.md new file mode 100644 index 0000000..2560c1e --- /dev/null +++ b/BertToSimple/BiGRU/README_zh.md @@ -0,0 +1,97 @@ +### albert_pytorch + +This repository contains a PyTorch implementation of the albert model from the paper + +[A Lite Bert For Self-Supervised Learning Language Representations](https://arxiv.org/pdf/1909.11942.pdf) + +by Zhenzhong Lan. Mingda Chen.... + +已经转好的pytorch版本权重，可以直接下载(谷歌盘): + +#### 中文权重下载 + +**google** + +- [albert_tiny_zh](https://drive.google.com/open?id=1qAykqB2OXIYXSVMQSt_EDEPCorTAkIvu) +- [albert_small_zh](https://drive.google.com/open?id=1t-DJKAqALgwlO8J_PZ3ZtOy0NwLipGxA) +- [albert_base_zh.zip](https://drive.google.com/open?id=1m_tnylngzEA94xVvBCc3I3DRQNbK32z3) +- [albert_large_zh.zip](https://drive.google.com/open?id=19UZPHfKJZY9BGS4mghuKcFyAIF3nJdlX) +- [albert_xlarge_zh.zip](https://drive.google.com/open?id=1DdZ3-AXaom10nrx8C99UhFYsx1BYYEJx) +- [albert_xxlarge_zh.zip](https://drive.google.com/open?id=1F-Mu9yWvj1XX5WN6gtyxbVLWr610ttgC) + +**bright** + +- [albert_tiny_bright](https://drive.google.com/open?id=1VBsUJ7R5eWF1VcUBQY6BEn1a9miEvlBr) +- [albert_base_bright.zip](https://drive.google.com/open?id=1HeijHGubWR-ElFnfxUf8IrRx7Ghm1S_Q) +- [albert_large_bright.zip](https://drive.google.com/open?id=1TAuv7OiFN8qbkT6S_VbfVbhkhg2GUF3q) +- [albert_xlarge_bright.zip](https://drive.google.com/open?id=1kMhogQRX0uGWIGdNhm7-3hsmHlrzY_gp) + +**说明**: 以上权重只适合该版本。**pytorch**版本为1.1.0 + +原始提供下载地址： + +- [brightmart](https://github.com/brightmart/albert_zh) +- [google albert](https://github.com/google-research/ALBERT) + +#### 使用方式 + +不同的模型权重加载不同的对应模型原始文件，如下(使用的时候加载对应模型文件即可): +```python +#google version +from model.modeling_albert import AlbertConfig, AlbertForSequenceClassification + +# bright version +from model.modeling_albert_bright import AlbertConfig, AlbertForSequenceClassification +``` +**注意**，如果需要运行该`run_classifier.py`脚本，需要将`config.json`和`vocab.txt`文件同时放入对应模型的目录中，比如: + +```text +├── prev_trained_model +| └── albert_base +| | └── pytorch_model.bin +| | └── config.json +| | └── vocab.txt +``` + +#### 预训练 + +**备注**：仅供参考 + +n-gram: 原始论文中按照以下分布随机生成n-gram，默认max_n为3 + +

+1. 将文本数据转化为一行一句格式，并且不同document之间使用`\n`分割 + +2. 运行以下命令： +```python +python prepare_lm_data_ngram.py \ + --data_dir=dataset/ \ + --vocab_path=vocab.txt \ + --data_name=albert \ + --max_ngram=3 \ + --do_data +``` +产生n-gram masking数据集，具体可根据对应数据进行修改代码 + +3. 运行以下命令： +```python +python run_pretraining.py \ + --data_dir=dataset/ \ + --vocab_path=configs/vocab.txt \ + --data_name=albert \ + --config_path=configs/albert_config_base.json \ + --output_dir=outputs/ \ + --data_name=albert \ + --share_type=all +``` +进行模型训练，具体可根据对应数据进行修改代码 + +#### 测试结果 + +问题匹配语任务：LCQMC(Sentence Pair Matching) + +| 模型 | 开发集(Dev) | 测试集(Test) | +| :------- | :---------: | :---------: | +| albert_base(tf) | 86.4 | 86.3 | +| albert_base(pytorch) | 87.4 | 86.4 | +| albert_tiny | 85.１ | 85.3 | diff --git a/BertToSimple/BiGRU/__init__.py b/BertToSimple/BiGRU/__init__.py new file mode 100644 index 0000000..661b40d --- /dev/null +++ b/BertToSimple/BiGRU/__init__.py @@ -0,0 +1 @@ +#encoding:utf-8 \ No newline at end of file diff --git a/BertToSimple/BiGRU/callback/__init__.py b/BertToSimple/BiGRU/callback/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/BertToSimple/BiGRU/callback/lr_scheduler.py b/BertToSimple/BiGRU/callback/lr_scheduler.py new file mode 100644 index 0000000..7334d56 --- /dev/null +++ b/BertToSimple/BiGRU/callback/lr_scheduler.py @@ -0,0 +1,544 @@ +import math +import numpy as np +import warnings +from torch.optim.optimizer import Optimizer +from torch.optim.lr_scheduler import LambdaLR + +__all__ = ['CustomDecayLR', + 'BertLR', + 'CyclicLR', + 'ReduceLROnPlateau', + 'ReduceLRWDOnPlateau', + 'CosineLRWithRestarts', + ] + +def get_constant_schedule(optimizer, last_epoch=-1): + """ Create a schedule with a constant learning rate. + """ + return LambdaLR(optimizer, lambda _: 1, last_epoch=last_epoch) + + +def get_constant_schedule_with_warmup(optimizer, num_warmup_steps, last_epoch=-1): + """ Create a schedule with a constant learning rate preceded by a warmup + period during which the learning rate increases linearly between 0 and 1. + """ + def lr_lambda(current_step): + if current_step < num_warmup_steps: + return float(current_step) / float(max(1.0, num_warmup_steps)) + return 1. + + return LambdaLR(optimizer, lr_lambda, last_epoch=last_epoch) + + +def get_linear_schedule_with_warmup(optimizer, num_warmup_steps, num_training_steps, last_epoch=-1): + """ Create a schedule with a learning rate that decreases linearly after + linearly increasing during a warmup period. + """ + def lr_lambda(current_step): + if current_step < num_warmup_steps: + return float(current_step) / float(max(1, num_warmup_steps)) + return max(0.0, float(num_training_steps - current_step) / float(max(1, num_training_steps - num_warmup_steps))) + + return LambdaLR(optimizer, lr_lambda, last_epoch) + + +def get_cosine_schedule_with_warmup(optimizer, num_warmup_steps, num_training_steps, num_cycles=.5, last_epoch=-1): + """ Create a schedule with a learning rate that decreases following the + values of the cosine function between 0 and `pi * cycles` after a warmup + period during which it increases linearly between 0 and 1. + """ + def lr_lambda(current_step): + if current_step < num_warmup_steps: + return float(current_step) / float(max(1, num_warmup_steps)) + progress = float(current_step - num_warmup_steps) / float(max(1, num_training_steps - num_warmup_steps)) + return max(0., 0.5 * (1. + math.cos(math.pi * float(num_cycles) * 2. * progress))) + + return LambdaLR(optimizer, lr_lambda, last_epoch) + + +def get_cosine_with_hard_restarts_schedule_with_warmup(optimizer, num_warmup_steps, num_training_steps, num_cycles=1., last_epoch=-1): + """ Create a schedule with a learning rate that decreases following the + values of the cosine function with several hard restarts, after a warmup + period during which it increases linearly between 0 and 1. + """ + def lr_lambda(current_step): + if current_step < num_warmup_steps: + return float(current_step) / float(max(1, num_warmup_steps)) + progress = float(current_step - num_warmup_steps) / float(max(1, num_training_steps - num_warmup_steps)) + if progress >= 1.: + return 0. + return max(0., 0.5 * (1. + math.cos(math.pi * ((float(num_cycles) * progress) % 1.)))) + + return LambdaLR(optimizer, lr_lambda, last_epoch) + + +class CustomDecayLR(object): + ''' + 自定义学习率变化机制 + Example: + >>> scheduler = CustomDecayLR(optimizer) + >>> for epoch in range(100): + >>> scheduler.epoch_step() + >>> train(...) + >>> ... + >>> optimizer.zero_grad() + >>> loss.backward() + >>> optimizer.step() + >>> validate(...) + ''' + def __init__(self,optimizer,lr): + self.optimizer = optimizer + self.lr = lr + + def epoch_step(self,epoch): + lr = self.lr + if epoch > 12: + lr = lr / 1000 + elif epoch > 8: + lr = lr / 100 + elif epoch > 4: + lr = lr / 10 + for param_group in self.optimizer.param_groups: + param_group['lr'] = lr + +class BertLR(object): + ''' + Bert模型内定的学习率变化机制 + Example: + >>> scheduler = BertLR(optimizer) + >>> for epoch in range(100): + >>> scheduler.step() + >>> train(...) + >>> ... + >>> optimizer.zero_grad() + >>> loss.backward() + >>> optimizer.step() + >>> scheduler.batch_step() + >>> validate(...) + ''' + def __init__(self,optimizer,learning_rate,t_total,warmup): + self.learning_rate = learning_rate + self.optimizer = optimizer + self.t_total = t_total + self.warmup = warmup + + # 线性预热方式 + def warmup_linear(self,x, warmup=0.002): + if x < warmup: + return x / warmup + return 1.0 - x + + def batch_step(self,training_step): + lr_this_step = self.learning_rate * self.warmup_linear(training_step / self.t_total,self.warmup) + for param_group in self.optimizer.param_groups: + param_group['lr'] = lr_this_step + +class CyclicLR(object): + ''' + Cyclical learning rates for training neural networks + Example: + >>> scheduler = CyclicLR(optimizer) + >>> for epoch in range(100): + >>> scheduler.step() + >>> train(...) + >>> ... + >>> optimizer.zero_grad() + >>> loss.backward() + >>> optimizer.step() + >>> scheduler.batch_step() + >>> validate(...) + ''' + def __init__(self, optimizer, base_lr=1e-3, max_lr=6e-3, + step_size=2000, mode='triangular', gamma=1., + scale_fn=None, scale_mode='cycle', last_batch_iteration=-1): + + if not isinstance(optimizer, Optimizer): + raise TypeError('{} is not an Optimizer'.format( + type(optimizer).__name__)) + + self.optimizer = optimizer + + if isinstance(base_lr, list) or isinstance(base_lr, tuple): + if len(base_lr) != len(optimizer.param_groups): + raise ValueError("expected {} base_lr, got {}".format( + len(optimizer.param_groups), len(base_lr))) + self.base_lrs = list(base_lr) + else: + self.base_lrs = [base_lr] * len(optimizer.param_groups) + + if isinstance(max_lr, list) or isinstance(max_lr, tuple): + if len(max_lr) != len(optimizer.param_groups): + raise ValueError("expected {} max_lr, got {}".format( + len(optimizer.param_groups), len(max_lr))) + self.max_lrs = list(max_lr) + else: + self.max_lrs = [max_lr] * len(optimizer.param_groups) + + self.step_size = step_size + + if mode not in ['triangular', 'triangular2', 'exp_range'] \ + and scale_fn is None: + raise ValueError('mode is invalid and scale_fn is None') + + self.mode = mode + self.gamma = gamma + + if scale_fn is None: + if self.mode == 'triangular': + self.scale_fn = self._triangular_scale_fn + self.scale_mode = 'cycle' + elif self.mode == 'triangular2': + self.scale_fn = self._triangular2_scale_fn + self.scale_mode = 'cycle' + elif self.mode == 'exp_range': + self.scale_fn = self._exp_range_scale_fn + self.scale_mode = 'iterations' + else: + self.scale_fn = scale_fn + self.scale_mode = scale_mode + + self.batch_step(last_batch_iteration + 1) + self.last_batch_iteration = last_batch_iteration + + def _triangular_scale_fn(self, x): + return 1. + + def _triangular2_scale_fn(self, x): + return 1 / (2. ** (x - 1)) + + def _exp_range_scale_fn(self, x): + return self.gamma**(x) + + def get_lr(self): + step_size = float(self.step_size) + cycle = np.floor(1 + self.last_batch_iteration / (2 * step_size)) + x = np.abs(self.last_batch_iteration / step_size - 2 * cycle + 1) + + lrs = [] + param_lrs = zip(self.optimizer.param_groups, self.base_lrs, self.max_lrs) + for param_group, base_lr, max_lr in param_lrs: + base_height = (max_lr - base_lr) * np.maximum(0, (1 - x)) + if self.scale_mode == 'cycle': + lr = base_lr + base_height * self.scale_fn(cycle) + else: + lr = base_lr + base_height * self.scale_fn(self.last_batch_iteration) + lrs.append(lr) + return lrs + + def batch_step(self, batch_iteration=None): + if batch_iteration is None: + batch_iteration = self.last_batch_iteration + 1 + self.last_batch_iteration = batch_iteration + for param_group, lr in zip(self.optimizer.param_groups, self.get_lr()): + param_group['lr'] = lr + +class ReduceLROnPlateau(object): + """Reduce learning rate when a metric has stopped improving. + Models often benefit from reducing the learning rate by a factor + of 2-10 once learning stagnates. This scheduler reads a metrics + quantity and if no improvement is seen for a 'patience' number + of epochs, the learning rate is reduced. + + Args: + factor: factor by which the learning rate will + be reduced. new_lr = lr * factor + patience: number of epochs with no improvement + after which learning rate will be reduced. + verbose: int. 0: quiet, 1: update messages. + mode: one of {min, max}. In `min` mode, + lr will be reduced when the quantity + monitored has stopped decreasing; in `max` + mode it will be reduced when the quantity + monitored has stopped increasing. + epsilon: threshold for measuring the new optimum, + to only focus on significant changes. + cooldown: number of epochs to wait before resuming + normal operation after lr has been reduced. + min_lr: lower bound on the learning rate. + + + Example: + >>> optimizer = torch.optim.SGD(model.parameters(), lr=0.1, momentum=0.9) + >>> scheduler = ReduceLROnPlateau(optimizer, 'min') + >>> for epoch in range(10): + >>> train(...) + >>> val_acc, val_loss = validate(...) + >>> scheduler.epoch_step(val_loss, epoch) + """ + + def __init__(self, optimizer, mode='min', factor=0.1, patience=10, + verbose=0, epsilon=1e-4, cooldown=0, min_lr=0,eps=1e-8): + + super(ReduceLROnPlateau, self).__init__() + assert isinstance(optimizer, Optimizer) + if factor >= 1.0: + raise ValueError('ReduceLROnPlateau ' + 'does not support a factor >= 1.0.') + self.factor = factor + self.min_lr = min_lr + self.epsilon = epsilon + self.patience = patience + self.verbose = verbose + self.cooldown = cooldown + self.cooldown_counter = 0 # Cooldown counter. + self.monitor_op = None + self.wait = 0 + self.best = 0 + self.mode = mode + self.optimizer = optimizer + self.eps = eps + self._reset() + + def _reset(self): + """Resets wait counter and cooldown counter. + """ + if self.mode not in ['min', 'max']: + raise RuntimeError('Learning Rate Plateau Reducing mode %s is unknown!') + if self.mode == 'min': + self.monitor_op = lambda a, b: np.less(a, b - self.epsilon) + self.best = np.Inf + else: + self.monitor_op = lambda a, b: np.greater(a, b + self.epsilon) + self.best = -np.Inf + self.cooldown_counter = 0 + self.wait = 0 + + def reset(self): + self._reset() + + def epoch_step(self, metrics, epoch): + current = metrics + if current is None: + warnings.warn('Learning Rate Plateau Reducing requires metrics available!', RuntimeWarning) + else: + if self.in_cooldown(): + self.cooldown_counter -= 1 + self.wait = 0 + + if self.monitor_op(current, self.best): + self.best = current + self.wait = 0 + elif not self.in_cooldown(): + if self.wait >= self.patience: + for param_group in self.optimizer.param_groups: + old_lr = float(param_group['lr']) + if old_lr > self.min_lr + self.eps: + new_lr = old_lr * self.factor + new_lr = max(new_lr, self.min_lr) + param_group['lr'] = new_lr + if self.verbose > 0: + print('\nEpoch %05d: reducing learning rate to %s.' % (epoch, new_lr)) + self.cooldown_counter = self.cooldown + self.wait = 0 + self.wait += 1 + + def in_cooldown(self): + return self.cooldown_counter > 0 + +class ReduceLRWDOnPlateau(ReduceLROnPlateau): + """Reduce learning rate and weight decay when a metric has stopped + improving. Models often benefit from reducing the learning rate by + a factor of 2-10 once learning stagnates. This scheduler reads a metric + quantity and if no improvement is seen for a 'patience' number + of epochs, the learning rate and weight decay factor is reduced for + optimizers that implement the the weight decay method from the paper + `Fixing Weight Decay Regularization in Adam`_. + + .. _Fixing Weight Decay Regularization in Adam: + https://arxiv.org/abs/1711.05101 + for AdamW or SGDW + Example: + >>> optimizer = AdamW(model.parameters(), lr=0.1, weight_decay=1e-3) + >>> scheduler = ReduceLRWDOnPlateau(optimizer, 'min') + >>> for epoch in range(10): + >>> train(...) + >>> val_loss = validate(...) + >>> # Note that step should be called after validate() + >>> scheduler.epoch_step(val_loss) + """ + def epoch_step(self, metrics, epoch): + current = metrics + if current is None: + warnings.warn('Learning Rate Plateau Reducing requires metrics available!', RuntimeWarning) + else: + if self.in_cooldown(): + self.cooldown_counter -= 1 + self.wait = 0 + + if self.monitor_op(current, self.best): + self.best = current + self.wait = 0 + elif not self.in_cooldown(): + if self.wait >= self.patience: + for param_group in self.optimizer.param_groups: + old_lr = float(param_group['lr']) + if old_lr > self.min_lr + self.eps: + new_lr = old_lr * self.factor + new_lr = max(new_lr, self.min_lr) + param_group['lr'] = new_lr + if self.verbose > 0: + print('\nEpoch %d: reducing learning rate to %s.' % (epoch, new_lr)) + if param_group['weight_decay'] != 0: + old_weight_decay = float(param_group['weight_decay']) + new_weight_decay = max(old_weight_decay * self.factor, self.min_lr) + if old_weight_decay > new_weight_decay + self.eps: + param_group['weight_decay'] = new_weight_decay + if self.verbose: + print('\nEpoch {epoch}: reducing weight decay factor of group {i} to {new_weight_decay:.4e}.') + self.cooldown_counter = self.cooldown + self.wait = 0 + self.wait += 1 + +class CosineLRWithRestarts(object): + """Decays learning rate with cosine annealing, normalizes weight decay + hyperparameter value, implements restarts. + https://arxiv.org/abs/1711.05101 + + Args: + optimizer (Optimizer): Wrapped optimizer. + batch_size: minibatch size + epoch_size: training samples per epoch + restart_period: epoch count in the first restart period + t_mult: multiplication factor by which the next restart period will extend/shrink + + Example: + >>> scheduler = CosineLRWithRestarts(optimizer, 32, 1024, restart_period=5, t_mult=1.2) + >>> for epoch in range(100): + >>> scheduler.step() + >>> train(...) + >>> ... + >>> optimizer.zero_grad() + >>> loss.backward() + >>> optimizer.step() + >>> scheduler.batch_step() + >>> validate(...) + """ + + def __init__(self, optimizer, batch_size, epoch_size, restart_period=100, + t_mult=2, last_epoch=-1, eta_threshold=1000, verbose=False): + if not isinstance(optimizer, Optimizer): + raise TypeError('{} is not an Optimizer'.format( + type(optimizer).__name__)) + self.optimizer = optimizer + if last_epoch == -1: + for group in optimizer.param_groups: + group.setdefault('initial_lr', group['lr']) + else: + for i, group in enumerate(optimizer.param_groups): + if 'initial_lr' not in group: + raise KeyError("param 'initial_lr' is not specified " + "in param_groups[{}] when resuming an" + " optimizer".format(i)) + self.base_lrs = list(map(lambda group: group['initial_lr'], + optimizer.param_groups)) + + self.last_epoch = last_epoch + self.batch_size = batch_size + self.iteration = 0 + self.epoch_size = epoch_size + self.eta_threshold = eta_threshold + self.t_mult = t_mult + self.verbose = verbose + self.base_weight_decays = list(map(lambda group: group['weight_decay'], + optimizer.param_groups)) + self.restart_period = restart_period + self.restarts = 0 + self.t_epoch = -1 + self.batch_increments = [] + self._set_batch_increment() + + def _schedule_eta(self): + """ + Threshold value could be adjusted to shrink eta_min and eta_max values. + """ + eta_min = 0 + eta_max = 1 + if self.restarts <= self.eta_threshold: + return eta_min, eta_max + else: + d = self.restarts - self.eta_threshold + k = d * 0.09 + return (eta_min + k, eta_max - k) + + def get_lr(self, t_cur): + eta_min, eta_max = self._schedule_eta() + + eta_t = (eta_min + 0.5 * (eta_max - eta_min) + * (1. + math.cos(math.pi * + (t_cur / self.restart_period)))) + + weight_decay_norm_multi = math.sqrt(self.batch_size / + (self.epoch_size * + self.restart_period)) + lrs = [base_lr * eta_t for base_lr in self.base_lrs] + weight_decays = [base_weight_decay * eta_t * weight_decay_norm_multi + for base_weight_decay in self.base_weight_decays] + + if self.t_epoch % self.restart_period < self.t_epoch: + if self.verbose: + print("Restart at epoch {}".format(self.last_epoch)) + self.restart_period *= self.t_mult + self.restarts += 1 + self.t_epoch = 0 + + return zip(lrs, weight_decays) + + def _set_batch_increment(self): + d, r = divmod(self.epoch_size, self.batch_size) + batches_in_epoch = d + 2 if r > 0 else d + 1 + self.iteration = 0 + self.batch_increments = list(np.linspace(0, 1, batches_in_epoch)) + + def batch_step(self): + self.last_epoch += 1 + self.t_epoch += 1 + self._set_batch_increment() + try: + t_cur = self.t_epoch + self.batch_increments[self.iteration] + self.iteration += 1 + except (IndexError): + raise RuntimeError("Epoch size and batch size used in the " + "training loop and while initializing " + "scheduler should be the same.") + + for param_group, (lr, weight_decay) in zip(self.optimizer.param_groups,self.get_lr(t_cur)): + param_group['lr'] = lr + param_group['weight_decay'] = weight_decay + + +class NoamLR(object): + ''' + 主要参考论文<< Attention Is All You Need>>中的学习更新方式 + Example: + >>> scheduler = NoamLR(d_model,factor,warm_up,optimizer) + >>> for epoch in range(100): + >>> scheduler.step() + >>> train(...) + >>> ... + >>> glopab_step += 1 + >>> optimizer.zero_grad() + >>> loss.backward() + >>> optimizer.step() + >>> scheduler.batch_step(global_step) + >>> validate(...) + ''' + def __init__(self,d_model,factor,warm_up,optimizer): + self.optimizer = optimizer + self.warm_up = warm_up + self.factor = factor + self.d_model = d_model + self._lr = 0 + + def get_lr(self,step): + lr = self.factor * (self.d_model ** (-0.5) * min(step ** (-0.5),step * self.warm_up ** (-1.5))) + return lr + + def batch_step(self,step): + ''' + update parameters and rate + :return: + ''' + lr = self.get_lr(step) + for p in self.optimizer.param_groups: + p['lr'] = lr + self._lr = lr diff --git a/BertToSimple/BiGRU/callback/modelcheckpoint.py b/BertToSimple/BiGRU/callback/modelcheckpoint.py new file mode 100644 index 0000000..534b7d6 --- /dev/null +++ b/BertToSimple/BiGRU/callback/modelcheckpoint.py @@ -0,0 +1,97 @@ +from pathlib import Path +import numpy as np +import torch +from ..tools.common import logger + +class ModelCheckpoint(object): + ''' + 模型保存，两种模式： + 1. 直接保存最好模型 + 2. 按照epoch频率保存模型 + ''' + def __init__(self, checkpoint_dir, + monitor, + arch,mode='min', + epoch_freq=1, + best = None, + save_best_only = True): + if isinstance(checkpoint_dir,Path): + checkpoint_dir = checkpoint_dir + else: + checkpoint_dir = Path(checkpoint_dir) + assert checkpoint_dir.is_dir() + checkpoint_dir.mkdir(exist_ok=True) + self.base_path = checkpoint_dir + self.arch = arch + self.monitor = monitor + self.epoch_freq = epoch_freq + self.save_best_only = save_best_only + + # 计算模式 + if mode == 'min': + self.monitor_op = np.less + self.best = np.Inf + + elif mode == 'max': + self.monitor_op = np.greater + self.best = -np.Inf + # 这里主要重新加载模型时候 + #对best重新赋值 + if best: + self.best = best + + if save_best_only: + self.model_name = f"BEST_{arch}_MODEL.pth" + + def epoch_step(self, state,current): + ''' + 正常模型 + :param state: 需要保存的信息 + :param current: 当前判断指标 + :return: + ''' + # 是否保存最好模型 + if self.save_best_only: + if self.monitor_op(current, self.best): + logger.info(f"\nEpoch {state['epoch']}: {self.monitor} improved from {self.best:.5f} to {current:.5f}") + self.best = current + state['best'] = self.best + best_path = self.base_path/ self.model_name + torch.save(state, str(best_path)) + # 每隔几个epoch保存下模型 + else: + filename = self.base_path / f"EPOCH_{state['epoch']}_{state[self.monitor]}_{self.arch}_MODEL.pth" + if state['epoch'] % self.epoch_freq == 0: + logger.info(f"\nEpoch {state['epoch']}: save model to disk.") + torch.save(state, str(filename)) + + def bert_epoch_step(self, state,current): + ''' + 适合bert类型模型，适合pytorch_transformer模块 + :param state: + :param current: + :return: + ''' + model_to_save = state['model'] + if self.save_best_only: + if self.monitor_op(current, self.best): + logger.info(f"\nEpoch {state['epoch']}: {self.monitor} improved from {self.best:.5f} to {current:.5f}") + self.best = current + state['best'] = self.best + model_to_save.save_pretrained(str(self.base_path)) + output_config_file = self.base_path / 'configs.json' + with open(str(output_config_file), 'w') as f: + f.write(model_to_save.config.to_json_string()) + state.pop("model") + torch.save(state,self.base_path / 'checkpoint_info.bin') + else: + if state['epoch'] % self.epoch_freq == 0: + save_path = self.base_path / f"checkpoint-epoch-{state['epoch']}" + save_path.mkdir(exist_ok=True) + logger.info(f"\nEpoch {state['epoch']}: save model to disk.") + model_to_save.save_pretrained(save_path) + output_config_file = save_path / 'configs.json' + with open(str(output_config_file), 'w') as f: + f.write(model_to_save.config.to_json_string()) + state.pop("model") + torch.save(state, save_path / 'checkpoint_info.bin') diff --git a/BertToSimple/BiGRU/callback/optimization/__init__.py b/BertToSimple/BiGRU/callback/optimization/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/BertToSimple/BiGRU/callback/optimization/adabound.py b/BertToSimple/BiGRU/callback/optimization/adabound.py new file mode 100644 index 0000000..52b3bd5 --- /dev/null +++ b/BertToSimple/BiGRU/callback/optimization/adabound.py @@ -0,0 +1,111 @@ +import torch +import math +from torch.optim.optimizer import Optimizer + +class AdaBound(Optimizer): + """Implements AdaBound algorithm. + It has been proposed in `Adaptive Gradient Methods with Dynamic Bound of Learning Rate`_. + Arguments: + params (iterable): iterable of parameters to optimize or dicts defining + parameter groups + lr (float, optional): Adam learning rate (default: 1e-3) + betas (Tuple[float, float], optional): coefficients used for computing + running averages of gradient and its square (default: (0.9, 0.999)) + final_lr (float, optional): final (SGD) learning rate (default: 0.1) + gamma (float, optional): convergence speed of the bound functions (default: 1e-3) + eps (float, optional): term added to the denominator to improve + numerical stability (default: 1e-8) + weight_decay (float, optional): weight decay (L2 penalty) (default: 0) + amsbound (boolean, optional): whether to use the AMSBound variant of this algorithm + .. Adaptive Gradient Methods with Dynamic Bound of Learning Rate: + https://openreview.net/forum?id=Bkg3g2R9FX + Example: + >>> model = LSTM() + >>> optimizer = AdaBound(model.parameters()) + """ + + def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), final_lr=0.1, gamma=1e-3, + eps=1e-8, weight_decay=0, amsbound=False): + if not 0.0 <= lr: + raise ValueError("Invalid learning rate: {}".format(lr)) + if not 0.0 <= eps: + raise ValueError("Invalid epsilon value: {}".format(eps)) + if not 0.0 <= betas[0] < 1.0: + raise ValueError("Invalid beta parameter at index 0: {}".format(betas[0])) + if not 0.0 <= betas[1] < 1.0: + raise ValueError("Invalid beta parameter at index 1: {}".format(betas[1])) + if not 0.0 <= final_lr: + raise ValueError("Invalid final learning rate: {}".format(final_lr)) + if not 0.0 <= gamma < 1.0: + raise ValueError("Invalid gamma parameter: {}".format(gamma)) + defaults = dict(lr=lr, betas=betas, final_lr=final_lr, gamma=gamma, eps=eps, + weight_decay=weight_decay, amsbound=amsbound) + super(AdaBound, self).__init__(params, defaults) + + self.base_lrs = list(map(lambda group: group['lr'], self.param_groups)) + + def __setstate__(self, state): + super(AdaBound, self).__setstate__(state) + for group in self.param_groups: + group.setdefault('amsbound', False) + + def step(self, closure=None): + """Performs a single optimization step. + Arguments: + closure (callable, optional): A closure that reevaluates the model + and returns the loss. + """ + loss = None + if closure is not None: + loss = closure() + for group, base_lr in zip(self.param_groups, self.base_lrs): + for p in group['params']: + if p.grad is None: + continue + grad = p.grad.data + if grad.is_sparse: + raise RuntimeError( + 'Adam does not support sparse gradients, please consider SparseAdam instead') + amsbound = group['amsbound'] + state = self.state[p] + # State initialization + if len(state) == 0: + state['step'] = 0 + # Exponential moving average of gradient values + state['exp_avg'] = torch.zeros_like(p.data) + # Exponential moving average of squared gradient values + state['exp_avg_sq'] = torch.zeros_like(p.data) + if amsbound: + # Maintains max of all exp. moving avg. of sq. grad. values + state['max_exp_avg_sq'] = torch.zeros_like(p.data) + exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq'] + if amsbound: + max_exp_avg_sq = state['max_exp_avg_sq'] + beta1, beta2 = group['betas'] + state['step'] += 1 + if group['weight_decay'] != 0: + grad = grad.add(group['weight_decay'], p.data) + # Decay the first and second moment running average coefficient + exp_avg.mul_(beta1).add_(1 - beta1, grad) + exp_avg_sq.mul_(beta2).addcmul_(1 - beta2, grad, grad) + if amsbound: + # Maintains the maximum of all 2nd moment running avg. till now + torch.max(max_exp_avg_sq, exp_avg_sq, out=max_exp_avg_sq) + # Use the max. for normalizing running avg. of gradient + denom = max_exp_avg_sq.sqrt().add_(group['eps']) + else: + denom = exp_avg_sq.sqrt().add_(group['eps']) + + bias_correction1 = 1 - beta1 ** state['step'] + bias_correction2 = 1 - beta2 ** state['step'] + step_size = group['lr'] * math.sqrt(bias_correction2) / bias_correction1 + + # Applies bounds on actual learning rate + # lr_scheduler cannot affect final_lr, this is a workaround to apply lr decay + final_lr = group['final_lr'] * group['lr'] / base_lr + lower_bound = final_lr * (1 - 1 / (group['gamma'] * state['step'] + 1)) + upper_bound = final_lr * (1 + 1 / (group['gamma'] * state['step'])) + step_size = torch.full_like(denom, step_size) + step_size.div_(denom).clamp_(lower_bound, upper_bound).mul_(exp_avg) + p.data.add_(-step_size) + return loss \ No newline at end of file diff --git a/BertToSimple/BiGRU/callback/optimization/adafactor.py b/BertToSimple/BiGRU/callback/optimization/adafactor.py new file mode 100644 index 0000000..f4f1851 --- /dev/null +++ b/BertToSimple/BiGRU/callback/optimization/adafactor.py @@ -0,0 +1,177 @@ +import operator +import torch +from copy import copy +import functools +from math import sqrt +from torch.optim.optimizer import Optimizer + + +class AdaFactor(Optimizer): + ''' + # Code below is an implementation of https://arxiv.org/pdf/1804.04235.pdf + # inspired but modified from https://github.com/DeadAt0m/adafactor-pytorch + Example: + >>> model = LSTM() + >>> optimizer = AdaFactor(model.parameters(),lr= lr) + ''' + + def __init__(self, params, lr=None, beta1=0.9, beta2=0.999, eps1=1e-30, + eps2=1e-3, cliping_threshold=1, non_constant_decay=True, + enable_factorization=True, ams_grad=True, weight_decay=0): + + enable_momentum = beta1 != 0 + if non_constant_decay: + ams_grad = False + + defaults = dict(lr=lr, beta1=beta1, beta2=beta2, eps1=eps1, + eps2=eps2, cliping_threshold=cliping_threshold, + weight_decay=weight_decay, ams_grad=ams_grad, + enable_factorization=enable_factorization, + enable_momentum=enable_momentum, + non_constant_decay=non_constant_decay) + + super(AdaFactor, self).__init__(params, defaults) + + def __setstate__(self, state): + super(AdaFactor, self).__setstate__(state) + + def _experimental_reshape(self, shape): + temp_shape = shape[2:] + if len(temp_shape) == 1: + new_shape = (shape[0], shape[1] * shape[2]) + else: + tmp_div = len(temp_shape) // 2 + len(temp_shape) % 2 + new_shape = (shape[0] * functools.reduce(operator.mul, + temp_shape[tmp_div:], 1), + shape[1] * functools.reduce(operator.mul, + temp_shape[:tmp_div], 1)) + return new_shape, copy(shape) + + def _check_shape(self, shape): + ''' + output1 - True - algorithm for matrix, False - vector; + output2 - need reshape + ''' + if len(shape) > 2: + return True, True + elif len(shape) == 2: + return True, False + elif len(shape) == 2 and (shape[0] == 1 or shape[1] == 1): + return False, False + else: + return False, False + + def _rms(self, x): + return sqrt(torch.mean(x.pow(2))) + + def step(self, closure=None): + loss = None + if closure is not None: + loss = closure() + for group in self.param_groups: + for p in group['params']: + if p.grad is None: + continue + grad = p.grad.data + + if grad.is_sparse: + raise RuntimeError('Adam does not support sparse \ + gradients, use SparseAdam instead') + + is_matrix, is_need_reshape = self._check_shape(grad.size()) + new_shape = p.data.size() + if is_need_reshape and group['enable_factorization']: + new_shape, old_shape = \ + self._experimental_reshape(p.data.size()) + grad = grad.view(new_shape) + + state = self.state[p] + if len(state) == 0: + state['step'] = 0 + if group['enable_momentum']: + state['exp_avg'] = torch.zeros(new_shape, + dtype=torch.float32, + device=p.grad.device) + + if is_matrix and group['enable_factorization']: + state['exp_avg_sq_R'] = \ + torch.zeros((1, new_shape[1]), + dtype=torch.float32, + device=p.grad.device) + state['exp_avg_sq_C'] = \ + torch.zeros((new_shape[0], 1), + dtype=torch.float32, + device=p.grad.device) + else: + state['exp_avg_sq'] = torch.zeros(new_shape, + dtype=torch.float32, + device=p.grad.device) + if group['ams_grad']: + state['exp_avg_sq_hat'] = \ + torch.zeros(new_shape, dtype=torch.float32, + device=p.grad.device) + + if group['enable_momentum']: + exp_avg = state['exp_avg'] + + if is_matrix and group['enable_factorization']: + exp_avg_sq_r = state['exp_avg_sq_R'] + exp_avg_sq_c = state['exp_avg_sq_C'] + else: + exp_avg_sq = state['exp_avg_sq'] + + if group['ams_grad']: + exp_avg_sq_hat = state['exp_avg_sq_hat'] + + state['step'] += 1 + lr_t = group['lr'] + lr_t *= max(group['eps2'], self._rms(p.data)) + + if group['enable_momentum']: + if group['non_constant_decay']: + beta1_t = group['beta1'] * \ + (1 - group['beta1'] ** (state['step'] - 1)) \ + / (1 - group['beta1'] ** state['step']) + else: + beta1_t = group['beta1'] + exp_avg.mul_(beta1_t).add_(1 - beta1_t, grad) + + if group['non_constant_decay']: + beta2_t = group['beta2'] * \ + (1 - group['beta2'] ** (state['step'] - 1)) / \ + (1 - group['beta2'] ** state['step']) + else: + beta2_t = group['beta2'] + + if is_matrix and group['enable_factorization']: + exp_avg_sq_r.mul_(beta2_t). \ + add_(1 - beta2_t, torch.sum(torch.mul(grad, grad). + add_(group['eps1']), + dim=0, keepdim=True)) + exp_avg_sq_c.mul_(beta2_t). \ + add_(1 - beta2_t, torch.sum(torch.mul(grad, grad). + add_(group['eps1']), + dim=1, keepdim=True)) + v = torch.mul(exp_avg_sq_c, + exp_avg_sq_r).div_(torch.sum(exp_avg_sq_r)) + else: + exp_avg_sq.mul_(beta2_t). \ + addcmul_(1 - beta2_t, grad, grad). \ + add_((1 - beta2_t) * group['eps1']) + v = exp_avg_sq + g = grad + if group['enable_momentum']: + g = torch.div(exp_avg, 1 - beta1_t ** state['step']) + if group['ams_grad']: + torch.max(exp_avg_sq_hat, v, out=exp_avg_sq_hat) + v = exp_avg_sq_hat + u = torch.div(g, (torch.div(v, 1 - beta2_t ** + state['step'])).sqrt().add_(group['eps1'])) + else: + u = torch.div(g, v.sqrt()) + u.div_(max(1, self._rms(u) / group['cliping_threshold'])) + p.data.add_(-lr_t * (u.view(old_shape) if is_need_reshape and + group['enable_factorization'] else u)) + if group['weight_decay'] != 0: + p.data.add_(-group['weight_decay'] * lr_t, p.data) + return loss diff --git a/BertToSimple/BiGRU/callback/optimization/adamw.py b/BertToSimple/BiGRU/callback/optimization/adamw.py new file mode 100644 index 0000000..be68851 --- /dev/null +++ b/BertToSimple/BiGRU/callback/optimization/adamw.py @@ -0,0 +1,90 @@ +import torch +import math +from torch.optim.optimizer import Optimizer + +class AdamW(Optimizer): + """ Implements Adam algorithm with weight decay fix. + + Parameters: + lr (float): learning rate. Default 1e-3. + betas (tuple of 2 floats): Adams beta parameters (b1, b2). Default: (0.9, 0.999) + eps (float): Adams epsilon. Default: 1e-6 + weight_decay (float): Weight decay. Default: 0.0 + correct_bias (bool): can be set to False to avoid correcting bias in Adam (e.g. like in Bert TF repository). Default True. + Example: + >>> model = LSTM() + >>> optimizer = AdamW(model.parameters(), lr=1e-3, weight_decay=1e-5) + """ + def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-6, weight_decay=0.0, correct_bias=True): + if lr < 0.0: + raise ValueError("Invalid learning rate: {} - should be >= 0.0".format(lr)) + if not 0.0 <= betas[0] < 1.0: + raise ValueError("Invalid beta parameter: {} - should be in [0.0, 1.0[".format(betas[0])) + if not 0.0 <= betas[1] < 1.0: + raise ValueError("Invalid beta parameter: {} - should be in [0.0, 1.0[".format(betas[1])) + if not 0.0 <= eps: + raise ValueError("Invalid epsilon value: {} - should be >= 0.0".format(eps)) + defaults = dict(lr=lr, betas=betas, eps=eps, weight_decay=weight_decay, + correct_bias=correct_bias) + super(AdamW, self).__init__(params, defaults) + + def step(self, closure=None): + """Performs a single optimization step. + + Arguments: + closure (callable, optional): A closure that reevaluates the model + and returns the loss. + """ + loss = None + if closure is not None: + loss = closure() + + for group in self.param_groups: + for p in group['params']: + if p.grad is None: + continue + grad = p.grad.data + if grad.is_sparse: + raise RuntimeError('Adam does not support sparse gradients, please consider SparseAdam instead') + + state = self.state[p] + + # State initialization + if len(state) == 0: + state['step'] = 0 + # Exponential moving average of gradient values + state['exp_avg'] = torch.zeros_like(p.data) + # Exponential moving average of squared gradient values + state['exp_avg_sq'] = torch.zeros_like(p.data) + + exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq'] + beta1, beta2 = group['betas'] + + state['step'] += 1 + + # Decay the first and second moment running average coefficient + # In-place operations to update the averages at the same time + exp_avg.mul_(beta1).add_(1.0 - beta1, grad) + exp_avg_sq.mul_(beta2).addcmul_(1.0 - beta2, grad, grad) + denom = exp_avg_sq.sqrt().add_(group['eps']) + + step_size = group['lr'] + if group['correct_bias']: # No bias correction for Bert + bias_correction1 = 1.0 - beta1 ** state['step'] + bias_correction2 = 1.0 - beta2 ** state['step'] + step_size = step_size * math.sqrt(bias_correction2) / bias_correction1 + + p.data.addcdiv_(-step_size, exp_avg, denom) + + # Just adding the square of the weights to the loss function is *not* + # the correct way of using L2 regularization/weight decay with Adam, + # since that will interact with the m and v parameters in strange ways. + # + # Instead we want to decay the weights in a manner that doesn't interact + # with the m/v parameters. This is equivalent to adding the square + # of the weights to the loss with plain (non-momentum) SGD. + # Add weight decay at the end (fixed version) + if group['weight_decay'] > 0.0: + p.data.add_(-group['lr'] * group['weight_decay'], p.data) + + return loss diff --git a/BertToSimple/BiGRU/callback/optimization/lamb.py b/BertToSimple/BiGRU/callback/optimization/lamb.py new file mode 100644 index 0000000..f8c4cd2 --- /dev/null +++ b/BertToSimple/BiGRU/callback/optimization/lamb.py @@ -0,0 +1,105 @@ +import torch +from torch.optim.optimizer import Optimizer + + +class Lamb(Optimizer): + r"""Implements Lamb algorithm. + It has been proposed in `Large Batch Optimization for Deep Learning: Training BERT in 76 minutes`_. + Arguments: + params (iterable): iterable of parameters to optimize or dicts defining + parameter groups + lr (float, optional): learning rate (default: 1e-3) + betas (Tuple[float, float], optional): coefficients used for computing + running averages of gradient and its square (default: (0.9, 0.999)) + eps (float, optional): term added to the denominator to improve + numerical stability (default: 1e-8) + weight_decay (float, optional): weight decay (L2 penalty) (default: 0) + adam (bool, optional): always use trust ratio = 1, which turns this into + Adam. Useful for comparison purposes. + .. _Large Batch Optimization for Deep Learning: Training BERT in 76 minutes: + https://arxiv.org/abs/1904.00962 + Example: + >>> model = ResNet() + >>> optimizer = Lamb(model.parameters(), lr=1e-2, weight_decay=1e-5) + """ + + def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-6, + weight_decay=0, adam=False): + if not 0.0 <= lr: + raise ValueError("Invalid learning rate: {}".format(lr)) + if not 0.0 <= eps: + raise ValueError("Invalid epsilon value: {}".format(eps)) + if not 0.0 <= betas[0] < 1.0: + raise ValueError("Invalid beta parameter at index 0: {}".format(betas[0])) + if not 0.0 <= betas[1] < 1.0: + raise ValueError("Invalid beta parameter at index 1: {}".format(betas[1])) + defaults = dict(lr=lr, betas=betas, eps=eps, + weight_decay=weight_decay) + self.adam = adam + super(Lamb, self).__init__(params, defaults) + + def step(self, closure=None): + """Performs a single optimization step. + Arguments: + closure (callable, optional): A closure that reevaluates the model + and returns the loss. + """ + loss = None + if closure is not None: + loss = closure() + + for group in self.param_groups: + for p in group['params']: + if p.grad is None: + continue + grad = p.grad.data + if grad.is_sparse: + raise RuntimeError('Lamb does not support sparse gradients, consider SparseAdam instad.') + + state = self.state[p] + + # State initialization + if len(state) == 0: + state['step'] = 0 + # Exponential moving average of gradient values + state['exp_avg'] = torch.zeros_like(p.data) + # Exponential moving average of squared gradient values + state['exp_avg_sq'] = torch.zeros_like(p.data) + + exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq'] + beta1, beta2 = group['betas'] + + state['step'] += 1 + + # Decay the first and second moment running average coefficient + # m_t + exp_avg.mul_(beta1).add_(1 - beta1, grad) + # v_t + exp_avg_sq.mul_(beta2).addcmul_(1 - beta2, grad, grad) + + # Paper v3 does not use debiasing. + # bias_correction1 = 1 - beta1 ** state['step'] + # bias_correction2 = 1 - beta2 ** state['step'] + # Apply bias to lr to avoid broadcast. + step_size = group['lr'] # * math.sqrt(bias_correction2) / bias_correction1 + + weight_norm = p.data.pow(2).sum().sqrt().clamp(0, 10) + + adam_step = exp_avg / exp_avg_sq.sqrt().add(group['eps']) + if group['weight_decay'] != 0: + adam_step.add_(group['weight_decay'], p.data) + + adam_norm = adam_step.pow(2).sum().sqrt() + if weight_norm == 0 or adam_norm == 0: + trust_ratio = 1 + else: + trust_ratio = weight_norm / adam_norm + state['weight_norm'] = weight_norm + state['adam_norm'] = adam_norm + state['trust_ratio'] = trust_ratio + if self.adam: + trust_ratio = 1 + + p.data.add_(-step_size * trust_ratio, adam_step) + + return loss \ No newline at end of file diff --git a/BertToSimple/BiGRU/callback/optimization/lars.py b/BertToSimple/BiGRU/callback/optimization/lars.py new file mode 100644 index 0000000..608e2e6 --- /dev/null +++ b/BertToSimple/BiGRU/callback/optimization/lars.py @@ -0,0 +1,91 @@ +import torch +from torch.optim.optimizer import Optimizer + +class Lars(Optimizer): + r"""Implements the LARS optimizer from https://arxiv.org/pdf/1708.03888.pdf + + Args: + params (iterable): iterable of parameters to optimize or dicts defining + parameter groups + lr (float): learning rate + momentum (float, optional): momentum factor (default: 0) + weight_decay (float, optional): weight decay (L2 penalty) (default: 0) + dampening (float, optional): dampening for momentum (default: 0) + nesterov (bool, optional): enables Nesterov momentum (default: False) + scale_clip (tuple, optional): the lower and upper bounds for the weight norm in local LR of LARS + Example: + >>> model = ResNet() + >>> optimizer = Lars(model.parameters(), lr=1e-2, weight_decay=1e-5) + """ + + def __init__(self, params, lr, momentum=0, dampening=0, + weight_decay=0, nesterov=False, scale_clip=None): + if lr < 0.0: + raise ValueError("Invalid learning rate: {}".format(lr)) + if momentum < 0.0: + raise ValueError("Invalid momentum value: {}".format(momentum)) + if weight_decay < 0.0: + raise ValueError("Invalid weight_decay value: {}".format(weight_decay)) + + defaults = dict(lr=lr, momentum=momentum, dampening=dampening, + weight_decay=weight_decay, nesterov=nesterov) + if nesterov and (momentum <= 0 or dampening != 0): + raise ValueError("Nesterov momentum requires a momentum and zero dampening") + super(Lars, self).__init__(params, defaults) + # LARS arguments + self.scale_clip = scale_clip + if self.scale_clip is None: + self.scale_clip = (0, 10) + + def __setstate__(self, state): + super(Lars, self).__setstate__(state) + for group in self.param_groups: + group.setdefault('nesterov', False) + + def step(self, closure=None): + """Performs a single optimization step. + + Arguments: + closure (callable, optional): A closure that reevaluates the model + and returns the loss. + """ + loss = None + if closure is not None: + loss = closure() + + for group in self.param_groups: + weight_decay = group['weight_decay'] + momentum = group['momentum'] + dampening = group['dampening'] + nesterov = group['nesterov'] + + for p in group['params']: + if p.grad is None: + continue + d_p = p.grad.data + if weight_decay != 0: + d_p.add_(weight_decay, p.data) + if momentum != 0: + param_state = self.state[p] + if 'momentum_buffer' not in param_state: + buf = param_state['momentum_buffer'] = torch.clone(d_p).detach() + else: + buf = param_state['momentum_buffer'] + buf.mul_(momentum).add_(1 - dampening, d_p) + if nesterov: + d_p = d_p.add(momentum, buf) + else: + d_p = buf + + # LARS + p_norm = p.data.pow(2).sum().sqrt() + update_norm = d_p.pow(2).sum().sqrt() + # Compute the local LR + if p_norm == 0 or update_norm == 0: + local_lr = 1 + else: + local_lr = p_norm / update_norm + + p.data.add_(-group['lr'] * local_lr, d_p) + + return loss \ No newline at end of file diff --git a/BertToSimple/BiGRU/callback/optimization/lookahead.py b/BertToSimple/BiGRU/callback/optimization/lookahead.py new file mode 100644 index 0000000..4244adb --- /dev/null +++ b/BertToSimple/BiGRU/callback/optimization/lookahead.py @@ -0,0 +1,105 @@ +import torch +from torch.optim import Optimizer +from collections import defaultdict + +class Lookahead(Optimizer): + ''' + PyTorch implementation of the lookahead wrapper. + Lookahead Optimizer: https://arxiv.org/abs/1907.08610 + + We found that evaluation performance is typically better using the slow weights. + This can be done in PyTorch with something like this in your eval loop: + if args.lookahead: + optimizer._backup_and_load_cache() + val_loss = eval_func(model) + optimizer._clear_and_load_backup() + ''' + def __init__(self, optimizer,alpha=0.5, k=6,pullback_momentum="none"): + ''' + :param optimizer:inner optimizer + :param k (int): number of lookahead steps + :param alpha(float): linear interpolation factor. 1.0 recovers the inner optimizer. + :param pullback_momentum (str): change to inner optimizer momentum on interpolation update + ''' + if not 0.0 <= alpha <= 1.0: + raise ValueError(f'Invalid slow update rate: {alpha}') + if not 1 <= k: + raise ValueError(f'Invalid lookahead steps: {k}') + self.optimizer = optimizer + self.param_groups = self.optimizer.param_groups + self.alpha = alpha + self.k = k + self.step_counter = 0 + assert pullback_momentum in ["reset", "pullback", "none"] + self.pullback_momentum = pullback_momentum + self.state = defaultdict(dict) + + # Cache the current optimizer parameters + for group in self.optimizer.param_groups: + for p in group['params']: + param_state = self.state[p] + param_state['cached_params'] = torch.zeros_like(p.data) + param_state['cached_params'].copy_(p.data) + + def __getstate__(self): + return { + 'state': self.state, + 'optimizer': self.optimizer, + 'alpha': self.alpha, + 'step_counter': self.step_counter, + 'k':self.k, + 'pullback_momentum': self.pullback_momentum + } + + def zero_grad(self): + self.optimizer.zero_grad() + + def state_dict(self): + return self.optimizer.state_dict() + + def load_state_dict(self, state_dict): + self.optimizer.load_state_dict(state_dict) + + def _backup_and_load_cache(self): + """Useful for performing evaluation on the slow weights (which typically generalize better) + """ + for group in self.optimizer.param_groups: + for p in group['params']: + param_state = self.state[p] + param_state['backup_params'] = torch.zeros_like(p.data) + param_state['backup_params'].copy_(p.data) + p.data.copy_(param_state['cached_params']) + + def _clear_and_load_backup(self): + for group in self.optimizer.param_groups: + for p in group['params']: + param_state = self.state[p] + p.data.copy_(param_state['backup_params']) + del param_state['backup_params'] + + def step(self, closure=None): + """Performs a single Lookahead optimization step. + Arguments: + closure (callable, optional): A closure that reevaluates the model + and returns the loss. + """ + loss = self.optimizer.step(closure) + self.step_counter += 1 + + if self.step_counter >= self.k: + self.step_counter = 0 + # Lookahead and cache the current optimizer parameters + for group in self.optimizer.param_groups: + for p in group['params']: + param_state = self.state[p] + p.data.mul_(self.alpha).add_(1.0 - self.alpha, param_state['cached_params']) # crucial line + param_state['cached_params'].copy_(p.data) + if self.pullback_momentum == "pullback": + internal_momentum = self.optimizer.state[p]["momentum_buffer"] + self.optimizer.state[p]["momentum_buffer"] = internal_momentum.mul_(self.alpha).add_( + 1.0 - self.alpha, param_state["cached_mom"]) + param_state["cached_mom"] = self.optimizer.state[p]["momentum_buffer"] + elif self.pullback_momentum == "reset": + self.optimizer.state[p]["momentum_buffer"] = torch.zeros_like(p.data) + + return loss diff --git a/BertToSimple/BiGRU/callback/optimization/nadam.py b/BertToSimple/BiGRU/callback/optimization/nadam.py new file mode 100644 index 0000000..c353063 --- /dev/null +++ b/BertToSimple/BiGRU/callback/optimization/nadam.py @@ -0,0 +1,91 @@ +import torch +import math +from torch.optim.optimizer import Optimizer + +class Nadam(Optimizer): + """Implements Nadam algorithm (a variant of Adam based on Nesterov momentum). + + It has been proposed in `Incorporating Nesterov Momentum into Adam`__. + + Arguments: + params (iterable): iterable of parameters to optimize or dicts defining + parameter groups + lr (float, optional): learning rate (default: 2e-3) + betas (Tuple[float, float], optional): coefficients used for computing + running averages of gradient and its square + eps (float, optional): term added to the denominator to improve + numerical stability (default: 1e-8) + weight_decay (float, optional): weight decay (L2 penalty) (default: 0) + schedule_decay (float, optional): momentum schedule decay (default: 4e-3) + + __ http://cs229.stanford.edu/proj2015/054_report.pdf + __ http://www.cs.toronto.edu/~fritz/absps/momentum.pdf + + Originally taken from: https://github.com/pytorch/pytorch/pull/1408 + NOTE: Has potential issues but does work well on some problems. + Example: + >>> model = LSTM() + >>> optimizer = Nadam(model.parameters()) + """ + + def __init__(self, params, lr=2e-3, betas=(0.9, 0.999), eps=1e-8, + weight_decay=0, schedule_decay=4e-3): + defaults = dict(lr=lr, betas=betas, eps=eps, + weight_decay=weight_decay, schedule_decay=schedule_decay) + super(Nadam, self).__init__(params, defaults) + + def step(self, closure=None): + """Performs a single optimization step. + + Arguments: + closure (callable, optional): A closure that reevaluates the model + and returns the loss. + """ + loss = None + if closure is not None: + loss = closure() + + for group in self.param_groups: + for p in group['params']: + if p.grad is None: + continue + grad = p.grad.data + state = self.state[p] + + # State initialization + if len(state) == 0: + state['step'] = 0 + state['m_schedule'] = 1. + state['exp_avg'] = grad.new().resize_as_(grad).zero_() + state['exp_avg_sq'] = grad.new().resize_as_(grad).zero_() + + # Warming momentum schedule + m_schedule = state['m_schedule'] + schedule_decay = group['schedule_decay'] + exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq'] + beta1, beta2 = group['betas'] + eps = group['eps'] + state['step'] += 1 + t = state['step'] + + if group['weight_decay'] != 0: + grad = grad.add(group['weight_decay'], p.data) + + momentum_cache_t = beta1 * \ + (1. - 0.5 * (0.96 ** (t * schedule_decay))) + momentum_cache_t_1 = beta1 * \ + (1. - 0.5 * (0.96 ** ((t + 1) * schedule_decay))) + m_schedule_new = m_schedule * momentum_cache_t + m_schedule_next = m_schedule * momentum_cache_t * momentum_cache_t_1 + state['m_schedule'] = m_schedule_new + + # Decay the first and second moment running average coefficient + exp_avg.mul_(beta1).add_(1. - beta1, grad) + exp_avg_sq.mul_(beta2).addcmul_(1. - beta2, grad, grad) + exp_avg_sq_prime = exp_avg_sq / (1. - beta2 ** t) + denom = exp_avg_sq_prime.sqrt_().add_(eps) + + p.data.addcdiv_(-group['lr'] * (1. - momentum_cache_t) / (1. - m_schedule_new), grad, denom) + p.data.addcdiv_(-group['lr'] * momentum_cache_t_1 / (1. - m_schedule_next), exp_avg, denom) + + return loss \ No newline at end of file diff --git a/BertToSimple/BiGRU/callback/optimization/novograd.py b/BertToSimple/BiGRU/callback/optimization/novograd.py new file mode 100644 index 0000000..ab22651 --- /dev/null +++ b/BertToSimple/BiGRU/callback/optimization/novograd.py @@ -0,0 +1,75 @@ +import torch +import math +from torch.optim.optimizer import Optimizer + + +class NovoGrad(Optimizer): + """Implements NovoGrad algorithm. + Arguments: + params (iterable): iterable of parameters to optimize or dicts defining + parameter groups + lr (float, optional): learning rate (default: 1e-2) + betas (Tuple[float, float], optional): coefficients used for computing + running averages of gradient and its square (default: (0.95, 0.98)) + eps (float, optional): term added to the denominator to improve + numerical stability (default: 1e-8) + weight_decay (float, optional): weight decay (L2 penalty) (default: 0) + Example: + >>> model = ResNet() + >>> optimizer = NovoGrad(model.parameters(), lr=1e-2, weight_decay=1e-5) + """ + + def __init__(self, params, lr=0.01, betas=(0.95, 0.98), eps=1e-8, + weight_decay=0, grad_averaging=False): + if lr < 0.0: + raise ValueError("Invalid learning rate: {}".format(lr)) + if not 0.0 <= betas[0] < 1.0: + raise ValueError("Invalid beta parameter at index 0: {}".format(betas[0])) + if not 0.0 <= betas[1] < 1.0: + raise ValueError("Invalid beta parameter at index 1: {}".format(betas[1])) + defaults = dict(lr=lr, betas=betas, eps=eps, weight_decay=weight_decay, grad_averaging=grad_averaging) + super().__init__(params, defaults) + + def step(self, closure=None): + loss = None + if closure is not None: + loss = closure() + for group in self.param_groups: + for p in group['params']: + if p.grad is None: + continue + grad = p.grad.data + if grad.is_sparse: + raise RuntimeError('NovoGrad does not support sparse gradients') + state = self.state[p] + g_2 = torch.sum(grad ** 2) + if len(state) == 0: + state['step'] = 0 + state['moments'] = grad.div(g_2.sqrt() + group['eps']) + \ + group['weight_decay'] * p.data + state['grads_ema'] = g_2 + moments = state['moments'] + grads_ema = state['grads_ema'] + beta1, beta2 = group['betas'] + state['step'] += 1 + grads_ema.mul_(beta2).add_(1 - beta2, g_2) + + denom = grads_ema.sqrt().add_(group['eps']) + grad.div_(denom) + # weight decay + if group['weight_decay'] != 0: + decayed_weights = torch.mul(p.data, group['weight_decay']) + grad.add_(decayed_weights) + + # Momentum --> SAG + if group['grad_averaging']: + grad.mul_(1.0 - beta1) + + moments.mul_(beta1).add_(grad) # velocity + + bias_correction1 = 1 - beta1 ** state['step'] + bias_correction2 = 1 - beta2 ** state['step'] + step_size = group['lr'] * math.sqrt(bias_correction2) / bias_correction1 + p.data.add_(-step_size, moments) + + return loss diff --git a/BertToSimple/BiGRU/callback/optimization/planradam.py b/BertToSimple/BiGRU/callback/optimization/planradam.py new file mode 100644 index 0000000..2b65e5c --- /dev/null +++ b/BertToSimple/BiGRU/callback/optimization/planradam.py @@ -0,0 +1,66 @@ +import torch +import math +from torch.optim.optimizer import Optimizer +class PlainRAdam(Optimizer): + + def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-8, weight_decay=0): + defaults = dict(lr=lr, betas=betas, eps=eps, weight_decay=weight_decay) + + super(PlainRAdam, self).__init__(params, defaults) + + def __setstate__(self, state): + super(PlainRAdam, self).__setstate__(state) + + def step(self, closure=None): + + loss = None + if closure is not None: + loss = closure() + + for group in self.param_groups: + + for p in group['params']: + if p.grad is None: + continue + grad = p.grad.data.float() + if grad.is_sparse: + raise RuntimeError('RAdam does not support sparse gradients') + + p_data_fp32 = p.data.float() + + state = self.state[p] + + if len(state) == 0: + state['step'] = 0 + state['exp_avg'] = torch.zeros_like(p_data_fp32) + state['exp_avg_sq'] = torch.zeros_like(p_data_fp32) + else: + state['exp_avg'] = state['exp_avg'].type_as(p_data_fp32) + state['exp_avg_sq'] = state['exp_avg_sq'].type_as(p_data_fp32) + + exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq'] + beta1, beta2 = group['betas'] + + exp_avg_sq.mul_(beta2).addcmul_(1 - beta2, grad, grad) + exp_avg.mul_(beta1).add_(1 - beta1, grad) + + state['step'] += 1 + beta2_t = beta2 ** state['step'] + N_sma_max = 2 / (1 - beta2) - 1 + N_sma = N_sma_max - 2 * state['step'] * beta2_t / (1 - beta2_t) + + if group['weight_decay'] != 0: + p_data_fp32.add_(-group['weight_decay'] * group['lr'], p_data_fp32) + + # more conservative since it's an approximated value + if N_sma >= 5: + step_size = group['lr'] * math.sqrt((1 - beta2_t) * (N_sma - 4) / (N_sma_max - 4) * (N_sma - 2) / N_sma * N_sma_max / (N_sma_max - 2)) / (1 - beta1 ** state['step']) + denom = exp_avg_sq.sqrt().add_(group['eps']) + p_data_fp32.addcdiv_(-step_size, exp_avg, denom) + else: + step_size = group['lr'] / (1 - beta1 ** state['step']) + p_data_fp32.add_(-step_size, exp_avg) + + p.data.copy_(p_data_fp32) + + return loss \ No newline at end of file diff --git a/BertToSimple/BiGRU/callback/optimization/radam.py b/BertToSimple/BiGRU/callback/optimization/radam.py new file mode 100644 index 0000000..0ee94fd --- /dev/null +++ b/BertToSimple/BiGRU/callback/optimization/radam.py @@ -0,0 +1,88 @@ +import torch +import math +from torch.optim.optimizer import Optimizer +class RAdam(Optimizer): + """Implements the RAdam optimizer from https://arxiv.org/pdf/1908.03265.pdf + Args: + params (iterable): iterable of parameters to optimize or dicts defining parameter groups + lr (float, optional): learning rate + betas (Tuple[float, float], optional): coefficients used for computing running averages of gradient and its square (default: (0.9, 0.999)) + eps (float, optional): term added to the denominator to improve numerical stability (default: 1e-8) + weight_decay (float, optional): weight decay (L2 penalty) (default: 0) + Example: + >>> model = ResNet() + >>> optimizer = RAdam(model.parameters(), lr=0.001) + """ + + def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-8, weight_decay=0): + defaults = dict(lr=lr, betas=betas, eps=eps, weight_decay=weight_decay) + self.buffer = [[None, None, None] for ind in range(10)] + super(RAdam, self).__init__(params, defaults) + + def __setstate__(self, state): + super(RAdam, self).__setstate__(state) + + def step(self, closure=None): + + loss = None + if closure is not None: + loss = closure() + + for group in self.param_groups: + + for p in group['params']: + if p.grad is None: + continue + grad = p.grad.data.float() + if grad.is_sparse: + raise RuntimeError('RAdam does not support sparse gradients') + + p_data_fp32 = p.data.float() + + state = self.state[p] + + if len(state) == 0: + state['step'] = 0 + state['exp_avg'] = torch.zeros_like(p_data_fp32) + state['exp_avg_sq'] = torch.zeros_like(p_data_fp32) + else: + state['exp_avg'] = state['exp_avg'].type_as(p_data_fp32) + state['exp_avg_sq'] = state['exp_avg_sq'].type_as(p_data_fp32) + + exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq'] + beta1, beta2 = group['betas'] + + exp_avg_sq.mul_(beta2).addcmul_(1 - beta2, grad, grad) + exp_avg.mul_(beta1).add_(1 - beta1, grad) + + state['step'] += 1 + buffered = self.buffer[int(state['step'] % 10)] + if state['step'] == buffered[0]: + N_sma, step_size = buffered[1], buffered[2] + else: + buffered[0] = state['step'] + beta2_t = beta2 ** state['step'] + N_sma_max = 2 / (1 - beta2) - 1 + N_sma = N_sma_max - 2 * state['step'] * beta2_t / (1 - beta2_t) + buffered[1] = N_sma + + # more conservative since it's an approximated value + if N_sma >= 5: + step_size = math.sqrt((1 - beta2_t) * (N_sma - 4) / (N_sma_max - 4) * (N_sma - 2) / N_sma * N_sma_max / (N_sma_max - 2)) / (1 - beta1 ** state['step']) + else: + step_size = 1.0 / (1 - beta1 ** state['step']) + buffered[2] = step_size + + if group['weight_decay'] != 0: + p_data_fp32.add_(-group['weight_decay'] * group['lr'], p_data_fp32) + + # more conservative since it's an approximated value + if N_sma >= 5: + denom = exp_avg_sq.sqrt().add_(group['eps']) + p_data_fp32.addcdiv_(-step_size * group['lr'], exp_avg, denom) + else: + p_data_fp32.add_(-step_size * group['lr'], exp_avg) + + p.data.copy_(p_data_fp32) + + return loss \ No newline at end of file diff --git a/BertToSimple/BiGRU/callback/optimization/ralamb.py b/BertToSimple/BiGRU/callback/optimization/ralamb.py new file mode 100644 index 0000000..79a2936 --- /dev/null +++ b/BertToSimple/BiGRU/callback/optimization/ralamb.py @@ -0,0 +1,104 @@ +import math +import torch +from torch.optim.optimizer import Optimizer + +class Ralamb(Optimizer): + ''' + RAdam + LARS + Example: + >>> model = ResNet() + >>> optimizer = Ralamb(model.parameters(), lr=0.001) + ''' + def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-8, weight_decay=0): + defaults = dict(lr=lr, betas=betas, eps=eps, weight_decay=weight_decay) + self.buffer = [[None, None, None] for ind in range(10)] + super(Ralamb, self).__init__(params, defaults) + + def __setstate__(self, state): + super(Ralamb, self).__setstate__(state) + + def step(self, closure=None): + + loss = None + if closure is not None: + loss = closure() + + for group in self.param_groups: + + for p in group['params']: + if p.grad is None: + continue + grad = p.grad.data.float() + if grad.is_sparse: + raise RuntimeError('Ralamb does not support sparse gradients') + + p_data_fp32 = p.data.float() + + state = self.state[p] + + if len(state) == 0: + state['step'] = 0 + state['exp_avg'] = torch.zeros_like(p_data_fp32) + state['exp_avg_sq'] = torch.zeros_like(p_data_fp32) + else: + state['exp_avg'] = state['exp_avg'].type_as(p_data_fp32) + state['exp_avg_sq'] = state['exp_avg_sq'].type_as(p_data_fp32) + + exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq'] + beta1, beta2 = group['betas'] + + # Decay the first and second moment running average coefficient + # m_t + exp_avg.mul_(beta1).add_(1 - beta1, grad) + # v_t + exp_avg_sq.mul_(beta2).addcmul_(1 - beta2, grad, grad) + + state['step'] += 1 + buffered = self.buffer[int(state['step'] % 10)] + + if state['step'] == buffered[0]: + N_sma, radam_step_size = buffered[1], buffered[2] + else: + buffered[0] = state['step'] + beta2_t = beta2 ** state['step'] + N_sma_max = 2 / (1 - beta2) - 1 + N_sma = N_sma_max - 2 * state['step'] * beta2_t / (1 - beta2_t) + buffered[1] = N_sma + + # more conservative since it's an approximated value + if N_sma >= 5: + radam_step_size = math.sqrt((1 - beta2_t) * (N_sma - 4) / (N_sma_max - 4) * (N_sma - 2) / N_sma * N_sma_max / (N_sma_max - 2)) / (1 - beta1 ** state['step']) + else: + radam_step_size = 1.0 / (1 - beta1 ** state['step']) + buffered[2] = radam_step_size + + if group['weight_decay'] != 0: + p_data_fp32.add_(-group['weight_decay'] * group['lr'], p_data_fp32) + + # more conservative since it's an approximated value + radam_step = p_data_fp32.clone() + if N_sma >= 5: + denom = exp_avg_sq.sqrt().add_(group['eps']) + radam_step.addcdiv_(-radam_step_size * group['lr'], exp_avg, denom) + else: + radam_step.add_(-radam_step_size * group['lr'], exp_avg) + + radam_norm = radam_step.pow(2).sum().sqrt() + weight_norm = p.data.pow(2).sum().sqrt().clamp(0, 10) + if weight_norm == 0 or radam_norm == 0: + trust_ratio = 1 + else: + trust_ratio = weight_norm / radam_norm + + state['weight_norm'] = weight_norm + state['adam_norm'] = radam_norm + state['trust_ratio'] = trust_ratio + + if N_sma >= 5: + p_data_fp32.addcdiv_(-radam_step_size * group['lr'] * trust_ratio, exp_avg, denom) + else: + p_data_fp32.add_(-radam_step_size * group['lr'] * trust_ratio, exp_avg) + + p.data.copy_(p_data_fp32) + + return loss \ No newline at end of file diff --git a/BertToSimple/BiGRU/callback/optimization/ralars.py b/BertToSimple/BiGRU/callback/optimization/ralars.py new file mode 100644 index 0000000..a6a34a9 --- /dev/null +++ b/BertToSimple/BiGRU/callback/optimization/ralars.py @@ -0,0 +1,120 @@ +import math +import torch +from torch.optim.optimizer import Optimizer + + +class RaLars(Optimizer): + """Implements the RAdam optimizer from https://arxiv.org/pdf/1908.03265.pdf + with optional Layer-wise adaptive Scaling from https://arxiv.org/pdf/1708.03888.pdf + + Args: + params (iterable): iterable of parameters to optimize or dicts defining parameter groups + lr (float, optional): learning rate + betas (Tuple[float, float], optional): coefficients used for computing running averages of gradient and its square (default: (0.9, 0.999)) + eps (float, optional): term added to the denominator to improve numerical stability (default: 1e-8) + weight_decay (float, optional): weight decay (L2 penalty) (default: 0) + scale_clip (float, optional): the maximal upper bound for the scale factor of LARS + Example: + >>> model = ResNet() + >>> optimizer = RaLars(model.parameters(), lr=0.001) + """ + + def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-8, weight_decay=0, + scale_clip=None): + if not 0.0 <= lr: + raise ValueError("Invalid learning rate: {}".format(lr)) + if not 0.0 <= eps: + raise ValueError("Invalid epsilon value: {}".format(eps)) + if not 0.0 <= betas[0] < 1.0: + raise ValueError("Invalid beta parameter at index 0: {}".format(betas[0])) + if not 0.0 <= betas[1] < 1.0: + raise ValueError("Invalid beta parameter at index 1: {}".format(betas[1])) + defaults = dict(lr=lr, betas=betas, eps=eps, weight_decay=weight_decay) + super(RaLars, self).__init__(params, defaults) + # LARS arguments + self.scale_clip = scale_clip + if self.scale_clip is None: + self.scale_clip = (0, 10) + + def step(self, closure=None): + """Performs a single optimization step. + Arguments: + closure (callable, optional): A closure that reevaluates the model + and returns the loss. + """ + loss = None + if closure is not None: + loss = closure() + + for group in self.param_groups: + + # Get group-shared variables + beta1, beta2 = group['betas'] + sma_inf = group.get('sma_inf') + # Compute max length of SMA on first step + if not isinstance(sma_inf, float): + group['sma_inf'] = 2 / (1 - beta2) - 1 + sma_inf = group.get('sma_inf') + + for p in group['params']: + if p.grad is None: + continue + grad = p.grad.data + if grad.is_sparse: + raise RuntimeError('RAdam does not support sparse gradients') + + state = self.state[p] + + # State initialization + if len(state) == 0: + state['step'] = 0 + # Exponential moving average of gradient values + state['exp_avg'] = torch.zeros_like(p.data) + # Exponential moving average of squared gradient values + state['exp_avg_sq'] = torch.zeros_like(p.data) + + exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq'] + + state['step'] += 1 + + # Decay the first and second moment running average coefficient + exp_avg.mul_(beta1).add_(1 - beta1, grad) + exp_avg_sq.mul_(beta2).addcmul_(1 - beta2, grad, grad) + + # Bias correction + bias_correction1 = 1 - beta1 ** state['step'] + bias_correction2 = 1 - beta2 ** state['step'] + + # Compute length of SMA + sma_t = sma_inf - 2 * state['step'] * (1 - bias_correction2) / bias_correction2 + + update = torch.zeros_like(p.data) + if sma_t > 4: + #  Variance rectification term + r_t = math.sqrt((sma_t - 4) * (sma_t - 2) * sma_inf / ((sma_inf - 4) * (sma_inf - 2) * sma_t)) + #  Adaptive momentum + update.addcdiv_(r_t, exp_avg / bias_correction1, + (exp_avg_sq / bias_correction2).sqrt().add_(group['eps'])) + else: + # Unadapted momentum + update.add_(exp_avg / bias_correction1) + + # Weight decay + if group['weight_decay'] != 0: + update.add_(group['weight_decay'], p.data) + + # LARS + p_norm = p.data.pow(2).sum().sqrt() + update_norm = update.pow(2).sum().sqrt() + phi_p = p_norm.clamp(*self.scale_clip) + # Compute the local LR + if phi_p == 0 or update_norm == 0: + local_lr = 1 + else: + local_lr = phi_p / update_norm + + state['local_lr'] = local_lr + + p.data.add_(-group['lr'] * local_lr, update) + + return loss diff --git a/BertToSimple/BiGRU/callback/optimization/sgdw.py b/BertToSimple/BiGRU/callback/optimization/sgdw.py new file mode 100644 index 0000000..6396eaf --- /dev/null +++ b/BertToSimple/BiGRU/callback/optimization/sgdw.py @@ -0,0 +1,82 @@ +import torch +from torch.optim.optimizer import Optimizer + +class SGDW(Optimizer): + r"""Implements stochastic gradient descent (optionally with momentum) with + weight decay from the paper `Fixing Weight Decay Regularization in Adam`_. + + Nesterov momentum is based on the formula from + `On the importance of initialization and momentum in deep learning`__. + + Args: + params (iterable): iterable of parameters to optimize or dicts defining + parameter groups + lr (float): learning rate + momentum (float, optional): momentum factor (default: 0) + weight_decay (float, optional): weight decay factor (default: 0) + dampening (float, optional): dampening for momentum (default: 0) + nesterov (bool, optional): enables Nesterov momentum (default: False) + + .. _Fixing Weight Decay Regularization in Adam: + https://arxiv.org/abs/1711.05101 + + Example: + >>> model = LSTM() + >>> optimizer = SGDW(model.parameters(), lr=0.1, momentum=0.9,weight_decay=1e-5) + """ + def __init__(self, params, lr=0.1, momentum=0, dampening=0, + weight_decay=0, nesterov=False): + if lr < 0.0: + raise ValueError(f"Invalid learning rate: {lr}") + if momentum < 0.0: + raise ValueError(f"Invalid momentum value: {momentum}") + if weight_decay < 0.0: + raise ValueError(f"Invalid weight_decay value: {weight_decay}") + + defaults = dict(lr=lr, momentum=momentum, dampening=dampening, + weight_decay=weight_decay, nesterov=nesterov) + if nesterov and (momentum <= 0 or dampening != 0): + raise ValueError("Nesterov momentum requires a momentum and zero dampening") + super(SGDW, self).__init__(params, defaults) + + def __setstate__(self, state): + super(SGDW, self).__setstate__(state) + for group in self.param_groups: + group.setdefault('nesterov', False) + + def step(self, closure=None): + """Performs a single optimization step. + + Arguments: + closure (callable, optional): A closure that reevaluates the model + and returns the loss. + """ + loss = None + if closure is not None: + loss = closure() + + for group in self.param_groups: + weight_decay = group['weight_decay'] + momentum = group['momentum'] + dampening = group['dampening'] + nesterov = group['nesterov'] + for p in group['params']: + if p.grad is None: + continue + d_p = p.grad.data + if momentum != 0: + param_state = self.state[p] + if 'momentum_buffer' not in param_state: + buf = param_state['momentum_buffer'] = torch.zeros_like(p.data) + buf.mul_(momentum).add_(d_p) + else: + buf = param_state['momentum_buffer'] + buf.mul_(momentum).add_(1 - dampening, d_p) + if nesterov: + d_p = d_p.add(momentum, buf) + else: + d_p = buf + if weight_decay != 0: + p.data.add_(-weight_decay, p.data) + p.data.add_(-group['lr'], d_p) + return loss \ No newline at end of file diff --git a/BertToSimple/BiGRU/callback/progressbar.py b/BertToSimple/BiGRU/callback/progressbar.py new file mode 100644 index 0000000..6040933 --- /dev/null +++ b/BertToSimple/BiGRU/callback/progressbar.py @@ -0,0 +1,58 @@ +import time +class ProgressBar(object): + ''' + custom progress bar + Example: + >>> pbar = ProgressBar(n_total=30,desc='training') + >>> step = 2 + >>> pbar(step=step) + ''' + def __init__(self, n_total,width=30,desc = 'Training'): + self.width = width + self.n_total = n_total + self.start_time = time.time() + self.desc = desc + + def __call__(self, step, info={}): + now = time.time() + current = step + 1 + recv_per = current / self.n_total + bar = f'[{self.desc}] {current}/{self.n_total} [' + if recv_per >= 1: + recv_per = 1 + prog_width = int(self.width * recv_per) + if prog_width > 0: + bar += '=' * (prog_width - 1) + if current< self.n_total: + bar += ">" + else: + bar += '=' + bar += '.' * (self.width - prog_width) + bar += ']' + show_bar = f"\r{bar}" + time_per_unit = (now - self.start_time) / current + if current < self.n_total: + eta = time_per_unit * (self.n_total - current) + if eta > 3600: + eta_format = ('%d:%02d:%02d' % + (eta // 3600, (eta % 3600) // 60, eta % 60)) + elif eta > 60: + eta_format = '%d:%02d' % (eta // 60, eta % 60) + else: + eta_format = '%ds' % eta + time_info = f' - ETA: {eta_format}' + else: + if time_per_unit >= 1: + time_info = f' {time_per_unit:.1f}s/step' + elif time_per_unit >= 1e-3: + time_info = f' {time_per_unit * 1e3:.1f}ms/step' + else: + time_info = f' {time_per_unit * 1e6:.1f}us/step' + + show_bar += time_info + if len(info) != 0: + show_info = f'{show_bar} ' + \ + "-".join([f' {key}: {value:.4f} ' for key, value in info.items()]) + print(show_info, end='') + else: + print(show_bar, end='') diff --git a/BertToSimple/BiGRU/callback/trainingmonitor.py b/BertToSimple/BiGRU/callback/trainingmonitor.py new file mode 100644 index 0000000..6aea128 --- /dev/null +++ b/BertToSimple/BiGRU/callback/trainingmonitor.py @@ -0,0 +1,69 @@ +# encoding:utf-8 +import numpy as np +from pathlib import Path +import matplotlib.pyplot as plt +from ..tools.common import load_json +from ..tools.common import save_json +plt.switch_backend('agg') + +class TrainingMonitor(): + def __init__(self, file_dir, arch, add_test=False): + ''' + :param startAt: 重新开始训练的epoch点 + ''' + if isinstance(file_dir, Path): + pass + else: + file_dir = Path(file_dir) + file_dir.mkdir(parents=True, exist_ok=True) + + self.arch = arch + self.file_dir = file_dir + self.H = {} + self.add_test = add_test + self.json_path = file_dir / (arch + "_training_monitor.json") + + def reset(self,start_at): + if start_at > 0: + if self.json_path is not None: + if self.json_path.exists(): + self.H = load_json(self.json_path) + for k in self.H.keys(): + self.H[k] = self.H[k][:start_at] + + def epoch_step(self, logs={}): + for (k, v) in logs.items(): + l = self.H.get(k, []) + # np.float32会报错 + if not isinstance(v, np.float): + v = round(float(v), 4) + l.append(v) + self.H[k] = l + + # 写入文件 + if self.json_path is not None: + save_json(data = self.H,file_path=self.json_path) + + # 保存train图像 + if len(self.H["loss"]) == 1: + self.paths = {key: self.file_dir / (self.arch + f'_{key.upper()}') for key in self.H.keys()} + + if len(self.H["loss"]) > 1: + # 指标变化 + # 曲线 + # 需要成对出现 + keys = [key for key, _ in self.H.items() if '_' not in key] + for key in keys: + N = np.arange(0, len(self.H[key])) + plt.style.use("ggplot") + plt.figure() + plt.plot(N, self.H[key], label=f"train_{key}") + plt.plot(N, self.H[f"valid_{key}"], label=f"valid_{key}") + if self.add_test: + plt.plot(N, self.H[f"test_{key}"], label=f"test_{key}") + plt.legend() + plt.xlabel("Epoch #") + plt.ylabel(key) + plt.title(f"Training {key} [Epoch {len(self.H[key])}]") + plt.savefig(str(self.paths[key])) + plt.close() diff --git a/BertToSimple/BiGRU/convert_albert_tf_checkpoint_to_pytorch.py b/BertToSimple/BiGRU/convert_albert_tf_checkpoint_to_pytorch.py new file mode 100644 index 0000000..1573176 --- /dev/null +++ b/BertToSimple/BiGRU/convert_albert_tf_checkpoint_to_pytorch.py @@ -0,0 +1,76 @@ +# coding=utf-8 +# Copyright 2018 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Convert BERT checkpoint.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import argparse +import torch +from model.modeling_albert import AlbertConfig, AlbertForPreTraining, load_tf_weights_in_albert +# from model.modeling_albert_bright import AlbertConfig, AlbertForPreTraining, load_tf_weights_in_albert # chinese version +import logging +logging.basicConfig(level=logging.INFO) + +def convert_tf_checkpoint_to_pytorch(tf_checkpoint_path, bert_config_file, pytorch_dump_path): + # Initialise PyTorch model + config = AlbertConfig.from_pretrained(bert_config_file) + # print("Building PyTorch model from configuration: {}".format(str(config))) + model = AlbertForPreTraining(config) + # Load weights from tf checkpoint + load_tf_weights_in_albert(model, config, tf_checkpoint_path) + + # Save pytorch-model + print("Save PyTorch model to {}".format(pytorch_dump_path)) + torch.save(model.state_dict(), pytorch_dump_path) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + ## Required parameters + parser.add_argument("--tf_checkpoint_path", + default = None, + type = str, + required = True, + help = "Path to the TensorFlow checkpoint path.") + parser.add_argument("--bert_config_file", + default = None, + type = str, + required = True, + help = "The config json file corresponding to the pre-trained BERT model. \n" + "This specifies the model architecture.") + parser.add_argument("--pytorch_dump_path", + default = None, + type = str, + required = True, + help = "Path to the output PyTorch model.") + args = parser.parse_args() + convert_tf_checkpoint_to_pytorch(args.tf_checkpoint_path,args.bert_config_file, + args.pytorch_dump_path) + +''' +python convert_albert_tf_checkpoint_to_pytorch.py \ + --tf_checkpoint_path=./prev_trained_model/albert_large_zh \ + --bert_config_file=./prev_trained_model/albert_large_zh/config.json \ + --pytorch_dump_path=./prev_trained_model/albert_large_zh/pytorch_model.bin + + +from model.modeling_albert_bright import AlbertConfig, AlbertForPreTraining, load_tf_weights_in_albert +python convert_albert_tf_checkpoint_to_pytorch.py \ + --tf_checkpoint_path=./prev_trained_model/albert_base_bright \ + --bert_config_file=./prev_trained_model/albert_base_bright/config.json \ + --pytorch_dump_path=./prev_trained_model/albert_base_bright/pytorch_model.bin +''' diff --git a/BertToSimple/BiGRU/label_and_pre.csv b/BertToSimple/BiGRU/label_and_pre.csv new file mode 100644 index 0000000..9de9f3a --- /dev/null +++ b/BertToSimple/BiGRU/label_and_pre.csv @@ -0,0 +1,17352 @@ +,label,pre +0,0,0 +1,2,2 +2,0,0 +3,2,2 +4,1,1 +5,2,2 +6,0,2 +7,2,2 +8,2,2 +9,0,0 +10,2,2 +11,0,0 +12,2,2 +13,2,2 +14,2,2 +15,2,2 +16,0,0 +17,2,2 +18,2,2 +19,0,0 +20,2,2 +21,0,0 +22,2,2 +23,2,2 +24,2,2 +25,2,2 +26,2,2 +27,0,0 +28,0,0 +29,2,2 +30,2,2 +31,2,2 +32,0,0 +33,0,0 +34,0,0 +35,2,2 +36,0,0 +37,2,2 +38,2,2 +39,2,2 +40,0,0 +41,2,2 +42,2,2 +43,1,1 +44,1,1 +45,0,0 +46,2,2 +47,2,2 +48,0,0 +49,2,2 +50,2,2 +51,1,1 +52,2,2 +53,2,2 +54,0,0 +55,0,0 +56,2,0 +57,2,2 +58,1,1 +59,2,2 +60,2,2 +61,0,0 +62,2,2 +63,2,2 +64,0,0 +65,2,2 +66,1,1 +67,2,2 +68,2,2 +69,2,2 +70,0,0 +71,2,2 +72,2,2 +73,2,2 +74,2,2 +75,2,2 +76,0,0 +77,1,1 +78,2,2 +79,0,0 +80,2,2 +81,2,2 +82,2,2 +83,2,2 +84,2,2 +85,2,2 +86,1,1 +87,2,2 +88,0,0 +89,2,2 +90,2,2 +91,2,2 +92,0,0 +93,2,2 +94,1,1 +95,0,0 +96,2,2 +97,2,2 +98,2,2 +99,2,0 +100,0,0 +101,2,2 +102,0,0 +103,2,1 +104,2,2 +105,2,2 +106,2,2 +107,2,2 +108,2,2 +109,2,2 +110,2,2 +111,2,2 +112,2,2 +113,2,2 +114,2,2 +115,2,2 +116,2,0 +117,2,2 +118,2,2 +119,0,0 +120,2,2 +121,2,2 +122,2,2 +123,0,0 +124,2,2 +125,2,0 +126,2,2 +127,2,2 +128,0,0 +129,2,2 +130,0,0 +131,2,2 +132,2,2 +133,0,0 +134,2,2 +135,2,2 +136,2,2 +137,2,2 +138,2,2 +139,2,2 +140,2,2 +141,0,0 +142,2,2 +143,2,2 +144,0,0 +145,0,0 +146,2,2 +147,0,0 +148,2,2 +149,0,0 +150,1,1 +151,2,2 +152,2,2 +153,0,0 +154,0,0 +155,2,2 +156,2,2 +157,2,2 +158,1,1 +159,2,2 +160,2,2 +161,0,0 +162,2,2 +163,0,0 +164,2,2 +165,1,1 +166,0,2 +167,2,2 +168,2,2 +169,1,1 +170,2,2 +171,2,2 +172,2,2 +173,0,0 +174,0,0 +175,2,2 +176,2,1 +177,2,2 +178,2,0 +179,2,2 +180,2,2 +181,2,2 +182,2,2 +183,2,2 +184,0,0 +185,2,2 +186,2,2 +187,0,0 +188,0,0 +189,2,2 +190,2,2 +191,2,2 +192,2,2 +193,2,2 +194,2,2 +195,2,2 +196,2,0 +197,2,2 +198,0,0 +199,1,1 +200,2,2 +201,1,1 +202,2,0 +203,0,0 +204,2,2 +205,2,2 +206,0,0 +207,0,0 +208,2,2 +209,2,2 +210,2,2 +211,2,2 +212,0,0 +213,2,2 +214,0,0 +215,0,0 +216,2,2 +217,2,2 +218,0,0 +219,2,2 +220,2,2 +221,2,2 +222,0,0 +223,2,2 +224,2,2 +225,2,2 +226,2,2 +227,2,2 +228,2,2 +229,0,0 +230,2,2 +231,2,2 +232,2,2 +233,2,2 +234,2,2 +235,1,1 +236,2,2 +237,0,0 +238,2,2 +239,2,2 +240,2,2 +241,2,2 +242,2,2 +243,0,0 +244,2,2 +245,2,2 +246,0,0 +247,0,0 +248,0,0 +249,2,2 +250,2,2 +251,2,2 +252,0,0 +253,0,0 +254,2,0 +255,2,0 +256,2,2 +257,2,2 +258,0,0 +259,0,0 +260,2,0 +261,0,0 +262,0,0 +263,2,2 +264,0,0 +265,2,2 +266,2,2 +267,2,2 +268,2,2 +269,2,2 +270,2,2 +271,2,2 +272,0,0 +273,2,2 +274,2,2 +275,0,0 +276,2,2 +277,2,2 +278,0,0 +279,2,2 +280,2,2 +281,0,0 +282,0,0 +283,0,2 +284,2,2 +285,2,2 +286,2,0 +287,2,2 +288,2,2 +289,2,2 +290,1,1 +291,2,2 +292,2,2 +293,2,2 +294,2,2 +295,2,2 +296,2,2 +297,2,2 +298,2,2 +299,2,2 +300,1,2 +301,2,2 +302,0,2 +303,2,2 +304,2,2 +305,2,2 +306,2,2 +307,2,2 +308,2,2 +309,2,2 +310,2,2 +311,2,2 +312,2,2 +313,0,0 +314,1,1 +315,0,0 +316,2,2 +317,2,1 +318,2,2 +319,2,2 +320,0,0 +321,2,2 +322,2,2 +323,2,2 +324,2,2 +325,2,2 +326,0,0 +327,2,2 +328,2,2 +329,0,0 +330,2,2 +331,0,0 +332,2,2 +333,2,2 +334,0,0 +335,2,2 +336,2,2 +337,2,2 +338,0,0 +339,0,0 +340,2,2 +341,2,2 +342,0,0 +343,2,2 +344,2,2 +345,2,2 +346,2,2 +347,2,2 +348,0,0 +349,1,1 +350,2,2 +351,0,0 +352,2,2 +353,2,2 +354,0,0 +355,0,0 +356,0,0 +357,2,0 +358,0,0 +359,2,2 +360,0,0 +361,2,2 +362,2,2 +363,2,2 +364,2,2 +365,2,2 +366,1,1 +367,2,2 +368,2,2 +369,1,1 +370,1,1 +371,2,2 +372,0,0 +373,2,0 +374,1,1 +375,2,2 +376,0,0 +377,0,0 +378,2,2 +379,2,2 +380,0,0 +381,2,2 +382,0,2 +383,2,2 +384,0,0 +385,0,0 +386,1,1 +387,0,0 +388,0,0 +389,2,2 +390,2,2 +391,0,0 +392,2,2 +393,0,0 +394,0,0 +395,2,2 +396,2,2 +397,0,0 +398,0,0 +399,0,0 +400,2,2 +401,2,2 +402,2,2 +403,2,2 +404,2,2 +405,0,0 +406,2,2 +407,0,0 +408,2,2 +409,0,0 +410,2,2 +411,2,2 +412,0,0 +413,2,2 +414,2,0 +415,2,2 +416,0,2 +417,2,2 +418,2,2 +419,2,1 +420,0,0 +421,2,2 +422,0,0 +423,2,2 +424,2,2 +425,0,0 +426,1,1 +427,2,2 +428,2,2 +429,0,0 +430,0,0 +431,2,2 +432,2,1 +433,2,2 +434,2,2 +435,0,0 +436,2,0 +437,0,0 +438,2,2 +439,1,1 +440,2,2 +441,2,2 +442,0,0 +443,2,2 +444,2,0 +445,2,2 +446,1,1 +447,0,0 +448,0,0 +449,0,0 +450,2,2 +451,2,0 +452,2,2 +453,2,2 +454,2,1 +455,2,2 +456,0,0 +457,1,1 +458,0,0 +459,0,0 +460,2,0 +461,2,2 +462,2,2 +463,0,0 +464,1,1 +465,0,0 +466,0,0 +467,2,2 +468,0,0 +469,0,0 +470,1,1 +471,2,2 +472,2,2 +473,1,1 +474,0,0 +475,2,0 +476,0,0 +477,2,0 +478,2,2 +479,2,2 +480,2,2 +481,0,0 +482,2,2 +483,2,2 +484,2,2 +485,0,0 +486,2,0 +487,2,2 +488,0,0 +489,0,0 +490,2,2 +491,1,1 +492,2,2 +493,2,2 +494,2,2 +495,2,2 +496,0,0 +497,0,0 +498,0,0 +499,2,2 +500,2,2 +501,0,0 +502,1,1 +503,2,2 +504,2,2 +505,2,2 +506,2,2 +507,2,2 +508,0,0 +509,2,2 +510,1,1 +511,2,2 +512,2,2 +513,2,2 +514,0,0 +515,2,2 +516,2,2 +517,2,2 +518,0,0 +519,2,2 +520,0,0 +521,2,1 +522,2,2 +523,2,2 +524,2,1 +525,2,2 +526,0,0 +527,0,0 +528,2,2 +529,2,2 +530,2,2 +531,2,2 +532,2,2 +533,2,2 +534,2,2 +535,1,1 +536,0,0 +537,0,0 +538,2,2 +539,2,2 +540,2,2 +541,0,0 +542,1,1 +543,0,0 +544,0,0 +545,2,2 +546,2,2 +547,2,2 +548,2,0 +549,2,2 +550,2,2 +551,1,1 +552,0,0 +553,2,2 +554,0,0 +555,2,2 +556,2,2 +557,2,2 +558,1,1 +559,0,0 +560,2,0 +561,2,2 +562,1,1 +563,2,2 +564,0,0 +565,2,0 +566,2,0 +567,1,1 +568,0,0 +569,2,2 +570,2,2 +571,0,0 +572,1,1 +573,2,2 +574,2,2 +575,2,2 +576,2,2 +577,2,1 +578,2,2 +579,2,2 +580,2,2 +581,2,2 +582,1,1 +583,2,2 +584,2,2 +585,2,2 +586,1,1 +587,2,1 +588,2,2 +589,0,0 +590,0,0 +591,0,0 +592,2,2 +593,2,2 +594,2,2 +595,2,2 +596,0,0 +597,0,0 +598,0,0 +599,1,1 +600,1,1 +601,2,2 +602,2,2 +603,2,2 +604,0,0 +605,2,2 +606,2,1 +607,2,2 +608,0,0 +609,2,2 +610,2,2 +611,2,2 +612,0,0 +613,2,2 +614,0,0 +615,0,0 +616,2,2 +617,2,2 +618,1,1 +619,2,2 +620,0,0 +621,0,0 +622,2,2 +623,2,2 +624,0,0 +625,2,2 +626,0,0 +627,2,2 +628,0,0 +629,2,2 +630,2,2 +631,0,0 +632,2,2 +633,2,2 +634,2,2 +635,0,0 +636,2,2 +637,2,1 +638,0,0 +639,0,0 +640,2,2 +641,2,2 +642,2,2 +643,0,0 +644,2,2 +645,2,2 +646,2,2 +647,0,0 +648,2,2 +649,2,0 +650,2,2 +651,2,2 +652,1,1 +653,2,2 +654,2,2 +655,2,2 +656,2,2 +657,2,2 +658,0,0 +659,2,2 +660,2,2 +661,2,2 +662,2,2 +663,0,0 +664,2,2 +665,0,0 +666,2,2 +667,2,2 +668,0,0 +669,1,1 +670,0,0 +671,2,2 +672,2,2 +673,2,2 +674,2,2 +675,1,1 +676,2,2 +677,2,2 +678,2,2 +679,2,2 +680,0,0 +681,2,0 +682,0,2 +683,2,2 +684,2,0 +685,0,0 +686,2,2 +687,0,2 +688,2,2 +689,2,2 +690,2,2 +691,0,0 +692,2,2 +693,2,2 +694,0,0 +695,2,0 +696,1,1 +697,0,0 +698,2,2 +699,2,0 +700,2,2 +701,2,2 +702,1,1 +703,0,0 +704,2,2 +705,2,2 +706,2,2 +707,2,2 +708,2,2 +709,1,1 +710,2,2 +711,0,0 +712,2,2 +713,0,0 +714,2,2 +715,2,2 +716,2,2 +717,0,0 +718,0,0 +719,2,2 +720,2,2 +721,0,0 +722,2,2 +723,2,2 +724,2,2 +725,2,2 +726,2,2 +727,2,2 +728,2,2 +729,2,2 +730,2,2 +731,2,2 +732,2,2 +733,1,1 +734,2,2 +735,2,2 +736,2,2 +737,2,2 +738,0,0 +739,2,2 +740,2,2 +741,2,0 +742,2,2 +743,2,2 +744,0,0 +745,2,2 +746,2,2 +747,1,1 +748,0,0 +749,2,2 +750,0,2 +751,2,0 +752,2,2 +753,2,2 +754,2,2 +755,0,0 +756,2,2 +757,2,2 +758,0,0 +759,0,0 +760,2,2 +761,2,2 +762,0,0 +763,0,0 +764,0,0 +765,2,2 +766,0,0 +767,0,0 +768,2,2 +769,2,2 +770,1,1 +771,2,2 +772,0,0 +773,2,2 +774,2,2 +775,0,0 +776,2,2 +777,2,2 +778,2,2 +779,0,0 +780,0,0 +781,2,2 +782,2,2 +783,2,2 +784,0,0 +785,2,2 +786,0,0 +787,2,2 +788,0,0 +789,2,2 +790,2,2 +791,0,0 +792,0,0 +793,0,0 +794,2,1 +795,2,2 +796,2,0 +797,0,0 +798,2,2 +799,2,2 +800,0,0 +801,2,2 +802,2,2 +803,1,1 +804,2,2 +805,2,2 +806,2,2 +807,2,2 +808,2,2 +809,2,2 +810,2,2 +811,2,2 +812,1,1 +813,2,2 +814,2,2 +815,2,2 +816,2,0 +817,2,0 +818,2,1 +819,2,2 +820,2,2 +821,1,1 +822,0,0 +823,2,2 +824,2,0 +825,2,2 +826,2,2 +827,2,2 +828,2,2 +829,2,2 +830,2,2 +831,2,2 +832,2,2 +833,2,2 +834,2,2 +835,2,2 +836,2,0 +837,2,2 +838,2,2 +839,0,0 +840,2,2 +841,2,0 +842,1,1 +843,2,2 +844,2,2 +845,2,2 +846,2,2 +847,0,0 +848,2,2 +849,2,2 +850,2,2 +851,2,2 +852,2,2 +853,2,2 +854,0,0 +855,2,2 +856,0,0 +857,2,2 +858,2,2 +859,1,1 +860,0,0 +861,2,2 +862,2,2 +863,2,0 +864,2,2 +865,0,0 +866,0,0 +867,1,1 +868,2,0 +869,2,2 +870,2,2 +871,2,2 +872,2,2 +873,2,2 +874,2,2 +875,2,2 +876,2,2 +877,2,2 +878,0,0 +879,2,2 +880,2,1 +881,0,0 +882,0,0 +883,2,2 +884,0,0 +885,0,0 +886,2,2 +887,0,0 +888,2,2 +889,2,2 +890,1,1 +891,2,2 +892,2,2 +893,2,2 +894,2,2 +895,2,2 +896,2,2 +897,2,2 +898,0,0 +899,2,2 +900,2,2 +901,0,0 +902,0,0 +903,0,0 +904,2,2 +905,2,2 +906,2,2 +907,2,2 +908,2,0 +909,2,2 +910,0,0 +911,0,0 +912,2,2 +913,2,2 +914,1,1 +915,2,2 +916,0,0 +917,2,2 +918,1,1 +919,2,2 +920,2,2 +921,2,2 +922,0,0 +923,2,2 +924,2,2 +925,2,2 +926,0,0 +927,2,2 +928,2,2 +929,0,0 +930,2,2 +931,2,2 +932,2,2 +933,2,2 +934,2,2 +935,2,2 +936,2,2 +937,2,2 +938,2,2 +939,1,1 +940,2,2 +941,2,2 +942,2,2 +943,2,2 +944,0,0 +945,2,2 +946,0,0 +947,2,2 +948,2,2 +949,2,2 +950,2,2 +951,2,2 +952,1,1 +953,2,2 +954,0,0 +955,2,1 +956,2,2 +957,0,0 +958,1,1 +959,2,2 +960,2,2 +961,2,2 +962,0,0 +963,0,0 +964,2,2 +965,2,2 +966,2,2 +967,2,2 +968,0,0 +969,2,2 +970,0,0 +971,0,0 +972,0,0 +973,0,0 +974,0,0 +975,2,2 +976,2,2 +977,2,2 +978,2,2 +979,2,2 +980,2,2 +981,2,2 +982,2,2 +983,0,0 +984,2,2 +985,2,2 +986,2,2 +987,2,2 +988,2,2 +989,2,2 +990,0,0 +991,2,2 +992,0,0 +993,2,2 +994,2,2 +995,0,0 +996,2,2 +997,0,0 +998,2,2 +999,2,2 +1000,2,2 +1001,0,0 +1002,2,2 +1003,2,0 +1004,0,0 +1005,2,2 +1006,2,2 +1007,2,2 +1008,2,2 +1009,2,2 +1010,0,0 +1011,2,2 +1012,2,2 +1013,2,2 +1014,0,0 +1015,2,1 +1016,0,0 +1017,2,2 +1018,2,2 +1019,2,2 +1020,2,2 +1021,2,2 +1022,2,2 +1023,2,2 +1024,0,0 +1025,2,2 +1026,2,2 +1027,2,2 +1028,0,0 +1029,2,2 +1030,2,0 +1031,2,2 +1032,1,1 +1033,2,2 +1034,2,2 +1035,2,2 +1036,2,2 +1037,2,2 +1038,2,0 +1039,2,2 +1040,0,0 +1041,0,0 +1042,2,2 +1043,2,2 +1044,2,2 +1045,2,2 +1046,1,1 +1047,2,2 +1048,2,2 +1049,0,0 +1050,0,0 +1051,2,2 +1052,2,2 +1053,1,1 +1054,2,2 +1055,0,0 +1056,2,2 +1057,2,2 +1058,0,0 +1059,2,0 +1060,0,0 +1061,2,1 +1062,2,2 +1063,0,0 +1064,2,2 +1065,1,1 +1066,2,2 +1067,2,2 +1068,2,2 +1069,2,2 +1070,2,2 +1071,2,2 +1072,2,2 +1073,2,2 +1074,2,2 +1075,2,2 +1076,2,2 +1077,0,0 +1078,1,1 +1079,2,2 +1080,0,0 +1081,0,0 +1082,2,0 +1083,0,0 +1084,0,0 +1085,0,0 +1086,0,0 +1087,0,0 +1088,2,2 +1089,0,0 +1090,2,2 +1091,2,2 +1092,2,2 +1093,0,0 +1094,2,2 +1095,2,2 +1096,2,2 +1097,2,2 +1098,2,2 +1099,2,2 +1100,2,2 +1101,0,0 +1102,0,0 +1103,2,2 +1104,0,0 +1105,0,0 +1106,2,2 +1107,2,2 +1108,2,2 +1109,0,0 +1110,2,2 +1111,1,1 +1112,2,2 +1113,1,1 +1114,2,2 +1115,2,2 +1116,2,2 +1117,1,1 +1118,2,2 +1119,1,1 +1120,0,0 +1121,2,2 +1122,1,1 +1123,2,2 +1124,2,2 +1125,2,2 +1126,2,2 +1127,0,0 +1128,2,2 +1129,2,2 +1130,2,2 +1131,2,2 +1132,0,0 +1133,2,2 +1134,1,2 +1135,2,2 +1136,2,2 +1137,2,2 +1138,0,0 +1139,0,0 +1140,2,0 +1141,2,2 +1142,0,0 +1143,2,2 +1144,1,1 +1145,2,2 +1146,2,2 +1147,0,0 +1148,2,2 +1149,1,1 +1150,0,0 +1151,2,2 +1152,2,2 +1153,2,2 +1154,2,2 +1155,2,2 +1156,2,2 +1157,2,2 +1158,2,2 +1159,2,2 +1160,2,2 +1161,0,0 +1162,2,2 +1163,2,2 +1164,2,2 +1165,2,2 +1166,0,0 +1167,2,2 +1168,2,2 +1169,2,2 +1170,2,1 +1171,0,2 +1172,2,2 +1173,2,2 +1174,0,2 +1175,0,0 +1176,2,2 +1177,2,2 +1178,2,2 +1179,2,2 +1180,2,2 +1181,2,2 +1182,2,2 +1183,2,2 +1184,2,2 +1185,2,2 +1186,2,2 +1187,2,2 +1188,2,2 +1189,1,1 +1190,2,1 +1191,1,1 +1192,2,0 +1193,2,2 +1194,2,2 +1195,2,0 +1196,2,2 +1197,2,2 +1198,0,0 +1199,2,2 +1200,2,2 +1201,2,2 +1202,2,2 +1203,2,2 +1204,2,2 +1205,2,2 +1206,2,2 +1207,0,0 +1208,2,2 +1209,2,1 +1210,2,2 +1211,0,0 +1212,2,2 +1213,2,2 +1214,0,0 +1215,2,2 +1216,2,2 +1217,2,2 +1218,2,2 +1219,2,2 +1220,0,0 +1221,0,0 +1222,0,0 +1223,0,0 +1224,2,2 +1225,2,2 +1226,2,2 +1227,0,0 +1228,2,2 +1229,0,0 +1230,2,2 +1231,2,2 +1232,2,2 +1233,0,0 +1234,0,0 +1235,2,2 +1236,1,1 +1237,2,2 +1238,2,2 +1239,2,0 +1240,2,2 +1241,2,2 +1242,1,1 +1243,2,2 +1244,2,2 +1245,2,0 +1246,0,0 +1247,2,2 +1248,0,0 +1249,2,2 +1250,2,2 +1251,2,2 +1252,2,0 +1253,2,2 +1254,2,2 +1255,2,2 +1256,2,2 +1257,2,2 +1258,2,2 +1259,2,2 +1260,2,2 +1261,2,2 +1262,2,2 +1263,2,2 +1264,2,2 +1265,0,0 +1266,2,0 +1267,2,2 +1268,0,0 +1269,2,2 +1270,2,2 +1271,2,2 +1272,2,2 +1273,2,2 +1274,2,2 +1275,0,0 +1276,2,2 +1277,2,2 +1278,2,2 +1279,0,0 +1280,2,2 +1281,0,0 +1282,0,0 +1283,2,2 +1284,0,0 +1285,2,2 +1286,2,2 +1287,2,1 +1288,0,0 +1289,2,0 +1290,2,2 +1291,2,2 +1292,2,2 +1293,2,2 +1294,1,1 +1295,0,0 +1296,2,0 +1297,2,2 +1298,0,0 +1299,2,2 +1300,2,2 +1301,2,2 +1302,2,2 +1303,0,0 +1304,0,0 +1305,2,2 +1306,2,2 +1307,2,2 +1308,2,2 +1309,1,1 +1310,2,2 +1311,0,0 +1312,2,2 +1313,2,2 +1314,2,2 +1315,2,0 +1316,2,2 +1317,0,0 +1318,2,2 +1319,2,2 +1320,1,1 +1321,1,1 +1322,2,2 +1323,2,2 +1324,2,2 +1325,2,2 +1326,2,2 +1327,2,0 +1328,2,2 +1329,2,2 +1330,2,2 +1331,2,2 +1332,2,2 +1333,2,2 +1334,0,0 +1335,2,2 +1336,0,0 +1337,2,2 +1338,0,0 +1339,2,2 +1340,2,2 +1341,2,2 +1342,2,1 +1343,0,0 +1344,2,2 +1345,2,2 +1346,0,0 +1347,0,0 +1348,2,2 +1349,2,2 +1350,2,2 +1351,2,2 +1352,2,2 +1353,0,0 +1354,2,2 +1355,0,0 +1356,2,2 +1357,2,0 +1358,2,0 +1359,2,2 +1360,2,2 +1361,2,2 +1362,0,0 +1363,2,2 +1364,1,1 +1365,2,2 +1366,2,2 +1367,0,0 +1368,2,2 +1369,2,2 +1370,2,2 +1371,1,1 +1372,2,2 +1373,2,1 +1374,0,0 +1375,1,1 +1376,2,2 +1377,2,2 +1378,2,2 +1379,0,0 +1380,2,2 +1381,2,1 +1382,2,2 +1383,0,0 +1384,0,0 +1385,2,0 +1386,2,2 +1387,2,0 +1388,0,0 +1389,2,2 +1390,2,0 +1391,2,2 +1392,2,2 +1393,0,0 +1394,2,2 +1395,2,2 +1396,2,2 +1397,2,2 +1398,2,2 +1399,2,2 +1400,2,2 +1401,2,2 +1402,2,2 +1403,1,1 +1404,2,2 +1405,0,0 +1406,2,2 +1407,0,0 +1408,0,0 +1409,2,2 +1410,0,0 +1411,2,2 +1412,2,2 +1413,2,2 +1414,2,2 +1415,0,0 +1416,2,2 +1417,1,1 +1418,2,2 +1419,2,2 +1420,2,2 +1421,2,2 +1422,2,2 +1423,2,2 +1424,0,2 +1425,2,0 +1426,0,0 +1427,1,1 +1428,2,2 +1429,2,2 +1430,2,2 +1431,2,2 +1432,2,2 +1433,2,2 +1434,0,0 +1435,2,0 +1436,2,0 +1437,2,2 +1438,2,2 +1439,0,0 +1440,0,0 +1441,2,2 +1442,1,1 +1443,0,0 +1444,2,2 +1445,2,2 +1446,2,2 +1447,0,0 +1448,2,2 +1449,2,2 +1450,0,0 +1451,0,0 +1452,2,2 +1453,2,1 +1454,2,2 +1455,2,2 +1456,0,0 +1457,0,0 +1458,2,2 +1459,2,2 +1460,2,2 +1461,1,1 +1462,2,2 +1463,2,2 +1464,2,2 +1465,2,2 +1466,0,0 +1467,0,0 +1468,2,2 +1469,2,2 +1470,2,2 +1471,2,2 +1472,2,2 +1473,2,2 +1474,2,2 +1475,2,2 +1476,2,2 +1477,2,2 +1478,2,2 +1479,2,2 +1480,2,2 +1481,0,0 +1482,0,0 +1483,2,2 +1484,0,0 +1485,2,2 +1486,2,0 +1487,0,0 +1488,2,2 +1489,0,0 +1490,2,2 +1491,2,2 +1492,0,0 +1493,2,2 +1494,0,0 +1495,2,2 +1496,0,0 +1497,2,0 +1498,2,2 +1499,0,0 +1500,2,2 +1501,2,2 +1502,2,2 +1503,0,0 +1504,2,2 +1505,0,0 +1506,2,2 +1507,2,2 +1508,2,2 +1509,2,2 +1510,2,2 +1511,1,1 +1512,0,0 +1513,2,2 +1514,2,2 +1515,0,0 +1516,2,0 +1517,0,0 +1518,2,2 +1519,2,2 +1520,0,0 +1521,2,2 +1522,2,2 +1523,0,0 +1524,0,0 +1525,2,2 +1526,2,2 +1527,2,2 +1528,2,2 +1529,2,2 +1530,2,2 +1531,2,2 +1532,0,0 +1533,2,2 +1534,2,2 +1535,2,2 +1536,2,2 +1537,1,1 +1538,2,2 +1539,1,1 +1540,2,2 +1541,0,0 +1542,1,1 +1543,1,1 +1544,2,2 +1545,2,2 +1546,0,0 +1547,2,2 +1548,2,2 +1549,0,0 +1550,2,2 +1551,2,2 +1552,2,2 +1553,0,0 +1554,2,2 +1555,1,1 +1556,0,0 +1557,1,1 +1558,0,0 +1559,2,2 +1560,2,2 +1561,2,2 +1562,2,2 +1563,0,0 +1564,0,0 +1565,0,0 +1566,0,0 +1567,0,0 +1568,0,0 +1569,2,2 +1570,2,2 +1571,2,2 +1572,2,2 +1573,2,2 +1574,2,2 +1575,2,2 +1576,2,2 +1577,2,2 +1578,2,2 +1579,2,0 +1580,0,0 +1581,2,2 +1582,2,2 +1583,0,0 +1584,2,2 +1585,1,1 +1586,1,1 +1587,2,2 +1588,0,0 +1589,0,0 +1590,2,0 +1591,0,0 +1592,2,2 +1593,0,0 +1594,0,0 +1595,2,2 +1596,2,2 +1597,0,0 +1598,2,2 +1599,0,0 +1600,2,2 +1601,0,0 +1602,1,1 +1603,2,0 +1604,0,0 +1605,0,0 +1606,2,2 +1607,2,2 +1608,2,1 +1609,2,2 +1610,0,0 +1611,2,0 +1612,0,0 +1613,2,2 +1614,2,2 +1615,2,2 +1616,2,2 +1617,1,1 +1618,0,0 +1619,2,2 +1620,0,0 +1621,0,0 +1622,2,0 +1623,2,2 +1624,2,2 +1625,2,2 +1626,2,2 +1627,1,1 +1628,2,2 +1629,2,2 +1630,2,0 +1631,2,2 +1632,2,2 +1633,2,2 +1634,0,0 +1635,2,2 +1636,2,2 +1637,0,0 +1638,2,2 +1639,2,2 +1640,2,2 +1641,0,0 +1642,2,1 +1643,2,2 +1644,2,2 +1645,0,0 +1646,0,0 +1647,2,2 +1648,0,0 +1649,0,0 +1650,0,0 +1651,0,0 +1652,2,2 +1653,2,2 +1654,2,2 +1655,2,2 +1656,2,2 +1657,2,2 +1658,2,2 +1659,2,2 +1660,2,2 +1661,2,2 +1662,1,1 +1663,2,2 +1664,2,2 +1665,2,2 +1666,2,2 +1667,2,0 +1668,1,1 +1669,0,0 +1670,0,0 +1671,0,0 +1672,2,0 +1673,0,0 +1674,2,2 +1675,0,0 +1676,2,2 +1677,0,0 +1678,0,0 +1679,2,2 +1680,2,2 +1681,2,2 +1682,0,0 +1683,2,1 +1684,2,2 +1685,2,2 +1686,0,0 +1687,2,2 +1688,0,0 +1689,2,2 +1690,2,0 +1691,0,0 +1692,2,2 +1693,2,2 +1694,2,2 +1695,2,2 +1696,2,2 +1697,0,0 +1698,2,2 +1699,2,2 +1700,2,2 +1701,2,2 +1702,2,2 +1703,2,2 +1704,2,2 +1705,1,1 +1706,2,2 +1707,1,1 +1708,0,0 +1709,0,0 +1710,2,2 +1711,0,0 +1712,2,2 +1713,2,2 +1714,0,0 +1715,2,2 +1716,2,2 +1717,2,2 +1718,0,0 +1719,0,0 +1720,2,2 +1721,2,2 +1722,0,0 +1723,1,1 +1724,2,2 +1725,0,0 +1726,2,0 +1727,2,2 +1728,1,1 +1729,2,2 +1730,0,0 +1731,0,0 +1732,2,2 +1733,2,2 +1734,0,0 +1735,2,2 +1736,2,2 +1737,2,2 +1738,2,2 +1739,2,2 +1740,2,2 +1741,2,2 +1742,1,1 +1743,0,0 +1744,0,0 +1745,0,0 +1746,2,2 +1747,0,0 +1748,0,2 +1749,2,2 +1750,2,2 +1751,2,2 +1752,2,2 +1753,0,0 +1754,1,1 +1755,2,2 +1756,0,0 +1757,0,0 +1758,2,2 +1759,2,2 +1760,2,0 +1761,2,2 +1762,2,2 +1763,0,2 +1764,2,2 +1765,2,2 +1766,2,2 +1767,2,2 +1768,0,0 +1769,2,2 +1770,2,2 +1771,0,0 +1772,1,1 +1773,2,0 +1774,2,2 +1775,2,2 +1776,2,2 +1777,2,2 +1778,2,2 +1779,2,2 +1780,2,2 +1781,0,0 +1782,2,2 +1783,0,0 +1784,2,0 +1785,2,2 +1786,0,0 +1787,2,2 +1788,2,2 +1789,2,2 +1790,2,2 +1791,2,2 +1792,2,2 +1793,2,2 +1794,2,2 +1795,2,2 +1796,2,2 +1797,2,2 +1798,2,2 +1799,0,0 +1800,2,2 +1801,2,2 +1802,2,2 +1803,2,2 +1804,2,2 +1805,1,1 +1806,2,2 +1807,2,2 +1808,0,0 +1809,2,2 +1810,0,0 +1811,2,2 +1812,2,2 +1813,2,2 +1814,2,2 +1815,2,2 +1816,2,2 +1817,0,0 +1818,2,0 +1819,0,0 +1820,2,2 +1821,2,2 +1822,2,1 +1823,1,1 +1824,2,2 +1825,0,0 +1826,1,1 +1827,2,2 +1828,2,2 +1829,2,2 +1830,2,2 +1831,2,2 +1832,2,2 +1833,0,0 +1834,1,1 +1835,2,2 +1836,0,0 +1837,0,0 +1838,1,1 +1839,2,2 +1840,1,1 +1841,2,2 +1842,2,2 +1843,2,2 +1844,0,0 +1845,0,0 +1846,0,0 +1847,2,2 +1848,2,2 +1849,2,2 +1850,1,1 +1851,2,2 +1852,0,0 +1853,2,2 +1854,2,2 +1855,2,2 +1856,0,0 +1857,1,1 +1858,2,2 +1859,2,2 +1860,2,2 +1861,0,0 +1862,2,2 +1863,0,0 +1864,2,1 +1865,2,2 +1866,2,2 +1867,0,0 +1868,0,0 +1869,2,2 +1870,2,2 +1871,2,0 +1872,2,2 +1873,2,2 +1874,2,2 +1875,2,2 +1876,2,2 +1877,2,2 +1878,2,2 +1879,2,0 +1880,1,1 +1881,2,2 +1882,2,1 +1883,2,2 +1884,2,2 +1885,2,2 +1886,0,0 +1887,2,2 +1888,0,0 +1889,2,2 +1890,1,1 +1891,0,0 +1892,2,2 +1893,0,0 +1894,2,2 +1895,0,0 +1896,0,0 +1897,0,0 +1898,2,2 +1899,2,2 +1900,2,2 +1901,2,2 +1902,2,2 +1903,0,0 +1904,0,0 +1905,2,2 +1906,2,2 +1907,0,0 +1908,2,2 +1909,2,2 +1910,0,0 +1911,2,2 +1912,2,2 +1913,0,0 +1914,2,0 +1915,2,2 +1916,2,2 +1917,1,1 +1918,0,0 +1919,0,0 +1920,1,1 +1921,2,2 +1922,2,2 +1923,2,2 +1924,2,2 +1925,0,0 +1926,2,2 +1927,0,0 +1928,2,2 +1929,2,2 +1930,2,2 +1931,2,2 +1932,2,2 +1933,2,2 +1934,0,0 +1935,2,0 +1936,2,2 +1937,2,2 +1938,2,2 +1939,2,0 +1940,0,0 +1941,2,2 +1942,0,0 +1943,2,2 +1944,2,0 +1945,2,2 +1946,2,2 +1947,2,0 +1948,2,2 +1949,2,2 +1950,0,0 +1951,1,1 +1952,1,1 +1953,0,0 +1954,2,2 +1955,2,2 +1956,0,0 +1957,2,2 +1958,2,2 +1959,2,2 +1960,0,0 +1961,2,2 +1962,2,2 +1963,2,2 +1964,2,0 +1965,2,2 +1966,2,2 +1967,2,2 +1968,2,2 +1969,2,2 +1970,2,2 +1971,0,0 +1972,2,2 +1973,2,2 +1974,2,2 +1975,2,2 +1976,2,2 +1977,2,2 +1978,2,1 +1979,0,0 +1980,1,1 +1981,0,0 +1982,2,2 +1983,2,2 +1984,2,2 +1985,1,1 +1986,2,2 +1987,0,0 +1988,2,2 +1989,2,2 +1990,0,0 +1991,2,2 +1992,0,0 +1993,1,1 +1994,2,2 +1995,2,2 +1996,2,1 +1997,2,2 +1998,2,2 +1999,2,2 +2000,2,2 +2001,0,0 +2002,2,2 +2003,2,2 +2004,0,0 +2005,0,0 +2006,2,2 +2007,0,0 +2008,0,0 +2009,2,2 +2010,2,2 +2011,2,2 +2012,2,2 +2013,0,0 +2014,1,1 +2015,2,2 +2016,2,2 +2017,0,0 +2018,0,0 +2019,2,2 +2020,0,0 +2021,2,2 +2022,2,2 +2023,2,2 +2024,2,2 +2025,2,2 +2026,2,2 +2027,0,0 +2028,2,2 +2029,0,0 +2030,0,0 +2031,2,2 +2032,2,2 +2033,2,2 +2034,2,2 +2035,2,1 +2036,2,2 +2037,2,0 +2038,1,1 +2039,2,2 +2040,2,2 +2041,2,2 +2042,2,1 +2043,2,2 +2044,2,2 +2045,2,2 +2046,2,2 +2047,2,2 +2048,2,2 +2049,2,2 +2050,0,0 +2051,2,2 +2052,2,2 +2053,2,2 +2054,2,2 +2055,2,2 +2056,2,2 +2057,2,2 +2058,0,0 +2059,2,2 +2060,2,2 +2061,2,2 +2062,2,2 +2063,2,2 +2064,2,2 +2065,0,0 +2066,2,2 +2067,2,2 +2068,0,0 +2069,0,0 +2070,0,0 +2071,2,2 +2072,2,0 +2073,2,2 +2074,2,1 +2075,2,2 +2076,0,0 +2077,2,2 +2078,2,2 +2079,2,0 +2080,0,0 +2081,2,2 +2082,0,0 +2083,2,2 +2084,0,0 +2085,0,0 +2086,2,1 +2087,0,0 +2088,1,1 +2089,2,2 +2090,2,2 +2091,0,0 +2092,0,1 +2093,2,2 +2094,2,2 +2095,2,2 +2096,2,2 +2097,0,0 +2098,2,2 +2099,2,2 +2100,1,1 +2101,1,1 +2102,2,2 +2103,2,2 +2104,2,0 +2105,2,2 +2106,2,2 +2107,2,2 +2108,2,2 +2109,2,2 +2110,2,2 +2111,2,2 +2112,2,2 +2113,2,2 +2114,2,2 +2115,2,2 +2116,2,2 +2117,2,2 +2118,2,2 +2119,2,0 +2120,0,2 +2121,2,2 +2122,0,0 +2123,2,2 +2124,0,0 +2125,2,0 +2126,2,2 +2127,2,2 +2128,2,2 +2129,0,0 +2130,0,0 +2131,2,2 +2132,2,1 +2133,2,2 +2134,0,0 +2135,1,1 +2136,2,2 +2137,0,0 +2138,2,2 +2139,2,2 +2140,2,2 +2141,0,0 +2142,2,2 +2143,2,2 +2144,2,2 +2145,2,2 +2146,0,0 +2147,2,2 +2148,0,0 +2149,2,2 +2150,2,2 +2151,2,2 +2152,2,2 +2153,2,2 +2154,0,0 +2155,2,2 +2156,0,0 +2157,2,0 +2158,2,2 +2159,0,0 +2160,2,2 +2161,2,2 +2162,2,2 +2163,2,2 +2164,2,2 +2165,2,2 +2166,2,2 +2167,2,2 +2168,2,2 +2169,0,0 +2170,2,2 +2171,2,2 +2172,2,2 +2173,0,0 +2174,2,2 +2175,2,2 +2176,0,0 +2177,2,2 +2178,2,0 +2179,2,2 +2180,1,1 +2181,2,2 +2182,2,2 +2183,2,2 +2184,0,0 +2185,2,2 +2186,2,2 +2187,2,2 +2188,2,2 +2189,2,2 +2190,2,2 +2191,2,2 +2192,2,2 +2193,0,0 +2194,0,0 +2195,2,2 +2196,0,0 +2197,0,0 +2198,2,1 +2199,0,0 +2200,2,2 +2201,2,2 +2202,2,2 +2203,2,0 +2204,2,2 +2205,0,0 +2206,2,2 +2207,2,2 +2208,2,1 +2209,2,2 +2210,2,2 +2211,2,2 +2212,2,2 +2213,2,2 +2214,2,2 +2215,2,2 +2216,0,0 +2217,2,2 +2218,2,2 +2219,2,2 +2220,0,0 +2221,0,0 +2222,2,2 +2223,1,1 +2224,2,2 +2225,2,2 +2226,2,0 +2227,1,1 +2228,2,2 +2229,2,0 +2230,0,0 +2231,2,2 +2232,0,0 +2233,2,2 +2234,2,2 +2235,1,1 +2236,0,0 +2237,2,2 +2238,2,2 +2239,2,2 +2240,2,2 +2241,2,2 +2242,1,1 +2243,1,1 +2244,2,2 +2245,1,2 +2246,0,0 +2247,2,2 +2248,2,2 +2249,0,0 +2250,2,2 +2251,2,2 +2252,2,0 +2253,2,2 +2254,0,0 +2255,2,2 +2256,1,1 +2257,2,2 +2258,2,2 +2259,2,2 +2260,2,2 +2261,2,2 +2262,2,0 +2263,2,2 +2264,2,2 +2265,1,1 +2266,2,2 +2267,0,0 +2268,2,2 +2269,2,2 +2270,2,2 +2271,2,1 +2272,2,2 +2273,1,1 +2274,2,0 +2275,2,2 +2276,0,0 +2277,2,0 +2278,2,2 +2279,2,2 +2280,2,2 +2281,1,1 +2282,2,2 +2283,2,2 +2284,2,2 +2285,0,0 +2286,2,2 +2287,2,2 +2288,0,0 +2289,2,2 +2290,0,0 +2291,1,1 +2292,2,2 +2293,0,0 +2294,0,0 +2295,0,0 +2296,1,1 +2297,2,1 +2298,2,2 +2299,2,2 +2300,1,1 +2301,2,2 +2302,2,2 +2303,0,0 +2304,2,2 +2305,2,2 +2306,2,2 +2307,0,0 +2308,0,0 +2309,2,2 +2310,2,2 +2311,2,2 +2312,2,2 +2313,2,2 +2314,2,2 +2315,0,0 +2316,2,2 +2317,1,1 +2318,2,2 +2319,2,2 +2320,2,2 +2321,2,2 +2322,0,0 +2323,0,0 +2324,0,0 +2325,0,0 +2326,2,2 +2327,0,0 +2328,0,0 +2329,2,2 +2330,1,1 +2331,0,0 +2332,1,1 +2333,2,2 +2334,2,2 +2335,0,0 +2336,2,2 +2337,0,0 +2338,2,2 +2339,1,1 +2340,2,2 +2341,0,0 +2342,2,2 +2343,2,2 +2344,2,2 +2345,2,2 +2346,2,2 +2347,0,0 +2348,2,2 +2349,2,2 +2350,0,0 +2351,0,0 +2352,2,2 +2353,2,2 +2354,2,2 +2355,1,1 +2356,2,2 +2357,2,2 +2358,2,2 +2359,2,2 +2360,2,2 +2361,2,2 +2362,2,2 +2363,0,0 +2364,2,2 +2365,2,1 +2366,2,2 +2367,2,2 +2368,0,0 +2369,2,2 +2370,2,2 +2371,2,2 +2372,2,0 +2373,0,0 +2374,2,2 +2375,0,0 +2376,0,0 +2377,2,2 +2378,2,2 +2379,2,2 +2380,0,0 +2381,2,2 +2382,2,2 +2383,2,0 +2384,2,2 +2385,2,2 +2386,2,2 +2387,2,2 +2388,2,2 +2389,1,1 +2390,0,0 +2391,2,2 +2392,2,2 +2393,1,1 +2394,2,2 +2395,2,2 +2396,2,2 +2397,0,0 +2398,0,0 +2399,2,2 +2400,2,2 +2401,2,2 +2402,2,2 +2403,2,2 +2404,0,0 +2405,2,2 +2406,0,0 +2407,2,2 +2408,2,2 +2409,2,2 +2410,0,0 +2411,1,1 +2412,1,1 +2413,0,0 +2414,2,2 +2415,0,0 +2416,0,0 +2417,2,2 +2418,2,2 +2419,2,2 +2420,0,0 +2421,2,2 +2422,2,2 +2423,0,0 +2424,0,0 +2425,2,2 +2426,2,2 +2427,2,2 +2428,2,2 +2429,2,2 +2430,2,2 +2431,2,2 +2432,0,0 +2433,0,0 +2434,2,2 +2435,0,0 +2436,2,2 +2437,2,2 +2438,2,2 +2439,2,2 +2440,2,2 +2441,0,0 +2442,0,0 +2443,2,2 +2444,0,2 +2445,2,2 +2446,0,0 +2447,2,2 +2448,1,1 +2449,2,2 +2450,2,2 +2451,2,2 +2452,2,2 +2453,2,2 +2454,2,2 +2455,2,2 +2456,0,2 +2457,2,2 +2458,2,2 +2459,0,0 +2460,2,2 +2461,2,2 +2462,2,2 +2463,2,2 +2464,2,2 +2465,0,0 +2466,2,2 +2467,0,0 +2468,2,2 +2469,1,1 +2470,0,0 +2471,2,2 +2472,2,1 +2473,2,0 +2474,2,2 +2475,2,2 +2476,2,2 +2477,0,0 +2478,2,2 +2479,2,2 +2480,0,0 +2481,2,2 +2482,2,2 +2483,0,0 +2484,0,0 +2485,2,2 +2486,2,0 +2487,0,0 +2488,0,0 +2489,2,1 +2490,0,0 +2491,0,0 +2492,2,2 +2493,2,2 +2494,0,0 +2495,2,2 +2496,2,0 +2497,0,0 +2498,1,1 +2499,2,2 +2500,2,2 +2501,2,2 +2502,2,2 +2503,2,2 +2504,0,0 +2505,2,2 +2506,2,2 +2507,2,2 +2508,0,0 +2509,2,2 +2510,2,2 +2511,2,2 +2512,2,2 +2513,0,0 +2514,1,1 +2515,2,2 +2516,2,2 +2517,0,0 +2518,2,2 +2519,2,2 +2520,2,2 +2521,2,2 +2522,2,2 +2523,2,2 +2524,2,2 +2525,0,0 +2526,2,2 +2527,2,2 +2528,2,2 +2529,2,2 +2530,2,2 +2531,2,2 +2532,2,2 +2533,2,2 +2534,0,0 +2535,1,1 +2536,2,2 +2537,0,0 +2538,1,1 +2539,2,2 +2540,2,2 +2541,2,2 +2542,1,2 +2543,0,0 +2544,0,0 +2545,0,0 +2546,2,2 +2547,2,2 +2548,2,2 +2549,2,2 +2550,2,2 +2551,0,0 +2552,0,0 +2553,2,2 +2554,2,2 +2555,2,2 +2556,2,2 +2557,0,0 +2558,2,2 +2559,0,0 +2560,2,2 +2561,2,2 +2562,0,0 +2563,2,2 +2564,0,0 +2565,2,2 +2566,2,2 +2567,0,0 +2568,0,0 +2569,2,2 +2570,1,1 +2571,2,2 +2572,2,2 +2573,2,2 +2574,2,2 +2575,0,0 +2576,2,2 +2577,2,2 +2578,2,2 +2579,0,0 +2580,2,2 +2581,2,2 +2582,2,2 +2583,2,2 +2584,1,1 +2585,2,2 +2586,2,2 +2587,2,1 +2588,0,0 +2589,2,2 +2590,2,2 +2591,2,2 +2592,2,2 +2593,2,2 +2594,2,2 +2595,2,1 +2596,2,2 +2597,2,2 +2598,2,2 +2599,2,2 +2600,2,2 +2601,2,2 +2602,0,0 +2603,2,2 +2604,0,0 +2605,0,0 +2606,2,2 +2607,2,2 +2608,0,0 +2609,2,2 +2610,2,2 +2611,2,2 +2612,2,2 +2613,2,2 +2614,2,2 +2615,0,0 +2616,2,2 +2617,0,0 +2618,2,2 +2619,2,2 +2620,2,2 +2621,2,2 +2622,2,2 +2623,2,2 +2624,0,0 +2625,0,0 +2626,0,0 +2627,0,0 +2628,2,2 +2629,1,1 +2630,2,2 +2631,2,0 +2632,2,2 +2633,2,2 +2634,2,2 +2635,2,0 +2636,2,2 +2637,2,0 +2638,2,2 +2639,0,0 +2640,0,0 +2641,2,2 +2642,1,2 +2643,2,2 +2644,2,2 +2645,2,2 +2646,0,0 +2647,2,2 +2648,0,0 +2649,2,2 +2650,2,2 +2651,2,2 +2652,2,0 +2653,0,0 +2654,2,2 +2655,1,1 +2656,2,2 +2657,2,2 +2658,0,0 +2659,1,1 +2660,2,2 +2661,0,0 +2662,0,0 +2663,2,2 +2664,0,0 +2665,2,2 +2666,2,0 +2667,2,2 +2668,2,2 +2669,0,0 +2670,2,2 +2671,2,0 +2672,1,1 +2673,0,0 +2674,2,2 +2675,2,2 +2676,2,2 +2677,2,2 +2678,2,2 +2679,2,2 +2680,0,2 +2681,2,2 +2682,1,1 +2683,2,2 +2684,0,0 +2685,2,2 +2686,0,0 +2687,2,2 +2688,2,2 +2689,2,2 +2690,2,2 +2691,0,0 +2692,2,2 +2693,0,0 +2694,2,2 +2695,1,1 +2696,0,0 +2697,2,2 +2698,1,1 +2699,2,2 +2700,2,2 +2701,2,2 +2702,2,2 +2703,2,2 +2704,2,2 +2705,2,0 +2706,0,0 +2707,2,2 +2708,0,0 +2709,0,0 +2710,2,2 +2711,1,1 +2712,0,0 +2713,1,1 +2714,0,0 +2715,2,2 +2716,2,2 +2717,2,2 +2718,2,2 +2719,2,2 +2720,2,2 +2721,0,0 +2722,0,0 +2723,2,0 +2724,2,2 +2725,0,0 +2726,2,0 +2727,2,2 +2728,2,2 +2729,2,2 +2730,2,2 +2731,2,2 +2732,2,2 +2733,2,2 +2734,2,2 +2735,2,2 +2736,2,2 +2737,2,2 +2738,0,0 +2739,2,2 +2740,2,2 +2741,0,0 +2742,2,2 +2743,2,2 +2744,0,0 +2745,0,0 +2746,0,0 +2747,2,2 +2748,2,2 +2749,2,2 +2750,1,1 +2751,2,2 +2752,2,2 +2753,1,1 +2754,0,0 +2755,2,2 +2756,2,2 +2757,0,0 +2758,0,0 +2759,2,2 +2760,2,2 +2761,2,2 +2762,0,0 +2763,2,2 +2764,2,2 +2765,2,2 +2766,2,2 +2767,2,0 +2768,2,2 +2769,2,2 +2770,0,0 +2771,2,2 +2772,2,2 +2773,2,2 +2774,0,0 +2775,2,2 +2776,2,2 +2777,1,1 +2778,2,1 +2779,1,1 +2780,0,0 +2781,2,2 +2782,2,2 +2783,2,2 +2784,2,2 +2785,1,1 +2786,2,0 +2787,2,2 +2788,2,2 +2789,0,0 +2790,2,2 +2791,2,2 +2792,0,0 +2793,2,2 +2794,0,0 +2795,2,2 +2796,0,0 +2797,0,0 +2798,0,0 +2799,0,0 +2800,1,1 +2801,0,0 +2802,0,0 +2803,2,2 +2804,0,0 +2805,2,2 +2806,2,2 +2807,2,2 +2808,2,2 +2809,2,2 +2810,2,2 +2811,2,2 +2812,2,1 +2813,2,2 +2814,2,2 +2815,0,0 +2816,2,2 +2817,0,0 +2818,1,1 +2819,0,0 +2820,2,2 +2821,0,0 +2822,0,0 +2823,0,0 +2824,2,2 +2825,2,2 +2826,2,2 +2827,2,2 +2828,2,2 +2829,2,2 +2830,2,0 +2831,1,1 +2832,2,2 +2833,2,2 +2834,0,0 +2835,2,2 +2836,2,2 +2837,0,0 +2838,2,2 +2839,2,2 +2840,2,2 +2841,0,0 +2842,2,2 +2843,2,2 +2844,2,2 +2845,0,0 +2846,2,0 +2847,1,1 +2848,2,0 +2849,0,0 +2850,2,2 +2851,0,0 +2852,1,1 +2853,0,0 +2854,0,0 +2855,2,2 +2856,2,2 +2857,2,0 +2858,2,2 +2859,2,2 +2860,1,1 +2861,2,2 +2862,0,0 +2863,2,2 +2864,2,2 +2865,2,2 +2866,0,2 +2867,2,2 +2868,2,2 +2869,2,2 +2870,2,0 +2871,2,0 +2872,0,0 +2873,2,2 +2874,0,0 +2875,2,2 +2876,0,0 +2877,2,2 +2878,2,2 +2879,2,2 +2880,2,0 +2881,2,2 +2882,2,0 +2883,2,2 +2884,2,2 +2885,0,0 +2886,2,2 +2887,2,2 +2888,2,2 +2889,0,0 +2890,0,0 +2891,2,2 +2892,2,2 +2893,1,1 +2894,1,1 +2895,2,2 +2896,0,0 +2897,2,2 +2898,2,2 +2899,2,2 +2900,2,2 +2901,2,2 +2902,2,2 +2903,0,2 +2904,2,2 +2905,2,2 +2906,0,0 +2907,2,2 +2908,2,2 +2909,2,2 +2910,2,2 +2911,2,2 +2912,0,0 +2913,1,1 +2914,0,0 +2915,2,1 +2916,2,2 +2917,1,1 +2918,2,2 +2919,2,2 +2920,2,2 +2921,0,0 +2922,2,2 +2923,2,2 +2924,2,2 +2925,1,1 +2926,1,1 +2927,2,2 +2928,0,0 +2929,2,2 +2930,2,2 +2931,0,0 +2932,2,2 +2933,0,0 +2934,0,0 +2935,2,2 +2936,0,0 +2937,2,2 +2938,0,0 +2939,0,0 +2940,2,2 +2941,2,2 +2942,2,1 +2943,1,1 +2944,2,2 +2945,2,2 +2946,2,2 +2947,0,0 +2948,0,0 +2949,2,2 +2950,2,2 +2951,2,2 +2952,2,2 +2953,1,1 +2954,0,0 +2955,2,2 +2956,2,2 +2957,0,0 +2958,2,2 +2959,2,2 +2960,2,2 +2961,0,2 +2962,2,2 +2963,2,2 +2964,2,2 +2965,2,2 +2966,2,2 +2967,0,0 +2968,2,2 +2969,2,2 +2970,0,0 +2971,2,2 +2972,1,1 +2973,2,1 +2974,2,2 +2975,2,2 +2976,2,2 +2977,2,2 +2978,0,0 +2979,2,2 +2980,0,0 +2981,0,0 +2982,2,2 +2983,0,0 +2984,2,2 +2985,1,1 +2986,2,0 +2987,2,2 +2988,2,2 +2989,0,0 +2990,2,2 +2991,0,0 +2992,0,0 +2993,2,2 +2994,0,0 +2995,2,2 +2996,2,0 +2997,0,2 +2998,2,2 +2999,0,0 +3000,0,0 +3001,0,0 +3002,0,0 +3003,2,2 +3004,2,2 +3005,0,0 +3006,2,2 +3007,2,2 +3008,2,2 +3009,2,2 +3010,2,2 +3011,2,2 +3012,1,1 +3013,2,2 +3014,1,1 +3015,2,2 +3016,2,2 +3017,2,2 +3018,0,0 +3019,2,2 +3020,2,2 +3021,2,2 +3022,2,2 +3023,2,2 +3024,1,2 +3025,0,0 +3026,2,2 +3027,2,2 +3028,0,0 +3029,2,2 +3030,2,2 +3031,2,2 +3032,1,1 +3033,0,0 +3034,2,2 +3035,2,2 +3036,2,2 +3037,2,2 +3038,2,2 +3039,0,0 +3040,0,2 +3041,2,2 +3042,0,0 +3043,2,0 +3044,2,2 +3045,0,0 +3046,2,2 +3047,2,2 +3048,2,2 +3049,2,2 +3050,2,2 +3051,1,1 +3052,2,2 +3053,0,0 +3054,0,0 +3055,2,2 +3056,0,0 +3057,0,0 +3058,2,2 +3059,2,2 +3060,2,2 +3061,2,2 +3062,2,2 +3063,0,0 +3064,2,2 +3065,0,0 +3066,0,0 +3067,2,2 +3068,2,2 +3069,0,0 +3070,0,0 +3071,2,2 +3072,2,2 +3073,2,2 +3074,2,2 +3075,2,2 +3076,2,2 +3077,2,2 +3078,2,0 +3079,2,0 +3080,2,2 +3081,0,0 +3082,2,2 +3083,2,2 +3084,2,2 +3085,2,2 +3086,0,0 +3087,0,0 +3088,2,2 +3089,2,2 +3090,0,0 +3091,2,2 +3092,2,2 +3093,2,2 +3094,2,2 +3095,2,2 +3096,2,2 +3097,2,2 +3098,0,0 +3099,0,0 +3100,2,2 +3101,2,2 +3102,2,2 +3103,2,2 +3104,2,2 +3105,2,2 +3106,2,2 +3107,2,2 +3108,2,2 +3109,2,2 +3110,2,2 +3111,2,2 +3112,2,2 +3113,2,2 +3114,2,2 +3115,0,0 +3116,2,2 +3117,0,0 +3118,0,0 +3119,0,0 +3120,2,2 +3121,0,0 +3122,2,2 +3123,0,0 +3124,2,2 +3125,2,0 +3126,2,2 +3127,2,2 +3128,0,2 +3129,2,2 +3130,0,0 +3131,0,0 +3132,2,0 +3133,2,2 +3134,0,0 +3135,2,2 +3136,2,2 +3137,2,2 +3138,2,2 +3139,2,2 +3140,0,0 +3141,2,2 +3142,0,0 +3143,2,2 +3144,2,2 +3145,2,2 +3146,0,0 +3147,0,0 +3148,2,2 +3149,2,0 +3150,0,0 +3151,2,2 +3152,2,2 +3153,2,0 +3154,2,2 +3155,2,1 +3156,1,1 +3157,0,0 +3158,2,1 +3159,2,2 +3160,2,2 +3161,2,2 +3162,0,0 +3163,2,2 +3164,1,1 +3165,2,2 +3166,2,2 +3167,2,2 +3168,0,0 +3169,2,2 +3170,2,2 +3171,2,2 +3172,2,2 +3173,0,0 +3174,0,0 +3175,2,2 +3176,2,2 +3177,0,0 +3178,2,2 +3179,2,2 +3180,2,2 +3181,0,0 +3182,2,2 +3183,2,2 +3184,1,1 +3185,2,2 +3186,2,2 +3187,0,0 +3188,2,2 +3189,0,0 +3190,0,0 +3191,0,0 +3192,0,0 +3193,0,0 +3194,0,0 +3195,1,1 +3196,2,2 +3197,2,2 +3198,2,2 +3199,2,2 +3200,0,0 +3201,0,0 +3202,0,0 +3203,0,0 +3204,2,2 +3205,2,2 +3206,1,2 +3207,2,2 +3208,1,1 +3209,2,2 +3210,2,2 +3211,0,0 +3212,2,2 +3213,0,0 +3214,1,1 +3215,2,2 +3216,2,2 +3217,0,0 +3218,2,2 +3219,2,2 +3220,2,2 +3221,2,2 +3222,2,1 +3223,2,2 +3224,0,0 +3225,2,0 +3226,0,0 +3227,1,1 +3228,2,2 +3229,2,2 +3230,2,2 +3231,2,2 +3232,2,0 +3233,0,0 +3234,2,0 +3235,2,2 +3236,2,2 +3237,0,0 +3238,2,2 +3239,2,2 +3240,2,2 +3241,0,0 +3242,2,0 +3243,2,2 +3244,2,0 +3245,2,2 +3246,2,0 +3247,2,2 +3248,2,2 +3249,2,2 +3250,0,0 +3251,0,0 +3252,0,0 +3253,2,2 +3254,2,2 +3255,2,2 +3256,2,2 +3257,2,2 +3258,2,2 +3259,1,1 +3260,2,2 +3261,2,2 +3262,2,2 +3263,2,2 +3264,2,2 +3265,2,2 +3266,2,0 +3267,2,2 +3268,1,1 +3269,0,0 +3270,0,0 +3271,2,2 +3272,0,0 +3273,2,2 +3274,0,0 +3275,2,0 +3276,2,2 +3277,0,0 +3278,2,2 +3279,2,2 +3280,2,2 +3281,2,2 +3282,0,0 +3283,0,0 +3284,0,2 +3285,2,2 +3286,2,2 +3287,0,0 +3288,1,1 +3289,2,2 +3290,1,1 +3291,1,1 +3292,1,1 +3293,2,2 +3294,2,2 +3295,2,2 +3296,2,2 +3297,1,1 +3298,2,2 +3299,2,2 +3300,2,2 +3301,2,2 +3302,1,1 +3303,2,2 +3304,2,2 +3305,2,2 +3306,2,2 +3307,2,2 +3308,2,2 +3309,2,2 +3310,2,2 +3311,2,2 +3312,2,2 +3313,2,1 +3314,0,0 +3315,2,2 +3316,0,0 +3317,2,2 +3318,2,2 +3319,2,0 +3320,2,2 +3321,2,2 +3322,2,2 +3323,2,2 +3324,2,2 +3325,2,2 +3326,2,2 +3327,2,2 +3328,1,1 +3329,2,2 +3330,2,2 +3331,1,1 +3332,2,2 +3333,2,2 +3334,0,0 +3335,2,2 +3336,0,0 +3337,2,2 +3338,2,2 +3339,2,2 +3340,2,2 +3341,0,0 +3342,2,2 +3343,2,2 +3344,2,2 +3345,0,0 +3346,2,2 +3347,2,2 +3348,2,2 +3349,2,2 +3350,2,2 +3351,2,2 +3352,2,2 +3353,0,0 +3354,0,0 +3355,2,2 +3356,2,2 +3357,2,2 +3358,2,2 +3359,2,2 +3360,0,0 +3361,2,2 +3362,2,2 +3363,0,0 +3364,2,2 +3365,2,2 +3366,2,2 +3367,0,0 +3368,2,2 +3369,2,2 +3370,0,0 +3371,2,2 +3372,2,2 +3373,2,2 +3374,0,0 +3375,2,2 +3376,2,2 +3377,2,2 +3378,2,2 +3379,0,0 +3380,2,2 +3381,2,2 +3382,0,0 +3383,2,2 +3384,2,2 +3385,1,1 +3386,2,2 +3387,2,2 +3388,2,2 +3389,0,0 +3390,2,0 +3391,2,2 +3392,2,2 +3393,2,2 +3394,2,2 +3395,2,2 +3396,2,2 +3397,0,0 +3398,2,2 +3399,1,1 +3400,2,2 +3401,2,2 +3402,0,0 +3403,2,2 +3404,2,2 +3405,2,2 +3406,2,2 +3407,2,2 +3408,0,0 +3409,2,2 +3410,0,0 +3411,1,1 +3412,2,2 +3413,0,0 +3414,0,0 +3415,2,2 +3416,2,2 +3417,2,2 +3418,0,0 +3419,2,2 +3420,2,2 +3421,1,1 +3422,2,2 +3423,0,0 +3424,2,2 +3425,2,2 +3426,2,2 +3427,0,0 +3428,1,1 +3429,2,2 +3430,1,1 +3431,2,2 +3432,2,2 +3433,0,0 +3434,0,0 +3435,2,2 +3436,2,2 +3437,0,0 +3438,0,0 +3439,2,0 +3440,0,0 +3441,2,2 +3442,2,2 +3443,2,2 +3444,0,0 +3445,2,2 +3446,2,2 +3447,0,0 +3448,2,2 +3449,2,2 +3450,2,2 +3451,2,2 +3452,0,0 +3453,2,2 +3454,2,2 +3455,0,0 +3456,2,0 +3457,0,0 +3458,2,2 +3459,2,2 +3460,0,0 +3461,1,1 +3462,2,2 +3463,0,0 +3464,2,2 +3465,2,2 +3466,2,2 +3467,1,1 +3468,0,0 +3469,2,2 +3470,0,0 +3471,2,2 +3472,2,0 +3473,0,0 +3474,2,2 +3475,0,0 +3476,1,1 +3477,2,2 +3478,2,2 +3479,1,1 +3480,1,1 +3481,2,2 +3482,2,2 +3483,0,0 +3484,2,2 +3485,0,0 +3486,2,2 +3487,0,0 +3488,0,0 +3489,0,0 +3490,2,2 +3491,2,2 +3492,1,1 +3493,0,0 +3494,0,0 +3495,2,2 +3496,0,0 +3497,0,0 +3498,2,2 +3499,2,2 +3500,0,0 +3501,0,0 +3502,0,0 +3503,2,0 +3504,0,0 +3505,2,2 +3506,0,0 +3507,2,2 +3508,2,2 +3509,0,2 +3510,2,2 +3511,2,2 +3512,2,1 +3513,0,0 +3514,0,0 +3515,2,2 +3516,1,2 +3517,2,2 +3518,2,1 +3519,2,2 +3520,2,0 +3521,1,1 +3522,2,2 +3523,2,2 +3524,2,2 +3525,2,2 +3526,2,0 +3527,0,0 +3528,2,2 +3529,2,2 +3530,2,2 +3531,2,2 +3532,0,0 +3533,2,2 +3534,2,2 +3535,0,0 +3536,2,2 +3537,2,2 +3538,2,2 +3539,0,0 +3540,0,0 +3541,2,2 +3542,1,1 +3543,2,2 +3544,2,2 +3545,0,0 +3546,0,0 +3547,2,2 +3548,2,2 +3549,1,1 +3550,2,2 +3551,0,0 +3552,2,2 +3553,2,2 +3554,1,1 +3555,0,0 +3556,2,2 +3557,2,2 +3558,0,0 +3559,2,2 +3560,2,2 +3561,1,1 +3562,2,2 +3563,2,2 +3564,2,2 +3565,2,2 +3566,2,0 +3567,2,0 +3568,2,2 +3569,2,2 +3570,2,2 +3571,1,1 +3572,2,2 +3573,2,2 +3574,2,2 +3575,2,2 +3576,0,0 +3577,2,2 +3578,0,0 +3579,1,1 +3580,1,1 +3581,0,0 +3582,0,0 +3583,2,2 +3584,2,2 +3585,2,0 +3586,2,2 +3587,2,2 +3588,2,2 +3589,2,2 +3590,0,0 +3591,2,2 +3592,2,2 +3593,2,2 +3594,0,0 +3595,0,0 +3596,0,0 +3597,0,0 +3598,2,2 +3599,0,0 +3600,2,2 +3601,2,2 +3602,2,2 +3603,2,2 +3604,0,0 +3605,2,2 +3606,0,0 +3607,0,0 +3608,1,1 +3609,2,2 +3610,0,0 +3611,2,2 +3612,2,2 +3613,2,2 +3614,0,0 +3615,2,2 +3616,0,0 +3617,2,2 +3618,2,2 +3619,2,1 +3620,1,1 +3621,2,2 +3622,2,2 +3623,2,2 +3624,0,0 +3625,2,2 +3626,2,2 +3627,2,2 +3628,2,2 +3629,2,2 +3630,1,1 +3631,2,2 +3632,2,2 +3633,0,0 +3634,2,2 +3635,1,1 +3636,2,2 +3637,2,2 +3638,0,0 +3639,2,2 +3640,2,2 +3641,1,1 +3642,0,0 +3643,1,1 +3644,1,1 +3645,2,2 +3646,2,2 +3647,0,0 +3648,2,2 +3649,2,2 +3650,2,2 +3651,0,0 +3652,0,0 +3653,2,2 +3654,2,2 +3655,2,2 +3656,2,2 +3657,2,2 +3658,2,2 +3659,2,2 +3660,2,2 +3661,0,0 +3662,2,2 +3663,0,0 +3664,2,2 +3665,0,0 +3666,0,0 +3667,2,2 +3668,2,2 +3669,2,2 +3670,2,2 +3671,2,2 +3672,2,2 +3673,2,2 +3674,2,2 +3675,1,1 +3676,2,2 +3677,2,2 +3678,2,2 +3679,0,0 +3680,2,2 +3681,1,1 +3682,2,2 +3683,2,2 +3684,2,2 +3685,0,0 +3686,2,2 +3687,2,2 +3688,2,2 +3689,2,2 +3690,0,0 +3691,0,0 +3692,2,0 +3693,2,2 +3694,0,0 +3695,0,0 +3696,0,0 +3697,2,2 +3698,2,2 +3699,2,2 +3700,2,2 +3701,0,0 +3702,2,2 +3703,2,2 +3704,2,2 +3705,2,2 +3706,2,2 +3707,2,2 +3708,2,2 +3709,2,2 +3710,2,2 +3711,0,0 +3712,2,2 +3713,2,2 +3714,2,2 +3715,0,0 +3716,2,2 +3717,1,1 +3718,2,2 +3719,2,2 +3720,2,2 +3721,2,2 +3722,0,0 +3723,2,2 +3724,2,1 +3725,2,2 +3726,0,0 +3727,2,2 +3728,2,2 +3729,0,0 +3730,0,0 +3731,0,0 +3732,2,2 +3733,1,1 +3734,0,0 +3735,2,2 +3736,2,2 +3737,1,1 +3738,2,2 +3739,0,0 +3740,2,2 +3741,2,2 +3742,2,2 +3743,1,1 +3744,2,2 +3745,2,2 +3746,0,0 +3747,2,2 +3748,2,2 +3749,2,1 +3750,2,0 +3751,2,2 +3752,2,2 +3753,2,2 +3754,0,0 +3755,0,0 +3756,2,2 +3757,2,2 +3758,2,2 +3759,0,0 +3760,2,2 +3761,1,1 +3762,0,0 +3763,2,2 +3764,2,2 +3765,2,2 +3766,2,2 +3767,2,2 +3768,2,2 +3769,1,1 +3770,2,2 +3771,2,2 +3772,2,2 +3773,0,0 +3774,1,1 +3775,2,2 +3776,0,0 +3777,0,0 +3778,2,2 +3779,2,2 +3780,2,2 +3781,2,2 +3782,2,2 +3783,2,2 +3784,2,2 +3785,0,0 +3786,0,0 +3787,2,2 +3788,0,0 +3789,2,2 +3790,2,2 +3791,0,0 +3792,0,0 +3793,2,2 +3794,2,2 +3795,0,2 +3796,2,2 +3797,2,2 +3798,0,0 +3799,2,2 +3800,2,2 +3801,0,0 +3802,2,2 +3803,2,2 +3804,2,2 +3805,2,1 +3806,1,1 +3807,2,2 +3808,2,0 +3809,2,2 +3810,2,2 +3811,2,2 +3812,2,2 +3813,0,2 +3814,2,2 +3815,0,0 +3816,0,0 +3817,0,0 +3818,2,2 +3819,1,1 +3820,2,2 +3821,0,0 +3822,2,2 +3823,1,1 +3824,2,2 +3825,2,2 +3826,0,0 +3827,2,2 +3828,2,2 +3829,2,2 +3830,2,2 +3831,2,2 +3832,2,1 +3833,2,2 +3834,2,0 +3835,1,1 +3836,2,2 +3837,2,2 +3838,1,1 +3839,2,2 +3840,0,0 +3841,2,2 +3842,1,1 +3843,2,2 +3844,2,2 +3845,2,2 +3846,2,2 +3847,2,2 +3848,1,1 +3849,2,2 +3850,2,2 +3851,2,2 +3852,2,2 +3853,0,0 +3854,2,2 +3855,2,2 +3856,0,0 +3857,2,2 +3858,2,2 +3859,0,0 +3860,2,2 +3861,0,0 +3862,0,0 +3863,0,0 +3864,1,1 +3865,2,2 +3866,2,2 +3867,2,2 +3868,2,2 +3869,0,0 +3870,2,2 +3871,2,1 +3872,2,2 +3873,2,2 +3874,2,1 +3875,2,2 +3876,2,2 +3877,0,0 +3878,2,2 +3879,2,2 +3880,0,0 +3881,2,2 +3882,2,2 +3883,2,2 +3884,0,0 +3885,2,2 +3886,2,2 +3887,2,2 +3888,2,2 +3889,2,2 +3890,2,2 +3891,2,2 +3892,2,2 +3893,0,0 +3894,1,1 +3895,2,2 +3896,0,0 +3897,0,0 +3898,2,2 +3899,2,2 +3900,2,2 +3901,0,0 +3902,2,2 +3903,2,2 +3904,2,2 +3905,2,2 +3906,2,2 +3907,2,2 +3908,2,2 +3909,2,2 +3910,2,2 +3911,0,0 +3912,2,2 +3913,2,2 +3914,1,1 +3915,0,0 +3916,0,0 +3917,2,2 +3918,0,0 +3919,2,2 +3920,2,2 +3921,2,2 +3922,0,0 +3923,2,2 +3924,2,2 +3925,2,2 +3926,2,2 +3927,2,2 +3928,2,2 +3929,2,2 +3930,2,2 +3931,2,2 +3932,2,2 +3933,2,2 +3934,2,2 +3935,2,2 +3936,1,1 +3937,0,0 +3938,2,2 +3939,0,0 +3940,2,0 +3941,0,0 +3942,2,2 +3943,2,2 +3944,2,2 +3945,0,0 +3946,0,0 +3947,2,2 +3948,2,2 +3949,2,2 +3950,2,2 +3951,0,0 +3952,0,0 +3953,0,0 +3954,2,2 +3955,2,2 +3956,2,2 +3957,0,0 +3958,2,2 +3959,1,1 +3960,0,0 +3961,2,2 +3962,2,2 +3963,2,2 +3964,0,0 +3965,2,2 +3966,0,0 +3967,2,2 +3968,2,0 +3969,2,2 +3970,2,2 +3971,2,2 +3972,2,2 +3973,2,2 +3974,2,2 +3975,2,2 +3976,1,1 +3977,2,1 +3978,2,2 +3979,2,2 +3980,0,0 +3981,2,2 +3982,2,2 +3983,2,2 +3984,0,0 +3985,2,2 +3986,2,2 +3987,2,2 +3988,2,1 +3989,2,2 +3990,2,2 +3991,2,2 +3992,2,2 +3993,2,2 +3994,2,2 +3995,2,2 +3996,2,2 +3997,2,2 +3998,2,2 +3999,2,0 +4000,0,0 +4001,2,2 +4002,1,1 +4003,2,2 +4004,0,0 +4005,2,2 +4006,2,2 +4007,2,2 +4008,2,2 +4009,2,2 +4010,2,2 +4011,0,0 +4012,0,0 +4013,2,2 +4014,0,0 +4015,2,2 +4016,2,2 +4017,2,2 +4018,2,0 +4019,2,2 +4020,2,2 +4021,2,2 +4022,2,2 +4023,2,2 +4024,2,2 +4025,0,0 +4026,0,0 +4027,0,0 +4028,2,2 +4029,0,0 +4030,0,0 +4031,2,2 +4032,2,2 +4033,2,2 +4034,2,2 +4035,2,2 +4036,1,1 +4037,2,2 +4038,2,2 +4039,0,0 +4040,0,0 +4041,2,2 +4042,2,2 +4043,2,2 +4044,2,2 +4045,0,0 +4046,2,2 +4047,2,2 +4048,2,2 +4049,2,2 +4050,2,2 +4051,2,2 +4052,2,2 +4053,0,0 +4054,1,1 +4055,0,0 +4056,0,0 +4057,2,2 +4058,2,2 +4059,2,2 +4060,2,2 +4061,0,0 +4062,0,0 +4063,2,2 +4064,2,2 +4065,0,0 +4066,0,0 +4067,2,2 +4068,0,2 +4069,2,2 +4070,0,0 +4071,2,2 +4072,2,2 +4073,2,2 +4074,1,1 +4075,2,2 +4076,2,2 +4077,2,2 +4078,2,2 +4079,0,0 +4080,1,1 +4081,1,1 +4082,2,2 +4083,2,2 +4084,2,2 +4085,2,2 +4086,2,0 +4087,2,2 +4088,2,2 +4089,2,2 +4090,2,2 +4091,0,0 +4092,2,2 +4093,2,2 +4094,2,1 +4095,2,2 +4096,0,0 +4097,2,2 +4098,2,2 +4099,2,2 +4100,0,0 +4101,2,2 +4102,2,2 +4103,2,2 +4104,2,0 +4105,0,0 +4106,0,0 +4107,2,2 +4108,2,2 +4109,1,1 +4110,0,0 +4111,1,1 +4112,2,2 +4113,0,0 +4114,2,2 +4115,1,1 +4116,2,2 +4117,0,0 +4118,0,0 +4119,2,0 +4120,1,1 +4121,2,2 +4122,2,2 +4123,2,2 +4124,2,2 +4125,2,2 +4126,2,1 +4127,0,0 +4128,2,2 +4129,0,0 +4130,0,0 +4131,0,0 +4132,1,1 +4133,2,2 +4134,2,0 +4135,2,2 +4136,2,2 +4137,0,0 +4138,2,2 +4139,2,2 +4140,2,2 +4141,2,2 +4142,0,0 +4143,2,2 +4144,2,2 +4145,2,2 +4146,2,2 +4147,2,2 +4148,2,2 +4149,2,2 +4150,2,2 +4151,2,2 +4152,2,2 +4153,2,2 +4154,2,2 +4155,2,2 +4156,2,2 +4157,2,0 +4158,0,0 +4159,2,2 +4160,0,0 +4161,2,1 +4162,2,2 +4163,2,2 +4164,2,2 +4165,2,2 +4166,2,2 +4167,0,0 +4168,2,2 +4169,0,0 +4170,1,1 +4171,0,0 +4172,0,0 +4173,1,1 +4174,2,2 +4175,2,2 +4176,2,2 +4177,2,2 +4178,0,0 +4179,2,2 +4180,2,2 +4181,2,0 +4182,2,1 +4183,2,2 +4184,2,0 +4185,1,1 +4186,0,0 +4187,2,2 +4188,0,0 +4189,2,2 +4190,2,2 +4191,2,2 +4192,2,2 +4193,0,0 +4194,0,0 +4195,2,0 +4196,2,2 +4197,2,2 +4198,2,2 +4199,2,2 +4200,2,2 +4201,0,0 +4202,2,2 +4203,1,1 +4204,2,2 +4205,1,1 +4206,2,2 +4207,2,2 +4208,0,2 +4209,0,2 +4210,2,2 +4211,0,0 +4212,2,0 +4213,2,2 +4214,2,2 +4215,2,2 +4216,2,2 +4217,0,0 +4218,2,0 +4219,2,2 +4220,2,2 +4221,2,2 +4222,1,1 +4223,2,2 +4224,0,0 +4225,2,2 +4226,2,2 +4227,2,2 +4228,2,2 +4229,2,2 +4230,2,2 +4231,1,1 +4232,2,2 +4233,0,0 +4234,0,0 +4235,2,2 +4236,2,2 +4237,2,2 +4238,2,2 +4239,0,0 +4240,2,2 +4241,2,2 +4242,2,2 +4243,1,2 +4244,2,2 +4245,2,0 +4246,2,2 +4247,0,0 +4248,2,2 +4249,2,2 +4250,2,2 +4251,0,0 +4252,2,2 +4253,2,2 +4254,2,2 +4255,2,2 +4256,2,2 +4257,0,0 +4258,2,2 +4259,2,2 +4260,2,2 +4261,2,1 +4262,2,0 +4263,2,2 +4264,1,1 +4265,2,2 +4266,2,2 +4267,2,2 +4268,2,2 +4269,2,2 +4270,0,2 +4271,0,0 +4272,0,0 +4273,2,2 +4274,2,2 +4275,2,2 +4276,2,2 +4277,2,2 +4278,2,2 +4279,0,0 +4280,0,0 +4281,2,2 +4282,2,2 +4283,2,2 +4284,2,2 +4285,0,0 +4286,0,0 +4287,0,0 +4288,2,2 +4289,1,1 +4290,0,0 +4291,2,0 +4292,2,2 +4293,2,2 +4294,0,0 +4295,2,2 +4296,2,2 +4297,2,2 +4298,0,0 +4299,2,2 +4300,2,2 +4301,0,0 +4302,0,0 +4303,0,0 +4304,2,2 +4305,0,0 +4306,2,2 +4307,2,2 +4308,2,2 +4309,2,2 +4310,2,2 +4311,2,2 +4312,2,2 +4313,2,0 +4314,0,0 +4315,2,2 +4316,0,0 +4317,2,2 +4318,0,0 +4319,2,2 +4320,2,2 +4321,2,2 +4322,2,2 +4323,2,2 +4324,2,1 +4325,0,0 +4326,0,0 +4327,2,2 +4328,2,2 +4329,2,2 +4330,2,2 +4331,0,0 +4332,2,2 +4333,2,2 +4334,2,2 +4335,2,2 +4336,2,2 +4337,2,0 +4338,2,0 +4339,2,2 +4340,0,0 +4341,2,2 +4342,0,0 +4343,0,0 +4344,2,2 +4345,0,0 +4346,2,2 +4347,0,0 +4348,2,2 +4349,1,1 +4350,2,2 +4351,2,2 +4352,2,2 +4353,0,0 +4354,2,2 +4355,2,2 +4356,0,0 +4357,0,0 +4358,2,2 +4359,0,0 +4360,2,0 +4361,2,0 +4362,2,2 +4363,2,2 +4364,2,2 +4365,0,0 +4366,2,2 +4367,2,0 +4368,2,2 +4369,2,2 +4370,0,0 +4371,2,2 +4372,2,2 +4373,2,2 +4374,2,2 +4375,0,0 +4376,2,0 +4377,2,2 +4378,0,0 +4379,2,2 +4380,2,0 +4381,0,0 +4382,2,2 +4383,2,2 +4384,2,2 +4385,0,0 +4386,2,2 +4387,2,0 +4388,0,0 +4389,2,2 +4390,2,2 +4391,0,0 +4392,2,2 +4393,1,1 +4394,2,2 +4395,2,2 +4396,1,1 +4397,0,0 +4398,0,0 +4399,0,0 +4400,0,0 +4401,2,2 +4402,2,0 +4403,0,0 +4404,2,2 +4405,2,2 +4406,0,0 +4407,2,0 +4408,2,2 +4409,2,2 +4410,2,2 +4411,2,2 +4412,2,2 +4413,0,0 +4414,2,2 +4415,0,2 +4416,2,2 +4417,1,1 +4418,2,2 +4419,2,2 +4420,2,2 +4421,2,2 +4422,2,2 +4423,0,0 +4424,2,2 +4425,0,0 +4426,0,0 +4427,2,2 +4428,2,0 +4429,2,2 +4430,2,2 +4431,2,2 +4432,2,2 +4433,2,2 +4434,2,2 +4435,2,2 +4436,2,2 +4437,2,2 +4438,2,0 +4439,0,0 +4440,2,0 +4441,2,1 +4442,2,2 +4443,0,2 +4444,2,0 +4445,2,2 +4446,0,0 +4447,2,2 +4448,2,2 +4449,0,0 +4450,2,2 +4451,0,0 +4452,0,0 +4453,2,2 +4454,0,0 +4455,0,0 +4456,0,0 +4457,1,1 +4458,0,0 +4459,0,0 +4460,2,2 +4461,2,2 +4462,2,2 +4463,2,2 +4464,1,1 +4465,2,2 +4466,2,2 +4467,2,2 +4468,2,2 +4469,2,2 +4470,2,2 +4471,2,0 +4472,0,0 +4473,2,2 +4474,2,2 +4475,0,2 +4476,0,0 +4477,0,0 +4478,2,0 +4479,2,2 +4480,2,2 +4481,2,1 +4482,2,2 +4483,2,2 +4484,2,2 +4485,2,2 +4486,2,2 +4487,2,2 +4488,1,1 +4489,2,2 +4490,2,2 +4491,2,2 +4492,2,2 +4493,2,2 +4494,2,2 +4495,0,0 +4496,2,2 +4497,2,2 +4498,2,2 +4499,2,2 +4500,1,1 +4501,2,0 +4502,2,2 +4503,2,2 +4504,0,0 +4505,2,0 +4506,2,2 +4507,1,1 +4508,0,0 +4509,2,2 +4510,0,0 +4511,0,0 +4512,1,1 +4513,1,2 +4514,2,2 +4515,2,2 +4516,0,0 +4517,0,0 +4518,2,2 +4519,2,2 +4520,1,1 +4521,0,0 +4522,0,0 +4523,0,0 +4524,0,0 +4525,2,2 +4526,2,2 +4527,1,1 +4528,0,0 +4529,2,2 +4530,2,2 +4531,2,2 +4532,2,2 +4533,0,0 +4534,2,2 +4535,0,0 +4536,2,2 +4537,2,2 +4538,0,0 +4539,2,2 +4540,2,2 +4541,2,2 +4542,2,2 +4543,0,0 +4544,2,2 +4545,2,2 +4546,0,0 +4547,2,0 +4548,0,0 +4549,0,0 +4550,1,1 +4551,0,0 +4552,2,2 +4553,2,2 +4554,2,2 +4555,2,2 +4556,1,1 +4557,0,0 +4558,2,0 +4559,2,2 +4560,2,2 +4561,2,2 +4562,0,0 +4563,0,2 +4564,0,0 +4565,2,2 +4566,2,2 +4567,2,2 +4568,0,0 +4569,2,2 +4570,2,0 +4571,2,2 +4572,2,2 +4573,2,2 +4574,1,1 +4575,0,0 +4576,2,2 +4577,2,2 +4578,0,0 +4579,0,0 +4580,2,2 +4581,2,2 +4582,2,2 +4583,0,0 +4584,2,2 +4585,2,2 +4586,2,2 +4587,0,0 +4588,2,2 +4589,2,2 +4590,2,2 +4591,2,2 +4592,2,0 +4593,2,2 +4594,2,2 +4595,0,0 +4596,2,2 +4597,1,1 +4598,2,2 +4599,2,2 +4600,2,2 +4601,2,2 +4602,0,0 +4603,2,2 +4604,1,1 +4605,2,2 +4606,2,2 +4607,0,2 +4608,0,0 +4609,2,0 +4610,2,2 +4611,0,0 +4612,2,2 +4613,0,0 +4614,2,2 +4615,0,0 +4616,2,2 +4617,2,0 +4618,0,0 +4619,0,0 +4620,2,2 +4621,2,2 +4622,0,0 +4623,2,2 +4624,0,0 +4625,2,2 +4626,2,2 +4627,2,2 +4628,0,0 +4629,0,0 +4630,2,2 +4631,0,0 +4632,0,0 +4633,0,0 +4634,0,2 +4635,2,2 +4636,0,0 +4637,2,2 +4638,2,0 +4639,2,2 +4640,2,1 +4641,0,0 +4642,2,2 +4643,2,2 +4644,2,2 +4645,2,2 +4646,2,2 +4647,2,1 +4648,1,1 +4649,1,1 +4650,2,2 +4651,2,2 +4652,2,2 +4653,2,2 +4654,2,2 +4655,1,1 +4656,1,1 +4657,2,2 +4658,2,2 +4659,2,2 +4660,2,2 +4661,2,2 +4662,0,0 +4663,2,2 +4664,2,2 +4665,2,2 +4666,2,2 +4667,2,2 +4668,2,0 +4669,2,2 +4670,2,2 +4671,2,2 +4672,2,2 +4673,2,2 +4674,1,1 +4675,2,2 +4676,2,2 +4677,2,2 +4678,2,2 +4679,1,1 +4680,2,2 +4681,0,0 +4682,0,0 +4683,0,0 +4684,2,2 +4685,0,0 +4686,2,0 +4687,2,2 +4688,2,2 +4689,2,2 +4690,2,2 +4691,2,2 +4692,0,0 +4693,2,2 +4694,2,2 +4695,2,0 +4696,0,2 +4697,2,2 +4698,0,0 +4699,2,2 +4700,2,2 +4701,0,0 +4702,2,2 +4703,0,0 +4704,0,0 +4705,0,0 +4706,2,2 +4707,2,0 +4708,2,2 +4709,0,0 +4710,0,0 +4711,1,1 +4712,2,2 +4713,0,0 +4714,2,2 +4715,2,0 +4716,2,2 +4717,2,2 +4718,2,2 +4719,2,2 +4720,2,2 +4721,2,2 +4722,0,0 +4723,0,0 +4724,2,2 +4725,2,2 +4726,1,1 +4727,2,2 +4728,2,1 +4729,0,0 +4730,2,2 +4731,0,0 +4732,2,2 +4733,2,2 +4734,2,2 +4735,2,2 +4736,0,0 +4737,2,2 +4738,1,1 +4739,0,0 +4740,0,0 +4741,0,0 +4742,2,0 +4743,2,2 +4744,1,1 +4745,2,2 +4746,2,2 +4747,2,2 +4748,2,2 +4749,2,2 +4750,2,2 +4751,1,1 +4752,0,0 +4753,2,2 +4754,2,0 +4755,2,2 +4756,0,0 +4757,0,0 +4758,2,2 +4759,2,2 +4760,2,2 +4761,0,0 +4762,0,0 +4763,0,0 +4764,0,0 +4765,0,0 +4766,2,2 +4767,2,0 +4768,0,0 +4769,0,0 +4770,2,2 +4771,2,2 +4772,2,2 +4773,2,2 +4774,2,2 +4775,2,2 +4776,2,2 +4777,2,2 +4778,2,2 +4779,2,2 +4780,0,0 +4781,1,1 +4782,0,0 +4783,0,0 +4784,0,0 +4785,2,2 +4786,2,2 +4787,0,0 +4788,2,2 +4789,1,1 +4790,0,0 +4791,2,2 +4792,2,2 +4793,2,2 +4794,2,2 +4795,2,2 +4796,2,2 +4797,0,0 +4798,2,2 +4799,2,2 +4800,0,0 +4801,2,0 +4802,2,2 +4803,1,1 +4804,0,0 +4805,2,2 +4806,2,2 +4807,1,1 +4808,0,0 +4809,2,2 +4810,2,2 +4811,2,2 +4812,0,0 +4813,2,2 +4814,0,0 +4815,0,0 +4816,0,0 +4817,2,2 +4818,1,1 +4819,0,0 +4820,2,2 +4821,2,2 +4822,0,0 +4823,2,2 +4824,0,0 +4825,2,2 +4826,0,0 +4827,0,0 +4828,2,2 +4829,0,0 +4830,2,2 +4831,2,2 +4832,0,0 +4833,0,0 +4834,2,2 +4835,2,2 +4836,2,2 +4837,0,0 +4838,0,0 +4839,2,2 +4840,0,0 +4841,0,0 +4842,2,2 +4843,0,0 +4844,2,2 +4845,0,0 +4846,2,2 +4847,2,2 +4848,2,2 +4849,2,2 +4850,0,0 +4851,0,0 +4852,0,0 +4853,2,2 +4854,2,1 +4855,2,2 +4856,2,2 +4857,1,1 +4858,2,2 +4859,0,0 +4860,2,2 +4861,2,2 +4862,2,2 +4863,0,0 +4864,0,0 +4865,2,2 +4866,2,1 +4867,0,0 +4868,0,0 +4869,2,2 +4870,2,2 +4871,2,2 +4872,2,2 +4873,2,0 +4874,2,2 +4875,2,2 +4876,2,2 +4877,2,2 +4878,0,0 +4879,2,2 +4880,0,0 +4881,2,2 +4882,0,0 +4883,1,1 +4884,0,0 +4885,2,2 +4886,2,2 +4887,2,0 +4888,2,2 +4889,2,2 +4890,2,2 +4891,2,2 +4892,2,2 +4893,0,0 +4894,0,0 +4895,2,2 +4896,0,0 +4897,0,0 +4898,2,2 +4899,0,0 +4900,2,2 +4901,2,2 +4902,2,2 +4903,2,0 +4904,2,2 +4905,2,2 +4906,2,2 +4907,0,0 +4908,0,0 +4909,2,2 +4910,2,2 +4911,2,2 +4912,2,2 +4913,2,2 +4914,2,2 +4915,2,2 +4916,1,1 +4917,2,2 +4918,2,2 +4919,2,2 +4920,2,2 +4921,0,0 +4922,2,2 +4923,2,2 +4924,2,2 +4925,2,2 +4926,2,2 +4927,2,2 +4928,2,2 +4929,2,2 +4930,2,2 +4931,2,2 +4932,2,2 +4933,2,2 +4934,2,2 +4935,1,1 +4936,0,0 +4937,0,0 +4938,2,2 +4939,2,2 +4940,2,2 +4941,2,2 +4942,0,0 +4943,2,2 +4944,2,2 +4945,2,2 +4946,2,2 +4947,2,2 +4948,0,0 +4949,2,2 +4950,0,0 +4951,1,1 +4952,0,0 +4953,0,0 +4954,2,2 +4955,2,2 +4956,2,2 +4957,2,2 +4958,2,2 +4959,0,0 +4960,2,2 +4961,2,2 +4962,2,2 +4963,2,2 +4964,2,2 +4965,2,2 +4966,2,2 +4967,2,2 +4968,2,2 +4969,0,0 +4970,2,2 +4971,0,0 +4972,2,2 +4973,2,0 +4974,2,2 +4975,0,0 +4976,2,2 +4977,1,1 +4978,2,2 +4979,2,2 +4980,2,2 +4981,1,1 +4982,2,2 +4983,2,2 +4984,2,2 +4985,0,0 +4986,0,0 +4987,0,0 +4988,2,2 +4989,2,2 +4990,0,0 +4991,2,2 +4992,2,2 +4993,2,2 +4994,1,1 +4995,0,0 +4996,2,2 +4997,2,2 +4998,2,2 +4999,2,2 +5000,2,2 +5001,2,2 +5002,2,2 +5003,2,2 +5004,2,2 +5005,2,2 +5006,2,2 +5007,0,0 +5008,2,2 +5009,2,2 +5010,1,1 +5011,0,0 +5012,2,2 +5013,2,2 +5014,2,2 +5015,2,2 +5016,0,0 +5017,2,2 +5018,2,2 +5019,1,1 +5020,2,2 +5021,0,0 +5022,2,2 +5023,2,2 +5024,2,2 +5025,0,0 +5026,0,0 +5027,1,1 +5028,2,2 +5029,2,2 +5030,2,2 +5031,2,2 +5032,1,1 +5033,2,2 +5034,2,2 +5035,0,0 +5036,0,0 +5037,2,2 +5038,0,0 +5039,2,2 +5040,1,1 +5041,2,2 +5042,2,2 +5043,1,1 +5044,0,0 +5045,2,2 +5046,2,2 +5047,2,2 +5048,2,2 +5049,2,2 +5050,2,2 +5051,0,0 +5052,2,2 +5053,2,2 +5054,0,0 +5055,1,1 +5056,2,2 +5057,2,2 +5058,0,0 +5059,0,0 +5060,0,0 +5061,2,0 +5062,2,0 +5063,0,0 +5064,2,2 +5065,2,2 +5066,0,0 +5067,2,2 +5068,0,0 +5069,2,2 +5070,2,2 +5071,2,2 +5072,0,0 +5073,2,2 +5074,1,1 +5075,0,0 +5076,0,0 +5077,2,0 +5078,2,2 +5079,2,2 +5080,0,0 +5081,0,0 +5082,1,1 +5083,2,2 +5084,1,1 +5085,2,2 +5086,0,0 +5087,2,2 +5088,0,0 +5089,2,2 +5090,2,2 +5091,0,0 +5092,2,2 +5093,1,1 +5094,2,2 +5095,2,2 +5096,2,2 +5097,2,2 +5098,0,0 +5099,0,0 +5100,0,0 +5101,2,2 +5102,0,0 +5103,0,0 +5104,0,0 +5105,0,0 +5106,2,2 +5107,2,2 +5108,2,2 +5109,2,0 +5110,2,2 +5111,2,2 +5112,0,0 +5113,2,2 +5114,2,2 +5115,2,0 +5116,1,1 +5117,0,0 +5118,0,0 +5119,0,0 +5120,2,2 +5121,2,2 +5122,2,2 +5123,2,2 +5124,1,1 +5125,0,0 +5126,0,0 +5127,0,0 +5128,1,1 +5129,2,2 +5130,2,0 +5131,0,2 +5132,2,2 +5133,2,2 +5134,2,2 +5135,2,2 +5136,2,2 +5137,2,0 +5138,0,0 +5139,2,2 +5140,2,2 +5141,2,2 +5142,2,2 +5143,2,2 +5144,0,0 +5145,2,2 +5146,2,1 +5147,2,2 +5148,1,1 +5149,2,2 +5150,0,0 +5151,2,2 +5152,2,2 +5153,2,2 +5154,2,2 +5155,2,2 +5156,2,2 +5157,2,2 +5158,2,2 +5159,0,0 +5160,1,1 +5161,0,0 +5162,2,2 +5163,0,0 +5164,1,1 +5165,2,2 +5166,1,0 +5167,2,0 +5168,0,0 +5169,2,2 +5170,0,0 +5171,2,2 +5172,1,1 +5173,2,2 +5174,2,2 +5175,0,0 +5176,0,0 +5177,1,1 +5178,2,2 +5179,2,2 +5180,2,1 +5181,1,1 +5182,0,0 +5183,2,2 +5184,0,0 +5185,0,0 +5186,2,2 +5187,2,2 +5188,0,0 +5189,0,0 +5190,0,0 +5191,2,2 +5192,2,2 +5193,0,2 +5194,0,0 +5195,2,2 +5196,2,2 +5197,1,1 +5198,0,0 +5199,0,0 +5200,2,2 +5201,0,0 +5202,2,2 +5203,2,2 +5204,2,2 +5205,0,0 +5206,2,2 +5207,2,0 +5208,2,2 +5209,2,2 +5210,2,2 +5211,2,2 +5212,2,2 +5213,2,2 +5214,2,2 +5215,2,2 +5216,2,2 +5217,2,2 +5218,0,0 +5219,2,2 +5220,2,2 +5221,0,2 +5222,2,2 +5223,1,1 +5224,0,0 +5225,2,2 +5226,2,2 +5227,2,2 +5228,0,0 +5229,1,2 +5230,0,0 +5231,0,0 +5232,2,2 +5233,2,2 +5234,2,2 +5235,0,0 +5236,2,2 +5237,2,2 +5238,2,2 +5239,1,1 +5240,2,2 +5241,2,2 +5242,2,2 +5243,0,0 +5244,0,0 +5245,2,2 +5246,2,2 +5247,0,2 +5248,0,0 +5249,2,2 +5250,2,2 +5251,0,2 +5252,2,2 +5253,2,2 +5254,2,2 +5255,2,2 +5256,2,2 +5257,0,0 +5258,2,2 +5259,2,2 +5260,2,2 +5261,2,2 +5262,2,2 +5263,0,0 +5264,2,2 +5265,1,1 +5266,0,0 +5267,1,1 +5268,2,2 +5269,2,2 +5270,2,2 +5271,2,2 +5272,2,2 +5273,0,0 +5274,2,2 +5275,2,2 +5276,2,2 +5277,2,2 +5278,2,2 +5279,2,2 +5280,2,0 +5281,2,2 +5282,2,2 +5283,2,0 +5284,2,2 +5285,0,0 +5286,2,2 +5287,2,0 +5288,2,2 +5289,2,2 +5290,2,2 +5291,0,0 +5292,0,0 +5293,2,2 +5294,1,1 +5295,2,2 +5296,2,2 +5297,2,2 +5298,0,0 +5299,2,0 +5300,2,2 +5301,0,0 +5302,2,2 +5303,2,0 +5304,2,2 +5305,2,0 +5306,2,2 +5307,2,2 +5308,2,2 +5309,2,2 +5310,2,2 +5311,2,2 +5312,2,2 +5313,2,2 +5314,2,2 +5315,0,0 +5316,2,2 +5317,0,0 +5318,0,2 +5319,0,0 +5320,2,2 +5321,2,2 +5322,2,2 +5323,2,2 +5324,2,0 +5325,2,0 +5326,2,2 +5327,2,2 +5328,2,2 +5329,2,2 +5330,2,2 +5331,2,2 +5332,0,0 +5333,2,2 +5334,0,0 +5335,2,2 +5336,2,0 +5337,0,0 +5338,0,0 +5339,2,2 +5340,2,0 +5341,0,0 +5342,1,1 +5343,0,0 +5344,2,2 +5345,0,0 +5346,2,2 +5347,2,2 +5348,0,0 +5349,2,2 +5350,2,2 +5351,0,0 +5352,2,2 +5353,2,2 +5354,0,0 +5355,2,2 +5356,0,0 +5357,2,2 +5358,0,0 +5359,1,1 +5360,2,2 +5361,0,0 +5362,0,0 +5363,2,2 +5364,0,0 +5365,2,1 +5366,2,2 +5367,0,0 +5368,2,2 +5369,2,2 +5370,0,0 +5371,2,2 +5372,2,2 +5373,2,2 +5374,2,2 +5375,0,0 +5376,0,0 +5377,2,2 +5378,2,2 +5379,2,2 +5380,2,2 +5381,2,2 +5382,2,2 +5383,0,0 +5384,2,2 +5385,2,2 +5386,2,2 +5387,2,2 +5388,2,2 +5389,2,2 +5390,2,2 +5391,2,2 +5392,2,2 +5393,2,2 +5394,0,0 +5395,2,2 +5396,2,2 +5397,0,0 +5398,0,0 +5399,2,0 +5400,0,0 +5401,2,2 +5402,2,2 +5403,2,2 +5404,1,1 +5405,2,2 +5406,2,2 +5407,2,2 +5408,2,2 +5409,2,2 +5410,2,2 +5411,2,2 +5412,2,0 +5413,0,0 +5414,2,2 +5415,2,2 +5416,2,2 +5417,0,0 +5418,0,0 +5419,2,2 +5420,2,2 +5421,2,2 +5422,0,0 +5423,0,0 +5424,2,2 +5425,0,0 +5426,2,2 +5427,2,2 +5428,2,2 +5429,2,2 +5430,0,0 +5431,2,2 +5432,2,2 +5433,2,2 +5434,2,0 +5435,2,2 +5436,0,0 +5437,2,0 +5438,0,0 +5439,0,0 +5440,2,2 +5441,2,2 +5442,2,2 +5443,2,2 +5444,2,2 +5445,2,2 +5446,2,1 +5447,0,0 +5448,1,1 +5449,2,2 +5450,2,0 +5451,0,0 +5452,2,2 +5453,2,2 +5454,2,2 +5455,2,2 +5456,0,0 +5457,2,2 +5458,0,0 +5459,2,2 +5460,1,1 +5461,0,0 +5462,2,0 +5463,0,0 +5464,2,2 +5465,2,2 +5466,0,0 +5467,2,2 +5468,2,2 +5469,2,2 +5470,0,0 +5471,2,2 +5472,0,0 +5473,2,2 +5474,1,1 +5475,2,2 +5476,2,2 +5477,2,2 +5478,2,2 +5479,0,0 +5480,0,0 +5481,2,2 +5482,2,2 +5483,2,2 +5484,0,0 +5485,2,2 +5486,2,2 +5487,2,2 +5488,2,0 +5489,2,2 +5490,2,2 +5491,2,2 +5492,0,0 +5493,2,2 +5494,2,2 +5495,2,2 +5496,2,2 +5497,2,2 +5498,0,0 +5499,2,2 +5500,0,0 +5501,2,2 +5502,0,2 +5503,1,1 +5504,0,0 +5505,0,0 +5506,2,1 +5507,2,0 +5508,0,0 +5509,1,1 +5510,0,0 +5511,0,0 +5512,0,0 +5513,2,2 +5514,2,2 +5515,0,0 +5516,2,2 +5517,0,1 +5518,2,2 +5519,0,0 +5520,0,0 +5521,0,2 +5522,0,0 +5523,2,2 +5524,2,2 +5525,2,2 +5526,2,2 +5527,2,2 +5528,2,2 +5529,0,0 +5530,2,2 +5531,0,0 +5532,2,2 +5533,2,2 +5534,2,2 +5535,0,0 +5536,0,0 +5537,2,2 +5538,2,2 +5539,0,0 +5540,2,2 +5541,2,2 +5542,2,2 +5543,2,2 +5544,2,1 +5545,2,2 +5546,0,0 +5547,0,0 +5548,0,0 +5549,2,2 +5550,2,2 +5551,2,2 +5552,0,0 +5553,2,2 +5554,2,2 +5555,2,2 +5556,0,0 +5557,2,2 +5558,2,2 +5559,2,0 +5560,2,2 +5561,0,0 +5562,0,0 +5563,2,2 +5564,2,2 +5565,1,1 +5566,2,2 +5567,2,2 +5568,0,0 +5569,0,0 +5570,2,2 +5571,0,0 +5572,0,0 +5573,1,1 +5574,2,0 +5575,0,2 +5576,1,1 +5577,2,2 +5578,2,2 +5579,1,1 +5580,2,2 +5581,2,2 +5582,2,2 +5583,2,0 +5584,2,2 +5585,0,0 +5586,2,2 +5587,2,2 +5588,2,1 +5589,2,2 +5590,0,0 +5591,2,2 +5592,0,0 +5593,2,2 +5594,2,2 +5595,2,2 +5596,0,0 +5597,2,2 +5598,0,0 +5599,0,0 +5600,1,1 +5601,2,2 +5602,2,2 +5603,2,2 +5604,2,2 +5605,0,0 +5606,0,0 +5607,2,2 +5608,0,0 +5609,2,2 +5610,2,2 +5611,2,0 +5612,2,2 +5613,2,2 +5614,2,2 +5615,0,0 +5616,2,2 +5617,2,2 +5618,2,2 +5619,2,2 +5620,2,2 +5621,2,2 +5622,1,1 +5623,2,0 +5624,0,0 +5625,0,0 +5626,0,0 +5627,0,0 +5628,0,0 +5629,2,2 +5630,0,0 +5631,2,2 +5632,2,2 +5633,2,2 +5634,2,0 +5635,0,0 +5636,1,1 +5637,0,0 +5638,2,2 +5639,2,2 +5640,2,2 +5641,0,0 +5642,2,2 +5643,2,2 +5644,0,0 +5645,0,0 +5646,2,2 +5647,2,2 +5648,2,2 +5649,1,1 +5650,2,2 +5651,2,2 +5652,2,0 +5653,2,2 +5654,2,2 +5655,0,0 +5656,0,0 +5657,0,0 +5658,2,2 +5659,0,0 +5660,2,2 +5661,0,0 +5662,2,2 +5663,0,0 +5664,0,0 +5665,2,2 +5666,0,0 +5667,0,0 +5668,2,2 +5669,2,2 +5670,2,2 +5671,2,2 +5672,0,0 +5673,1,1 +5674,2,2 +5675,0,0 +5676,2,2 +5677,0,0 +5678,0,0 +5679,2,2 +5680,2,2 +5681,0,0 +5682,0,0 +5683,2,2 +5684,0,0 +5685,0,0 +5686,1,1 +5687,2,2 +5688,2,2 +5689,0,0 +5690,0,0 +5691,2,2 +5692,2,0 +5693,2,2 +5694,2,2 +5695,2,2 +5696,2,2 +5697,2,2 +5698,2,2 +5699,0,0 +5700,2,2 +5701,2,1 +5702,0,0 +5703,2,2 +5704,0,0 +5705,2,2 +5706,2,2 +5707,0,0 +5708,2,2 +5709,2,2 +5710,2,2 +5711,2,2 +5712,2,2 +5713,0,0 +5714,1,1 +5715,2,2 +5716,2,2 +5717,0,0 +5718,2,2 +5719,2,2 +5720,0,0 +5721,2,2 +5722,2,2 +5723,0,0 +5724,2,2 +5725,2,2 +5726,2,2 +5727,0,0 +5728,2,2 +5729,2,2 +5730,2,2 +5731,1,1 +5732,2,2 +5733,2,2 +5734,2,2 +5735,2,2 +5736,2,2 +5737,0,0 +5738,2,2 +5739,0,0 +5740,2,2 +5741,2,2 +5742,2,2 +5743,0,0 +5744,2,2 +5745,2,2 +5746,2,2 +5747,2,2 +5748,0,0 +5749,2,2 +5750,2,2 +5751,2,2 +5752,2,2 +5753,2,2 +5754,2,2 +5755,2,1 +5756,2,2 +5757,0,0 +5758,2,2 +5759,0,0 +5760,2,0 +5761,2,2 +5762,2,2 +5763,0,0 +5764,0,0 +5765,0,0 +5766,2,2 +5767,2,2 +5768,2,2 +5769,2,2 +5770,2,2 +5771,2,1 +5772,2,2 +5773,2,2 +5774,2,2 +5775,2,0 +5776,2,2 +5777,2,2 +5778,2,2 +5779,0,0 +5780,2,2 +5781,2,2 +5782,2,2 +5783,0,0 +5784,2,2 +5785,0,0 +5786,2,2 +5787,2,0 +5788,2,2 +5789,2,0 +5790,2,2 +5791,0,0 +5792,2,2 +5793,2,2 +5794,2,2 +5795,2,2 +5796,2,2 +5797,2,2 +5798,2,2 +5799,2,2 +5800,0,0 +5801,0,0 +5802,0,0 +5803,1,1 +5804,2,2 +5805,2,2 +5806,0,0 +5807,2,2 +5808,2,2 +5809,2,2 +5810,2,2 +5811,0,0 +5812,0,0 +5813,2,2 +5814,0,0 +5815,0,0 +5816,2,2 +5817,2,2 +5818,2,2 +5819,1,1 +5820,2,2 +5821,2,2 +5822,2,2 +5823,0,0 +5824,0,0 +5825,2,2 +5826,2,2 +5827,2,2 +5828,0,0 +5829,2,2 +5830,2,2 +5831,2,2 +5832,2,2 +5833,2,2 +5834,2,0 +5835,1,1 +5836,0,0 +5837,0,0 +5838,2,2 +5839,2,1 +5840,0,0 +5841,1,1 +5842,2,2 +5843,2,2 +5844,2,2 +5845,2,2 +5846,0,0 +5847,2,2 +5848,2,2 +5849,2,2 +5850,2,2 +5851,0,0 +5852,2,2 +5853,0,0 +5854,2,2 +5855,2,2 +5856,0,0 +5857,2,0 +5858,2,2 +5859,2,2 +5860,2,0 +5861,2,2 +5862,2,2 +5863,2,2 +5864,2,2 +5865,0,0 +5866,2,2 +5867,2,2 +5868,2,2 +5869,0,0 +5870,2,2 +5871,2,2 +5872,2,2 +5873,0,0 +5874,0,0 +5875,0,0 +5876,0,0 +5877,2,2 +5878,2,2 +5879,2,2 +5880,2,2 +5881,2,0 +5882,2,2 +5883,0,0 +5884,2,2 +5885,1,1 +5886,0,0 +5887,2,2 +5888,1,1 +5889,0,0 +5890,2,2 +5891,2,2 +5892,0,0 +5893,2,2 +5894,2,2 +5895,2,2 +5896,0,0 +5897,2,2 +5898,0,0 +5899,2,2 +5900,2,0 +5901,2,2 +5902,2,2 +5903,0,0 +5904,2,2 +5905,2,2 +5906,2,2 +5907,0,0 +5908,2,2 +5909,2,0 +5910,2,0 +5911,2,2 +5912,2,2 +5913,1,1 +5914,2,2 +5915,2,2 +5916,2,2 +5917,1,1 +5918,0,0 +5919,0,0 +5920,0,0 +5921,2,2 +5922,2,2 +5923,2,2 +5924,2,2 +5925,2,2 +5926,2,2 +5927,1,2 +5928,0,0 +5929,2,2 +5930,1,1 +5931,0,0 +5932,0,0 +5933,2,2 +5934,2,0 +5935,2,2 +5936,1,1 +5937,2,2 +5938,2,2 +5939,2,2 +5940,2,2 +5941,0,0 +5942,2,2 +5943,2,2 +5944,2,2 +5945,0,0 +5946,2,2 +5947,2,2 +5948,2,2 +5949,1,1 +5950,2,2 +5951,2,0 +5952,2,2 +5953,0,0 +5954,2,2 +5955,2,2 +5956,2,2 +5957,2,2 +5958,2,2 +5959,2,2 +5960,2,0 +5961,0,0 +5962,2,2 +5963,0,0 +5964,2,2 +5965,2,2 +5966,2,2 +5967,2,2 +5968,0,0 +5969,2,2 +5970,2,2 +5971,0,0 +5972,1,1 +5973,2,2 +5974,1,1 +5975,2,2 +5976,0,0 +5977,2,2 +5978,0,0 +5979,0,0 +5980,2,2 +5981,2,2 +5982,2,0 +5983,0,0 +5984,2,2 +5985,0,0 +5986,2,2 +5987,0,0 +5988,2,2 +5989,2,2 +5990,0,0 +5991,2,2 +5992,2,2 +5993,2,2 +5994,2,2 +5995,2,2 +5996,2,2 +5997,2,2 +5998,0,0 +5999,0,0 +6000,2,2 +6001,0,0 +6002,0,0 +6003,2,2 +6004,2,2 +6005,2,2 +6006,0,0 +6007,1,1 +6008,2,2 +6009,2,0 +6010,1,1 +6011,2,2 +6012,2,2 +6013,0,0 +6014,0,0 +6015,0,0 +6016,2,2 +6017,2,2 +6018,2,2 +6019,2,2 +6020,2,2 +6021,0,0 +6022,0,0 +6023,2,2 +6024,2,2 +6025,2,2 +6026,2,2 +6027,0,0 +6028,0,0 +6029,0,0 +6030,0,0 +6031,2,2 +6032,2,2 +6033,0,0 +6034,2,2 +6035,2,2 +6036,0,0 +6037,2,2 +6038,2,2 +6039,2,0 +6040,2,1 +6041,2,2 +6042,0,0 +6043,2,2 +6044,2,2 +6045,2,2 +6046,2,2 +6047,2,2 +6048,2,2 +6049,0,0 +6050,2,2 +6051,2,2 +6052,0,0 +6053,2,2 +6054,0,0 +6055,2,2 +6056,2,2 +6057,2,0 +6058,2,2 +6059,0,0 +6060,2,2 +6061,0,0 +6062,1,1 +6063,2,2 +6064,0,0 +6065,2,2 +6066,2,2 +6067,2,2 +6068,2,2 +6069,0,0 +6070,2,2 +6071,0,0 +6072,2,2 +6073,2,2 +6074,2,2 +6075,1,1 +6076,0,0 +6077,2,2 +6078,2,2 +6079,2,1 +6080,0,0 +6081,2,2 +6082,0,0 +6083,0,0 +6084,2,2 +6085,2,2 +6086,0,0 +6087,0,0 +6088,2,2 +6089,2,2 +6090,2,2 +6091,2,2 +6092,1,1 +6093,2,2 +6094,0,0 +6095,2,2 +6096,0,0 +6097,2,2 +6098,0,0 +6099,2,0 +6100,2,2 +6101,2,2 +6102,2,2 +6103,2,2 +6104,0,0 +6105,2,2 +6106,0,0 +6107,1,1 +6108,1,1 +6109,1,1 +6110,2,2 +6111,2,2 +6112,2,2 +6113,2,2 +6114,2,2 +6115,0,0 +6116,2,2 +6117,2,2 +6118,2,2 +6119,2,2 +6120,2,0 +6121,2,2 +6122,2,2 +6123,2,2 +6124,0,0 +6125,0,0 +6126,0,0 +6127,2,2 +6128,1,1 +6129,2,2 +6130,2,2 +6131,2,2 +6132,2,2 +6133,0,0 +6134,2,2 +6135,2,2 +6136,2,2 +6137,2,2 +6138,2,2 +6139,0,0 +6140,0,2 +6141,2,2 +6142,2,2 +6143,2,2 +6144,2,2 +6145,2,2 +6146,1,1 +6147,0,0 +6148,0,0 +6149,2,2 +6150,2,2 +6151,0,0 +6152,2,2 +6153,0,0 +6154,2,2 +6155,2,2 +6156,0,0 +6157,2,2 +6158,2,2 +6159,2,2 +6160,1,1 +6161,2,2 +6162,2,2 +6163,0,0 +6164,2,0 +6165,1,2 +6166,0,0 +6167,2,2 +6168,2,2 +6169,0,0 +6170,2,2 +6171,2,2 +6172,2,2 +6173,2,2 +6174,2,2 +6175,2,2 +6176,2,2 +6177,2,2 +6178,2,2 +6179,1,1 +6180,2,2 +6181,2,2 +6182,2,2 +6183,2,2 +6184,0,2 +6185,2,2 +6186,0,0 +6187,0,2 +6188,2,2 +6189,0,0 +6190,0,0 +6191,0,0 +6192,0,0 +6193,2,2 +6194,2,2 +6195,0,0 +6196,2,2 +6197,2,2 +6198,0,2 +6199,2,2 +6200,2,2 +6201,2,2 +6202,2,2 +6203,2,2 +6204,2,2 +6205,2,2 +6206,2,2 +6207,2,2 +6208,2,2 +6209,2,2 +6210,2,2 +6211,2,2 +6212,1,1 +6213,2,2 +6214,0,0 +6215,0,0 +6216,0,0 +6217,2,2 +6218,2,2 +6219,0,0 +6220,0,0 +6221,0,0 +6222,2,2 +6223,2,2 +6224,0,0 +6225,2,2 +6226,0,0 +6227,2,2 +6228,0,0 +6229,2,2 +6230,2,2 +6231,0,0 +6232,0,0 +6233,2,2 +6234,2,2 +6235,2,2 +6236,2,2 +6237,0,0 +6238,2,2 +6239,2,2 +6240,0,0 +6241,2,2 +6242,0,0 +6243,0,0 +6244,2,2 +6245,2,2 +6246,2,2 +6247,2,2 +6248,2,1 +6249,2,2 +6250,0,0 +6251,0,0 +6252,0,0 +6253,2,2 +6254,2,2 +6255,2,2 +6256,2,0 +6257,0,0 +6258,2,2 +6259,2,2 +6260,2,2 +6261,2,2 +6262,2,2 +6263,2,2 +6264,0,0 +6265,0,0 +6266,1,1 +6267,0,0 +6268,2,0 +6269,2,2 +6270,2,2 +6271,2,2 +6272,0,0 +6273,2,2 +6274,2,2 +6275,1,1 +6276,2,2 +6277,2,2 +6278,0,0 +6279,2,2 +6280,2,2 +6281,2,2 +6282,2,2 +6283,2,2 +6284,2,2 +6285,2,2 +6286,2,2 +6287,2,2 +6288,0,0 +6289,0,0 +6290,0,0 +6291,0,0 +6292,2,2 +6293,1,1 +6294,2,2 +6295,2,2 +6296,0,0 +6297,2,2 +6298,0,0 +6299,2,2 +6300,0,0 +6301,2,2 +6302,2,0 +6303,0,0 +6304,2,2 +6305,2,2 +6306,2,2 +6307,2,2 +6308,2,2 +6309,0,0 +6310,2,2 +6311,2,2 +6312,2,2 +6313,2,2 +6314,0,0 +6315,2,2 +6316,2,2 +6317,0,0 +6318,2,2 +6319,0,0 +6320,2,2 +6321,0,0 +6322,2,2 +6323,1,1 +6324,1,1 +6325,2,2 +6326,0,0 +6327,0,0 +6328,2,2 +6329,2,2 +6330,2,2 +6331,2,2 +6332,2,2 +6333,0,0 +6334,2,2 +6335,2,2 +6336,2,2 +6337,2,2 +6338,1,1 +6339,0,0 +6340,2,2 +6341,2,2 +6342,0,0 +6343,1,1 +6344,2,2 +6345,2,0 +6346,2,2 +6347,2,2 +6348,2,2 +6349,1,1 +6350,0,0 +6351,0,0 +6352,1,1 +6353,2,2 +6354,2,2 +6355,2,2 +6356,0,0 +6357,2,2 +6358,0,0 +6359,2,2 +6360,2,2 +6361,2,2 +6362,2,2 +6363,2,2 +6364,0,0 +6365,2,1 +6366,0,0 +6367,2,2 +6368,2,2 +6369,0,0 +6370,2,2 +6371,2,2 +6372,2,2 +6373,2,2 +6374,2,2 +6375,0,0 +6376,2,2 +6377,2,2 +6378,2,2 +6379,0,0 +6380,2,0 +6381,2,2 +6382,1,1 +6383,2,2 +6384,2,2 +6385,2,2 +6386,2,2 +6387,0,0 +6388,0,0 +6389,2,2 +6390,2,2 +6391,0,0 +6392,0,0 +6393,0,0 +6394,2,2 +6395,0,0 +6396,2,2 +6397,2,2 +6398,2,2 +6399,2,2 +6400,0,0 +6401,1,1 +6402,2,2 +6403,1,1 +6404,0,0 +6405,2,2 +6406,2,2 +6407,2,2 +6408,2,2 +6409,0,0 +6410,0,0 +6411,0,0 +6412,2,2 +6413,1,1 +6414,0,0 +6415,2,2 +6416,2,2 +6417,0,0 +6418,2,2 +6419,2,2 +6420,2,2 +6421,0,0 +6422,2,2 +6423,0,0 +6424,2,2 +6425,0,0 +6426,2,2 +6427,2,0 +6428,0,0 +6429,0,0 +6430,0,0 +6431,2,2 +6432,2,1 +6433,2,2 +6434,2,2 +6435,2,2 +6436,0,0 +6437,2,2 +6438,0,0 +6439,2,2 +6440,2,2 +6441,2,2 +6442,2,2 +6443,2,2 +6444,2,2 +6445,2,2 +6446,0,0 +6447,2,2 +6448,2,2 +6449,2,2 +6450,0,0 +6451,2,2 +6452,2,2 +6453,0,0 +6454,2,2 +6455,2,2 +6456,2,0 +6457,2,2 +6458,2,2 +6459,0,0 +6460,0,0 +6461,2,0 +6462,0,0 +6463,2,2 +6464,1,1 +6465,2,2 +6466,2,2 +6467,2,2 +6468,0,0 +6469,2,2 +6470,2,2 +6471,0,0 +6472,0,0 +6473,2,2 +6474,2,0 +6475,2,2 +6476,2,2 +6477,2,2 +6478,2,2 +6479,0,0 +6480,0,0 +6481,2,2 +6482,2,2 +6483,2,2 +6484,2,2 +6485,0,0 +6486,2,2 +6487,0,0 +6488,2,2 +6489,1,1 +6490,2,2 +6491,2,2 +6492,2,2 +6493,0,0 +6494,2,2 +6495,2,2 +6496,2,2 +6497,0,0 +6498,2,2 +6499,2,2 +6500,2,2 +6501,2,2 +6502,0,0 +6503,1,1 +6504,0,0 +6505,0,0 +6506,2,2 +6507,2,2 +6508,2,2 +6509,2,2 +6510,2,2 +6511,0,0 +6512,2,2 +6513,0,0 +6514,0,0 +6515,0,0 +6516,0,0 +6517,2,2 +6518,2,2 +6519,2,2 +6520,0,0 +6521,2,2 +6522,2,2 +6523,2,2 +6524,2,2 +6525,2,2 +6526,2,2 +6527,2,2 +6528,2,2 +6529,2,0 +6530,0,0 +6531,2,2 +6532,2,0 +6533,2,2 +6534,2,2 +6535,2,2 +6536,2,2 +6537,2,2 +6538,2,2 +6539,2,2 +6540,2,2 +6541,2,2 +6542,2,2 +6543,2,2 +6544,2,2 +6545,0,0 +6546,2,2 +6547,2,2 +6548,0,0 +6549,2,0 +6550,2,2 +6551,2,2 +6552,2,2 +6553,0,0 +6554,2,2 +6555,0,0 +6556,0,0 +6557,2,2 +6558,2,2 +6559,1,1 +6560,2,2 +6561,2,2 +6562,2,0 +6563,2,2 +6564,0,0 +6565,2,2 +6566,0,0 +6567,2,2 +6568,2,2 +6569,0,0 +6570,2,2 +6571,2,2 +6572,0,0 +6573,1,1 +6574,2,2 +6575,0,0 +6576,2,2 +6577,2,2 +6578,2,2 +6579,0,0 +6580,2,2 +6581,2,2 +6582,2,2 +6583,0,0 +6584,2,2 +6585,2,2 +6586,0,0 +6587,2,2 +6588,2,2 +6589,1,1 +6590,0,0 +6591,2,2 +6592,2,2 +6593,0,0 +6594,0,0 +6595,0,0 +6596,2,2 +6597,2,2 +6598,2,2 +6599,0,0 +6600,2,2 +6601,2,2 +6602,2,2 +6603,2,2 +6604,2,2 +6605,1,1 +6606,2,2 +6607,2,2 +6608,2,2 +6609,0,0 +6610,2,2 +6611,1,1 +6612,2,2 +6613,0,0 +6614,2,2 +6615,2,2 +6616,2,2 +6617,0,0 +6618,2,2 +6619,2,2 +6620,2,2 +6621,2,2 +6622,2,2 +6623,2,2 +6624,2,2 +6625,2,2 +6626,2,2 +6627,2,2 +6628,2,2 +6629,2,2 +6630,2,2 +6631,2,2 +6632,0,0 +6633,2,2 +6634,2,2 +6635,0,0 +6636,0,0 +6637,2,2 +6638,2,2 +6639,2,2 +6640,2,2 +6641,2,2 +6642,0,0 +6643,2,2 +6644,2,2 +6645,2,2 +6646,2,0 +6647,2,2 +6648,0,0 +6649,0,0 +6650,2,2 +6651,2,2 +6652,2,2 +6653,2,2 +6654,2,2 +6655,0,0 +6656,2,2 +6657,2,2 +6658,2,2 +6659,2,1 +6660,0,0 +6661,2,2 +6662,2,2 +6663,2,2 +6664,2,2 +6665,2,2 +6666,2,2 +6667,0,0 +6668,2,2 +6669,2,2 +6670,2,2 +6671,2,2 +6672,2,2 +6673,2,2 +6674,2,2 +6675,2,2 +6676,0,0 +6677,2,1 +6678,2,2 +6679,2,2 +6680,2,2 +6681,0,0 +6682,1,1 +6683,1,1 +6684,0,0 +6685,2,2 +6686,2,2 +6687,0,0 +6688,2,2 +6689,2,2 +6690,1,1 +6691,2,2 +6692,2,2 +6693,2,2 +6694,2,2 +6695,2,2 +6696,0,0 +6697,0,0 +6698,1,2 +6699,2,2 +6700,2,2 +6701,2,2 +6702,2,2 +6703,0,0 +6704,2,2 +6705,1,1 +6706,0,0 +6707,2,2 +6708,0,0 +6709,2,2 +6710,2,2 +6711,0,0 +6712,2,2 +6713,2,2 +6714,2,2 +6715,2,2 +6716,2,2 +6717,2,2 +6718,0,0 +6719,2,2 +6720,2,2 +6721,0,0 +6722,2,2 +6723,0,0 +6724,0,0 +6725,0,2 +6726,1,1 +6727,2,0 +6728,2,2 +6729,2,2 +6730,2,0 +6731,2,1 +6732,2,2 +6733,2,0 +6734,0,0 +6735,2,2 +6736,2,2 +6737,2,2 +6738,2,2 +6739,0,0 +6740,0,0 +6741,0,0 +6742,2,2 +6743,2,2 +6744,0,0 +6745,0,0 +6746,2,2 +6747,2,0 +6748,2,2 +6749,2,2 +6750,2,2 +6751,2,0 +6752,2,2 +6753,2,2 +6754,1,2 +6755,2,2 +6756,2,2 +6757,2,2 +6758,0,0 +6759,2,2 +6760,2,2 +6761,2,0 +6762,2,2 +6763,2,2 +6764,1,1 +6765,0,0 +6766,2,2 +6767,2,2 +6768,2,2 +6769,2,2 +6770,2,2 +6771,2,2 +6772,2,2 +6773,2,2 +6774,0,0 +6775,0,0 +6776,2,2 +6777,2,2 +6778,0,0 +6779,2,2 +6780,1,1 +6781,2,2 +6782,0,0 +6783,2,0 +6784,2,2 +6785,0,0 +6786,0,0 +6787,2,0 +6788,2,0 +6789,2,2 +6790,2,2 +6791,0,0 +6792,2,2 +6793,2,2 +6794,2,2 +6795,2,2 +6796,2,2 +6797,2,2 +6798,2,2 +6799,2,2 +6800,2,2 +6801,2,1 +6802,0,0 +6803,0,0 +6804,0,0 +6805,0,0 +6806,2,2 +6807,2,2 +6808,1,2 +6809,2,2 +6810,2,1 +6811,2,2 +6812,1,1 +6813,2,2 +6814,0,0 +6815,2,2 +6816,2,2 +6817,2,2 +6818,2,2 +6819,2,2 +6820,2,2 +6821,2,2 +6822,2,2 +6823,2,2 +6824,2,2 +6825,2,0 +6826,1,1 +6827,2,2 +6828,0,0 +6829,2,2 +6830,0,0 +6831,2,2 +6832,2,0 +6833,2,2 +6834,2,2 +6835,2,2 +6836,0,0 +6837,2,2 +6838,2,2 +6839,2,2 +6840,1,1 +6841,0,0 +6842,2,2 +6843,2,2 +6844,2,0 +6845,2,2 +6846,2,0 +6847,0,0 +6848,2,2 +6849,2,2 +6850,2,2 +6851,2,2 +6852,0,0 +6853,0,0 +6854,2,2 +6855,2,1 +6856,0,0 +6857,2,2 +6858,2,2 +6859,2,2 +6860,0,0 +6861,0,0 +6862,0,0 +6863,2,2 +6864,0,0 +6865,2,2 +6866,0,0 +6867,2,2 +6868,2,2 +6869,2,0 +6870,2,2 +6871,2,0 +6872,2,2 +6873,2,2 +6874,2,2 +6875,2,2 +6876,1,1 +6877,0,0 +6878,2,2 +6879,2,2 +6880,1,1 +6881,1,1 +6882,2,2 +6883,2,2 +6884,2,2 +6885,2,2 +6886,0,0 +6887,2,2 +6888,2,2 +6889,2,2 +6890,2,2 +6891,2,2 +6892,2,2 +6893,2,2 +6894,0,0 +6895,2,2 +6896,2,0 +6897,2,2 +6898,2,2 +6899,2,2 +6900,0,0 +6901,2,2 +6902,2,0 +6903,2,2 +6904,2,2 +6905,0,0 +6906,2,0 +6907,2,2 +6908,0,0 +6909,0,0 +6910,2,2 +6911,0,0 +6912,2,2 +6913,0,0 +6914,2,0 +6915,0,0 +6916,2,2 +6917,2,2 +6918,2,2 +6919,2,2 +6920,2,2 +6921,0,0 +6922,2,2 +6923,2,2 +6924,2,2 +6925,2,2 +6926,2,2 +6927,2,2 +6928,2,2 +6929,2,2 +6930,2,2 +6931,0,0 +6932,2,2 +6933,2,2 +6934,2,2 +6935,0,0 +6936,2,2 +6937,0,0 +6938,1,1 +6939,2,2 +6940,2,2 +6941,2,2 +6942,2,2 +6943,0,0 +6944,0,0 +6945,2,2 +6946,2,2 +6947,2,2 +6948,2,2 +6949,0,0 +6950,0,0 +6951,2,2 +6952,0,0 +6953,0,0 +6954,0,0 +6955,2,2 +6956,2,2 +6957,2,2 +6958,2,0 +6959,2,0 +6960,2,2 +6961,2,2 +6962,0,0 +6963,2,2 +6964,0,0 +6965,2,2 +6966,0,0 +6967,0,0 +6968,0,0 +6969,0,0 +6970,0,0 +6971,2,2 +6972,2,2 +6973,0,0 +6974,2,1 +6975,0,0 +6976,0,0 +6977,2,2 +6978,1,1 +6979,2,2 +6980,2,2 +6981,2,1 +6982,1,1 +6983,0,0 +6984,2,2 +6985,2,2 +6986,0,0 +6987,2,2 +6988,2,2 +6989,0,0 +6990,0,0 +6991,2,2 +6992,2,2 +6993,2,2 +6994,0,0 +6995,2,2 +6996,0,0 +6997,2,2 +6998,2,2 +6999,2,2 +7000,0,0 +7001,2,0 +7002,0,0 +7003,0,0 +7004,0,0 +7005,0,0 +7006,0,0 +7007,2,2 +7008,0,0 +7009,2,2 +7010,0,0 +7011,2,2 +7012,2,2 +7013,0,0 +7014,2,2 +7015,0,0 +7016,2,2 +7017,2,2 +7018,0,0 +7019,0,2 +7020,2,2 +7021,2,2 +7022,2,2 +7023,0,0 +7024,2,2 +7025,2,2 +7026,2,2 +7027,2,2 +7028,2,2 +7029,0,0 +7030,2,2 +7031,2,0 +7032,0,0 +7033,2,2 +7034,0,0 +7035,2,2 +7036,2,2 +7037,2,2 +7038,0,0 +7039,0,0 +7040,2,2 +7041,2,2 +7042,2,0 +7043,0,0 +7044,2,2 +7045,2,2 +7046,2,2 +7047,2,2 +7048,2,1 +7049,2,2 +7050,2,2 +7051,2,2 +7052,0,0 +7053,0,0 +7054,2,2 +7055,0,0 +7056,2,2 +7057,2,2 +7058,2,2 +7059,1,1 +7060,2,2 +7061,2,2 +7062,0,0 +7063,0,0 +7064,1,1 +7065,2,2 +7066,2,2 +7067,2,2 +7068,2,2 +7069,2,2 +7070,1,1 +7071,1,1 +7072,2,2 +7073,1,1 +7074,2,0 +7075,2,2 +7076,2,2 +7077,2,2 +7078,2,2 +7079,2,2 +7080,2,2 +7081,2,2 +7082,2,2 +7083,2,2 +7084,2,2 +7085,2,2 +7086,2,1 +7087,2,2 +7088,2,2 +7089,1,1 +7090,2,2 +7091,2,2 +7092,2,2 +7093,0,0 +7094,2,2 +7095,0,0 +7096,2,2 +7097,2,2 +7098,2,2 +7099,0,0 +7100,2,0 +7101,2,2 +7102,2,2 +7103,2,2 +7104,2,2 +7105,2,2 +7106,2,2 +7107,2,2 +7108,0,0 +7109,2,2 +7110,2,2 +7111,2,2 +7112,0,0 +7113,0,0 +7114,2,2 +7115,2,2 +7116,0,0 +7117,0,0 +7118,0,0 +7119,2,2 +7120,0,0 +7121,0,0 +7122,2,2 +7123,2,2 +7124,2,2 +7125,2,2 +7126,2,0 +7127,2,2 +7128,2,2 +7129,2,2 +7130,2,2 +7131,2,0 +7132,2,2 +7133,0,0 +7134,0,0 +7135,2,2 +7136,2,2 +7137,0,0 +7138,2,2 +7139,0,0 +7140,2,2 +7141,2,2 +7142,0,0 +7143,2,2 +7144,0,0 +7145,2,2 +7146,2,2 +7147,2,2 +7148,2,2 +7149,2,0 +7150,0,0 +7151,0,2 +7152,0,0 +7153,2,2 +7154,2,2 +7155,2,2 +7156,2,2 +7157,2,2 +7158,2,2 +7159,0,0 +7160,1,1 +7161,2,2 +7162,2,2 +7163,2,2 +7164,1,1 +7165,2,2 +7166,2,2 +7167,0,0 +7168,2,2 +7169,2,2 +7170,1,1 +7171,2,2 +7172,1,1 +7173,0,0 +7174,2,2 +7175,2,2 +7176,2,2 +7177,2,2 +7178,2,2 +7179,0,0 +7180,0,0 +7181,2,2 +7182,2,2 +7183,2,2 +7184,0,0 +7185,2,2 +7186,2,2 +7187,2,2 +7188,0,0 +7189,0,0 +7190,2,0 +7191,2,2 +7192,0,0 +7193,2,2 +7194,2,2 +7195,2,2 +7196,2,1 +7197,2,2 +7198,2,2 +7199,2,2 +7200,0,0 +7201,2,2 +7202,2,2 +7203,2,1 +7204,2,0 +7205,0,0 +7206,0,0 +7207,2,2 +7208,2,2 +7209,0,0 +7210,0,0 +7211,2,2 +7212,0,0 +7213,2,2 +7214,2,2 +7215,1,1 +7216,2,2 +7217,2,2 +7218,2,2 +7219,2,2 +7220,2,2 +7221,2,2 +7222,2,2 +7223,0,0 +7224,2,2 +7225,0,0 +7226,0,0 +7227,0,0 +7228,2,2 +7229,2,2 +7230,2,2 +7231,2,2 +7232,2,2 +7233,2,2 +7234,2,2 +7235,2,2 +7236,2,2 +7237,1,1 +7238,2,2 +7239,1,1 +7240,2,2 +7241,2,2 +7242,2,2 +7243,2,0 +7244,2,2 +7245,2,2 +7246,2,2 +7247,0,0 +7248,0,0 +7249,2,2 +7250,0,0 +7251,0,0 +7252,0,0 +7253,2,2 +7254,0,0 +7255,0,0 +7256,2,2 +7257,2,2 +7258,0,0 +7259,2,2 +7260,0,0 +7261,2,2 +7262,0,0 +7263,1,1 +7264,2,2 +7265,2,2 +7266,2,2 +7267,2,2 +7268,2,2 +7269,2,2 +7270,0,0 +7271,2,2 +7272,0,0 +7273,1,1 +7274,2,2 +7275,2,2 +7276,1,1 +7277,2,2 +7278,2,2 +7279,0,0 +7280,2,2 +7281,2,1 +7282,2,2 +7283,0,0 +7284,2,2 +7285,2,2 +7286,2,0 +7287,0,0 +7288,1,1 +7289,2,2 +7290,0,0 +7291,2,0 +7292,2,2 +7293,0,0 +7294,2,0 +7295,0,0 +7296,2,2 +7297,0,0 +7298,0,0 +7299,2,2 +7300,2,2 +7301,2,2 +7302,2,2 +7303,2,2 +7304,2,2 +7305,0,0 +7306,2,2 +7307,2,2 +7308,0,0 +7309,2,2 +7310,2,2 +7311,2,2 +7312,2,2 +7313,2,2 +7314,0,0 +7315,0,0 +7316,2,2 +7317,0,0 +7318,2,2 +7319,2,2 +7320,2,2 +7321,0,0 +7322,2,2 +7323,0,0 +7324,0,0 +7325,1,1 +7326,0,0 +7327,2,2 +7328,2,2 +7329,2,2 +7330,0,0 +7331,2,2 +7332,2,2 +7333,2,2 +7334,0,0 +7335,2,2 +7336,2,2 +7337,2,2 +7338,2,2 +7339,0,0 +7340,2,2 +7341,0,0 +7342,2,2 +7343,0,0 +7344,2,2 +7345,0,0 +7346,0,0 +7347,2,2 +7348,2,2 +7349,2,2 +7350,0,0 +7351,0,0 +7352,0,0 +7353,1,1 +7354,0,0 +7355,2,2 +7356,0,0 +7357,2,2 +7358,2,2 +7359,0,0 +7360,2,2 +7361,2,2 +7362,2,2 +7363,2,2 +7364,1,1 +7365,2,2 +7366,2,2 +7367,2,2 +7368,1,1 +7369,2,0 +7370,2,0 +7371,2,2 +7372,2,2 +7373,1,1 +7374,2,2 +7375,2,0 +7376,2,2 +7377,2,2 +7378,2,0 +7379,2,2 +7380,0,0 +7381,2,2 +7382,2,2 +7383,2,2 +7384,2,0 +7385,0,0 +7386,0,0 +7387,2,2 +7388,2,2 +7389,2,2 +7390,2,1 +7391,0,0 +7392,2,2 +7393,2,2 +7394,2,2 +7395,2,2 +7396,2,2 +7397,2,1 +7398,2,2 +7399,2,2 +7400,0,0 +7401,2,2 +7402,2,2 +7403,0,0 +7404,0,0 +7405,2,2 +7406,2,2 +7407,1,1 +7408,2,2 +7409,2,2 +7410,2,2 +7411,2,2 +7412,0,0 +7413,2,2 +7414,2,2 +7415,2,2 +7416,0,0 +7417,2,2 +7418,0,0 +7419,2,2 +7420,0,0 +7421,2,2 +7422,2,2 +7423,0,0 +7424,2,2 +7425,2,2 +7426,0,0 +7427,2,2 +7428,2,2 +7429,2,2 +7430,0,0 +7431,2,2 +7432,0,0 +7433,0,0 +7434,2,2 +7435,1,1 +7436,2,2 +7437,0,0 +7438,2,2 +7439,0,0 +7440,0,0 +7441,2,2 +7442,2,2 +7443,2,2 +7444,2,2 +7445,2,2 +7446,2,2 +7447,2,2 +7448,1,1 +7449,2,2 +7450,2,2 +7451,0,0 +7452,2,2 +7453,2,2 +7454,2,2 +7455,2,2 +7456,2,2 +7457,2,2 +7458,0,0 +7459,0,0 +7460,0,0 +7461,2,0 +7462,2,2 +7463,0,0 +7464,2,2 +7465,0,0 +7466,2,2 +7467,2,2 +7468,2,2 +7469,2,2 +7470,2,2 +7471,0,0 +7472,2,1 +7473,0,0 +7474,2,2 +7475,2,2 +7476,2,2 +7477,0,0 +7478,1,1 +7479,0,0 +7480,2,2 +7481,2,2 +7482,2,2 +7483,0,0 +7484,2,2 +7485,2,2 +7486,2,2 +7487,0,0 +7488,2,2 +7489,2,0 +7490,0,0 +7491,0,0 +7492,1,1 +7493,0,0 +7494,2,2 +7495,2,2 +7496,2,2 +7497,0,0 +7498,2,2 +7499,2,0 +7500,0,2 +7501,2,2 +7502,2,2 +7503,2,2 +7504,2,2 +7505,0,0 +7506,0,0 +7507,2,2 +7508,2,2 +7509,2,2 +7510,2,2 +7511,2,2 +7512,2,2 +7513,0,0 +7514,2,2 +7515,2,2 +7516,2,2 +7517,2,2 +7518,2,1 +7519,2,2 +7520,2,0 +7521,0,0 +7522,2,2 +7523,2,2 +7524,2,2 +7525,0,0 +7526,2,2 +7527,0,0 +7528,0,0 +7529,0,0 +7530,0,0 +7531,2,2 +7532,2,2 +7533,2,2 +7534,2,2 +7535,2,2 +7536,2,2 +7537,0,0 +7538,2,2 +7539,0,0 +7540,2,2 +7541,2,2 +7542,0,0 +7543,2,0 +7544,0,0 +7545,2,2 +7546,0,0 +7547,2,2 +7548,0,0 +7549,0,0 +7550,2,2 +7551,0,0 +7552,2,2 +7553,2,2 +7554,2,2 +7555,2,2 +7556,0,0 +7557,2,2 +7558,2,2 +7559,1,1 +7560,2,2 +7561,2,2 +7562,2,2 +7563,2,2 +7564,2,2 +7565,2,2 +7566,0,0 +7567,0,0 +7568,0,0 +7569,2,2 +7570,1,1 +7571,2,2 +7572,0,0 +7573,2,2 +7574,2,2 +7575,2,1 +7576,1,2 +7577,2,2 +7578,2,2 +7579,0,0 +7580,2,2 +7581,2,2 +7582,0,0 +7583,0,0 +7584,2,2 +7585,0,0 +7586,2,2 +7587,2,2 +7588,2,2 +7589,2,2 +7590,0,0 +7591,2,2 +7592,2,2 +7593,2,2 +7594,2,0 +7595,2,2 +7596,0,0 +7597,2,2 +7598,2,2 +7599,2,2 +7600,2,2 +7601,2,2 +7602,0,0 +7603,2,2 +7604,2,2 +7605,2,2 +7606,2,2 +7607,2,2 +7608,0,0 +7609,0,0 +7610,0,0 +7611,2,2 +7612,2,2 +7613,2,1 +7614,2,2 +7615,2,2 +7616,2,2 +7617,2,2 +7618,2,2 +7619,0,0 +7620,1,1 +7621,2,2 +7622,0,0 +7623,2,2 +7624,2,2 +7625,0,0 +7626,2,2 +7627,2,2 +7628,2,2 +7629,2,2 +7630,2,2 +7631,0,0 +7632,2,2 +7633,2,0 +7634,2,2 +7635,2,2 +7636,2,2 +7637,2,2 +7638,0,0 +7639,1,1 +7640,2,2 +7641,2,2 +7642,2,2 +7643,2,2 +7644,2,2 +7645,0,0 +7646,2,2 +7647,2,2 +7648,0,0 +7649,0,0 +7650,1,1 +7651,2,2 +7652,2,2 +7653,2,2 +7654,2,2 +7655,0,0 +7656,2,2 +7657,2,2 +7658,2,2 +7659,2,2 +7660,1,1 +7661,2,2 +7662,2,2 +7663,2,2 +7664,0,0 +7665,2,2 +7666,2,2 +7667,0,0 +7668,0,0 +7669,2,2 +7670,2,2 +7671,1,1 +7672,2,0 +7673,1,1 +7674,2,2 +7675,2,2 +7676,1,1 +7677,2,0 +7678,2,2 +7679,2,2 +7680,2,2 +7681,0,0 +7682,2,2 +7683,2,2 +7684,2,2 +7685,2,2 +7686,2,2 +7687,2,2 +7688,2,2 +7689,2,2 +7690,2,2 +7691,2,2 +7692,2,2 +7693,2,2 +7694,2,2 +7695,1,1 +7696,0,0 +7697,2,2 +7698,2,2 +7699,2,2 +7700,2,2 +7701,1,1 +7702,2,2 +7703,2,2 +7704,1,1 +7705,0,0 +7706,0,0 +7707,0,2 +7708,0,0 +7709,2,2 +7710,0,0 +7711,2,2 +7712,2,2 +7713,2,2 +7714,1,1 +7715,2,0 +7716,0,0 +7717,0,0 +7718,0,0 +7719,2,2 +7720,0,0 +7721,0,0 +7722,2,2 +7723,2,2 +7724,2,2 +7725,2,2 +7726,0,0 +7727,0,0 +7728,2,2 +7729,2,2 +7730,2,2 +7731,2,2 +7732,2,2 +7733,2,2 +7734,0,0 +7735,2,2 +7736,2,2 +7737,2,2 +7738,2,2 +7739,0,0 +7740,2,2 +7741,2,2 +7742,2,2 +7743,2,2 +7744,2,2 +7745,2,2 +7746,2,2 +7747,0,0 +7748,2,2 +7749,0,0 +7750,1,1 +7751,2,2 +7752,2,2 +7753,0,0 +7754,2,2 +7755,2,2 +7756,2,0 +7757,2,1 +7758,2,2 +7759,0,0 +7760,2,0 +7761,2,2 +7762,2,2 +7763,0,0 +7764,2,2 +7765,0,0 +7766,2,2 +7767,0,0 +7768,2,2 +7769,2,2 +7770,2,2 +7771,2,2 +7772,2,2 +7773,2,2 +7774,2,2 +7775,0,0 +7776,2,2 +7777,0,2 +7778,2,2 +7779,1,1 +7780,2,0 +7781,2,2 +7782,2,2 +7783,0,0 +7784,0,0 +7785,2,2 +7786,2,2 +7787,0,0 +7788,2,2 +7789,2,2 +7790,2,2 +7791,2,2 +7792,2,2 +7793,2,2 +7794,0,0 +7795,2,2 +7796,2,2 +7797,2,2 +7798,0,0 +7799,2,2 +7800,0,0 +7801,2,2 +7802,2,2 +7803,2,2 +7804,0,0 +7805,0,0 +7806,2,0 +7807,2,2 +7808,2,2 +7809,0,0 +7810,0,0 +7811,2,2 +7812,2,2 +7813,0,0 +7814,1,1 +7815,2,2 +7816,2,2 +7817,2,2 +7818,2,2 +7819,2,2 +7820,1,2 +7821,0,0 +7822,2,2 +7823,0,0 +7824,2,2 +7825,2,2 +7826,0,0 +7827,2,2 +7828,0,0 +7829,1,1 +7830,0,0 +7831,1,1 +7832,2,2 +7833,0,0 +7834,0,0 +7835,2,2 +7836,2,2 +7837,0,0 +7838,2,2 +7839,2,2 +7840,2,2 +7841,0,0 +7842,0,2 +7843,0,0 +7844,0,0 +7845,2,2 +7846,2,0 +7847,2,2 +7848,2,2 +7849,2,2 +7850,0,0 +7851,2,2 +7852,2,1 +7853,2,0 +7854,2,0 +7855,2,2 +7856,2,2 +7857,0,0 +7858,2,2 +7859,2,0 +7860,0,0 +7861,0,0 +7862,0,0 +7863,2,2 +7864,2,2 +7865,0,0 +7866,2,1 +7867,2,2 +7868,0,0 +7869,2,2 +7870,0,0 +7871,2,2 +7872,0,0 +7873,2,2 +7874,2,2 +7875,2,2 +7876,2,2 +7877,2,2 +7878,2,2 +7879,2,0 +7880,2,2 +7881,2,2 +7882,2,2 +7883,2,0 +7884,2,2 +7885,0,0 +7886,2,1 +7887,0,0 +7888,2,2 +7889,2,2 +7890,0,0 +7891,2,2 +7892,2,2 +7893,0,0 +7894,2,2 +7895,2,2 +7896,2,2 +7897,0,0 +7898,0,0 +7899,0,0 +7900,0,0 +7901,2,2 +7902,2,2 +7903,2,0 +7904,0,0 +7905,2,2 +7906,2,2 +7907,1,1 +7908,2,2 +7909,2,2 +7910,2,0 +7911,0,0 +7912,0,0 +7913,2,2 +7914,0,0 +7915,2,2 +7916,2,2 +7917,2,2 +7918,2,0 +7919,2,2 +7920,0,0 +7921,0,0 +7922,2,2 +7923,2,2 +7924,2,2 +7925,2,2 +7926,2,2 +7927,2,2 +7928,0,0 +7929,2,2 +7930,0,0 +7931,2,2 +7932,2,2 +7933,1,1 +7934,0,0 +7935,2,2 +7936,2,2 +7937,0,0 +7938,2,2 +7939,2,2 +7940,2,2 +7941,0,0 +7942,2,2 +7943,2,2 +7944,2,2 +7945,1,1 +7946,2,2 +7947,2,2 +7948,2,2 +7949,0,0 +7950,2,2 +7951,0,0 +7952,1,1 +7953,1,1 +7954,2,2 +7955,0,0 +7956,2,2 +7957,0,2 +7958,2,2 +7959,2,0 +7960,0,0 +7961,2,2 +7962,1,1 +7963,1,1 +7964,2,2 +7965,2,2 +7966,2,2 +7967,2,2 +7968,2,2 +7969,2,2 +7970,2,2 +7971,2,2 +7972,2,2 +7973,0,0 +7974,0,0 +7975,0,0 +7976,2,2 +7977,0,0 +7978,2,2 +7979,2,2 +7980,2,2 +7981,0,0 +7982,2,2 +7983,2,2 +7984,0,0 +7985,2,2 +7986,2,2 +7987,2,2 +7988,2,2 +7989,2,2 +7990,2,2 +7991,0,0 +7992,2,2 +7993,2,2 +7994,0,0 +7995,0,0 +7996,2,2 +7997,0,0 +7998,2,2 +7999,2,2 +8000,0,0 +8001,1,1 +8002,2,2 +8003,0,0 +8004,2,2 +8005,2,2 +8006,2,2 +8007,2,0 +8008,2,2 +8009,2,2 +8010,2,2 +8011,2,1 +8012,1,1 +8013,0,0 +8014,1,2 +8015,2,2 +8016,2,2 +8017,2,2 +8018,0,0 +8019,2,2 +8020,2,2 +8021,2,2 +8022,2,2 +8023,2,2 +8024,2,2 +8025,2,2 +8026,2,2 +8027,2,2 +8028,2,2 +8029,2,2 +8030,0,0 +8031,1,2 +8032,0,0 +8033,2,2 +8034,2,0 +8035,2,2 +8036,2,2 +8037,2,2 +8038,0,0 +8039,2,2 +8040,0,0 +8041,2,2 +8042,2,2 +8043,2,2 +8044,0,0 +8045,2,2 +8046,2,2 +8047,2,2 +8048,2,2 +8049,1,1 +8050,2,2 +8051,2,2 +8052,2,2 +8053,2,2 +8054,2,2 +8055,2,2 +8056,0,0 +8057,2,2 +8058,0,0 +8059,2,1 +8060,2,2 +8061,2,2 +8062,2,2 +8063,0,0 +8064,0,0 +8065,2,2 +8066,2,2 +8067,2,2 +8068,0,0 +8069,2,2 +8070,2,2 +8071,0,0 +8072,1,1 +8073,2,2 +8074,2,2 +8075,2,2 +8076,0,0 +8077,0,0 +8078,2,2 +8079,2,2 +8080,2,2 +8081,2,2 +8082,0,0 +8083,2,2 +8084,2,2 +8085,2,2 +8086,1,1 +8087,2,2 +8088,2,2 +8089,1,1 +8090,2,2 +8091,2,2 +8092,2,2 +8093,0,0 +8094,2,2 +8095,0,0 +8096,0,0 +8097,2,2 +8098,2,2 +8099,0,0 +8100,0,0 +8101,2,2 +8102,2,2 +8103,1,1 +8104,0,0 +8105,2,2 +8106,2,2 +8107,0,0 +8108,2,2 +8109,2,2 +8110,2,2 +8111,2,2 +8112,0,0 +8113,0,0 +8114,0,0 +8115,2,1 +8116,2,2 +8117,2,2 +8118,2,2 +8119,2,2 +8120,2,2 +8121,2,2 +8122,1,1 +8123,2,2 +8124,2,2 +8125,0,0 +8126,2,2 +8127,2,2 +8128,2,2 +8129,2,2 +8130,2,2 +8131,2,2 +8132,2,2 +8133,0,0 +8134,2,2 +8135,0,0 +8136,2,2 +8137,2,2 +8138,2,2 +8139,2,2 +8140,0,0 +8141,2,2 +8142,2,2 +8143,2,2 +8144,2,2 +8145,2,2 +8146,2,2 +8147,0,0 +8148,0,0 +8149,2,2 +8150,0,0 +8151,0,0 +8152,2,2 +8153,2,2 +8154,0,0 +8155,2,2 +8156,0,2 +8157,2,2 +8158,0,0 +8159,2,2 +8160,2,2 +8161,2,2 +8162,2,2 +8163,2,2 +8164,2,0 +8165,2,2 +8166,0,0 +8167,2,2 +8168,2,2 +8169,2,2 +8170,2,2 +8171,2,2 +8172,0,0 +8173,2,0 +8174,2,1 +8175,2,2 +8176,2,2 +8177,0,0 +8178,2,2 +8179,2,2 +8180,2,2 +8181,2,2 +8182,2,0 +8183,0,0 +8184,1,1 +8185,0,0 +8186,2,2 +8187,2,2 +8188,2,2 +8189,2,2 +8190,2,2 +8191,2,2 +8192,0,0 +8193,2,2 +8194,0,0 +8195,2,0 +8196,0,0 +8197,1,1 +8198,2,2 +8199,0,0 +8200,0,0 +8201,2,2 +8202,0,0 +8203,1,1 +8204,0,0 +8205,0,0 +8206,2,0 +8207,0,0 +8208,2,2 +8209,2,2 +8210,2,2 +8211,2,2 +8212,2,2 +8213,2,2 +8214,0,0 +8215,2,2 +8216,1,1 +8217,0,0 +8218,2,2 +8219,1,1 +8220,2,2 +8221,1,1 +8222,2,2 +8223,0,0 +8224,1,1 +8225,2,2 +8226,2,2 +8227,2,2 +8228,1,1 +8229,2,2 +8230,1,1 +8231,2,2 +8232,2,2 +8233,0,0 +8234,2,2 +8235,2,2 +8236,0,0 +8237,2,0 +8238,0,0 +8239,2,2 +8240,0,2 +8241,0,0 +8242,2,2 +8243,2,2 +8244,1,1 +8245,2,1 +8246,2,0 +8247,0,0 +8248,2,2 +8249,2,2 +8250,2,2 +8251,2,0 +8252,2,2 +8253,2,2 +8254,0,0 +8255,2,2 +8256,0,0 +8257,2,2 +8258,0,0 +8259,2,2 +8260,2,2 +8261,2,2 +8262,2,2 +8263,2,2 +8264,2,1 +8265,0,0 +8266,2,2 +8267,2,2 +8268,2,2 +8269,0,0 +8270,2,2 +8271,2,2 +8272,0,0 +8273,2,2 +8274,0,0 +8275,0,0 +8276,2,2 +8277,2,2 +8278,2,2 +8279,0,0 +8280,2,2 +8281,2,2 +8282,2,2 +8283,2,2 +8284,2,2 +8285,0,0 +8286,0,0 +8287,2,2 +8288,0,0 +8289,2,2 +8290,2,2 +8291,2,2 +8292,0,0 +8293,0,0 +8294,0,0 +8295,1,1 +8296,0,0 +8297,2,2 +8298,2,2 +8299,2,2 +8300,2,2 +8301,2,0 +8302,2,2 +8303,2,2 +8304,2,2 +8305,2,2 +8306,2,2 +8307,2,2 +8308,2,2 +8309,0,0 +8310,2,0 +8311,2,2 +8312,1,1 +8313,2,2 +8314,2,2 +8315,0,0 +8316,2,2 +8317,0,0 +8318,2,2 +8319,2,2 +8320,2,2 +8321,2,2 +8322,0,0 +8323,0,0 +8324,2,2 +8325,2,2 +8326,2,2 +8327,2,2 +8328,2,2 +8329,2,2 +8330,0,0 +8331,0,0 +8332,1,1 +8333,2,2 +8334,2,2 +8335,0,0 +8336,2,2 +8337,2,2 +8338,2,2 +8339,2,1 +8340,2,1 +8341,1,1 +8342,0,0 +8343,2,2 +8344,2,2 +8345,2,2 +8346,2,2 +8347,2,2 +8348,0,0 +8349,2,2 +8350,2,2 +8351,0,0 +8352,2,2 +8353,2,2 +8354,2,0 +8355,2,2 +8356,0,0 +8357,2,2 +8358,2,2 +8359,2,2 +8360,2,2 +8361,2,2 +8362,0,0 +8363,2,2 +8364,2,2 +8365,2,2 +8366,0,0 +8367,2,2 +8368,0,0 +8369,2,2 +8370,2,2 +8371,2,2 +8372,0,0 +8373,2,2 +8374,2,2 +8375,0,0 +8376,2,2 +8377,1,1 +8378,2,2 +8379,0,0 +8380,0,0 +8381,2,2 +8382,0,0 +8383,2,2 +8384,2,2 +8385,2,2 +8386,0,0 +8387,0,0 +8388,2,2 +8389,0,0 +8390,2,2 +8391,1,1 +8392,2,2 +8393,0,0 +8394,2,2 +8395,2,2 +8396,2,2 +8397,0,0 +8398,2,2 +8399,2,2 +8400,2,2 +8401,0,0 +8402,0,0 +8403,2,2 +8404,2,2 +8405,2,2 +8406,0,0 +8407,2,2 +8408,2,0 +8409,2,2 +8410,0,0 +8411,0,0 +8412,2,2 +8413,2,2 +8414,2,2 +8415,2,2 +8416,0,0 +8417,2,2 +8418,2,2 +8419,0,0 +8420,2,2 +8421,2,2 +8422,2,2 +8423,2,2 +8424,2,2 +8425,2,2 +8426,1,1 +8427,2,2 +8428,2,2 +8429,2,2 +8430,2,2 +8431,2,2 +8432,1,1 +8433,2,2 +8434,1,1 +8435,0,0 +8436,2,2 +8437,2,2 +8438,1,1 +8439,2,2 +8440,2,2 +8441,2,2 +8442,1,1 +8443,2,2 +8444,2,2 +8445,0,0 +8446,2,2 +8447,2,2 +8448,2,2 +8449,0,0 +8450,0,0 +8451,2,2 +8452,0,1 +8453,2,2 +8454,2,2 +8455,0,0 +8456,0,0 +8457,2,2 +8458,0,0 +8459,2,2 +8460,2,2 +8461,0,0 +8462,2,2 +8463,0,0 +8464,2,2 +8465,2,2 +8466,2,0 +8467,0,0 +8468,2,2 +8469,2,2 +8470,2,2 +8471,2,2 +8472,0,0 +8473,2,2 +8474,2,2 +8475,2,2 +8476,2,2 +8477,1,1 +8478,2,2 +8479,2,2 +8480,0,0 +8481,0,0 +8482,2,2 +8483,0,0 +8484,0,0 +8485,2,2 +8486,0,0 +8487,2,2 +8488,2,2 +8489,2,2 +8490,1,1 +8491,2,2 +8492,2,2 +8493,2,2 +8494,0,0 +8495,1,1 +8496,0,0 +8497,2,2 +8498,2,2 +8499,2,2 +8500,2,2 +8501,2,2 +8502,2,1 +8503,0,0 +8504,2,2 +8505,2,2 +8506,2,2 +8507,2,2 +8508,2,2 +8509,2,2 +8510,2,0 +8511,0,0 +8512,2,2 +8513,2,2 +8514,2,2 +8515,2,1 +8516,2,2 +8517,2,2 +8518,2,2 +8519,2,2 +8520,2,2 +8521,2,2 +8522,0,0 +8523,0,2 +8524,0,0 +8525,2,2 +8526,2,2 +8527,0,0 +8528,0,0 +8529,0,0 +8530,2,2 +8531,1,1 +8532,2,0 +8533,2,2 +8534,2,0 +8535,2,2 +8536,2,2 +8537,2,2 +8538,2,2 +8539,2,2 +8540,1,1 +8541,2,2 +8542,2,0 +8543,0,0 +8544,2,2 +8545,2,1 +8546,2,1 +8547,2,2 +8548,2,2 +8549,0,0 +8550,0,0 +8551,0,0 +8552,2,2 +8553,2,2 +8554,0,0 +8555,2,2 +8556,2,2 +8557,0,0 +8558,2,2 +8559,1,1 +8560,2,2 +8561,2,2 +8562,0,0 +8563,2,2 +8564,2,0 +8565,2,2 +8566,2,2 +8567,2,2 +8568,2,2 +8569,2,2 +8570,2,2 +8571,0,0 +8572,2,2 +8573,2,2 +8574,2,0 +8575,0,0 +8576,2,2 +8577,2,2 +8578,2,2 +8579,2,2 +8580,2,2 +8581,2,2 +8582,2,2 +8583,1,1 +8584,2,2 +8585,2,2 +8586,0,0 +8587,0,0 +8588,0,0 +8589,2,2 +8590,0,0 +8591,2,2 +8592,0,0 +8593,2,2 +8594,2,2 +8595,0,0 +8596,2,2 +8597,2,0 +8598,2,2 +8599,1,1 +8600,0,0 +8601,2,2 +8602,2,2 +8603,0,0 +8604,2,2 +8605,2,2 +8606,2,2 +8607,2,0 +8608,2,0 +8609,0,0 +8610,2,2 +8611,0,0 +8612,0,0 +8613,2,2 +8614,0,0 +8615,1,1 +8616,2,2 +8617,2,0 +8618,2,2 +8619,2,2 +8620,2,2 +8621,5,5 +8622,5,5 +8623,4,4 +8624,3,3 +8625,5,5 +8626,3,3 +8627,4,4 +8628,5,3 +8629,5,5 +8630,5,5 +8631,5,5 +8632,5,5 +8633,5,5 +8634,3,3 +8635,3,3 +8636,5,5 +8637,5,5 +8638,3,5 +8639,3,3 +8640,3,3 +8641,3,3 +8642,5,5 +8643,4,4 +8644,3,3 +8645,5,5 +8646,3,3 +8647,5,5 +8648,5,5 +8649,3,3 +8650,3,3 +8651,5,5 +8652,5,5 +8653,5,5 +8654,3,3 +8655,5,5 +8656,3,3 +8657,5,3 +8658,3,3 +8659,5,5 +8660,3,3 +8661,5,5 +8662,5,5 +8663,5,4 +8664,5,5 +8665,5,5 +8666,3,3 +8667,5,5 +8668,5,5 +8669,5,5 +8670,3,5 +8671,5,5 +8672,3,3 +8673,3,3 +8674,5,5 +8675,3,3 +8676,5,5 +8677,5,5 +8678,5,5 +8679,5,3 +8680,3,3 +8681,3,3 +8682,5,4 +8683,5,5 +8684,5,5 +8685,5,5 +8686,5,4 +8687,4,4 +8688,5,5 +8689,5,5 +8690,5,5 +8691,5,5 +8692,3,3 +8693,5,5 +8694,5,5 +8695,3,3 +8696,3,3 +8697,5,5 +8698,3,3 +8699,5,5 +8700,3,3 +8701,5,5 +8702,5,5 +8703,5,5 +8704,5,5 +8705,3,3 +8706,3,3 +8707,5,5 +8708,3,3 +8709,5,5 +8710,5,5 +8711,4,4 +8712,5,5 +8713,5,5 +8714,3,3 +8715,5,5 +8716,5,5 +8717,4,4 +8718,5,5 +8719,5,5 +8720,5,4 +8721,3,3 +8722,5,5 +8723,3,3 +8724,3,3 +8725,3,3 +8726,4,4 +8727,3,3 +8728,5,5 +8729,3,3 +8730,5,5 +8731,5,5 +8732,5,5 +8733,5,5 +8734,5,5 +8735,3,3 +8736,3,3 +8737,3,3 +8738,5,5 +8739,5,5 +8740,3,3 +8741,3,3 +8742,3,3 +8743,5,4 +8744,5,5 +8745,5,5 +8746,5,5 +8747,5,5 +8748,5,5 +8749,3,3 +8750,5,5 +8751,5,5 +8752,4,5 +8753,3,3 +8754,5,5 +8755,3,3 +8756,5,5 +8757,5,5 +8758,5,5 +8759,5,5 +8760,3,3 +8761,5,5 +8762,4,4 +8763,5,5 +8764,3,3 +8765,5,5 +8766,5,5 +8767,5,5 +8768,5,5 +8769,3,3 +8770,5,5 +8771,5,5 +8772,5,5 +8773,5,5 +8774,3,3 +8775,5,5 +8776,3,3 +8777,5,5 +8778,3,3 +8779,5,5 +8780,5,5 +8781,5,3 +8782,5,5 +8783,3,3 +8784,3,3 +8785,3,3 +8786,3,3 +8787,3,3 +8788,3,3 +8789,3,3 +8790,3,3 +8791,5,5 +8792,5,5 +8793,3,3 +8794,5,5 +8795,5,5 +8796,3,3 +8797,3,3 +8798,5,5 +8799,5,5 +8800,3,3 +8801,5,5 +8802,5,5 +8803,5,5 +8804,5,5 +8805,3,3 +8806,3,3 +8807,5,5 +8808,3,3 +8809,3,3 +8810,5,5 +8811,5,5 +8812,3,3 +8813,3,3 +8814,4,4 +8815,5,5 +8816,4,4 +8817,3,3 +8818,5,5 +8819,3,3 +8820,3,3 +8821,3,5 +8822,3,3 +8823,5,5 +8824,4,4 +8825,3,3 +8826,5,5 +8827,3,3 +8828,5,4 +8829,3,3 +8830,3,3 +8831,5,5 +8832,3,3 +8833,3,3 +8834,3,3 +8835,5,5 +8836,5,5 +8837,5,5 +8838,5,5 +8839,3,3 +8840,5,5 +8841,3,3 +8842,5,5 +8843,3,3 +8844,4,4 +8845,3,3 +8846,3,3 +8847,3,3 +8848,5,5 +8849,4,4 +8850,3,3 +8851,5,5 +8852,5,5 +8853,5,5 +8854,5,5 +8855,3,3 +8856,3,3 +8857,3,3 +8858,5,5 +8859,5,5 +8860,3,3 +8861,3,3 +8862,5,4 +8863,3,3 +8864,4,4 +8865,5,5 +8866,5,5 +8867,5,5 +8868,5,5 +8869,5,5 +8870,5,5 +8871,5,5 +8872,3,3 +8873,4,4 +8874,3,3 +8875,4,4 +8876,5,5 +8877,5,5 +8878,4,4 +8879,3,3 +8880,5,5 +8881,3,3 +8882,5,4 +8883,4,4 +8884,5,5 +8885,4,3 +8886,3,3 +8887,5,5 +8888,5,5 +8889,5,5 +8890,3,3 +8891,3,3 +8892,3,3 +8893,5,5 +8894,5,3 +8895,3,5 +8896,3,3 +8897,5,5 +8898,4,4 +8899,5,5 +8900,3,3 +8901,3,3 +8902,5,5 +8903,4,3 +8904,5,5 +8905,5,5 +8906,3,3 +8907,3,3 +8908,5,5 +8909,3,3 +8910,5,5 +8911,5,5 +8912,3,3 +8913,5,5 +8914,5,5 +8915,4,4 +8916,3,3 +8917,5,4 +8918,5,5 +8919,5,5 +8920,5,3 +8921,4,5 +8922,3,3 +8923,5,5 +8924,5,5 +8925,3,3 +8926,5,5 +8927,5,5 +8928,5,5 +8929,5,5 +8930,4,4 +8931,3,3 +8932,3,3 +8933,5,5 +8934,5,5 +8935,4,4 +8936,5,5 +8937,4,4 +8938,3,3 +8939,3,5 +8940,3,3 +8941,3,3 +8942,3,3 +8943,3,3 +8944,3,3 +8945,3,3 +8946,5,5 +8947,3,3 +8948,3,3 +8949,3,3 +8950,3,3 +8951,3,3 +8952,5,5 +8953,3,3 +8954,3,3 +8955,5,5 +8956,5,5 +8957,5,5 +8958,3,3 +8959,5,5 +8960,3,3 +8961,5,5 +8962,5,5 +8963,3,3 +8964,5,5 +8965,5,5 +8966,5,5 +8967,3,3 +8968,3,3 +8969,3,3 +8970,3,3 +8971,5,5 +8972,5,5 +8973,3,3 +8974,5,5 +8975,3,3 +8976,5,5 +8977,3,3 +8978,5,5 +8979,3,3 +8980,5,5 +8981,3,3 +8982,3,3 +8983,3,3 +8984,5,5 +8985,5,5 +8986,5,5 +8987,5,5 +8988,5,5 +8989,5,5 +8990,5,5 +8991,4,4 +8992,3,3 +8993,3,3 +8994,3,3 +8995,3,3 +8996,3,3 +8997,3,3 +8998,3,3 +8999,4,4 +9000,3,3 +9001,5,5 +9002,3,3 +9003,3,3 +9004,4,4 +9005,5,5 +9006,3,3 +9007,5,5 +9008,3,3 +9009,3,3 +9010,3,3 +9011,5,5 +9012,3,3 +9013,5,5 +9014,3,3 +9015,5,5 +9016,5,5 +9017,5,5 +9018,5,5 +9019,3,3 +9020,5,4 +9021,5,3 +9022,5,5 +9023,3,3 +9024,3,3 +9025,3,3 +9026,5,5 +9027,3,3 +9028,5,3 +9029,3,3 +9030,5,5 +9031,5,5 +9032,5,5 +9033,5,5 +9034,3,3 +9035,5,5 +9036,4,4 +9037,5,5 +9038,3,3 +9039,3,3 +9040,5,5 +9041,4,4 +9042,5,5 +9043,5,5 +9044,5,5 +9045,3,3 +9046,5,5 +9047,3,3 +9048,5,5 +9049,3,3 +9050,5,5 +9051,3,3 +9052,5,5 +9053,5,5 +9054,5,5 +9055,3,3 +9056,3,3 +9057,3,3 +9058,3,3 +9059,5,5 +9060,5,5 +9061,4,5 +9062,4,4 +9063,5,3 +9064,5,5 +9065,4,4 +9066,5,5 +9067,3,5 +9068,5,5 +9069,5,5 +9070,3,3 +9071,5,5 +9072,3,3 +9073,3,3 +9074,5,5 +9075,5,5 +9076,4,4 +9077,3,3 +9078,5,3 +9079,5,5 +9080,3,3 +9081,5,5 +9082,5,5 +9083,5,5 +9084,3,3 +9085,5,5 +9086,5,5 +9087,3,3 +9088,5,5 +9089,5,5 +9090,5,5 +9091,5,4 +9092,3,3 +9093,3,3 +9094,5,5 +9095,5,5 +9096,5,5 +9097,4,4 +9098,4,4 +9099,5,4 +9100,3,3 +9101,3,3 +9102,3,3 +9103,3,3 +9104,4,4 +9105,3,3 +9106,5,5 +9107,3,5 +9108,3,3 +9109,5,5 +9110,3,3 +9111,3,3 +9112,3,3 +9113,5,5 +9114,3,3 +9115,4,4 +9116,3,3 +9117,5,5 +9118,3,3 +9119,3,3 +9120,5,3 +9121,5,5 +9122,3,3 +9123,5,5 +9124,5,5 +9125,3,3 +9126,5,5 +9127,3,3 +9128,3,3 +9129,3,3 +9130,3,3 +9131,3,3 +9132,3,3 +9133,5,5 +9134,4,4 +9135,4,4 +9136,4,4 +9137,5,5 +9138,3,3 +9139,5,5 +9140,5,5 +9141,3,3 +9142,5,5 +9143,5,5 +9144,5,5 +9145,3,3 +9146,5,5 +9147,3,3 +9148,5,5 +9149,5,5 +9150,5,5 +9151,5,5 +9152,3,3 +9153,3,3 +9154,5,5 +9155,5,5 +9156,5,5 +9157,3,3 +9158,3,3 +9159,5,5 +9160,5,5 +9161,3,3 +9162,3,3 +9163,5,5 +9164,3,3 +9165,3,3 +9166,5,5 +9167,3,3 +9168,5,5 +9169,4,4 +9170,4,4 +9171,3,3 +9172,4,4 +9173,5,5 +9174,4,4 +9175,3,3 +9176,5,5 +9177,5,5 +9178,5,4 +9179,5,5 +9180,3,3 +9181,3,3 +9182,5,4 +9183,3,3 +9184,5,5 +9185,5,5 +9186,5,5 +9187,3,3 +9188,3,3 +9189,3,3 +9190,5,5 +9191,3,3 +9192,3,3 +9193,5,5 +9194,5,5 +9195,3,3 +9196,4,4 +9197,5,3 +9198,5,5 +9199,5,5 +9200,5,5 +9201,5,5 +9202,5,5 +9203,3,3 +9204,5,5 +9205,5,5 +9206,5,3 +9207,3,3 +9208,4,4 +9209,5,5 +9210,5,5 +9211,3,3 +9212,5,5 +9213,3,3 +9214,5,3 +9215,5,5 +9216,3,3 +9217,3,3 +9218,5,5 +9219,4,4 +9220,5,5 +9221,5,5 +9222,3,3 +9223,5,5 +9224,3,3 +9225,3,3 +9226,4,4 +9227,3,3 +9228,5,5 +9229,5,5 +9230,3,3 +9231,5,5 +9232,5,5 +9233,5,5 +9234,5,5 +9235,5,5 +9236,5,5 +9237,3,3 +9238,5,5 +9239,5,3 +9240,5,5 +9241,3,3 +9242,3,3 +9243,5,5 +9244,5,5 +9245,4,4 +9246,5,5 +9247,3,3 +9248,3,3 +9249,5,5 +9250,3,3 +9251,5,5 +9252,5,5 +9253,5,5 +9254,5,5 +9255,5,5 +9256,5,5 +9257,5,5 +9258,5,5 +9259,5,5 +9260,5,5 +9261,5,5 +9262,5,5 +9263,5,5 +9264,5,3 +9265,5,5 +9266,3,3 +9267,3,3 +9268,3,3 +9269,5,5 +9270,4,5 +9271,5,5 +9272,3,4 +9273,5,5 +9274,3,3 +9275,4,4 +9276,5,5 +9277,5,5 +9278,5,5 +9279,5,5 +9280,3,3 +9281,3,5 +9282,5,5 +9283,5,5 +9284,3,3 +9285,3,3 +9286,5,5 +9287,5,5 +9288,5,5 +9289,5,5 +9290,3,3 +9291,5,5 +9292,5,5 +9293,5,5 +9294,3,3 +9295,5,3 +9296,5,5 +9297,3,3 +9298,3,3 +9299,3,3 +9300,5,5 +9301,4,4 +9302,3,3 +9303,5,5 +9304,3,3 +9305,5,5 +9306,5,5 +9307,4,4 +9308,5,5 +9309,3,3 +9310,3,3 +9311,5,5 +9312,5,3 +9313,3,3 +9314,4,4 +9315,3,3 +9316,5,5 +9317,5,5 +9318,5,5 +9319,3,5 +9320,5,5 +9321,5,5 +9322,5,5 +9323,5,4 +9324,5,5 +9325,3,3 +9326,4,4 +9327,3,3 +9328,5,5 +9329,3,5 +9330,3,3 +9331,5,5 +9332,5,5 +9333,3,3 +9334,5,5 +9335,3,3 +9336,5,5 +9337,3,3 +9338,5,5 +9339,3,3 +9340,3,3 +9341,3,3 +9342,3,3 +9343,3,3 +9344,5,5 +9345,3,3 +9346,5,5 +9347,5,5 +9348,3,3 +9349,5,5 +9350,4,4 +9351,5,5 +9352,3,3 +9353,5,5 +9354,3,3 +9355,3,3 +9356,3,3 +9357,3,3 +9358,3,5 +9359,5,5 +9360,3,3 +9361,3,3 +9362,5,5 +9363,3,3 +9364,5,5 +9365,3,3 +9366,3,3 +9367,3,3 +9368,5,3 +9369,5,5 +9370,3,3 +9371,5,5 +9372,5,5 +9373,5,5 +9374,3,3 +9375,3,3 +9376,3,3 +9377,5,5 +9378,3,3 +9379,3,3 +9380,5,5 +9381,5,5 +9382,3,4 +9383,3,3 +9384,5,5 +9385,3,3 +9386,3,3 +9387,5,5 +9388,3,3 +9389,5,5 +9390,3,3 +9391,5,5 +9392,3,3 +9393,3,5 +9394,5,5 +9395,5,5 +9396,4,4 +9397,5,5 +9398,3,3 +9399,3,3 +9400,5,4 +9401,5,5 +9402,3,3 +9403,5,5 +9404,4,4 +9405,5,5 +9406,5,5 +9407,3,3 +9408,3,3 +9409,5,5 +9410,3,3 +9411,3,3 +9412,5,5 +9413,3,3 +9414,5,3 +9415,3,3 +9416,4,4 +9417,5,5 +9418,5,5 +9419,5,5 +9420,3,3 +9421,3,3 +9422,3,3 +9423,3,5 +9424,3,3 +9425,3,3 +9426,5,5 +9427,3,3 +9428,5,5 +9429,3,3 +9430,3,3 +9431,5,5 +9432,5,5 +9433,3,3 +9434,3,3 +9435,3,3 +9436,5,5 +9437,3,3 +9438,5,5 +9439,5,5 +9440,4,4 +9441,5,5 +9442,3,3 +9443,5,3 +9444,5,5 +9445,3,3 +9446,3,3 +9447,5,5 +9448,5,5 +9449,5,5 +9450,3,3 +9451,5,5 +9452,3,3 +9453,3,3 +9454,5,5 +9455,5,4 +9456,5,5 +9457,5,5 +9458,3,3 +9459,5,5 +9460,3,3 +9461,5,5 +9462,3,3 +9463,5,5 +9464,3,3 +9465,5,5 +9466,3,3 +9467,4,4 +9468,5,5 +9469,3,3 +9470,3,3 +9471,5,5 +9472,3,3 +9473,3,3 +9474,3,3 +9475,3,3 +9476,3,3 +9477,5,5 +9478,3,3 +9479,5,5 +9480,5,5 +9481,4,4 +9482,3,3 +9483,5,5 +9484,5,5 +9485,3,5 +9486,5,5 +9487,5,3 +9488,5,5 +9489,5,5 +9490,4,4 +9491,5,5 +9492,3,3 +9493,4,4 +9494,3,3 +9495,5,5 +9496,5,5 +9497,3,3 +9498,3,3 +9499,5,5 +9500,5,5 +9501,3,3 +9502,3,3 +9503,5,5 +9504,4,4 +9505,5,5 +9506,3,3 +9507,5,5 +9508,3,5 +9509,4,4 +9510,5,5 +9511,4,5 +9512,3,3 +9513,3,3 +9514,5,5 +9515,5,5 +9516,3,3 +9517,5,5 +9518,5,5 +9519,5,5 +9520,3,3 +9521,5,5 +9522,3,3 +9523,5,5 +9524,5,5 +9525,3,3 +9526,3,3 +9527,5,5 +9528,3,3 +9529,5,5 +9530,4,4 +9531,5,5 +9532,3,3 +9533,5,5 +9534,3,3 +9535,5,5 +9536,3,3 +9537,5,5 +9538,5,5 +9539,3,3 +9540,4,4 +9541,5,5 +9542,5,5 +9543,5,5 +9544,3,3 +9545,5,5 +9546,5,5 +9547,5,5 +9548,5,5 +9549,4,4 +9550,3,3 +9551,5,5 +9552,3,3 +9553,3,3 +9554,5,5 +9555,3,3 +9556,3,3 +9557,5,5 +9558,5,5 +9559,4,4 +9560,5,5 +9561,5,5 +9562,3,3 +9563,5,5 +9564,3,5 +9565,5,5 +9566,5,5 +9567,3,3 +9568,5,5 +9569,5,5 +9570,3,3 +9571,5,5 +9572,5,5 +9573,3,3 +9574,3,3 +9575,3,3 +9576,3,3 +9577,5,5 +9578,5,5 +9579,3,3 +9580,5,3 +9581,3,3 +9582,3,3 +9583,3,3 +9584,5,5 +9585,5,5 +9586,5,5 +9587,3,4 +9588,4,4 +9589,3,3 +9590,3,3 +9591,3,3 +9592,4,4 +9593,3,3 +9594,5,5 +9595,3,3 +9596,3,3 +9597,5,5 +9598,4,4 +9599,5,5 +9600,4,4 +9601,5,5 +9602,5,5 +9603,5,5 +9604,5,5 +9605,5,5 +9606,5,5 +9607,3,3 +9608,4,4 +9609,4,4 +9610,3,3 +9611,3,3 +9612,5,5 +9613,5,5 +9614,3,3 +9615,5,5 +9616,5,3 +9617,5,3 +9618,3,3 +9619,3,3 +9620,5,5 +9621,5,5 +9622,3,3 +9623,3,5 +9624,5,5 +9625,5,5 +9626,3,3 +9627,5,5 +9628,5,5 +9629,4,4 +9630,5,5 +9631,3,3 +9632,3,3 +9633,5,5 +9634,5,5 +9635,3,3 +9636,5,5 +9637,5,5 +9638,5,5 +9639,5,5 +9640,5,5 +9641,3,3 +9642,5,5 +9643,5,5 +9644,3,3 +9645,3,3 +9646,3,3 +9647,5,5 +9648,3,3 +9649,5,3 +9650,5,5 +9651,5,3 +9652,3,3 +9653,5,5 +9654,3,3 +9655,3,3 +9656,3,5 +9657,3,3 +9658,5,5 +9659,3,3 +9660,3,3 +9661,5,5 +9662,3,3 +9663,3,3 +9664,5,5 +9665,4,4 +9666,5,5 +9667,5,5 +9668,5,5 +9669,5,5 +9670,5,5 +9671,5,5 +9672,5,5 +9673,3,3 +9674,5,5 +9675,5,5 +9676,3,3 +9677,4,4 +9678,3,3 +9679,3,3 +9680,5,5 +9681,3,3 +9682,5,3 +9683,5,5 +9684,5,5 +9685,5,5 +9686,5,5 +9687,3,3 +9688,5,5 +9689,5,5 +9690,5,5 +9691,3,3 +9692,3,3 +9693,5,5 +9694,3,3 +9695,5,5 +9696,3,3 +9697,5,5 +9698,5,5 +9699,4,4 +9700,5,5 +9701,5,5 +9702,5,5 +9703,3,3 +9704,5,5 +9705,3,3 +9706,4,4 +9707,3,3 +9708,3,3 +9709,4,4 +9710,3,3 +9711,3,3 +9712,3,3 +9713,3,3 +9714,3,3 +9715,3,3 +9716,5,5 +9717,5,5 +9718,3,3 +9719,3,3 +9720,5,5 +9721,5,5 +9722,3,3 +9723,3,3 +9724,5,5 +9725,5,5 +9726,5,5 +9727,5,5 +9728,3,3 +9729,5,5 +9730,3,3 +9731,5,5 +9732,5,5 +9733,3,5 +9734,5,5 +9735,5,5 +9736,5,5 +9737,5,5 +9738,3,3 +9739,3,3 +9740,3,3 +9741,3,3 +9742,3,3 +9743,4,4 +9744,5,5 +9745,5,5 +9746,3,3 +9747,3,3 +9748,5,3 +9749,5,5 +9750,4,4 +9751,3,5 +9752,5,5 +9753,3,3 +9754,3,3 +9755,5,3 +9756,5,5 +9757,5,5 +9758,5,5 +9759,5,5 +9760,5,5 +9761,5,5 +9762,5,5 +9763,5,5 +9764,5,5 +9765,3,3 +9766,5,5 +9767,3,3 +9768,3,3 +9769,3,3 +9770,5,5 +9771,5,5 +9772,3,3 +9773,3,3 +9774,5,5 +9775,3,3 +9776,5,5 +9777,5,5 +9778,5,3 +9779,5,5 +9780,5,5 +9781,5,5 +9782,3,3 +9783,5,5 +9784,3,3 +9785,3,3 +9786,5,5 +9787,5,3 +9788,5,5 +9789,3,3 +9790,3,3 +9791,5,5 +9792,3,3 +9793,5,5 +9794,3,3 +9795,3,3 +9796,3,3 +9797,5,3 +9798,3,3 +9799,3,3 +9800,3,3 +9801,5,5 +9802,3,3 +9803,3,3 +9804,5,5 +9805,3,3 +9806,5,5 +9807,3,3 +9808,3,5 +9809,3,3 +9810,5,5 +9811,3,3 +9812,5,5 +9813,3,3 +9814,5,5 +9815,3,3 +9816,3,3 +9817,3,3 +9818,5,5 +9819,3,3 +9820,3,3 +9821,5,5 +9822,3,3 +9823,3,3 +9824,5,5 +9825,5,5 +9826,5,5 +9827,5,5 +9828,3,3 +9829,5,5 +9830,5,5 +9831,3,3 +9832,5,5 +9833,3,3 +9834,5,5 +9835,3,3 +9836,5,5 +9837,3,3 +9838,3,3 +9839,5,5 +9840,4,4 +9841,5,5 +9842,5,5 +9843,3,3 +9844,5,5 +9845,4,4 +9846,5,5 +9847,3,3 +9848,5,5 +9849,3,3 +9850,5,5 +9851,3,3 +9852,3,3 +9853,3,3 +9854,3,3 +9855,5,5 +9856,5,5 +9857,3,3 +9858,5,5 +9859,4,4 +9860,5,4 +9861,5,5 +9862,3,3 +9863,4,4 +9864,4,4 +9865,5,5 +9866,4,4 +9867,3,3 +9868,3,3 +9869,5,3 +9870,3,5 +9871,3,3 +9872,5,5 +9873,4,4 +9874,3,3 +9875,5,5 +9876,3,3 +9877,5,5 +9878,3,3 +9879,3,3 +9880,5,5 +9881,5,5 +9882,3,3 +9883,5,5 +9884,5,5 +9885,3,3 +9886,5,5 +9887,5,5 +9888,3,3 +9889,3,3 +9890,3,3 +9891,5,3 +9892,3,3 +9893,5,5 +9894,5,5 +9895,4,4 +9896,5,5 +9897,5,5 +9898,3,3 +9899,5,5 +9900,3,5 +9901,5,5 +9902,5,5 +9903,5,5 +9904,5,5 +9905,5,5 +9906,5,5 +9907,5,4 +9908,3,3 +9909,5,5 +9910,5,5 +9911,5,5 +9912,4,3 +9913,5,5 +9914,5,5 +9915,3,3 +9916,5,5 +9917,5,5 +9918,5,5 +9919,3,3 +9920,4,4 +9921,3,3 +9922,5,5 +9923,5,5 +9924,3,3 +9925,5,5 +9926,5,5 +9927,5,5 +9928,3,3 +9929,3,3 +9930,5,5 +9931,3,3 +9932,5,5 +9933,5,5 +9934,4,4 +9935,3,3 +9936,3,3 +9937,3,3 +9938,5,5 +9939,5,5 +9940,3,3 +9941,3,3 +9942,3,3 +9943,3,3 +9944,3,3 +9945,3,3 +9946,5,5 +9947,3,3 +9948,5,5 +9949,5,5 +9950,3,3 +9951,5,5 +9952,3,3 +9953,5,5 +9954,4,3 +9955,5,5 +9956,3,3 +9957,3,3 +9958,5,3 +9959,5,5 +9960,5,5 +9961,5,5 +9962,5,5 +9963,3,3 +9964,5,5 +9965,4,4 +9966,3,3 +9967,5,3 +9968,5,5 +9969,5,3 +9970,3,3 +9971,3,3 +9972,5,5 +9973,5,5 +9974,5,5 +9975,5,5 +9976,3,3 +9977,4,4 +9978,5,5 +9979,5,5 +9980,5,5 +9981,3,3 +9982,4,5 +9983,5,5 +9984,3,3 +9985,3,3 +9986,3,3 +9987,5,5 +9988,5,5 +9989,3,3 +9990,3,3 +9991,3,3 +9992,3,3 +9993,5,5 +9994,5,5 +9995,5,5 +9996,5,5 +9997,4,4 +9998,5,5 +9999,5,5 +10000,5,5 +10001,5,5 +10002,4,4 +10003,3,3 +10004,3,3 +10005,5,5 +10006,5,5 +10007,3,3 +10008,5,5 +10009,5,5 +10010,3,3 +10011,3,5 +10012,5,5 +10013,5,5 +10014,5,5 +10015,3,3 +10016,5,5 +10017,5,5 +10018,3,3 +10019,5,5 +10020,5,5 +10021,5,5 +10022,5,5 +10023,5,5 +10024,3,3 +10025,3,3 +10026,5,5 +10027,4,4 +10028,3,3 +10029,4,4 +10030,3,3 +10031,5,5 +10032,3,3 +10033,3,3 +10034,3,3 +10035,3,3 +10036,3,3 +10037,5,5 +10038,5,5 +10039,3,3 +10040,5,5 +10041,3,3 +10042,3,3 +10043,5,5 +10044,3,3 +10045,5,5 +10046,3,3 +10047,3,3 +10048,3,3 +10049,5,5 +10050,3,3 +10051,3,3 +10052,3,3 +10053,5,5 +10054,3,3 +10055,5,5 +10056,5,5 +10057,3,3 +10058,5,5 +10059,3,3 +10060,5,5 +10061,4,4 +10062,5,5 +10063,3,3 +10064,3,3 +10065,5,5 +10066,3,3 +10067,3,3 +10068,3,3 +10069,5,5 +10070,3,3 +10071,5,5 +10072,5,5 +10073,5,5 +10074,5,5 +10075,5,5 +10076,5,5 +10077,5,5 +10078,3,3 +10079,3,3 +10080,3,3 +10081,3,3 +10082,3,3 +10083,5,5 +10084,5,5 +10085,5,5 +10086,5,5 +10087,5,5 +10088,3,5 +10089,5,5 +10090,3,3 +10091,5,5 +10092,3,3 +10093,5,5 +10094,5,5 +10095,5,5 +10096,5,5 +10097,4,4 +10098,3,3 +10099,5,5 +10100,5,5 +10101,3,3 +10102,3,3 +10103,5,5 +10104,3,3 +10105,5,5 +10106,5,5 +10107,3,3 +10108,5,5 +10109,3,3 +10110,3,3 +10111,5,5 +10112,3,3 +10113,5,5 +10114,5,5 +10115,5,5 +10116,3,3 +10117,5,5 +10118,5,5 +10119,5,5 +10120,4,4 +10121,5,3 +10122,5,5 +10123,3,3 +10124,5,4 +10125,5,5 +10126,5,5 +10127,5,5 +10128,5,3 +10129,3,3 +10130,3,3 +10131,5,5 +10132,3,4 +10133,5,5 +10134,5,5 +10135,3,3 +10136,3,3 +10137,3,5 +10138,3,3 +10139,5,5 +10140,3,3 +10141,5,5 +10142,5,5 +10143,5,5 +10144,3,3 +10145,5,5 +10146,5,5 +10147,3,3 +10148,5,5 +10149,3,3 +10150,5,5 +10151,3,3 +10152,5,5 +10153,3,3 +10154,5,5 +10155,4,4 +10156,3,3 +10157,3,3 +10158,3,3 +10159,3,3 +10160,5,3 +10161,3,3 +10162,3,3 +10163,3,3 +10164,3,3 +10165,3,3 +10166,4,4 +10167,5,5 +10168,4,5 +10169,4,4 +10170,3,3 +10171,3,3 +10172,3,3 +10173,5,5 +10174,3,3 +10175,3,3 +10176,3,3 +10177,5,5 +10178,3,3 +10179,3,3 +10180,5,5 +10181,3,3 +10182,3,3 +10183,3,3 +10184,3,3 +10185,3,3 +10186,3,3 +10187,5,5 +10188,3,3 +10189,5,5 +10190,5,5 +10191,5,5 +10192,5,5 +10193,5,5 +10194,5,5 +10195,4,4 +10196,4,4 +10197,5,5 +10198,5,5 +10199,4,4 +10200,3,3 +10201,5,5 +10202,4,4 +10203,3,3 +10204,3,3 +10205,3,3 +10206,5,5 +10207,5,3 +10208,3,5 +10209,5,3 +10210,5,5 +10211,4,4 +10212,5,5 +10213,5,5 +10214,3,3 +10215,5,5 +10216,3,3 +10217,5,5 +10218,5,5 +10219,5,5 +10220,5,5 +10221,3,3 +10222,5,5 +10223,3,3 +10224,4,4 +10225,5,5 +10226,5,3 +10227,5,5 +10228,5,5 +10229,5,5 +10230,5,5 +10231,3,3 +10232,4,4 +10233,3,3 +10234,3,3 +10235,5,5 +10236,5,5 +10237,5,5 +10238,3,3 +10239,5,5 +10240,3,3 +10241,3,3 +10242,5,5 +10243,5,5 +10244,5,5 +10245,5,5 +10246,5,3 +10247,5,5 +10248,5,5 +10249,5,5 +10250,3,3 +10251,3,3 +10252,5,5 +10253,5,5 +10254,5,5 +10255,5,5 +10256,5,5 +10257,3,3 +10258,4,4 +10259,5,5 +10260,4,4 +10261,5,5 +10262,3,3 +10263,3,3 +10264,3,3 +10265,3,3 +10266,4,4 +10267,3,3 +10268,5,4 +10269,3,3 +10270,5,5 +10271,3,3 +10272,5,5 +10273,3,3 +10274,5,5 +10275,5,5 +10276,5,5 +10277,5,5 +10278,3,3 +10279,3,3 +10280,5,5 +10281,3,3 +10282,5,5 +10283,5,5 +10284,5,5 +10285,5,5 +10286,5,5 +10287,3,3 +10288,3,3 +10289,5,5 +10290,3,3 +10291,5,5 +10292,4,4 +10293,5,5 +10294,5,5 +10295,5,5 +10296,5,5 +10297,3,3 +10298,5,5 +10299,5,5 +10300,3,3 +10301,5,5 +10302,5,5 +10303,5,5 +10304,5,5 +10305,5,5 +10306,5,5 +10307,3,3 +10308,5,5 +10309,5,5 +10310,5,5 +10311,3,3 +10312,3,3 +10313,5,5 +10314,5,5 +10315,3,3 +10316,5,5 +10317,5,5 +10318,3,3 +10319,3,3 +10320,4,4 +10321,3,5 +10322,5,5 +10323,5,5 +10324,4,4 +10325,5,5 +10326,3,3 +10327,5,5 +10328,5,5 +10329,5,5 +10330,5,5 +10331,3,3 +10332,3,3 +10333,3,3 +10334,5,5 +10335,3,3 +10336,5,5 +10337,5,5 +10338,4,4 +10339,5,5 +10340,5,5 +10341,3,3 +10342,4,4 +10343,3,3 +10344,5,5 +10345,3,3 +10346,4,4 +10347,5,5 +10348,3,3 +10349,5,5 +10350,5,5 +10351,5,5 +10352,3,3 +10353,3,3 +10354,3,3 +10355,5,5 +10356,5,5 +10357,3,3 +10358,5,5 +10359,5,3 +10360,3,3 +10361,5,3 +10362,3,3 +10363,5,5 +10364,5,5 +10365,3,3 +10366,5,5 +10367,3,3 +10368,5,5 +10369,5,5 +10370,5,5 +10371,3,3 +10372,3,3 +10373,3,3 +10374,5,5 +10375,3,3 +10376,3,5 +10377,5,5 +10378,3,3 +10379,3,3 +10380,3,3 +10381,3,3 +10382,3,3 +10383,5,3 +10384,3,3 +10385,5,5 +10386,5,5 +10387,5,5 +10388,3,3 +10389,3,3 +10390,5,5 +10391,3,5 +10392,5,5 +10393,5,5 +10394,5,5 +10395,5,3 +10396,4,4 +10397,3,3 +10398,5,5 +10399,4,4 +10400,3,3 +10401,5,5 +10402,5,5 +10403,5,5 +10404,5,3 +10405,5,5 +10406,4,4 +10407,3,3 +10408,3,3 +10409,5,5 +10410,5,5 +10411,3,3 +10412,3,3 +10413,5,5 +10414,4,3 +10415,3,3 +10416,5,5 +10417,3,3 +10418,3,3 +10419,5,5 +10420,4,4 +10421,5,5 +10422,5,5 +10423,5,5 +10424,3,3 +10425,5,5 +10426,5,5 +10427,3,3 +10428,3,5 +10429,3,3 +10430,3,3 +10431,3,3 +10432,3,3 +10433,3,3 +10434,5,3 +10435,5,5 +10436,3,3 +10437,3,3 +10438,5,5 +10439,5,5 +10440,5,5 +10441,5,5 +10442,5,5 +10443,3,3 +10444,3,3 +10445,4,4 +10446,5,5 +10447,5,5 +10448,5,5 +10449,5,5 +10450,5,5 +10451,5,5 +10452,3,3 +10453,5,5 +10454,5,5 +10455,5,5 +10456,5,5 +10457,3,3 +10458,5,5 +10459,5,5 +10460,4,4 +10461,3,3 +10462,3,3 +10463,3,3 +10464,4,4 +10465,3,3 +10466,5,3 +10467,5,5 +10468,5,5 +10469,5,5 +10470,5,5 +10471,3,3 +10472,5,5 +10473,5,5 +10474,3,3 +10475,5,5 +10476,5,5 +10477,3,3 +10478,5,5 +10479,5,4 +10480,3,3 +10481,3,3 +10482,5,5 +10483,5,5 +10484,3,3 +10485,5,5 +10486,3,3 +10487,3,3 +10488,5,3 +10489,3,3 +10490,3,3 +10491,5,5 +10492,5,5 +10493,5,5 +10494,3,3 +10495,3,3 +10496,5,5 +10497,3,3 +10498,3,3 +10499,3,5 +10500,3,3 +10501,3,3 +10502,5,5 +10503,5,4 +10504,3,3 +10505,3,3 +10506,4,4 +10507,3,3 +10508,5,5 +10509,3,3 +10510,5,5 +10511,5,5 +10512,5,5 +10513,3,5 +10514,4,5 +10515,5,5 +10516,3,3 +10517,3,3 +10518,3,4 +10519,3,3 +10520,3,3 +10521,5,5 +10522,5,5 +10523,3,3 +10524,3,3 +10525,5,5 +10526,5,5 +10527,4,5 +10528,5,5 +10529,5,5 +10530,3,3 +10531,3,3 +10532,5,5 +10533,3,3 +10534,3,3 +10535,3,3 +10536,5,5 +10537,5,5 +10538,3,3 +10539,5,5 +10540,5,5 +10541,3,3 +10542,5,5 +10543,5,5 +10544,5,5 +10545,3,3 +10546,5,3 +10547,5,5 +10548,3,3 +10549,5,5 +10550,5,5 +10551,5,5 +10552,5,5 +10553,5,5 +10554,5,5 +10555,3,3 +10556,3,3 +10557,5,5 +10558,3,3 +10559,5,5 +10560,5,5 +10561,3,3 +10562,3,3 +10563,3,3 +10564,3,5 +10565,3,3 +10566,5,5 +10567,5,5 +10568,5,5 +10569,5,5 +10570,5,5 +10571,4,4 +10572,5,5 +10573,5,5 +10574,5,5 +10575,5,5 +10576,5,5 +10577,5,5 +10578,5,5 +10579,5,5 +10580,5,3 +10581,3,3 +10582,5,5 +10583,3,3 +10584,4,3 +10585,3,3 +10586,3,5 +10587,5,5 +10588,5,5 +10589,5,5 +10590,5,5 +10591,3,3 +10592,3,3 +10593,3,3 +10594,5,5 +10595,3,3 +10596,5,3 +10597,3,3 +10598,3,3 +10599,5,3 +10600,5,5 +10601,3,5 +10602,4,4 +10603,5,5 +10604,5,5 +10605,3,3 +10606,5,5 +10607,3,3 +10608,5,3 +10609,5,5 +10610,5,5 +10611,3,3 +10612,5,5 +10613,3,3 +10614,5,5 +10615,3,3 +10616,3,3 +10617,3,3 +10618,3,3 +10619,3,3 +10620,5,5 +10621,5,5 +10622,5,5 +10623,5,3 +10624,3,3 +10625,5,5 +10626,3,5 +10627,5,5 +10628,5,5 +10629,3,3 +10630,5,5 +10631,3,3 +10632,5,5 +10633,5,3 +10634,3,3 +10635,5,5 +10636,5,5 +10637,5,5 +10638,4,4 +10639,5,5 +10640,3,3 +10641,5,5 +10642,5,5 +10643,3,3 +10644,3,3 +10645,5,5 +10646,5,5 +10647,3,3 +10648,3,3 +10649,3,3 +10650,5,5 +10651,3,3 +10652,3,3 +10653,5,5 +10654,3,3 +10655,3,3 +10656,5,5 +10657,3,3 +10658,5,5 +10659,3,3 +10660,5,5 +10661,3,3 +10662,5,5 +10663,5,5 +10664,5,5 +10665,5,5 +10666,5,5 +10667,5,5 +10668,3,3 +10669,3,3 +10670,5,5 +10671,5,5 +10672,5,3 +10673,5,5 +10674,5,5 +10675,3,3 +10676,5,5 +10677,3,3 +10678,3,3 +10679,5,5 +10680,3,3 +10681,3,3 +10682,5,5 +10683,4,4 +10684,3,3 +10685,5,5 +10686,5,5 +10687,3,3 +10688,5,5 +10689,3,3 +10690,3,3 +10691,3,3 +10692,5,3 +10693,5,5 +10694,5,5 +10695,5,5 +10696,3,3 +10697,5,5 +10698,5,5 +10699,5,5 +10700,5,5 +10701,3,3 +10702,3,3 +10703,3,3 +10704,3,3 +10705,5,5 +10706,3,3 +10707,3,3 +10708,5,5 +10709,5,5 +10710,5,5 +10711,3,3 +10712,5,5 +10713,5,5 +10714,5,5 +10715,5,5 +10716,5,5 +10717,5,5 +10718,3,3 +10719,3,3 +10720,3,3 +10721,5,5 +10722,3,3 +10723,5,5 +10724,5,5 +10725,3,3 +10726,5,5 +10727,3,3 +10728,3,3 +10729,5,5 +10730,3,3 +10731,3,3 +10732,5,5 +10733,5,3 +10734,3,3 +10735,5,5 +10736,5,5 +10737,5,5 +10738,5,5 +10739,4,4 +10740,5,5 +10741,5,3 +10742,3,3 +10743,4,4 +10744,3,3 +10745,3,3 +10746,3,3 +10747,5,5 +10748,5,5 +10749,5,5 +10750,3,3 +10751,5,5 +10752,5,5 +10753,5,5 +10754,3,3 +10755,5,5 +10756,5,5 +10757,5,5 +10758,3,3 +10759,5,5 +10760,3,3 +10761,5,5 +10762,5,5 +10763,3,3 +10764,5,5 +10765,3,3 +10766,3,3 +10767,5,5 +10768,5,5 +10769,3,3 +10770,3,3 +10771,3,3 +10772,3,3 +10773,5,5 +10774,5,5 +10775,5,5 +10776,5,5 +10777,3,3 +10778,5,5 +10779,3,3 +10780,5,5 +10781,3,3 +10782,3,3 +10783,3,3 +10784,3,3 +10785,5,5 +10786,3,3 +10787,5,5 +10788,3,3 +10789,5,5 +10790,5,5 +10791,3,3 +10792,3,3 +10793,3,3 +10794,5,5 +10795,5,5 +10796,3,3 +10797,5,5 +10798,5,5 +10799,5,5 +10800,4,4 +10801,3,3 +10802,5,5 +10803,5,5 +10804,5,5 +10805,5,5 +10806,4,4 +10807,3,3 +10808,5,5 +10809,3,3 +10810,5,5 +10811,3,3 +10812,3,3 +10813,3,3 +10814,3,3 +10815,5,5 +10816,3,3 +10817,5,5 +10818,3,3 +10819,3,3 +10820,3,3 +10821,5,3 +10822,5,5 +10823,3,3 +10824,3,3 +10825,5,5 +10826,5,5 +10827,3,3 +10828,5,5 +10829,3,3 +10830,5,5 +10831,5,5 +10832,5,5 +10833,5,5 +10834,5,5 +10835,5,5 +10836,3,3 +10837,5,5 +10838,5,5 +10839,3,3 +10840,5,5 +10841,4,4 +10842,5,5 +10843,5,5 +10844,5,5 +10845,5,5 +10846,3,3 +10847,5,5 +10848,3,3 +10849,5,5 +10850,5,5 +10851,3,3 +10852,3,3 +10853,5,5 +10854,5,5 +10855,4,4 +10856,5,5 +10857,3,3 +10858,5,5 +10859,3,3 +10860,3,3 +10861,3,3 +10862,5,4 +10863,4,4 +10864,3,3 +10865,4,4 +10866,3,3 +10867,3,3 +10868,3,3 +10869,5,5 +10870,5,5 +10871,5,4 +10872,4,4 +10873,5,5 +10874,5,5 +10875,3,5 +10876,5,5 +10877,5,5 +10878,5,5 +10879,3,3 +10880,5,5 +10881,5,5 +10882,3,3 +10883,3,3 +10884,3,3 +10885,3,3 +10886,5,5 +10887,3,3 +10888,3,3 +10889,3,3 +10890,3,3 +10891,5,5 +10892,4,4 +10893,3,5 +10894,5,5 +10895,3,3 +10896,3,3 +10897,3,3 +10898,5,5 +10899,5,5 +10900,5,5 +10901,3,3 +10902,3,3 +10903,5,5 +10904,5,5 +10905,5,5 +10906,5,5 +10907,5,5 +10908,5,5 +10909,5,5 +10910,5,5 +10911,3,5 +10912,3,3 +10913,5,5 +10914,5,5 +10915,5,5 +10916,3,3 +10917,3,3 +10918,5,5 +10919,3,3 +10920,5,5 +10921,5,5 +10922,5,5 +10923,5,5 +10924,5,5 +10925,3,3 +10926,3,3 +10927,5,5 +10928,5,5 +10929,5,5 +10930,5,5 +10931,5,5 +10932,5,3 +10933,3,3 +10934,3,3 +10935,5,5 +10936,5,5 +10937,5,5 +10938,4,4 +10939,3,3 +10940,5,5 +10941,3,3 +10942,3,3 +10943,5,5 +10944,3,3 +10945,4,4 +10946,5,5 +10947,5,5 +10948,5,5 +10949,5,5 +10950,5,5 +10951,4,4 +10952,5,5 +10953,3,3 +10954,3,3 +10955,5,5 +10956,5,5 +10957,5,5 +10958,5,5 +10959,5,5 +10960,3,3 +10961,5,5 +10962,3,3 +10963,5,5 +10964,5,3 +10965,5,5 +10966,5,5 +10967,5,5 +10968,3,3 +10969,5,5 +10970,5,5 +10971,3,3 +10972,5,5 +10973,3,3 +10974,5,5 +10975,3,3 +10976,3,3 +10977,3,3 +10978,5,5 +10979,3,3 +10980,3,3 +10981,3,5 +10982,5,5 +10983,5,5 +10984,3,3 +10985,5,5 +10986,3,3 +10987,3,3 +10988,5,3 +10989,5,5 +10990,5,5 +10991,5,5 +10992,3,3 +10993,5,5 +10994,5,5 +10995,5,3 +10996,5,5 +10997,5,5 +10998,5,5 +10999,5,5 +11000,3,3 +11001,3,3 +11002,3,3 +11003,5,5 +11004,5,5 +11005,4,4 +11006,3,3 +11007,3,3 +11008,3,3 +11009,5,5 +11010,3,3 +11011,5,5 +11012,3,3 +11013,3,3 +11014,3,3 +11015,5,5 +11016,4,5 +11017,3,3 +11018,5,5 +11019,4,4 +11020,3,3 +11021,3,3 +11022,5,5 +11023,4,4 +11024,3,3 +11025,3,3 +11026,3,3 +11027,3,3 +11028,3,3 +11029,3,3 +11030,5,5 +11031,5,5 +11032,5,5 +11033,3,3 +11034,5,5 +11035,5,5 +11036,5,5 +11037,5,5 +11038,5,5 +11039,4,4 +11040,5,3 +11041,5,5 +11042,4,4 +11043,5,5 +11044,3,3 +11045,5,5 +11046,5,5 +11047,5,5 +11048,5,5 +11049,5,3 +11050,5,5 +11051,4,4 +11052,3,5 +11053,5,5 +11054,4,5 +11055,3,3 +11056,3,3 +11057,3,3 +11058,5,5 +11059,4,3 +11060,3,3 +11061,5,5 +11062,5,5 +11063,5,5 +11064,5,5 +11065,5,5 +11066,5,5 +11067,3,3 +11068,5,5 +11069,5,5 +11070,5,5 +11071,3,3 +11072,3,3 +11073,5,5 +11074,3,3 +11075,3,3 +11076,4,4 +11077,3,3 +11078,5,5 +11079,3,3 +11080,5,5 +11081,4,4 +11082,5,5 +11083,3,3 +11084,5,5 +11085,3,3 +11086,3,3 +11087,3,3 +11088,5,5 +11089,5,5 +11090,5,5 +11091,3,3 +11092,5,5 +11093,5,3 +11094,3,3 +11095,5,5 +11096,5,5 +11097,4,4 +11098,5,5 +11099,5,5 +11100,4,4 +11101,4,4 +11102,3,3 +11103,5,5 +11104,3,3 +11105,5,5 +11106,5,5 +11107,5,3 +11108,3,3 +11109,3,3 +11110,5,5 +11111,5,5 +11112,5,5 +11113,3,3 +11114,4,4 +11115,3,3 +11116,5,5 +11117,3,3 +11118,5,5 +11119,4,4 +11120,5,5 +11121,5,5 +11122,5,5 +11123,5,5 +11124,5,3 +11125,3,3 +11126,5,5 +11127,3,3 +11128,3,3 +11129,5,5 +11130,5,5 +11131,5,5 +11132,3,3 +11133,3,3 +11134,3,3 +11135,3,3 +11136,3,3 +11137,5,5 +11138,3,3 +11139,5,5 +11140,5,5 +11141,3,3 +11142,4,4 +11143,3,3 +11144,3,3 +11145,5,5 +11146,5,5 +11147,5,5 +11148,3,4 +11149,5,5 +11150,5,5 +11151,5,5 +11152,3,3 +11153,3,3 +11154,5,5 +11155,3,3 +11156,5,3 +11157,5,5 +11158,5,5 +11159,3,3 +11160,5,3 +11161,3,3 +11162,3,3 +11163,3,3 +11164,5,3 +11165,5,5 +11166,4,4 +11167,5,5 +11168,5,5 +11169,5,5 +11170,5,5 +11171,3,3 +11172,5,5 +11173,3,3 +11174,5,5 +11175,5,5 +11176,4,4 +11177,4,4 +11178,3,3 +11179,5,4 +11180,5,5 +11181,5,5 +11182,5,5 +11183,5,5 +11184,3,3 +11185,3,3 +11186,3,3 +11187,5,5 +11188,3,3 +11189,3,3 +11190,5,5 +11191,3,3 +11192,3,3 +11193,3,3 +11194,5,5 +11195,5,5 +11196,3,3 +11197,5,5 +11198,4,4 +11199,3,5 +11200,3,3 +11201,3,3 +11202,3,3 +11203,5,5 +11204,3,3 +11205,5,5 +11206,3,3 +11207,3,3 +11208,4,5 +11209,3,3 +11210,5,5 +11211,5,5 +11212,5,3 +11213,5,5 +11214,5,5 +11215,3,3 +11216,4,4 +11217,5,5 +11218,5,5 +11219,3,3 +11220,3,3 +11221,5,5 +11222,5,3 +11223,3,3 +11224,5,5 +11225,5,3 +11226,5,5 +11227,5,5 +11228,3,3 +11229,5,4 +11230,5,5 +11231,3,3 +11232,5,5 +11233,5,5 +11234,5,3 +11235,5,5 +11236,5,5 +11237,5,3 +11238,5,5 +11239,5,5 +11240,5,5 +11241,4,4 +11242,3,3 +11243,5,5 +11244,3,3 +11245,5,5 +11246,5,5 +11247,4,4 +11248,3,3 +11249,5,5 +11250,3,3 +11251,3,3 +11252,5,5 +11253,3,3 +11254,5,5 +11255,5,5 +11256,3,3 +11257,5,5 +11258,3,3 +11259,3,3 +11260,5,5 +11261,3,3 +11262,5,5 +11263,5,5 +11264,5,5 +11265,3,3 +11266,3,3 +11267,5,5 +11268,5,5 +11269,5,5 +11270,3,3 +11271,5,5 +11272,5,5 +11273,3,3 +11274,5,5 +11275,3,3 +11276,3,3 +11277,3,3 +11278,3,3 +11279,5,5 +11280,4,4 +11281,3,3 +11282,3,3 +11283,5,5 +11284,3,3 +11285,5,3 +11286,5,3 +11287,3,5 +11288,3,3 +11289,3,3 +11290,5,5 +11291,5,5 +11292,5,5 +11293,3,3 +11294,3,3 +11295,5,5 +11296,3,3 +11297,3,3 +11298,5,5 +11299,5,5 +11300,3,3 +11301,3,3 +11302,3,3 +11303,3,5 +11304,5,5 +11305,5,5 +11306,5,5 +11307,5,5 +11308,5,5 +11309,3,3 +11310,3,3 +11311,5,5 +11312,3,3 +11313,5,5 +11314,5,5 +11315,4,4 +11316,3,3 +11317,5,5 +11318,4,4 +11319,5,3 +11320,5,5 +11321,3,3 +11322,4,4 +11323,5,5 +11324,5,5 +11325,3,3 +11326,3,3 +11327,5,5 +11328,5,5 +11329,3,3 +11330,5,5 +11331,3,3 +11332,5,5 +11333,3,3 +11334,4,4 +11335,5,5 +11336,5,5 +11337,3,3 +11338,5,5 +11339,4,4 +11340,4,4 +11341,5,5 +11342,5,5 +11343,5,5 +11344,3,3 +11345,3,5 +11346,5,5 +11347,3,5 +11348,3,3 +11349,4,4 +11350,5,5 +11351,4,4 +11352,3,3 +11353,5,5 +11354,3,3 +11355,3,3 +11356,3,3 +11357,3,3 +11358,5,5 +11359,5,5 +11360,5,5 +11361,3,3 +11362,3,3 +11363,5,5 +11364,3,3 +11365,3,3 +11366,5,5 +11367,5,5 +11368,5,5 +11369,5,5 +11370,3,3 +11371,3,3 +11372,3,3 +11373,5,5 +11374,5,5 +11375,5,4 +11376,4,4 +11377,5,3 +11378,3,3 +11379,5,5 +11380,5,5 +11381,5,5 +11382,5,5 +11383,5,5 +11384,5,5 +11385,3,3 +11386,3,3 +11387,3,3 +11388,3,3 +11389,5,5 +11390,5,5 +11391,3,5 +11392,5,5 +11393,5,5 +11394,5,5 +11395,5,3 +11396,3,3 +11397,5,3 +11398,3,3 +11399,3,3 +11400,5,5 +11401,5,5 +11402,5,5 +11403,3,3 +11404,5,5 +11405,5,5 +11406,3,3 +11407,5,5 +11408,5,5 +11409,3,3 +11410,3,3 +11411,3,3 +11412,5,5 +11413,5,5 +11414,3,3 +11415,3,3 +11416,5,5 +11417,5,5 +11418,5,3 +11419,3,3 +11420,4,4 +11421,5,5 +11422,5,5 +11423,5,5 +11424,5,5 +11425,3,3 +11426,5,5 +11427,3,3 +11428,5,5 +11429,5,5 +11430,5,5 +11431,5,5 +11432,5,5 +11433,3,5 +11434,5,5 +11435,5,5 +11436,4,4 +11437,3,3 +11438,5,5 +11439,5,5 +11440,3,3 +11441,5,5 +11442,5,5 +11443,5,5 +11444,5,5 +11445,5,5 +11446,3,3 +11447,3,3 +11448,3,3 +11449,5,5 +11450,3,3 +11451,3,5 +11452,3,3 +11453,3,3 +11454,3,5 +11455,3,3 +11456,3,3 +11457,5,5 +11458,3,3 +11459,5,5 +11460,3,3 +11461,5,5 +11462,3,3 +11463,5,5 +11464,5,5 +11465,4,4 +11466,3,3 +11467,3,3 +11468,5,5 +11469,3,3 +11470,3,3 +11471,3,3 +11472,4,4 +11473,3,3 +11474,5,5 +11475,5,5 +11476,3,3 +11477,3,3 +11478,5,5 +11479,3,3 +11480,5,5 +11481,5,5 +11482,3,3 +11483,3,3 +11484,3,3 +11485,4,4 +11486,5,5 +11487,5,5 +11488,3,3 +11489,5,5 +11490,3,3 +11491,3,3 +11492,4,4 +11493,5,5 +11494,5,5 +11495,3,3 +11496,5,5 +11497,5,5 +11498,5,5 +11499,4,5 +11500,3,3 +11501,5,5 +11502,5,5 +11503,5,5 +11504,5,5 +11505,5,5 +11506,3,3 +11507,4,5 +11508,5,3 +11509,5,5 +11510,3,3 +11511,5,5 +11512,5,5 +11513,5,5 +11514,5,5 +11515,5,3 +11516,5,5 +11517,3,3 +11518,5,5 +11519,5,5 +11520,5,5 +11521,3,5 +11522,5,5 +11523,3,5 +11524,3,5 +11525,4,3 +11526,3,3 +11527,3,3 +11528,3,3 +11529,3,3 +11530,5,5 +11531,5,3 +11532,5,5 +11533,5,5 +11534,5,5 +11535,5,5 +11536,3,5 +11537,5,5 +11538,5,5 +11539,3,3 +11540,3,3 +11541,5,5 +11542,5,5 +11543,3,3 +11544,5,4 +11545,3,3 +11546,3,3 +11547,3,3 +11548,5,5 +11549,3,3 +11550,3,3 +11551,5,5 +11552,5,5 +11553,5,5 +11554,3,3 +11555,5,5 +11556,3,3 +11557,4,5 +11558,5,5 +11559,5,5 +11560,3,3 +11561,5,5 +11562,5,5 +11563,4,5 +11564,3,3 +11565,5,5 +11566,5,5 +11567,3,3 +11568,5,5 +11569,5,5 +11570,3,3 +11571,5,5 +11572,5,5 +11573,4,4 +11574,3,3 +11575,5,5 +11576,5,5 +11577,5,5 +11578,5,5 +11579,3,3 +11580,5,5 +11581,5,3 +11582,5,5 +11583,5,5 +11584,5,5 +11585,5,5 +11586,5,5 +11587,5,5 +11588,5,4 +11589,3,3 +11590,5,5 +11591,5,5 +11592,3,3 +11593,3,3 +11594,5,5 +11595,3,3 +11596,3,3 +11597,3,3 +11598,5,5 +11599,5,5 +11600,5,5 +11601,5,5 +11602,3,3 +11603,3,3 +11604,3,3 +11605,5,5 +11606,3,3 +11607,5,5 +11608,3,3 +11609,3,3 +11610,5,5 +11611,3,3 +11612,4,4 +11613,3,3 +11614,5,5 +11615,5,5 +11616,3,3 +11617,5,5 +11618,5,5 +11619,5,5 +11620,5,5 +11621,3,3 +11622,3,3 +11623,3,3 +11624,5,5 +11625,5,5 +11626,5,5 +11627,5,5 +11628,3,3 +11629,3,3 +11630,3,3 +11631,5,5 +11632,5,5 +11633,5,5 +11634,3,3 +11635,5,5 +11636,3,3 +11637,3,3 +11638,5,5 +11639,5,5 +11640,5,5 +11641,3,3 +11642,3,3 +11643,3,3 +11644,5,5 +11645,5,5 +11646,3,3 +11647,5,5 +11648,3,3 +11649,5,5 +11650,3,3 +11651,3,3 +11652,4,5 +11653,5,5 +11654,5,5 +11655,3,5 +11656,5,5 +11657,3,3 +11658,3,3 +11659,5,5 +11660,3,3 +11661,4,4 +11662,5,5 +11663,3,3 +11664,4,5 +11665,5,5 +11666,5,5 +11667,5,5 +11668,5,5 +11669,3,3 +11670,5,5 +11671,5,5 +11672,5,5 +11673,5,5 +11674,5,5 +11675,3,3 +11676,3,3 +11677,5,5 +11678,5,5 +11679,5,4 +11680,4,5 +11681,4,5 +11682,3,3 +11683,5,5 +11684,3,3 +11685,3,3 +11686,5,5 +11687,5,5 +11688,5,5 +11689,3,3 +11690,5,5 +11691,4,4 +11692,5,5 +11693,3,3 +11694,5,5 +11695,3,3 +11696,5,5 +11697,5,3 +11698,3,5 +11699,5,5 +11700,3,3 +11701,5,5 +11702,3,3 +11703,3,5 +11704,5,4 +11705,3,3 +11706,5,5 +11707,3,3 +11708,3,3 +11709,3,5 +11710,5,5 +11711,3,3 +11712,3,3 +11713,5,3 +11714,5,5 +11715,5,5 +11716,3,3 +11717,3,3 +11718,3,3 +11719,5,5 +11720,3,3 +11721,5,5 +11722,5,5 +11723,4,4 +11724,5,5 +11725,5,5 +11726,3,3 +11727,3,3 +11728,5,5 +11729,3,3 +11730,3,3 +11731,5,5 +11732,3,3 +11733,5,5 +11734,5,5 +11735,3,3 +11736,3,3 +11737,5,5 +11738,5,5 +11739,3,3 +11740,5,5 +11741,3,3 +11742,5,5 +11743,5,3 +11744,4,5 +11745,3,3 +11746,4,3 +11747,3,3 +11748,5,5 +11749,3,3 +11750,5,4 +11751,3,3 +11752,3,3 +11753,4,4 +11754,3,3 +11755,3,3 +11756,3,3 +11757,5,5 +11758,3,5 +11759,5,5 +11760,5,4 +11761,3,3 +11762,5,5 +11763,5,5 +11764,5,3 +11765,4,5 +11766,5,5 +11767,3,3 +11768,5,5 +11769,5,5 +11770,5,5 +11771,3,3 +11772,5,5 +11773,5,5 +11774,3,3 +11775,3,3 +11776,5,5 +11777,5,5 +11778,5,5 +11779,3,3 +11780,5,5 +11781,3,5 +11782,5,5 +11783,5,5 +11784,3,3 +11785,5,5 +11786,5,5 +11787,5,5 +11788,5,3 +11789,3,3 +11790,5,5 +11791,3,3 +11792,3,3 +11793,4,4 +11794,3,3 +11795,5,5 +11796,3,3 +11797,5,5 +11798,3,3 +11799,3,3 +11800,5,5 +11801,5,5 +11802,5,5 +11803,5,5 +11804,3,3 +11805,5,5 +11806,3,3 +11807,3,3 +11808,5,5 +11809,3,3 +11810,4,4 +11811,4,4 +11812,4,4 +11813,5,5 +11814,3,3 +11815,5,5 +11816,3,3 +11817,4,4 +11818,5,5 +11819,4,4 +11820,4,4 +11821,4,4 +11822,3,3 +11823,5,5 +11824,5,5 +11825,3,3 +11826,3,3 +11827,3,3 +11828,4,4 +11829,5,5 +11830,3,3 +11831,5,5 +11832,3,3 +11833,3,3 +11834,5,5 +11835,3,3 +11836,5,5 +11837,3,3 +11838,5,5 +11839,3,3 +11840,5,5 +11841,5,3 +11842,5,5 +11843,5,5 +11844,5,5 +11845,4,4 +11846,3,3 +11847,5,5 +11848,5,5 +11849,5,5 +11850,5,5 +11851,5,5 +11852,3,3 +11853,5,5 +11854,3,3 +11855,3,3 +11856,5,5 +11857,5,3 +11858,3,3 +11859,3,3 +11860,5,5 +11861,3,3 +11862,5,4 +11863,5,3 +11864,3,3 +11865,5,5 +11866,5,5 +11867,5,5 +11868,5,5 +11869,5,5 +11870,4,4 +11871,5,5 +11872,3,3 +11873,3,3 +11874,3,3 +11875,5,5 +11876,3,3 +11877,5,5 +11878,3,4 +11879,3,5 +11880,5,4 +11881,3,3 +11882,3,3 +11883,5,5 +11884,3,3 +11885,3,3 +11886,5,5 +11887,5,5 +11888,3,3 +11889,5,5 +11890,5,5 +11891,5,5 +11892,3,3 +11893,5,5 +11894,3,3 +11895,5,5 +11896,5,3 +11897,5,5 +11898,3,3 +11899,3,3 +11900,3,3 +11901,3,3 +11902,5,4 +11903,5,5 +11904,5,5 +11905,5,5 +11906,5,5 +11907,5,5 +11908,4,5 +11909,5,5 +11910,5,5 +11911,5,5 +11912,5,5 +11913,3,3 +11914,5,5 +11915,5,5 +11916,3,3 +11917,4,4 +11918,3,3 +11919,5,5 +11920,3,3 +11921,4,5 +11922,3,3 +11923,3,3 +11924,3,3 +11925,5,5 +11926,5,4 +11927,5,5 +11928,3,3 +11929,5,5 +11930,5,5 +11931,5,5 +11932,3,3 +11933,3,3 +11934,4,4 +11935,5,5 +11936,5,5 +11937,5,5 +11938,3,3 +11939,3,3 +11940,5,5 +11941,5,5 +11942,5,5 +11943,5,5 +11944,3,3 +11945,3,3 +11946,3,3 +11947,5,5 +11948,5,5 +11949,5,5 +11950,3,3 +11951,5,5 +11952,5,5 +11953,5,5 +11954,5,5 +11955,3,3 +11956,5,5 +11957,3,5 +11958,5,5 +11959,3,3 +11960,3,3 +11961,5,5 +11962,5,5 +11963,3,3 +11964,3,3 +11965,3,3 +11966,5,5 +11967,5,5 +11968,5,5 +11969,5,5 +11970,5,5 +11971,3,3 +11972,5,5 +11973,5,5 +11974,5,5 +11975,5,5 +11976,3,3 +11977,3,3 +11978,5,5 +11979,4,4 +11980,3,3 +11981,5,5 +11982,3,3 +11983,3,3 +11984,3,3 +11985,5,5 +11986,5,5 +11987,5,3 +11988,3,3 +11989,3,3 +11990,3,3 +11991,5,5 +11992,5,5 +11993,5,5 +11994,5,5 +11995,3,3 +11996,5,5 +11997,3,3 +11998,5,5 +11999,5,5 +12000,5,5 +12001,5,5 +12002,3,3 +12003,5,5 +12004,5,5 +12005,5,5 +12006,3,3 +12007,5,5 +12008,3,3 +12009,5,5 +12010,3,3 +12011,3,3 +12012,5,5 +12013,3,3 +12014,5,5 +12015,3,3 +12016,5,5 +12017,3,3 +12018,3,3 +12019,4,4 +12020,5,5 +12021,3,3 +12022,5,5 +12023,5,5 +12024,3,3 +12025,4,4 +12026,5,5 +12027,3,3 +12028,5,5 +12029,5,5 +12030,3,3 +12031,5,5 +12032,5,5 +12033,5,5 +12034,3,3 +12035,3,3 +12036,5,5 +12037,5,5 +12038,5,5 +12039,3,3 +12040,5,5 +12041,3,5 +12042,3,3 +12043,4,4 +12044,5,5 +12045,5,5 +12046,5,3 +12047,3,3 +12048,3,3 +12049,3,3 +12050,3,3 +12051,5,5 +12052,3,3 +12053,5,5 +12054,3,3 +12055,3,3 +12056,5,5 +12057,3,3 +12058,4,5 +12059,3,3 +12060,5,5 +12061,4,4 +12062,5,5 +12063,5,5 +12064,3,3 +12065,5,5 +12066,3,3 +12067,5,5 +12068,5,5 +12069,3,3 +12070,5,5 +12071,5,5 +12072,5,5 +12073,5,3 +12074,5,5 +12075,3,3 +12076,3,3 +12077,3,3 +12078,3,3 +12079,3,3 +12080,5,5 +12081,5,5 +12082,3,3 +12083,3,3 +12084,3,3 +12085,3,3 +12086,3,3 +12087,4,5 +12088,5,5 +12089,5,5 +12090,3,4 +12091,5,5 +12092,5,5 +12093,5,5 +12094,3,3 +12095,3,3 +12096,5,3 +12097,3,3 +12098,5,5 +12099,5,5 +12100,5,5 +12101,5,5 +12102,3,3 +12103,3,3 +12104,5,5 +12105,3,5 +12106,3,3 +12107,3,3 +12108,5,5 +12109,5,5 +12110,3,3 +12111,3,3 +12112,5,5 +12113,3,3 +12114,5,5 +12115,5,5 +12116,5,5 +12117,5,5 +12118,3,3 +12119,3,3 +12120,5,5 +12121,4,4 +12122,3,3 +12123,3,3 +12124,5,5 +12125,5,5 +12126,5,5 +12127,5,5 +12128,3,3 +12129,5,5 +12130,5,5 +12131,5,5 +12132,5,5 +12133,5,5 +12134,5,5 +12135,5,5 +12136,3,3 +12137,3,3 +12138,5,3 +12139,4,4 +12140,3,3 +12141,4,4 +12142,3,5 +12143,5,5 +12144,5,5 +12145,5,5 +12146,5,5 +12147,3,3 +12148,5,5 +12149,3,3 +12150,3,3 +12151,5,5 +12152,3,3 +12153,3,3 +12154,5,5 +12155,5,5 +12156,3,3 +12157,5,5 +12158,3,3 +12159,3,3 +12160,5,5 +12161,3,3 +12162,5,5 +12163,5,3 +12164,3,3 +12165,5,5 +12166,5,5 +12167,3,3 +12168,5,5 +12169,3,3 +12170,3,3 +12171,3,3 +12172,5,5 +12173,5,5 +12174,5,5 +12175,5,5 +12176,5,5 +12177,5,5 +12178,4,5 +12179,5,3 +12180,3,3 +12181,5,5 +12182,4,4 +12183,3,3 +12184,3,3 +12185,5,3 +12186,5,5 +12187,5,5 +12188,3,3 +12189,3,3 +12190,3,3 +12191,5,5 +12192,4,4 +12193,3,3 +12194,3,3 +12195,5,5 +12196,3,3 +12197,5,3 +12198,5,5 +12199,5,5 +12200,3,3 +12201,5,5 +12202,3,3 +12203,5,5 +12204,3,3 +12205,3,3 +12206,5,5 +12207,5,5 +12208,5,5 +12209,5,5 +12210,5,5 +12211,5,5 +12212,3,3 +12213,5,5 +12214,5,5 +12215,5,3 +12216,3,3 +12217,3,3 +12218,3,3 +12219,3,3 +12220,5,5 +12221,3,3 +12222,3,3 +12223,5,5 +12224,3,5 +12225,3,3 +12226,5,5 +12227,5,5 +12228,5,5 +12229,4,4 +12230,5,5 +12231,5,5 +12232,4,4 +12233,5,5 +12234,5,5 +12235,5,5 +12236,5,5 +12237,5,5 +12238,3,3 +12239,3,5 +12240,5,5 +12241,3,3 +12242,3,3 +12243,5,5 +12244,5,5 +12245,5,5 +12246,3,3 +12247,5,5 +12248,5,5 +12249,5,5 +12250,3,3 +12251,5,5 +12252,3,3 +12253,3,3 +12254,5,5 +12255,3,3 +12256,5,5 +12257,5,5 +12258,3,3 +12259,5,4 +12260,3,3 +12261,5,5 +12262,5,5 +12263,5,5 +12264,3,3 +12265,5,5 +12266,5,3 +12267,3,3 +12268,3,3 +12269,3,3 +12270,5,5 +12271,3,3 +12272,5,5 +12273,5,5 +12274,5,5 +12275,3,3 +12276,5,5 +12277,3,3 +12278,5,5 +12279,5,5 +12280,5,5 +12281,3,3 +12282,3,3 +12283,5,5 +12284,4,5 +12285,3,3 +12286,3,3 +12287,5,5 +12288,3,3 +12289,3,3 +12290,3,3 +12291,5,5 +12292,5,5 +12293,5,5 +12294,5,5 +12295,3,3 +12296,5,5 +12297,5,5 +12298,3,3 +12299,3,3 +12300,3,3 +12301,3,3 +12302,5,5 +12303,5,5 +12304,5,5 +12305,3,3 +12306,3,3 +12307,5,5 +12308,5,5 +12309,4,4 +12310,5,5 +12311,3,3 +12312,5,5 +12313,3,3 +12314,5,5 +12315,3,3 +12316,5,5 +12317,5,5 +12318,3,3 +12319,3,5 +12320,5,5 +12321,4,4 +12322,5,5 +12323,3,3 +12324,5,5 +12325,4,4 +12326,3,3 +12327,5,5 +12328,5,4 +12329,3,3 +12330,3,5 +12331,5,4 +12332,3,3 +12333,3,3 +12334,3,3 +12335,4,4 +12336,4,4 +12337,5,5 +12338,5,5 +12339,3,3 +12340,3,3 +12341,5,5 +12342,5,5 +12343,3,3 +12344,5,5 +12345,5,5 +12346,3,3 +12347,5,5 +12348,5,5 +12349,5,5 +12350,5,5 +12351,5,5 +12352,5,5 +12353,5,3 +12354,5,5 +12355,3,3 +12356,5,5 +12357,3,3 +12358,3,3 +12359,3,3 +12360,5,5 +12361,5,5 +12362,3,3 +12363,5,5 +12364,3,3 +12365,5,5 +12366,3,3 +12367,3,3 +12368,5,5 +12369,3,3 +12370,3,3 +12371,5,5 +12372,3,3 +12373,5,3 +12374,3,3 +12375,3,3 +12376,3,3 +12377,3,3 +12378,3,3 +12379,3,3 +12380,5,5 +12381,3,5 +12382,3,3 +12383,3,3 +12384,5,5 +12385,5,5 +12386,5,3 +12387,3,3 +12388,5,5 +12389,5,5 +12390,3,3 +12391,5,4 +12392,5,5 +12393,4,4 +12394,3,3 +12395,5,5 +12396,3,3 +12397,3,3 +12398,5,5 +12399,5,5 +12400,5,5 +12401,3,3 +12402,5,5 +12403,5,5 +12404,3,5 +12405,3,3 +12406,5,5 +12407,3,3 +12408,3,3 +12409,5,5 +12410,5,5 +12411,3,3 +12412,3,3 +12413,3,3 +12414,3,3 +12415,5,3 +12416,5,5 +12417,5,5 +12418,5,5 +12419,3,3 +12420,5,5 +12421,5,5 +12422,3,3 +12423,3,3 +12424,3,3 +12425,3,3 +12426,3,3 +12427,3,3 +12428,3,3 +12429,3,3 +12430,3,5 +12431,4,4 +12432,5,5 +12433,3,3 +12434,3,3 +12435,5,5 +12436,5,5 +12437,5,5 +12438,5,5 +12439,5,5 +12440,4,4 +12441,3,3 +12442,5,5 +12443,3,3 +12444,3,3 +12445,3,3 +12446,4,4 +12447,3,3 +12448,5,5 +12449,5,5 +12450,5,5 +12451,5,5 +12452,5,5 +12453,5,5 +12454,4,4 +12455,3,3 +12456,5,5 +12457,3,3 +12458,5,5 +12459,3,5 +12460,3,3 +12461,3,3 +12462,3,3 +12463,3,3 +12464,3,3 +12465,3,3 +12466,3,3 +12467,3,3 +12468,3,3 +12469,3,3 +12470,3,3 +12471,5,5 +12472,3,3 +12473,5,5 +12474,5,5 +12475,5,5 +12476,5,4 +12477,5,5 +12478,5,5 +12479,5,5 +12480,3,3 +12481,3,3 +12482,5,5 +12483,5,5 +12484,5,3 +12485,5,5 +12486,5,5 +12487,5,5 +12488,5,5 +12489,5,5 +12490,5,5 +12491,3,3 +12492,5,5 +12493,5,5 +12494,3,3 +12495,5,5 +12496,5,3 +12497,3,3 +12498,3,3 +12499,3,3 +12500,5,5 +12501,3,3 +12502,5,5 +12503,5,5 +12504,5,5 +12505,3,3 +12506,5,5 +12507,5,5 +12508,5,5 +12509,5,5 +12510,4,4 +12511,4,4 +12512,3,3 +12513,3,3 +12514,3,3 +12515,5,5 +12516,5,5 +12517,5,5 +12518,4,4 +12519,3,3 +12520,5,3 +12521,3,3 +12522,3,3 +12523,3,3 +12524,3,3 +12525,5,5 +12526,5,5 +12527,5,5 +12528,5,5 +12529,3,3 +12530,5,5 +12531,5,5 +12532,3,3 +12533,5,5 +12534,5,5 +12535,3,3 +12536,3,3 +12537,5,5 +12538,3,3 +12539,3,3 +12540,5,5 +12541,5,5 +12542,5,5 +12543,3,3 +12544,5,5 +12545,3,3 +12546,3,3 +12547,3,3 +12548,5,5 +12549,3,3 +12550,5,5 +12551,3,5 +12552,3,3 +12553,5,4 +12554,3,3 +12555,5,5 +12556,5,5 +12557,5,4 +12558,3,3 +12559,5,5 +12560,5,5 +12561,5,5 +12562,3,3 +12563,5,5 +12564,5,5 +12565,3,3 +12566,5,3 +12567,3,3 +12568,5,5 +12569,5,5 +12570,3,3 +12571,3,3 +12572,5,5 +12573,5,5 +12574,5,5 +12575,5,5 +12576,5,5 +12577,5,5 +12578,5,5 +12579,5,5 +12580,3,3 +12581,4,4 +12582,4,4 +12583,3,3 +12584,3,3 +12585,3,3 +12586,5,5 +12587,5,5 +12588,5,5 +12589,5,5 +12590,3,3 +12591,5,5 +12592,5,3 +12593,3,3 +12594,3,3 +12595,5,5 +12596,3,3 +12597,5,5 +12598,5,5 +12599,5,5 +12600,3,3 +12601,5,5 +12602,3,3 +12603,3,3 +12604,3,3 +12605,5,4 +12606,3,3 +12607,5,5 +12608,4,4 +12609,3,3 +12610,3,3 +12611,3,3 +12612,3,3 +12613,5,5 +12614,3,3 +12615,3,3 +12616,5,5 +12617,5,5 +12618,3,3 +12619,3,3 +12620,5,5 +12621,5,5 +12622,5,5 +12623,3,3 +12624,3,3 +12625,3,3 +12626,3,3 +12627,3,3 +12628,5,5 +12629,3,3 +12630,5,5 +12631,5,4 +12632,3,3 +12633,3,3 +12634,3,3 +12635,3,3 +12636,3,3 +12637,3,3 +12638,5,5 +12639,3,5 +12640,5,5 +12641,5,5 +12642,5,5 +12643,5,5 +12644,3,3 +12645,4,4 +12646,5,4 +12647,5,5 +12648,4,4 +12649,5,5 +12650,5,5 +12651,5,5 +12652,5,5 +12653,5,3 +12654,3,3 +12655,3,3 +12656,5,5 +12657,5,5 +12658,5,5 +12659,5,5 +12660,3,3 +12661,3,3 +12662,5,5 +12663,3,3 +12664,5,5 +12665,5,5 +12666,5,5 +12667,3,3 +12668,3,3 +12669,5,5 +12670,3,3 +12671,5,5 +12672,3,4 +12673,3,3 +12674,5,5 +12675,3,3 +12676,4,4 +12677,5,5 +12678,5,5 +12679,3,3 +12680,3,3 +12681,3,3 +12682,5,5 +12683,3,3 +12684,5,5 +12685,5,3 +12686,3,3 +12687,3,3 +12688,5,5 +12689,5,5 +12690,5,4 +12691,5,5 +12692,5,5 +12693,5,5 +12694,5,5 +12695,3,3 +12696,5,5 +12697,3,3 +12698,5,5 +12699,5,5 +12700,3,3 +12701,3,3 +12702,3,3 +12703,5,5 +12704,5,5 +12705,5,5 +12706,5,5 +12707,5,5 +12708,5,5 +12709,3,3 +12710,5,3 +12711,4,4 +12712,5,5 +12713,3,3 +12714,3,3 +12715,3,3 +12716,5,5 +12717,5,5 +12718,3,3 +12719,3,3 +12720,5,5 +12721,5,5 +12722,5,5 +12723,5,5 +12724,5,5 +12725,3,3 +12726,5,5 +12727,5,5 +12728,5,5 +12729,3,3 +12730,5,5 +12731,3,3 +12732,5,5 +12733,4,4 +12734,5,5 +12735,5,5 +12736,5,5 +12737,5,5 +12738,5,5 +12739,5,5 +12740,5,5 +12741,5,5 +12742,5,5 +12743,3,3 +12744,3,3 +12745,3,3 +12746,3,3 +12747,5,5 +12748,5,5 +12749,5,5 +12750,3,3 +12751,5,3 +12752,5,5 +12753,5,3 +12754,3,3 +12755,3,3 +12756,4,4 +12757,5,4 +12758,5,5 +12759,4,4 +12760,3,3 +12761,3,3 +12762,5,5 +12763,3,5 +12764,3,3 +12765,3,3 +12766,5,5 +12767,3,3 +12768,5,3 +12769,5,5 +12770,5,5 +12771,3,3 +12772,5,5 +12773,3,3 +12774,5,5 +12775,5,5 +12776,3,3 +12777,3,3 +12778,5,5 +12779,5,5 +12780,4,4 +12781,3,3 +12782,5,5 +12783,5,5 +12784,3,3 +12785,5,5 +12786,5,5 +12787,5,5 +12788,3,3 +12789,5,5 +12790,3,3 +12791,5,5 +12792,5,5 +12793,3,3 +12794,5,5 +12795,5,5 +12796,3,3 +12797,3,3 +12798,3,3 +12799,5,5 +12800,5,5 +12801,3,3 +12802,5,5 +12803,5,5 +12804,3,3 +12805,3,3 +12806,5,4 +12807,3,3 +12808,5,5 +12809,3,3 +12810,5,5 +12811,3,3 +12812,5,5 +12813,3,3 +12814,5,5 +12815,5,5 +12816,5,5 +12817,5,5 +12818,3,3 +12819,3,3 +12820,5,5 +12821,4,4 +12822,5,5 +12823,4,4 +12824,5,5 +12825,5,5 +12826,5,5 +12827,5,5 +12828,5,3 +12829,3,3 +12830,3,3 +12831,5,5 +12832,5,5 +12833,4,4 +12834,3,3 +12835,3,3 +12836,3,3 +12837,3,4 +12838,5,4 +12839,5,5 +12840,5,5 +12841,3,3 +12842,3,3 +12843,3,3 +12844,3,5 +12845,5,5 +12846,3,3 +12847,5,5 +12848,5,5 +12849,5,5 +12850,4,4 +12851,5,5 +12852,3,3 +12853,3,3 +12854,5,5 +12855,5,5 +12856,3,3 +12857,3,3 +12858,5,5 +12859,5,5 +12860,5,5 +12861,4,4 +12862,3,3 +12863,5,5 +12864,5,5 +12865,3,3 +12866,3,3 +12867,5,5 +12868,3,3 +12869,5,5 +12870,3,3 +12871,3,3 +12872,3,3 +12873,4,5 +12874,3,3 +12875,5,5 +12876,5,5 +12877,3,3 +12878,5,5 +12879,3,3 +12880,5,5 +12881,3,3 +12882,3,3 +12883,3,3 +12884,3,3 +12885,5,5 +12886,5,4 +12887,5,5 +12888,3,3 +12889,3,3 +12890,3,3 +12891,5,5 +12892,5,5 +12893,5,3 +12894,5,5 +12895,5,5 +12896,3,3 +12897,5,5 +12898,3,3 +12899,5,5 +12900,5,5 +12901,3,3 +12902,5,5 +12903,5,3 +12904,5,5 +12905,3,3 +12906,5,5 +12907,3,3 +12908,5,5 +12909,5,5 +12910,3,3 +12911,5,5 +12912,3,5 +12913,5,5 +12914,3,3 +12915,5,5 +12916,3,5 +12917,3,3 +12918,4,4 +12919,3,3 +12920,3,3 +12921,3,3 +12922,5,5 +12923,3,5 +12924,5,4 +12925,3,5 +12926,3,3 +12927,5,5 +12928,3,3 +12929,5,5 +12930,3,3 +12931,3,3 +12932,3,3 +12933,5,5 +12934,5,5 +12935,3,3 +12936,5,5 +12937,5,5 +12938,5,5 +12939,5,3 +12940,5,5 +12941,3,5 +12942,3,3 +12943,3,3 +12944,3,3 +12945,4,4 +12946,5,5 +12947,5,5 +12948,5,5 +12949,5,5 +12950,3,3 +12951,5,3 +12952,3,3 +12953,3,3 +12954,5,5 +12955,5,4 +12956,5,5 +12957,5,5 +12958,5,5 +12959,3,3 +12960,3,3 +12961,5,5 +12962,3,3 +12963,5,5 +12964,5,5 +12965,4,4 +12966,3,3 +12967,3,3 +12968,5,5 +12969,5,5 +12970,3,5 +12971,5,5 +12972,5,5 +12973,5,5 +12974,5,5 +12975,4,4 +12976,5,5 +12977,5,5 +12978,3,3 +12979,3,3 +12980,5,5 +12981,3,3 +12982,5,5 +12983,5,5 +12984,3,3 +12985,5,5 +12986,3,3 +12987,5,5 +12988,5,5 +12989,3,3 +12990,5,5 +12991,3,3 +12992,3,3 +12993,5,5 +12994,5,5 +12995,3,3 +12996,5,5 +12997,3,3 +12998,3,3 +12999,3,3 +13000,3,3 +13001,5,5 +13002,5,5 +13003,3,3 +13004,3,3 +13005,3,3 +13006,5,5 +13007,5,3 +13008,3,3 +13009,5,5 +13010,5,3 +13011,5,5 +13012,3,3 +13013,3,3 +13014,5,4 +13015,5,5 +13016,4,4 +13017,5,5 +13018,5,5 +13019,3,3 +13020,3,3 +13021,5,5 +13022,3,3 +13023,3,3 +13024,3,5 +13025,5,5 +13026,5,5 +13027,5,5 +13028,5,5 +13029,3,3 +13030,5,5 +13031,3,3 +13032,3,3 +13033,3,3 +13034,5,5 +13035,5,5 +13036,4,4 +13037,5,5 +13038,5,5 +13039,5,5 +13040,5,5 +13041,5,5 +13042,5,5 +13043,3,3 +13044,5,5 +13045,5,5 +13046,3,3 +13047,5,5 +13048,5,5 +13049,5,5 +13050,3,3 +13051,5,5 +13052,5,5 +13053,5,5 +13054,4,4 +13055,3,3 +13056,3,3 +13057,5,5 +13058,5,5 +13059,5,5 +13060,5,5 +13061,5,5 +13062,5,5 +13063,3,3 +13064,3,3 +13065,5,5 +13066,5,5 +13067,3,3 +13068,5,5 +13069,3,3 +13070,5,5 +13071,5,5 +13072,3,3 +13073,3,3 +13074,5,5 +13075,5,5 +13076,3,3 +13077,5,5 +13078,4,4 +13079,3,3 +13080,3,3 +13081,5,5 +13082,5,5 +13083,5,5 +13084,4,4 +13085,4,4 +13086,3,3 +13087,4,4 +13088,5,5 +13089,3,3 +13090,3,3 +13091,3,3 +13092,5,5 +13093,5,5 +13094,5,5 +13095,4,4 +13096,3,3 +13097,3,3 +13098,3,3 +13099,5,5 +13100,4,5 +13101,3,3 +13102,5,5 +13103,5,5 +13104,5,5 +13105,3,3 +13106,3,3 +13107,4,4 +13108,3,3 +13109,5,5 +13110,3,3 +13111,3,5 +13112,3,3 +13113,4,4 +13114,5,5 +13115,5,5 +13116,3,3 +13117,3,3 +13118,5,5 +13119,5,5 +13120,3,3 +13121,3,3 +13122,3,3 +13123,3,3 +13124,3,3 +13125,3,3 +13126,3,3 +13127,5,5 +13128,5,5 +13129,4,4 +13130,5,5 +13131,5,5 +13132,5,5 +13133,5,5 +13134,5,5 +13135,5,5 +13136,3,3 +13137,3,3 +13138,3,3 +13139,3,3 +13140,5,5 +13141,5,5 +13142,5,5 +13143,5,5 +13144,5,5 +13145,3,3 +13146,3,3 +13147,5,5 +13148,3,3 +13149,5,5 +13150,5,5 +13151,3,3 +13152,5,5 +13153,4,4 +13154,5,5 +13155,4,4 +13156,3,3 +13157,5,5 +13158,5,5 +13159,3,3 +13160,5,5 +13161,5,5 +13162,4,5 +13163,3,3 +13164,5,5 +13165,4,5 +13166,3,3 +13167,5,5 +13168,5,5 +13169,3,3 +13170,4,4 +13171,3,5 +13172,3,3 +13173,5,5 +13174,5,3 +13175,5,5 +13176,3,3 +13177,5,5 +13178,5,5 +13179,5,5 +13180,5,5 +13181,5,5 +13182,4,4 +13183,5,3 +13184,3,3 +13185,3,3 +13186,5,5 +13187,5,3 +13188,3,3 +13189,4,4 +13190,5,5 +13191,4,4 +13192,3,3 +13193,5,5 +13194,5,5 +13195,3,3 +13196,3,3 +13197,5,5 +13198,3,5 +13199,3,3 +13200,5,5 +13201,5,3 +13202,3,3 +13203,3,3 +13204,3,3 +13205,5,5 +13206,5,5 +13207,3,3 +13208,4,4 +13209,5,5 +13210,5,5 +13211,3,3 +13212,3,3 +13213,3,3 +13214,5,5 +13215,5,5 +13216,4,4 +13217,5,5 +13218,5,5 +13219,3,3 +13220,3,3 +13221,4,4 +13222,5,5 +13223,5,5 +13224,3,3 +13225,3,3 +13226,4,5 +13227,5,5 +13228,3,3 +13229,3,3 +13230,5,5 +13231,5,5 +13232,5,5 +13233,3,3 +13234,5,5 +13235,5,5 +13236,3,3 +13237,3,3 +13238,3,3 +13239,5,5 +13240,5,5 +13241,3,3 +13242,5,5 +13243,5,5 +13244,3,3 +13245,5,5 +13246,4,4 +13247,4,4 +13248,5,5 +13249,5,5 +13250,3,3 +13251,5,5 +13252,5,5 +13253,3,3 +13254,4,4 +13255,3,3 +13256,5,5 +13257,5,5 +13258,3,3 +13259,3,3 +13260,5,5 +13261,3,3 +13262,3,5 +13263,5,5 +13264,5,5 +13265,5,5 +13266,3,3 +13267,5,5 +13268,5,5 +13269,5,5 +13270,3,3 +13271,3,3 +13272,5,4 +13273,3,3 +13274,3,3 +13275,5,5 +13276,5,5 +13277,3,3 +13278,3,3 +13279,3,3 +13280,4,4 +13281,3,3 +13282,5,4 +13283,5,5 +13284,5,5 +13285,5,5 +13286,3,3 +13287,5,5 +13288,5,5 +13289,5,5 +13290,3,3 +13291,5,5 +13292,5,5 +13293,3,5 +13294,4,4 +13295,5,5 +13296,5,3 +13297,5,5 +13298,5,5 +13299,5,5 +13300,5,3 +13301,3,3 +13302,3,3 +13303,3,3 +13304,5,5 +13305,5,5 +13306,5,5 +13307,5,5 +13308,5,5 +13309,5,3 +13310,5,5 +13311,3,3 +13312,5,5 +13313,5,5 +13314,5,5 +13315,3,3 +13316,5,5 +13317,5,5 +13318,5,5 +13319,3,3 +13320,5,5 +13321,5,5 +13322,5,3 +13323,5,5 +13324,3,3 +13325,3,3 +13326,5,5 +13327,3,3 +13328,3,3 +13329,3,3 +13330,3,3 +13331,5,5 +13332,5,5 +13333,5,4 +13334,3,3 +13335,3,3 +13336,5,5 +13337,3,3 +13338,3,3 +13339,5,5 +13340,5,5 +13341,3,3 +13342,5,5 +13343,3,3 +13344,3,3 +13345,5,5 +13346,4,4 +13347,4,4 +13348,3,3 +13349,5,5 +13350,3,3 +13351,4,5 +13352,4,4 +13353,5,5 +13354,5,5 +13355,5,5 +13356,4,4 +13357,3,3 +13358,5,5 +13359,3,3 +13360,3,3 +13361,5,5 +13362,3,3 +13363,5,5 +13364,5,5 +13365,3,3 +13366,3,3 +13367,5,5 +13368,5,5 +13369,5,5 +13370,3,3 +13371,5,5 +13372,5,4 +13373,3,3 +13374,4,4 +13375,5,5 +13376,5,5 +13377,3,3 +13378,5,5 +13379,3,3 +13380,3,3 +13381,5,5 +13382,5,5 +13383,3,3 +13384,5,5 +13385,5,5 +13386,5,5 +13387,5,5 +13388,5,5 +13389,5,5 +13390,5,5 +13391,4,4 +13392,5,5 +13393,5,5 +13394,4,4 +13395,5,5 +13396,3,3 +13397,3,3 +13398,3,3 +13399,5,5 +13400,5,5 +13401,3,3 +13402,5,5 +13403,5,5 +13404,3,3 +13405,5,5 +13406,3,3 +13407,5,5 +13408,5,5 +13409,5,5 +13410,5,5 +13411,5,5 +13412,3,3 +13413,5,5 +13414,3,3 +13415,5,5 +13416,5,5 +13417,3,3 +13418,5,5 +13419,5,5 +13420,3,3 +13421,5,3 +13422,3,3 +13423,3,3 +13424,3,3 +13425,3,3 +13426,4,4 +13427,3,3 +13428,5,4 +13429,5,5 +13430,5,3 +13431,5,5 +13432,5,5 +13433,5,5 +13434,5,5 +13435,5,5 +13436,5,5 +13437,3,3 +13438,5,5 +13439,3,3 +13440,3,3 +13441,3,3 +13442,5,5 +13443,3,3 +13444,5,5 +13445,5,5 +13446,3,3 +13447,3,3 +13448,3,3 +13449,3,3 +13450,3,3 +13451,5,5 +13452,3,3 +13453,4,5 +13454,5,5 +13455,5,3 +13456,3,3 +13457,5,5 +13458,5,5 +13459,5,5 +13460,3,3 +13461,5,5 +13462,3,3 +13463,4,4 +13464,5,5 +13465,5,5 +13466,5,5 +13467,3,3 +13468,3,3 +13469,5,5 +13470,5,3 +13471,3,3 +13472,3,3 +13473,4,4 +13474,5,5 +13475,3,3 +13476,5,5 +13477,3,3 +13478,3,3 +13479,3,3 +13480,3,3 +13481,3,3 +13482,5,5 +13483,3,3 +13484,4,4 +13485,4,4 +13486,3,5 +13487,3,3 +13488,5,5 +13489,5,5 +13490,3,3 +13491,3,3 +13492,3,3 +13493,5,5 +13494,5,5 +13495,3,3 +13496,3,3 +13497,3,3 +13498,5,5 +13499,5,5 +13500,3,3 +13501,5,5 +13502,3,3 +13503,5,5 +13504,3,3 +13505,3,3 +13506,4,5 +13507,3,3 +13508,5,5 +13509,3,3 +13510,4,4 +13511,3,3 +13512,3,5 +13513,5,5 +13514,5,5 +13515,5,5 +13516,5,5 +13517,5,5 +13518,3,3 +13519,5,5 +13520,5,5 +13521,5,5 +13522,3,3 +13523,3,3 +13524,5,5 +13525,5,5 +13526,3,3 +13527,5,5 +13528,3,3 +13529,4,5 +13530,5,5 +13531,3,3 +13532,3,3 +13533,5,5 +13534,3,5 +13535,5,5 +13536,3,3 +13537,3,3 +13538,5,5 +13539,5,5 +13540,3,3 +13541,3,3 +13542,5,5 +13543,5,5 +13544,3,3 +13545,5,5 +13546,3,5 +13547,5,5 +13548,3,3 +13549,3,3 +13550,5,5 +13551,5,5 +13552,3,3 +13553,5,5 +13554,3,3 +13555,5,5 +13556,3,3 +13557,3,3 +13558,3,3 +13559,3,3 +13560,5,5 +13561,3,3 +13562,3,3 +13563,4,4 +13564,5,5 +13565,3,3 +13566,5,5 +13567,5,5 +13568,3,3 +13569,5,5 +13570,5,5 +13571,5,5 +13572,5,5 +13573,5,5 +13574,4,4 +13575,3,3 +13576,3,3 +13577,5,5 +13578,3,3 +13579,3,3 +13580,5,5 +13581,5,5 +13582,5,5 +13583,4,4 +13584,5,5 +13585,5,5 +13586,5,5 +13587,3,3 +13588,5,5 +13589,5,5 +13590,5,5 +13591,5,5 +13592,3,3 +13593,3,3 +13594,5,5 +13595,3,3 +13596,5,5 +13597,3,3 +13598,3,5 +13599,5,5 +13600,3,3 +13601,5,5 +13602,5,5 +13603,4,4 +13604,5,5 +13605,5,5 +13606,3,3 +13607,5,5 +13608,5,3 +13609,5,5 +13610,3,3 +13611,5,3 +13612,5,5 +13613,5,5 +13614,3,3 +13615,3,3 +13616,5,5 +13617,3,3 +13618,5,5 +13619,3,3 +13620,3,3 +13621,3,3 +13622,5,5 +13623,5,5 +13624,5,5 +13625,5,5 +13626,5,5 +13627,5,5 +13628,5,3 +13629,5,5 +13630,5,5 +13631,5,5 +13632,3,3 +13633,5,5 +13634,5,5 +13635,5,5 +13636,5,5 +13637,3,3 +13638,3,3 +13639,5,5 +13640,5,5 +13641,5,5 +13642,3,3 +13643,5,5 +13644,5,5 +13645,3,3 +13646,5,3 +13647,3,3 +13648,3,3 +13649,3,3 +13650,4,4 +13651,3,3 +13652,5,5 +13653,3,3 +13654,5,3 +13655,3,3 +13656,3,3 +13657,3,3 +13658,4,5 +13659,3,3 +13660,5,5 +13661,5,5 +13662,5,5 +13663,5,5 +13664,5,5 +13665,3,5 +13666,3,3 +13667,5,5 +13668,3,3 +13669,5,5 +13670,3,3 +13671,5,5 +13672,5,5 +13673,3,3 +13674,3,3 +13675,5,4 +13676,3,3 +13677,3,5 +13678,5,5 +13679,3,3 +13680,5,5 +13681,5,5 +13682,5,5 +13683,3,3 +13684,5,5 +13685,5,5 +13686,5,5 +13687,3,3 +13688,5,3 +13689,3,5 +13690,3,3 +13691,5,5 +13692,5,5 +13693,5,5 +13694,3,3 +13695,5,5 +13696,5,5 +13697,3,3 +13698,5,5 +13699,5,5 +13700,3,3 +13701,5,5 +13702,5,5 +13703,3,5 +13704,5,3 +13705,3,3 +13706,5,5 +13707,5,5 +13708,5,5 +13709,5,5 +13710,5,5 +13711,3,3 +13712,3,3 +13713,5,5 +13714,3,3 +13715,3,3 +13716,3,3 +13717,3,3 +13718,3,3 +13719,5,5 +13720,5,5 +13721,4,5 +13722,3,3 +13723,5,5 +13724,3,3 +13725,5,5 +13726,3,3 +13727,4,4 +13728,5,5 +13729,3,3 +13730,5,5 +13731,5,5 +13732,5,5 +13733,5,5 +13734,5,5 +13735,3,3 +13736,4,4 +13737,5,5 +13738,5,3 +13739,3,3 +13740,3,3 +13741,5,5 +13742,5,5 +13743,3,3 +13744,3,3 +13745,4,5 +13746,3,3 +13747,5,3 +13748,4,4 +13749,5,5 +13750,3,3 +13751,3,3 +13752,5,5 +13753,5,5 +13754,3,3 +13755,5,5 +13756,5,5 +13757,5,3 +13758,5,5 +13759,4,4 +13760,3,3 +13761,3,3 +13762,5,5 +13763,4,5 +13764,3,3 +13765,3,3 +13766,3,3 +13767,5,5 +13768,3,3 +13769,5,3 +13770,5,5 +13771,5,5 +13772,3,3 +13773,5,5 +13774,3,3 +13775,3,3 +13776,5,5 +13777,4,4 +13778,5,5 +13779,5,5 +13780,4,4 +13781,5,3 +13782,5,5 +13783,5,5 +13784,5,5 +13785,3,5 +13786,5,5 +13787,3,3 +13788,3,3 +13789,4,4 +13790,3,3 +13791,3,3 +13792,5,5 +13793,3,3 +13794,5,5 +13795,5,5 +13796,3,3 +13797,3,3 +13798,5,5 +13799,5,5 +13800,5,5 +13801,3,3 +13802,5,5 +13803,5,5 +13804,3,3 +13805,5,5 +13806,3,3 +13807,3,3 +13808,3,3 +13809,5,5 +13810,5,5 +13811,3,3 +13812,4,4 +13813,3,3 +13814,5,5 +13815,5,5 +13816,5,5 +13817,3,3 +13818,5,5 +13819,5,5 +13820,3,3 +13821,3,3 +13822,5,5 +13823,5,5 +13824,5,4 +13825,5,5 +13826,3,3 +13827,3,3 +13828,5,4 +13829,8,8 +13830,6,6 +13831,8,8 +13832,8,8 +13833,8,8 +13834,8,8 +13835,6,6 +13836,8,8 +13837,8,8 +13838,8,8 +13839,8,8 +13840,8,8 +13841,6,6 +13842,8,8 +13843,8,8 +13844,8,8 +13845,7,7 +13846,8,8 +13847,8,8 +13848,8,8 +13849,8,8 +13850,6,6 +13851,7,7 +13852,8,8 +13853,8,8 +13854,8,8 +13855,8,8 +13856,8,8 +13857,8,8 +13858,7,7 +13859,8,8 +13860,6,6 +13861,7,7 +13862,7,7 +13863,8,8 +13864,6,6 +13865,8,8 +13866,8,8 +13867,8,8 +13868,8,8 +13869,6,6 +13870,8,8 +13871,8,6 +13872,8,8 +13873,8,8 +13874,8,8 +13875,8,8 +13876,6,6 +13877,8,6 +13878,8,8 +13879,8,8 +13880,6,6 +13881,8,8 +13882,8,8 +13883,8,8 +13884,8,8 +13885,8,8 +13886,8,8 +13887,7,7 +13888,6,6 +13889,8,8 +13890,8,8 +13891,6,6 +13892,6,6 +13893,7,7 +13894,8,8 +13895,6,6 +13896,8,8 +13897,8,8 +13898,8,8 +13899,8,8 +13900,8,8 +13901,8,8 +13902,8,6 +13903,6,6 +13904,8,8 +13905,6,8 +13906,8,8 +13907,7,7 +13908,8,8 +13909,8,8 +13910,8,6 +13911,8,8 +13912,8,8 +13913,8,8 +13914,8,8 +13915,8,8 +13916,8,8 +13917,8,8 +13918,8,8 +13919,6,6 +13920,6,6 +13921,8,8 +13922,8,8 +13923,8,8 +13924,8,8 +13925,8,8 +13926,8,8 +13927,7,7 +13928,8,8 +13929,8,8 +13930,7,7 +13931,8,8 +13932,8,8 +13933,7,7 +13934,8,8 +13935,8,8 +13936,8,8 +13937,8,8 +13938,8,8 +13939,8,8 +13940,8,8 +13941,6,6 +13942,8,8 +13943,8,8 +13944,8,8 +13945,7,7 +13946,6,6 +13947,8,8 +13948,6,6 +13949,8,8 +13950,8,8 +13951,8,8 +13952,6,6 +13953,6,6 +13954,8,8 +13955,8,8 +13956,8,8 +13957,7,7 +13958,8,8 +13959,6,6 +13960,8,8 +13961,8,8 +13962,7,7 +13963,8,8 +13964,7,7 +13965,8,8 +13966,8,8 +13967,8,8 +13968,8,8 +13969,8,8 +13970,6,6 +13971,8,8 +13972,7,7 +13973,8,6 +13974,8,8 +13975,7,7 +13976,8,8 +13977,8,8 +13978,8,8 +13979,8,8 +13980,8,7 +13981,8,6 +13982,7,7 +13983,6,6 +13984,8,8 +13985,7,7 +13986,8,8 +13987,8,8 +13988,8,6 +13989,8,8 +13990,8,8 +13991,6,6 +13992,8,8 +13993,8,8 +13994,8,8 +13995,8,6 +13996,8,8 +13997,8,8 +13998,8,8 +13999,7,7 +14000,7,7 +14001,8,8 +14002,8,8 +14003,8,8 +14004,8,8 +14005,8,8 +14006,8,8 +14007,8,8 +14008,6,6 +14009,8,8 +14010,8,8 +14011,6,6 +14012,8,8 +14013,8,8 +14014,8,6 +14015,8,8 +14016,7,7 +14017,8,8 +14018,8,8 +14019,8,8 +14020,6,6 +14021,6,6 +14022,8,8 +14023,6,6 +14024,8,8 +14025,6,6 +14026,8,8 +14027,8,8 +14028,8,8 +14029,8,8 +14030,8,6 +14031,8,8 +14032,8,8 +14033,6,6 +14034,8,8 +14035,6,6 +14036,8,8 +14037,8,8 +14038,8,8 +14039,8,8 +14040,6,6 +14041,8,8 +14042,8,8 +14043,8,8 +14044,6,6 +14045,6,6 +14046,8,8 +14047,8,8 +14048,8,8 +14049,8,8 +14050,8,8 +14051,8,8 +14052,8,8 +14053,8,8 +14054,8,8 +14055,8,8 +14056,8,8 +14057,6,6 +14058,7,7 +14059,8,8 +14060,8,8 +14061,6,6 +14062,8,8 +14063,8,8 +14064,8,8 +14065,8,8 +14066,8,7 +14067,8,8 +14068,8,8 +14069,8,8 +14070,8,8 +14071,7,7 +14072,6,6 +14073,8,8 +14074,8,8 +14075,8,8 +14076,8,8 +14077,6,8 +14078,8,8 +14079,7,7 +14080,8,8 +14081,6,6 +14082,8,8 +14083,8,8 +14084,8,8 +14085,8,8 +14086,8,6 +14087,8,8 +14088,8,8 +14089,7,7 +14090,8,8 +14091,8,8 +14092,8,8 +14093,8,8 +14094,7,7 +14095,8,8 +14096,8,7 +14097,6,6 +14098,6,6 +14099,8,8 +14100,8,8 +14101,8,8 +14102,6,6 +14103,6,6 +14104,8,6 +14105,8,8 +14106,8,8 +14107,6,8 +14108,8,8 +14109,8,8 +14110,8,8 +14111,8,6 +14112,6,6 +14113,8,8 +14114,8,8 +14115,6,6 +14116,6,6 +14117,8,8 +14118,8,8 +14119,8,8 +14120,8,8 +14121,8,8 +14122,6,6 +14123,8,8 +14124,8,8 +14125,8,8 +14126,8,8 +14127,8,8 +14128,8,8 +14129,8,8 +14130,6,6 +14131,8,8 +14132,8,8 +14133,8,8 +14134,6,6 +14135,8,8 +14136,8,8 +14137,8,8 +14138,8,8 +14139,8,8 +14140,8,8 +14141,8,8 +14142,8,8 +14143,8,8 +14144,6,6 +14145,8,8 +14146,7,7 +14147,6,6 +14148,6,6 +14149,8,8 +14150,8,8 +14151,6,6 +14152,8,8 +14153,8,8 +14154,7,7 +14155,8,8 +14156,8,8 +14157,8,8 +14158,8,8 +14159,8,8 +14160,8,8 +14161,6,6 +14162,8,8 +14163,8,8 +14164,8,8 +14165,8,8 +14166,8,8 +14167,6,6 +14168,8,8 +14169,8,8 +14170,6,6 +14171,6,6 +14172,8,8 +14173,7,7 +14174,6,6 +14175,6,6 +14176,6,6 +14177,8,8 +14178,8,8 +14179,7,7 +14180,6,6 +14181,8,8 +14182,6,6 +14183,8,6 +14184,8,8 +14185,6,6 +14186,8,8 +14187,6,6 +14188,8,8 +14189,8,8 +14190,8,8 +14191,8,8 +14192,6,6 +14193,8,8 +14194,6,6 +14195,6,6 +14196,8,8 +14197,8,8 +14198,8,8 +14199,8,8 +14200,8,8 +14201,8,8 +14202,6,6 +14203,8,8 +14204,8,8 +14205,8,8 +14206,8,8 +14207,8,8 +14208,8,8 +14209,6,6 +14210,8,8 +14211,7,7 +14212,8,8 +14213,8,8 +14214,7,7 +14215,8,8 +14216,8,6 +14217,8,8 +14218,6,6 +14219,7,7 +14220,6,6 +14221,8,8 +14222,7,7 +14223,8,8 +14224,8,8 +14225,7,7 +14226,8,8 +14227,8,8 +14228,8,8 +14229,8,8 +14230,8,8 +14231,8,8 +14232,8,8 +14233,7,6 +14234,8,8 +14235,6,8 +14236,8,8 +14237,8,8 +14238,8,8 +14239,8,8 +14240,8,8 +14241,8,8 +14242,8,8 +14243,8,8 +14244,8,8 +14245,8,8 +14246,6,6 +14247,6,6 +14248,8,8 +14249,6,6 +14250,8,8 +14251,8,8 +14252,7,7 +14253,6,6 +14254,7,7 +14255,8,8 +14256,8,8 +14257,8,8 +14258,6,6 +14259,8,8 +14260,8,8 +14261,6,6 +14262,8,8 +14263,8,8 +14264,8,8 +14265,8,6 +14266,8,8 +14267,6,6 +14268,8,8 +14269,8,8 +14270,8,8 +14271,6,6 +14272,6,6 +14273,6,6 +14274,8,8 +14275,8,8 +14276,8,8 +14277,8,8 +14278,8,8 +14279,8,8 +14280,8,8 +14281,6,6 +14282,8,8 +14283,8,8 +14284,8,8 +14285,7,7 +14286,8,8 +14287,8,8 +14288,8,6 +14289,6,6 +14290,8,8 +14291,8,8 +14292,6,6 +14293,8,8 +14294,8,8 +14295,7,7 +14296,8,7 +14297,8,8 +14298,8,8 +14299,8,8 +14300,8,8 +14301,8,8 +14302,8,8 +14303,8,8 +14304,8,8 +14305,8,8 +14306,6,6 +14307,6,6 +14308,8,8 +14309,8,8 +14310,7,7 +14311,8,8 +14312,8,8 +14313,8,8 +14314,8,8 +14315,8,6 +14316,8,8 +14317,8,8 +14318,8,8 +14319,7,7 +14320,6,8 +14321,8,8 +14322,8,8 +14323,8,8 +14324,8,8 +14325,8,8 +14326,8,6 +14327,8,8 +14328,8,8 +14329,8,8 +14330,6,6 +14331,6,6 +14332,7,7 +14333,7,7 +14334,6,6 +14335,8,8 +14336,8,8 +14337,8,8 +14338,6,6 +14339,8,8 +14340,8,8 +14341,8,8 +14342,8,8 +14343,7,7 +14344,6,6 +14345,8,8 +14346,7,7 +14347,6,6 +14348,8,8 +14349,8,8 +14350,8,8 +14351,6,6 +14352,8,8 +14353,8,8 +14354,8,8 +14355,8,8 +14356,8,6 +14357,6,6 +14358,8,8 +14359,8,8 +14360,8,8 +14361,8,8 +14362,8,8 +14363,8,8 +14364,8,6 +14365,8,6 +14366,6,6 +14367,8,8 +14368,8,8 +14369,8,8 +14370,8,8 +14371,8,6 +14372,8,8 +14373,8,8 +14374,7,7 +14375,6,6 +14376,8,8 +14377,6,6 +14378,6,6 +14379,7,7 +14380,8,8 +14381,8,8 +14382,7,7 +14383,8,8 +14384,8,8 +14385,6,6 +14386,6,6 +14387,8,8 +14388,8,7 +14389,8,8 +14390,8,8 +14391,8,8 +14392,7,7 +14393,8,8 +14394,8,8 +14395,8,8 +14396,8,8 +14397,8,8 +14398,8,8 +14399,8,8 +14400,8,8 +14401,8,8 +14402,8,8 +14403,8,8 +14404,8,8 +14405,8,6 +14406,8,8 +14407,6,8 +14408,8,8 +14409,6,6 +14410,7,7 +14411,6,6 +14412,6,6 +14413,8,8 +14414,8,8 +14415,8,8 +14416,6,6 +14417,8,8 +14418,8,8 +14419,8,8 +14420,8,8 +14421,6,6 +14422,7,7 +14423,8,6 +14424,6,6 +14425,8,8 +14426,7,7 +14427,8,8 +14428,8,8 +14429,8,8 +14430,8,6 +14431,8,8 +14432,8,8 +14433,6,6 +14434,8,8 +14435,8,8 +14436,7,7 +14437,6,6 +14438,8,8 +14439,8,8 +14440,8,8 +14441,6,8 +14442,6,6 +14443,8,8 +14444,8,8 +14445,6,6 +14446,8,8 +14447,8,8 +14448,8,8 +14449,6,6 +14450,8,8 +14451,8,6 +14452,8,8 +14453,8,6 +14454,6,6 +14455,8,8 +14456,8,6 +14457,8,8 +14458,8,8 +14459,8,8 +14460,6,8 +14461,7,7 +14462,8,8 +14463,8,8 +14464,8,8 +14465,8,6 +14466,8,8 +14467,8,8 +14468,6,6 +14469,8,8 +14470,6,8 +14471,8,8 +14472,8,8 +14473,7,7 +14474,6,6 +14475,8,8 +14476,8,8 +14477,6,6 +14478,8,8 +14479,8,8 +14480,6,6 +14481,8,8 +14482,8,8 +14483,8,8 +14484,7,7 +14485,8,8 +14486,8,8 +14487,8,8 +14488,8,8 +14489,8,8 +14490,8,8 +14491,8,8 +14492,6,6 +14493,8,8 +14494,8,8 +14495,8,8 +14496,6,6 +14497,8,8 +14498,7,7 +14499,8,6 +14500,6,6 +14501,8,8 +14502,8,6 +14503,8,8 +14504,6,6 +14505,6,6 +14506,8,8 +14507,6,6 +14508,8,6 +14509,7,7 +14510,8,8 +14511,8,8 +14512,6,6 +14513,8,8 +14514,8,8 +14515,8,8 +14516,8,8 +14517,7,7 +14518,6,6 +14519,8,8 +14520,8,8 +14521,7,7 +14522,6,8 +14523,8,8 +14524,8,8 +14525,8,8 +14526,8,8 +14527,8,8 +14528,8,8 +14529,8,8 +14530,6,6 +14531,8,7 +14532,8,8 +14533,8,8 +14534,8,8 +14535,6,6 +14536,8,8 +14537,8,8 +14538,8,8 +14539,8,8 +14540,8,8 +14541,8,8 +14542,8,8 +14543,8,8 +14544,8,8 +14545,8,8 +14546,6,6 +14547,6,6 +14548,6,8 +14549,8,8 +14550,8,8 +14551,8,8 +14552,6,6 +14553,8,8 +14554,8,8 +14555,8,8 +14556,8,8 +14557,8,8 +14558,8,8 +14559,7,7 +14560,8,8 +14561,8,8 +14562,8,8 +14563,6,6 +14564,8,8 +14565,8,8 +14566,8,8 +14567,8,8 +14568,8,8 +14569,6,6 +14570,8,6 +14571,8,8 +14572,8,8 +14573,8,8 +14574,7,7 +14575,8,8 +14576,8,8 +14577,7,7 +14578,8,8 +14579,8,8 +14580,8,8 +14581,6,8 +14582,6,6 +14583,8,8 +14584,6,6 +14585,6,6 +14586,8,8 +14587,8,8 +14588,8,8 +14589,8,8 +14590,6,6 +14591,8,8 +14592,8,8 +14593,8,8 +14594,8,8 +14595,6,6 +14596,6,6 +14597,8,8 +14598,6,6 +14599,7,7 +14600,8,8 +14601,8,8 +14602,8,8 +14603,6,6 +14604,8,8 +14605,8,8 +14606,6,6 +14607,6,6 +14608,7,7 +14609,7,7 +14610,8,8 +14611,8,8 +14612,8,8 +14613,8,8 +14614,8,6 +14615,6,6 +14616,8,8 +14617,8,8 +14618,8,8 +14619,8,8 +14620,8,8 +14621,8,8 +14622,6,6 +14623,8,8 +14624,8,8 +14625,8,8 +14626,8,8 +14627,8,6 +14628,8,8 +14629,6,6 +14630,8,8 +14631,8,8 +14632,8,8 +14633,8,8 +14634,6,6 +14635,8,8 +14636,8,8 +14637,8,8 +14638,6,6 +14639,8,8 +14640,8,8 +14641,8,8 +14642,6,6 +14643,8,8 +14644,8,8 +14645,6,6 +14646,6,6 +14647,8,8 +14648,7,7 +14649,8,8 +14650,6,6 +14651,8,8 +14652,8,8 +14653,8,8 +14654,8,8 +14655,8,8 +14656,8,8 +14657,7,7 +14658,8,8 +14659,6,6 +14660,8,8 +14661,8,8 +14662,8,8 +14663,8,8 +14664,8,8 +14665,8,8 +14666,8,8 +14667,8,8 +14668,7,7 +14669,8,8 +14670,8,8 +14671,8,8 +14672,8,8 +14673,8,8 +14674,8,8 +14675,8,6 +14676,8,8 +14677,8,8 +14678,8,8 +14679,8,8 +14680,8,8 +14681,8,8 +14682,8,8 +14683,8,6 +14684,8,8 +14685,8,8 +14686,8,8 +14687,6,6 +14688,8,8 +14689,6,6 +14690,8,8 +14691,8,8 +14692,6,6 +14693,7,7 +14694,8,8 +14695,8,8 +14696,8,8 +14697,8,8 +14698,8,8 +14699,6,6 +14700,8,8 +14701,8,8 +14702,8,8 +14703,8,8 +14704,6,6 +14705,8,8 +14706,8,8 +14707,7,7 +14708,6,6 +14709,6,6 +14710,6,6 +14711,8,8 +14712,8,8 +14713,8,8 +14714,8,8 +14715,8,8 +14716,6,6 +14717,8,8 +14718,7,7 +14719,6,6 +14720,8,8 +14721,6,6 +14722,8,8 +14723,8,8 +14724,8,8 +14725,6,6 +14726,8,8 +14727,8,8 +14728,8,8 +14729,8,8 +14730,8,8 +14731,8,8 +14732,8,6 +14733,6,6 +14734,8,8 +14735,8,8 +14736,6,6 +14737,8,8 +14738,8,8 +14739,8,8 +14740,8,8 +14741,8,8 +14742,8,8 +14743,6,6 +14744,8,7 +14745,6,6 +14746,6,6 +14747,8,8 +14748,8,8 +14749,8,8 +14750,8,8 +14751,8,8 +14752,8,6 +14753,8,8 +14754,8,8 +14755,8,8 +14756,8,8 +14757,8,8 +14758,8,8 +14759,6,6 +14760,6,6 +14761,6,6 +14762,8,8 +14763,8,8 +14764,8,8 +14765,8,8 +14766,8,8 +14767,8,8 +14768,8,8 +14769,8,8 +14770,8,8 +14771,6,6 +14772,8,8 +14773,8,8 +14774,8,8 +14775,8,8 +14776,8,8 +14777,6,6 +14778,6,6 +14779,8,8 +14780,8,8 +14781,10,10 +14782,9,9 +14783,11,11 +14784,9,9 +14785,9,9 +14786,9,9 +14787,11,11 +14788,11,11 +14789,11,11 +14790,9,11 +14791,11,11 +14792,9,9 +14793,9,9 +14794,9,9 +14795,11,11 +14796,11,11 +14797,9,9 +14798,9,9 +14799,10,10 +14800,11,11 +14801,9,9 +14802,10,10 +14803,9,9 +14804,11,11 +14805,9,9 +14806,11,9 +14807,11,11 +14808,9,9 +14809,11,11 +14810,9,9 +14811,9,11 +14812,11,11 +14813,9,9 +14814,9,9 +14815,11,11 +14816,9,9 +14817,11,11 +14818,11,11 +14819,10,10 +14820,11,11 +14821,11,11 +14822,11,11 +14823,9,9 +14824,10,10 +14825,9,9 +14826,9,9 +14827,11,9 +14828,9,9 +14829,11,11 +14830,11,11 +14831,11,11 +14832,9,9 +14833,11,11 +14834,11,11 +14835,11,11 +14836,9,9 +14837,9,9 +14838,9,9 +14839,10,10 +14840,10,10 +14841,11,11 +14842,11,11 +14843,9,9 +14844,10,10 +14845,10,10 +14846,10,10 +14847,11,11 +14848,9,9 +14849,11,11 +14850,11,11 +14851,11,11 +14852,11,11 +14853,9,11 +14854,11,11 +14855,9,9 +14856,11,11 +14857,10,10 +14858,9,9 +14859,11,9 +14860,11,11 +14861,9,9 +14862,10,10 +14863,10,10 +14864,9,9 +14865,9,9 +14866,11,11 +14867,11,11 +14868,11,11 +14869,11,11 +14870,11,11 +14871,11,11 +14872,11,11 +14873,10,10 +14874,9,11 +14875,10,10 +14876,9,9 +14877,10,10 +14878,11,11 +14879,11,11 +14880,11,11 +14881,11,11 +14882,11,9 +14883,10,10 +14884,10,10 +14885,11,11 +14886,11,11 +14887,9,9 +14888,9,9 +14889,10,10 +14890,11,11 +14891,11,11 +14892,11,11 +14893,9,9 +14894,11,11 +14895,9,9 +14896,10,10 +14897,11,11 +14898,10,10 +14899,9,9 +14900,10,10 +14901,11,11 +14902,9,9 +14903,9,9 +14904,10,10 +14905,11,11 +14906,11,11 +14907,9,9 +14908,11,11 +14909,9,9 +14910,11,11 +14911,9,9 +14912,11,11 +14913,10,10 +14914,11,11 +14915,11,11 +14916,11,11 +14917,11,11 +14918,10,10 +14919,11,11 +14920,11,11 +14921,10,10 +14922,9,9 +14923,10,10 +14924,11,11 +14925,10,10 +14926,10,10 +14927,10,10 +14928,9,9 +14929,9,9 +14930,10,10 +14931,11,11 +14932,11,11 +14933,10,10 +14934,11,11 +14935,11,11 +14936,10,10 +14937,10,10 +14938,10,10 +14939,11,11 +14940,10,10 +14941,11,11 +14942,11,11 +14943,10,10 +14944,11,11 +14945,11,11 +14946,10,10 +14947,10,10 +14948,11,11 +14949,11,11 +14950,11,11 +14951,9,9 +14952,11,11 +14953,9,9 +14954,11,11 +14955,10,10 +14956,9,9 +14957,11,11 +14958,9,9 +14959,9,11 +14960,11,11 +14961,10,11 +14962,11,11 +14963,11,11 +14964,11,11 +14965,11,11 +14966,10,10 +14967,10,10 +14968,11,11 +14969,11,11 +14970,11,11 +14971,10,10 +14972,10,10 +14973,9,9 +14974,11,11 +14975,11,11 +14976,9,9 +14977,10,10 +14978,11,11 +14979,11,11 +14980,11,11 +14981,11,11 +14982,11,11 +14983,9,9 +14984,9,9 +14985,11,11 +14986,10,10 +14987,11,11 +14988,10,10 +14989,9,9 +14990,9,9 +14991,10,10 +14992,10,10 +14993,11,11 +14994,11,11 +14995,11,11 +14996,10,11 +14997,9,9 +14998,10,10 +14999,9,9 +15000,11,11 +15001,9,9 +15002,11,11 +15003,9,9 +15004,10,10 +15005,9,9 +15006,9,9 +15007,9,9 +15008,10,10 +15009,9,9 +15010,11,11 +15011,10,10 +15012,9,9 +15013,11,11 +15014,11,11 +15015,9,9 +15016,11,11 +15017,11,11 +15018,11,11 +15019,11,11 +15020,10,10 +15021,11,11 +15022,10,10 +15023,11,11 +15024,9,10 +15025,11,11 +15026,9,11 +15027,11,11 +15028,11,11 +15029,9,9 +15030,10,10 +15031,11,11 +15032,11,11 +15033,10,10 +15034,9,9 +15035,9,9 +15036,10,10 +15037,11,11 +15038,9,9 +15039,9,9 +15040,11,11 +15041,10,10 +15042,9,9 +15043,11,11 +15044,11,11 +15045,10,10 +15046,11,11 +15047,10,10 +15048,9,9 +15049,11,11 +15050,11,11 +15051,9,9 +15052,11,11 +15053,9,9 +15054,9,9 +15055,11,11 +15056,11,11 +15057,11,11 +15058,9,9 +15059,9,9 +15060,11,11 +15061,9,9 +15062,9,9 +15063,9,9 +15064,10,10 +15065,11,11 +15066,11,11 +15067,10,10 +15068,11,11 +15069,11,11 +15070,9,9 +15071,11,11 +15072,11,11 +15073,11,11 +15074,10,10 +15075,11,11 +15076,11,11 +15077,9,11 +15078,11,11 +15079,11,11 +15080,10,10 +15081,11,11 +15082,11,11 +15083,9,9 +15084,10,10 +15085,9,9 +15086,11,11 +15087,11,11 +15088,9,9 +15089,9,9 +15090,10,11 +15091,9,9 +15092,11,11 +15093,9,9 +15094,11,11 +15095,11,11 +15096,10,10 +15097,11,11 +15098,9,9 +15099,9,9 +15100,10,10 +15101,9,9 +15102,9,9 +15103,9,9 +15104,9,9 +15105,11,11 +15106,11,11 +15107,11,11 +15108,11,11 +15109,10,10 +15110,10,10 +15111,9,9 +15112,11,11 +15113,11,11 +15114,10,10 +15115,11,11 +15116,9,9 +15117,11,11 +15118,11,11 +15119,11,11 +15120,9,9 +15121,9,9 +15122,11,11 +15123,9,9 +15124,9,9 +15125,11,11 +15126,9,9 +15127,10,10 +15128,11,11 +15129,9,9 +15130,9,9 +15131,10,10 +15132,11,11 +15133,10,10 +15134,9,9 +15135,9,9 +15136,11,11 +15137,11,9 +15138,11,11 +15139,11,11 +15140,9,9 +15141,10,10 +15142,9,9 +15143,11,11 +15144,9,10 +15145,9,9 +15146,11,11 +15147,9,9 +15148,9,9 +15149,9,9 +15150,11,11 +15151,11,11 +15152,9,9 +15153,9,9 +15154,9,9 +15155,11,11 +15156,10,10 +15157,10,10 +15158,11,11 +15159,10,11 +15160,11,11 +15161,9,9 +15162,11,11 +15163,11,11 +15164,9,9 +15165,9,9 +15166,9,9 +15167,11,11 +15168,11,11 +15169,10,10 +15170,9,9 +15171,10,10 +15172,11,11 +15173,11,11 +15174,9,11 +15175,10,10 +15176,10,10 +15177,11,11 +15178,10,10 +15179,11,11 +15180,10,10 +15181,11,11 +15182,11,11 +15183,11,11 +15184,11,11 +15185,10,10 +15186,11,11 +15187,11,11 +15188,9,9 +15189,11,11 +15190,11,11 +15191,9,9 +15192,11,11 +15193,10,10 +15194,11,11 +15195,9,9 +15196,11,11 +15197,10,10 +15198,11,11 +15199,11,10 +15200,9,9 +15201,11,11 +15202,11,11 +15203,11,11 +15204,9,9 +15205,11,11 +15206,10,10 +15207,11,11 +15208,9,9 +15209,10,11 +15210,10,10 +15211,9,9 +15212,10,11 +15213,10,10 +15214,10,10 +15215,11,11 +15216,11,11 +15217,9,9 +15218,11,11 +15219,10,11 +15220,9,9 +15221,11,11 +15222,11,11 +15223,9,11 +15224,10,10 +15225,9,9 +15226,10,10 +15227,11,11 +15228,11,11 +15229,9,9 +15230,11,11 +15231,9,9 +15232,11,11 +15233,11,11 +15234,9,9 +15235,10,10 +15236,9,9 +15237,11,11 +15238,10,10 +15239,9,9 +15240,11,11 +15241,11,11 +15242,11,11 +15243,11,11 +15244,9,9 +15245,11,11 +15246,11,11 +15247,9,9 +15248,11,11 +15249,11,11 +15250,9,9 +15251,9,9 +15252,9,9 +15253,11,11 +15254,11,11 +15255,11,11 +15256,9,9 +15257,10,10 +15258,9,9 +15259,9,9 +15260,11,11 +15261,9,9 +15262,10,10 +15263,11,11 +15264,11,11 +15265,11,11 +15266,11,11 +15267,9,9 +15268,11,11 +15269,10,11 +15270,9,9 +15271,11,11 +15272,9,9 +15273,10,10 +15274,11,11 +15275,10,10 +15276,11,11 +15277,11,11 +15278,9,9 +15279,11,11 +15280,10,10 +15281,11,11 +15282,9,9 +15283,11,11 +15284,11,9 +15285,10,11 +15286,9,9 +15287,11,11 +15288,11,11 +15289,9,9 +15290,9,9 +15291,9,9 +15292,11,11 +15293,9,9 +15294,11,11 +15295,10,10 +15296,9,9 +15297,11,11 +15298,11,11 +15299,11,11 +15300,10,10 +15301,11,11 +15302,11,11 +15303,9,9 +15304,11,11 +15305,9,9 +15306,11,11 +15307,11,11 +15308,10,10 +15309,11,11 +15310,11,11 +15311,11,9 +15312,11,11 +15313,9,9 +15314,11,11 +15315,11,11 +15316,11,11 +15317,9,9 +15318,11,11 +15319,9,9 +15320,10,10 +15321,11,11 +15322,11,11 +15323,11,11 +15324,11,11 +15325,11,11 +15326,10,10 +15327,11,11 +15328,10,10 +15329,9,9 +15330,10,10 +15331,11,11 +15332,10,10 +15333,9,9 +15334,9,10 +15335,11,11 +15336,10,10 +15337,11,11 +15338,9,9 +15339,9,9 +15340,11,11 +15341,11,11 +15342,11,11 +15343,11,11 +15344,11,11 +15345,11,11 +15346,10,10 +15347,9,9 +15348,9,11 +15349,11,11 +15350,9,9 +15351,10,11 +15352,9,9 +15353,9,9 +15354,11,11 +15355,9,9 +15356,9,9 +15357,11,11 +15358,11,11 +15359,11,11 +15360,10,10 +15361,11,11 +15362,9,9 +15363,11,11 +15364,11,11 +15365,11,11 +15366,11,11 +15367,9,9 +15368,10,10 +15369,9,9 +15370,9,9 +15371,11,11 +15372,10,10 +15373,11,11 +15374,11,11 +15375,11,11 +15376,11,11 +15377,9,9 +15378,11,11 +15379,10,10 +15380,11,11 +15381,11,10 +15382,11,11 +15383,11,11 +15384,11,11 +15385,11,11 +15386,11,11 +15387,9,9 +15388,11,11 +15389,11,11 +15390,11,11 +15391,11,11 +15392,9,9 +15393,11,11 +15394,11,11 +15395,11,11 +15396,9,9 +15397,9,9 +15398,9,9 +15399,10,10 +15400,11,11 +15401,11,11 +15402,11,11 +15403,11,11 +15404,11,11 +15405,11,11 +15406,11,9 +15407,9,9 +15408,9,9 +15409,9,9 +15410,9,9 +15411,11,11 +15412,11,11 +15413,10,10 +15414,11,11 +15415,11,11 +15416,11,11 +15417,11,11 +15418,11,11 +15419,11,11 +15420,9,9 +15421,9,9 +15422,11,11 +15423,11,11 +15424,9,9 +15425,9,9 +15426,9,9 +15427,11,11 +15428,11,11 +15429,11,11 +15430,11,11 +15431,11,11 +15432,9,9 +15433,11,11 +15434,9,9 +15435,11,11 +15436,11,11 +15437,11,11 +15438,11,11 +15439,11,11 +15440,9,9 +15441,11,11 +15442,10,10 +15443,11,11 +15444,11,11 +15445,9,9 +15446,11,11 +15447,11,11 +15448,11,11 +15449,9,9 +15450,11,11 +15451,11,11 +15452,11,11 +15453,9,9 +15454,9,11 +15455,11,11 +15456,11,11 +15457,11,11 +15458,10,10 +15459,10,10 +15460,9,9 +15461,11,11 +15462,9,9 +15463,10,10 +15464,10,10 +15465,9,9 +15466,10,10 +15467,9,9 +15468,11,11 +15469,10,10 +15470,11,11 +15471,11,11 +15472,10,10 +15473,9,9 +15474,11,11 +15475,11,11 +15476,11,11 +15477,9,9 +15478,11,9 +15479,11,11 +15480,11,11 +15481,10,10 +15482,10,10 +15483,11,11 +15484,9,9 +15485,11,11 +15486,11,11 +15487,10,10 +15488,9,9 +15489,11,11 +15490,11,11 +15491,11,11 +15492,11,11 +15493,11,11 +15494,11,11 +15495,9,9 +15496,10,10 +15497,9,11 +15498,10,10 +15499,9,9 +15500,11,11 +15501,11,11 +15502,11,11 +15503,11,11 +15504,11,11 +15505,10,10 +15506,11,11 +15507,9,9 +15508,10,10 +15509,11,11 +15510,11,11 +15511,9,9 +15512,9,9 +15513,11,11 +15514,9,9 +15515,11,11 +15516,9,9 +15517,9,9 +15518,11,11 +15519,11,11 +15520,10,10 +15521,10,10 +15522,10,10 +15523,9,9 +15524,10,10 +15525,9,9 +15526,9,9 +15527,9,9 +15528,11,11 +15529,11,11 +15530,11,11 +15531,10,10 +15532,9,9 +15533,10,10 +15534,10,10 +15535,11,11 +15536,9,9 +15537,9,9 +15538,11,11 +15539,11,11 +15540,11,11 +15541,11,11 +15542,10,10 +15543,9,9 +15544,10,10 +15545,11,11 +15546,11,11 +15547,10,10 +15548,10,10 +15549,9,9 +15550,10,10 +15551,10,10 +15552,10,10 +15553,10,10 +15554,11,11 +15555,11,11 +15556,11,11 +15557,9,9 +15558,10,10 +15559,11,9 +15560,10,10 +15561,10,10 +15562,11,11 +15563,11,11 +15564,9,11 +15565,10,10 +15566,11,11 +15567,11,11 +15568,9,9 +15569,11,11 +15570,11,11 +15571,11,11 +15572,11,11 +15573,10,10 +15574,11,11 +15575,11,11 +15576,10,10 +15577,11,11 +15578,11,11 +15579,9,9 +15580,11,11 +15581,9,9 +15582,11,11 +15583,9,9 +15584,10,10 +15585,11,11 +15586,10,9 +15587,10,10 +15588,10,10 +15589,9,9 +15590,11,11 +15591,9,9 +15592,11,11 +15593,9,9 +15594,9,9 +15595,11,11 +15596,11,11 +15597,11,11 +15598,9,9 +15599,11,11 +15600,9,9 +15601,11,11 +15602,11,11 +15603,9,9 +15604,9,9 +15605,11,11 +15606,9,9 +15607,9,9 +15608,11,11 +15609,11,11 +15610,11,11 +15611,11,11 +15612,11,9 +15613,11,11 +15614,9,9 +15615,9,9 +15616,9,9 +15617,9,9 +15618,11,11 +15619,9,9 +15620,11,11 +15621,10,10 +15622,9,9 +15623,11,11 +15624,11,11 +15625,10,10 +15626,10,10 +15627,11,11 +15628,11,11 +15629,11,11 +15630,11,11 +15631,10,10 +15632,10,10 +15633,10,10 +15634,10,10 +15635,9,9 +15636,9,9 +15637,9,9 +15638,11,11 +15639,11,11 +15640,9,9 +15641,9,9 +15642,11,11 +15643,11,11 +15644,11,11 +15645,10,10 +15646,10,10 +15647,10,10 +15648,11,11 +15649,11,11 +15650,11,11 +15651,10,10 +15652,10,10 +15653,10,10 +15654,11,10 +15655,9,9 +15656,9,9 +15657,11,11 +15658,10,10 +15659,9,9 +15660,11,11 +15661,14,14 +15662,14,14 +15663,12,12 +15664,14,14 +15665,12,12 +15666,14,14 +15667,12,12 +15668,12,12 +15669,14,14 +15670,14,12 +15671,14,13 +15672,12,12 +15673,14,14 +15674,14,14 +15675,14,14 +15676,14,14 +15677,14,14 +15678,14,14 +15679,14,14 +15680,14,12 +15681,14,14 +15682,13,13 +15683,13,14 +15684,14,14 +15685,13,13 +15686,14,14 +15687,14,14 +15688,14,12 +15689,12,12 +15690,12,12 +15691,12,12 +15692,14,14 +15693,14,14 +15694,14,14 +15695,14,14 +15696,12,12 +15697,14,14 +15698,14,12 +15699,14,14 +15700,14,14 +15701,14,14 +15702,14,14 +15703,14,14 +15704,14,14 +15705,14,14 +15706,14,14 +15707,14,14 +15708,14,14 +15709,14,14 +15710,14,14 +15711,14,14 +15712,12,12 +15713,14,14 +15714,14,14 +15715,14,14 +15716,13,13 +15717,14,14 +15718,14,13 +15719,14,14 +15720,12,14 +15721,14,14 +15722,14,14 +15723,14,14 +15724,12,12 +15725,14,14 +15726,14,14 +15727,12,12 +15728,14,14 +15729,14,14 +15730,14,14 +15731,12,12 +15732,14,14 +15733,12,12 +15734,14,14 +15735,14,14 +15736,14,14 +15737,14,14 +15738,14,14 +15739,14,14 +15740,12,12 +15741,12,12 +15742,13,13 +15743,12,12 +15744,14,14 +15745,12,12 +15746,13,13 +15747,14,12 +15748,14,14 +15749,13,13 +15750,14,14 +15751,14,14 +15752,12,12 +15753,13,13 +15754,14,14 +15755,12,12 +15756,14,14 +15757,13,13 +15758,14,14 +15759,12,12 +15760,13,13 +15761,14,14 +15762,14,14 +15763,14,14 +15764,12,12 +15765,14,14 +15766,12,12 +15767,14,14 +15768,13,13 +15769,14,14 +15770,14,14 +15771,14,14 +15772,14,14 +15773,14,14 +15774,14,14 +15775,13,13 +15776,14,14 +15777,14,14 +15778,14,14 +15779,12,12 +15780,12,12 +15781,13,13 +15782,12,12 +15783,14,14 +15784,14,14 +15785,14,14 +15786,14,14 +15787,14,12 +15788,14,14 +15789,12,12 +15790,14,14 +15791,12,12 +15792,12,12 +15793,14,14 +15794,13,14 +15795,14,14 +15796,13,13 +15797,14,14 +15798,14,14 +15799,14,13 +15800,14,14 +15801,14,14 +15802,14,14 +15803,12,12 +15804,14,12 +15805,12,12 +15806,14,14 +15807,14,14 +15808,14,14 +15809,14,14 +15810,12,12 +15811,12,12 +15812,14,14 +15813,13,13 +15814,14,14 +15815,12,12 +15816,14,14 +15817,14,13 +15818,14,14 +15819,14,14 +15820,12,12 +15821,12,12 +15822,14,14 +15823,14,14 +15824,12,12 +15825,14,14 +15826,12,12 +15827,14,14 +15828,12,12 +15829,12,12 +15830,14,14 +15831,14,14 +15832,12,12 +15833,12,12 +15834,12,12 +15835,12,12 +15836,14,12 +15837,14,12 +15838,14,14 +15839,14,13 +15840,14,14 +15841,14,14 +15842,12,12 +15843,14,14 +15844,14,14 +15845,14,14 +15846,12,12 +15847,14,14 +15848,12,12 +15849,12,12 +15850,12,12 +15851,12,12 +15852,12,12 +15853,12,12 +15854,14,14 +15855,14,14 +15856,12,12 +15857,14,14 +15858,14,14 +15859,12,12 +15860,14,14 +15861,14,14 +15862,14,12 +15863,14,14 +15864,14,14 +15865,14,14 +15866,14,14 +15867,14,13 +15868,14,14 +15869,14,14 +15870,12,12 +15871,14,14 +15872,14,14 +15873,12,12 +15874,14,14 +15875,12,12 +15876,13,13 +15877,14,14 +15878,14,14 +15879,14,14 +15880,12,12 +15881,14,14 +15882,14,14 +15883,14,13 +15884,12,14 +15885,14,14 +15886,12,12 +15887,14,14 +15888,14,14 +15889,13,14 +15890,13,13 +15891,12,12 +15892,12,14 +15893,14,14 +15894,12,12 +15895,12,14 +15896,12,12 +15897,12,12 +15898,14,14 +15899,14,14 +15900,13,13 +15901,12,12 +15902,12,12 +15903,14,14 +15904,12,12 +15905,14,14 +15906,14,14 +15907,13,13 +15908,12,12 +15909,14,14 +15910,14,14 +15911,14,14 +15912,12,12 +15913,14,12 +15914,14,14 +15915,12,12 +15916,14,14 +15917,12,12 +15918,14,14 +15919,14,14 +15920,14,14 +15921,12,12 +15922,13,13 +15923,14,14 +15924,12,12 +15925,12,12 +15926,14,14 +15927,14,14 +15928,14,14 +15929,14,14 +15930,12,12 +15931,12,12 +15932,14,14 +15933,13,12 +15934,14,14 +15935,14,14 +15936,12,12 +15937,14,14 +15938,14,14 +15939,12,12 +15940,12,12 +15941,14,14 +15942,14,14 +15943,12,12 +15944,14,14 +15945,14,14 +15946,14,14 +15947,12,12 +15948,14,14 +15949,14,14 +15950,13,13 +15951,12,14 +15952,14,14 +15953,12,12 +15954,12,12 +15955,12,12 +15956,12,12 +15957,14,14 +15958,14,14 +15959,14,14 +15960,14,14 +15961,14,14 +15962,14,14 +15963,14,14 +15964,14,14 +15965,13,13 +15966,14,14 +15967,14,14 +15968,14,14 +15969,14,14 +15970,12,12 +15971,14,14 +15972,14,14 +15973,14,12 +15974,14,14 +15975,13,14 +15976,12,12 +15977,12,12 +15978,12,14 +15979,12,12 +15980,14,14 +15981,14,14 +15982,14,14 +15983,14,14 +15984,14,14 +15985,12,12 +15986,14,14 +15987,12,12 +15988,14,14 +15989,12,12 +15990,12,12 +15991,12,12 +15992,14,14 +15993,12,12 +15994,14,14 +15995,14,14 +15996,14,14 +15997,14,12 +15998,14,14 +15999,12,12 +16000,14,14 +16001,12,12 +16002,14,14 +16003,12,12 +16004,12,12 +16005,14,12 +16006,14,14 +16007,12,12 +16008,12,12 +16009,12,12 +16010,12,12 +16011,14,14 +16012,14,14 +16013,12,12 +16014,14,14 +16015,14,14 +16016,12,12 +16017,12,12 +16018,14,14 +16019,13,13 +16020,12,12 +16021,14,14 +16022,14,14 +16023,14,14 +16024,12,12 +16025,14,14 +16026,12,12 +16027,12,12 +16028,14,14 +16029,14,14 +16030,14,14 +16031,12,12 +16032,14,14 +16033,12,12 +16034,12,12 +16035,14,14 +16036,14,14 +16037,14,14 +16038,13,14 +16039,12,12 +16040,12,12 +16041,14,14 +16042,14,14 +16043,14,12 +16044,14,14 +16045,14,14 +16046,12,12 +16047,12,14 +16048,14,14 +16049,14,14 +16050,14,14 +16051,14,14 +16052,14,14 +16053,12,14 +16054,12,12 +16055,12,12 +16056,14,14 +16057,14,14 +16058,14,14 +16059,12,12 +16060,14,14 +16061,12,12 +16062,14,14 +16063,12,12 +16064,12,13 +16065,12,12 +16066,14,14 +16067,12,12 +16068,14,12 +16069,12,12 +16070,12,12 +16071,14,14 +16072,14,14 +16073,14,14 +16074,14,14 +16075,14,14 +16076,14,14 +16077,14,13 +16078,12,12 +16079,14,14 +16080,14,14 +16081,13,14 +16082,14,14 +16083,14,14 +16084,14,14 +16085,14,14 +16086,12,12 +16087,14,14 +16088,12,12 +16089,13,13 +16090,14,14 +16091,14,14 +16092,14,14 +16093,14,14 +16094,12,12 +16095,14,14 +16096,12,12 +16097,14,12 +16098,14,14 +16099,14,14 +16100,12,12 +16101,12,12 +16102,14,14 +16103,14,14 +16104,14,14 +16105,14,14 +16106,14,13 +16107,14,14 +16108,14,14 +16109,12,12 +16110,14,14 +16111,12,12 +16112,14,14 +16113,12,12 +16114,14,14 +16115,12,12 +16116,14,14 +16117,14,14 +16118,14,12 +16119,14,14 +16120,14,14 +16121,13,14 +16122,14,14 +16123,14,14 +16124,14,14 +16125,13,13 +16126,14,14 +16127,12,12 +16128,12,12 +16129,14,14 +16130,14,14 +16131,14,14 +16132,14,14 +16133,12,12 +16134,13,13 +16135,12,12 +16136,14,14 +16137,14,14 +16138,14,14 +16139,12,12 +16140,12,12 +16141,14,14 +16142,14,14 +16143,14,14 +16144,14,14 +16145,14,14 +16146,14,14 +16147,12,12 +16148,14,14 +16149,14,14 +16150,14,14 +16151,14,12 +16152,14,14 +16153,14,14 +16154,14,14 +16155,14,14 +16156,12,12 +16157,14,14 +16158,12,12 +16159,14,14 +16160,14,14 +16161,14,14 +16162,12,12 +16163,14,14 +16164,12,12 +16165,14,14 +16166,14,14 +16167,12,12 +16168,12,14 +16169,14,14 +16170,14,14 +16171,14,14 +16172,12,12 +16173,12,12 +16174,12,12 +16175,12,12 +16176,14,14 +16177,13,13 +16178,12,12 +16179,14,14 +16180,13,13 +16181,12,14 +16182,12,12 +16183,14,12 +16184,12,12 +16185,14,14 +16186,12,12 +16187,14,14 +16188,14,14 +16189,13,14 +16190,14,14 +16191,12,12 +16192,12,12 +16193,14,14 +16194,14,12 +16195,14,14 +16196,12,12 +16197,12,12 +16198,12,14 +16199,12,12 +16200,12,12 +16201,14,14 +16202,14,14 +16203,14,14 +16204,14,14 +16205,14,13 +16206,14,14 +16207,12,12 +16208,14,14 +16209,14,14 +16210,12,14 +16211,12,12 +16212,14,12 +16213,14,14 +16214,14,14 +16215,12,12 +16216,14,14 +16217,13,13 +16218,13,13 +16219,14,14 +16220,14,14 +16221,14,14 +16222,14,14 +16223,14,14 +16224,14,14 +16225,14,14 +16226,14,14 +16227,14,14 +16228,13,13 +16229,12,12 +16230,12,12 +16231,12,12 +16232,14,14 +16233,14,12 +16234,12,12 +16235,12,12 +16236,14,14 +16237,12,12 +16238,14,14 +16239,12,12 +16240,12,12 +16241,14,14 +16242,13,12 +16243,14,14 +16244,14,14 +16245,17,17 +16246,15,15 +16247,17,17 +16248,17,17 +16249,17,17 +16250,17,17 +16251,15,15 +16252,17,17 +16253,17,17 +16254,17,17 +16255,17,17 +16256,17,17 +16257,17,17 +16258,17,17 +16259,15,15 +16260,17,17 +16261,17,17 +16262,17,17 +16263,17,17 +16264,15,15 +16265,17,17 +16266,17,17 +16267,17,17 +16268,15,15 +16269,16,16 +16270,15,15 +16271,17,17 +16272,17,17 +16273,17,17 +16274,15,15 +16275,15,15 +16276,17,17 +16277,17,17 +16278,17,17 +16279,17,17 +16280,17,17 +16281,17,17 +16282,17,17 +16283,17,17 +16284,17,17 +16285,15,17 +16286,15,15 +16287,15,15 +16288,15,15 +16289,17,17 +16290,17,17 +16291,17,17 +16292,17,17 +16293,15,15 +16294,17,17 +16295,17,17 +16296,17,17 +16297,17,17 +16298,15,15 +16299,17,17 +16300,17,17 +16301,17,17 +16302,17,17 +16303,16,16 +16304,15,15 +16305,16,16 +16306,15,15 +16307,15,15 +16308,17,17 +16309,17,17 +16310,17,17 +16311,17,17 +16312,15,15 +16313,15,15 +16314,17,17 +16315,15,15 +16316,17,17 +16317,15,15 +16318,16,16 +16319,17,17 +16320,15,15 +16321,17,17 +16322,17,17 +16323,17,17 +16324,17,17 +16325,17,17 +16326,15,15 +16327,15,15 +16328,15,15 +16329,17,17 +16330,17,17 +16331,17,17 +16332,17,17 +16333,17,17 +16334,17,17 +16335,15,15 +16336,17,17 +16337,17,15 +16338,17,17 +16339,17,17 +16340,17,17 +16341,17,17 +16342,17,17 +16343,17,17 +16344,17,17 +16345,17,17 +16346,17,17 +16347,17,17 +16348,17,17 +16349,15,15 +16350,17,17 +16351,17,17 +16352,17,17 +16353,17,17 +16354,17,17 +16355,15,15 +16356,17,17 +16357,15,15 +16358,15,15 +16359,17,17 +16360,17,17 +16361,17,17 +16362,17,15 +16363,17,17 +16364,17,17 +16365,15,15 +16366,17,17 +16367,15,15 +16368,17,17 +16369,15,15 +16370,15,15 +16371,17,17 +16372,17,17 +16373,17,17 +16374,17,17 +16375,15,15 +16376,15,15 +16377,15,15 +16378,17,17 +16379,17,17 +16380,17,17 +16381,17,17 +16382,15,15 +16383,15,15 +16384,17,17 +16385,17,17 +16386,15,15 +16387,16,16 +16388,17,17 +16389,15,15 +16390,17,17 +16391,17,17 +16392,17,17 +16393,17,17 +16394,17,17 +16395,17,17 +16396,17,15 +16397,17,17 +16398,17,17 +16399,17,17 +16400,17,17 +16401,17,17 +16402,17,17 +16403,17,17 +16404,15,15 +16405,17,17 +16406,17,17 +16407,17,17 +16408,17,16 +16409,15,15 +16410,17,17 +16411,16,16 +16412,17,17 +16413,17,17 +16414,15,15 +16415,16,16 +16416,17,17 +16417,17,17 +16418,17,17 +16419,17,17 +16420,17,17 +16421,17,17 +16422,17,17 +16423,17,17 +16424,17,17 +16425,15,15 +16426,17,17 +16427,17,17 +16428,17,17 +16429,17,17 +16430,17,17 +16431,15,15 +16432,15,15 +16433,17,17 +16434,17,17 +16435,15,15 +16436,17,16 +16437,17,17 +16438,17,17 +16439,17,17 +16440,15,15 +16441,17,17 +16442,17,17 +16443,15,15 +16444,17,17 +16445,17,17 +16446,17,17 +16447,15,15 +16448,17,17 +16449,17,17 +16450,17,17 +16451,15,15 +16452,17,17 +16453,17,17 +16454,17,17 +16455,17,17 +16456,17,17 +16457,17,17 +16458,17,17 +16459,17,17 +16460,17,17 +16461,16,16 +16462,17,17 +16463,15,15 +16464,17,17 +16465,17,17 +16466,17,17 +16467,17,17 +16468,17,17 +16469,17,17 +16470,17,17 +16471,17,17 +16472,17,16 +16473,15,15 +16474,17,17 +16475,15,15 +16476,16,16 +16477,17,17 +16478,17,17 +16479,15,15 +16480,15,15 +16481,17,17 +16482,15,15 +16483,17,17 +16484,17,17 +16485,17,17 +16486,17,17 +16487,17,17 +16488,17,17 +16489,17,17 +16490,15,15 +16491,15,15 +16492,17,17 +16493,17,17 +16494,15,15 +16495,17,17 +16496,17,17 +16497,17,17 +16498,17,17 +16499,17,17 +16500,17,17 +16501,16,16 +16502,17,17 +16503,17,17 +16504,17,17 +16505,15,15 +16506,15,15 +16507,16,16 +16508,17,17 +16509,17,17 +16510,15,15 +16511,17,17 +16512,15,17 +16513,17,17 +16514,17,17 +16515,17,17 +16516,17,17 +16517,17,17 +16518,17,17 +16519,17,17 +16520,15,15 +16521,17,17 +16522,17,17 +16523,17,17 +16524,17,17 +16525,17,17 +16526,15,15 +16527,15,15 +16528,17,17 +16529,17,17 +16530,17,17 +16531,15,15 +16532,15,15 +16533,17,15 +16534,17,17 +16535,17,17 +16536,17,17 +16537,15,15 +16538,17,17 +16539,17,17 +16540,15,15 +16541,17,17 +16542,17,17 +16543,17,17 +16544,17,17 +16545,17,17 +16546,17,17 +16547,17,17 +16548,17,17 +16549,17,17 +16550,17,17 +16551,17,17 +16552,15,15 +16553,17,17 +16554,17,17 +16555,17,17 +16556,17,17 +16557,17,17 +16558,17,17 +16559,17,17 +16560,17,17 +16561,17,17 +16562,15,15 +16563,17,17 +16564,17,17 +16565,17,17 +16566,15,15 +16567,17,17 +16568,15,15 +16569,15,15 +16570,15,15 +16571,15,17 +16572,15,15 +16573,15,15 +16574,17,17 +16575,17,17 +16576,17,17 +16577,17,17 +16578,17,17 +16579,17,17 +16580,15,15 +16581,17,17 +16582,17,17 +16583,15,15 +16584,17,17 +16585,17,17 +16586,17,17 +16587,16,16 +16588,17,17 +16589,17,17 +16590,17,17 +16591,17,17 +16592,17,17 +16593,17,17 +16594,17,15 +16595,17,17 +16596,17,17 +16597,17,17 +16598,17,16 +16599,17,17 +16600,17,17 +16601,17,17 +16602,17,17 +16603,17,17 +16604,15,15 +16605,17,17 +16606,17,17 +16607,17,17 +16608,15,15 +16609,15,15 +16610,17,17 +16611,17,17 +16612,17,17 +16613,15,15 +16614,17,17 +16615,15,15 +16616,17,17 +16617,17,17 +16618,15,15 +16619,17,17 +16620,17,17 +16621,17,17 +16622,17,17 +16623,15,15 +16624,17,17 +16625,17,17 +16626,17,16 +16627,17,17 +16628,17,16 +16629,15,15 +16630,17,17 +16631,17,17 +16632,15,15 +16633,17,17 +16634,17,17 +16635,17,17 +16636,15,15 +16637,17,17 +16638,17,17 +16639,17,17 +16640,17,17 +16641,17,17 +16642,17,17 +16643,15,17 +16644,17,17 +16645,17,17 +16646,15,15 +16647,16,16 +16648,17,17 +16649,17,17 +16650,17,17 +16651,15,15 +16652,17,17 +16653,17,17 +16654,17,17 +16655,17,17 +16656,17,17 +16657,17,17 +16658,17,17 +16659,17,17 +16660,17,17 +16661,17,17 +16662,15,15 +16663,16,16 +16664,17,17 +16665,17,15 +16666,17,17 +16667,15,15 +16668,17,17 +16669,17,17 +16670,17,17 +16671,15,15 +16672,17,17 +16673,17,17 +16674,17,17 +16675,17,17 +16676,17,17 +16677,17,17 +16678,17,17 +16679,17,17 +16680,17,17 +16681,17,17 +16682,17,17 +16683,15,15 +16684,17,17 +16685,15,15 +16686,17,17 +16687,17,17 +16688,15,15 +16689,17,17 +16690,17,17 +16691,15,15 +16692,17,17 +16693,17,17 +16694,17,17 +16695,17,17 +16696,15,15 +16697,17,16 +16698,17,17 +16699,16,16 +16700,15,15 +16701,17,17 +16702,17,17 +16703,15,15 +16704,15,15 +16705,17,17 +16706,17,17 +16707,17,17 +16708,17,17 +16709,17,17 +16710,17,17 +16711,16,16 +16712,15,15 +16713,17,17 +16714,17,17 +16715,15,15 +16716,15,15 +16717,17,17 +16718,17,17 +16719,17,17 +16720,17,17 +16721,15,15 +16722,17,17 +16723,15,15 +16724,17,17 +16725,15,15 +16726,15,15 +16727,17,17 +16728,17,17 +16729,17,17 +16730,17,17 +16731,17,17 +16732,15,15 +16733,17,17 +16734,17,17 +16735,17,17 +16736,15,15 +16737,17,17 +16738,17,17 +16739,17,17 +16740,17,17 +16741,15,15 +16742,17,17 +16743,17,17 +16744,17,17 +16745,15,15 +16746,15,15 +16747,15,15 +16748,17,17 +16749,17,16 +16750,17,17 +16751,15,15 +16752,17,17 +16753,16,16 +16754,17,17 +16755,17,17 +16756,17,17 +16757,17,17 +16758,17,17 +16759,17,17 +16760,17,17 +16761,17,17 +16762,17,17 +16763,17,17 +16764,15,15 +16765,17,17 +16766,16,16 +16767,17,17 +16768,17,17 +16769,17,17 +16770,17,16 +16771,15,17 +16772,17,17 +16773,17,17 +16774,15,15 +16775,17,17 +16776,17,17 +16777,17,17 +16778,17,15 +16779,17,17 +16780,15,15 +16781,17,17 +16782,15,15 +16783,15,15 +16784,17,15 +16785,17,17 +16786,17,17 +16787,17,17 +16788,17,17 +16789,17,17 +16790,17,17 +16791,15,15 +16792,17,17 +16793,17,17 +16794,15,15 +16795,17,17 +16796,17,17 +16797,17,17 +16798,15,15 +16799,17,17 +16800,17,17 +16801,15,15 +16802,15,15 +16803,17,17 +16804,17,17 +16805,17,17 +16806,17,17 +16807,17,17 +16808,15,15 +16809,17,17 +16810,17,17 +16811,17,17 +16812,15,15 +16813,17,17 +16814,17,17 +16815,16,16 +16816,17,17 +16817,17,17 +16818,17,17 +16819,17,17 +16820,18,18 +16821,20,20 +16822,19,19 +16823,20,20 +16824,18,18 +16825,20,20 +16826,20,20 +16827,20,20 +16828,20,20 +16829,18,18 +16830,18,18 +16831,18,18 +16832,18,18 +16833,18,18 +16834,19,19 +16835,18,20 +16836,19,19 +16837,18,18 +16838,20,20 +16839,18,18 +16840,20,20 +16841,20,18 +16842,18,18 +16843,18,18 +16844,18,18 +16845,20,20 +16846,18,18 +16847,20,20 +16848,18,18 +16849,20,20 +16850,20,20 +16851,20,20 +16852,20,20 +16853,20,20 +16854,18,20 +16855,18,18 +16856,20,20 +16857,20,20 +16858,20,20 +16859,18,18 +16860,20,20 +16861,20,20 +16862,18,18 +16863,18,18 +16864,20,20 +16865,18,18 +16866,18,18 +16867,18,20 +16868,20,20 +16869,20,20 +16870,18,18 +16871,20,20 +16872,19,19 +16873,20,20 +16874,20,20 +16875,18,18 +16876,19,19 +16877,20,20 +16878,20,20 +16879,20,20 +16880,18,18 +16881,18,18 +16882,20,20 +16883,18,18 +16884,20,20 +16885,18,18 +16886,20,20 +16887,20,18 +16888,18,18 +16889,19,19 +16890,18,18 +16891,20,20 +16892,18,18 +16893,20,20 +16894,19,19 +16895,20,20 +16896,18,20 +16897,18,18 +16898,19,19 +16899,20,20 +16900,19,19 +16901,18,18 +16902,18,18 +16903,20,18 +16904,20,20 +16905,20,20 +16906,20,20 +16907,19,20 +16908,18,18 +16909,18,18 +16910,20,20 +16911,18,18 +16912,20,20 +16913,20,18 +16914,20,20 +16915,20,20 +16916,18,18 +16917,20,18 +16918,19,19 +16919,18,18 +16920,19,19 +16921,18,18 +16922,19,19 +16923,20,20 +16924,20,20 +16925,18,18 +16926,18,18 +16927,20,20 +16928,20,20 +16929,20,20 +16930,20,20 +16931,20,20 +16932,20,20 +16933,20,20 +16934,18,18 +16935,18,18 +16936,18,20 +16937,20,20 +16938,19,19 +16939,18,18 +16940,20,20 +16941,18,18 +16942,20,20 +16943,18,18 +16944,20,20 +16945,20,20 +16946,20,20 +16947,20,20 +16948,20,19 +16949,20,20 +16950,20,20 +16951,20,20 +16952,18,18 +16953,18,19 +16954,20,20 +16955,18,18 +16956,20,20 +16957,20,20 +16958,20,18 +16959,20,20 +16960,18,18 +16961,20,20 +16962,18,18 +16963,18,18 +16964,20,20 +16965,19,19 +16966,20,20 +16967,20,20 +16968,18,18 +16969,20,20 +16970,20,20 +16971,19,19 +16972,20,19 +16973,20,20 +16974,20,20 +16975,20,20 +16976,19,19 +16977,20,20 +16978,18,18 +16979,20,18 +16980,20,20 +16981,20,20 +16982,18,18 +16983,18,18 +16984,18,18 +16985,20,20 +16986,18,18 +16987,18,18 +16988,19,19 +16989,18,18 +16990,20,20 +16991,18,18 +16992,20,20 +16993,20,20 +16994,20,20 +16995,20,18 +16996,20,20 +16997,18,18 +16998,20,20 +16999,20,20 +17000,18,18 +17001,20,20 +17002,20,20 +17003,18,18 +17004,18,18 +17005,18,18 +17006,19,19 +17007,19,19 +17008,20,20 +17009,18,18 +17010,18,18 +17011,18,18 +17012,20,20 +17013,18,18 +17014,19,19 +17015,18,18 +17016,20,20 +17017,20,20 +17018,18,18 +17019,20,20 +17020,20,20 +17021,19,19 +17022,20,20 +17023,18,18 +17024,20,20 +17025,18,18 +17026,20,20 +17027,18,18 +17028,20,20 +17029,19,19 +17030,20,20 +17031,18,18 +17032,18,18 +17033,20,20 +17034,20,20 +17035,18,18 +17036,20,20 +17037,20,18 +17038,18,18 +17039,18,18 +17040,20,20 +17041,19,19 +17042,20,20 +17043,18,18 +17044,20,20 +17045,18,18 +17046,20,20 +17047,18,18 +17048,20,20 +17049,20,20 +17050,20,20 +17051,20,20 +17052,20,20 +17053,18,18 +17054,18,18 +17055,18,18 +17056,20,20 +17057,20,20 +17058,18,18 +17059,20,20 +17060,18,19 +17061,19,19 +17062,20,20 +17063,19,19 +17064,20,20 +17065,18,18 +17066,18,18 +17067,20,20 +17068,18,18 +17069,19,19 +17070,18,18 +17071,18,18 +17072,19,19 +17073,18,18 +17074,20,20 +17075,20,20 +17076,20,20 +17077,20,20 +17078,20,20 +17079,20,20 +17080,20,18 +17081,18,18 +17082,18,18 +17083,19,19 +17084,20,20 +17085,20,20 +17086,20,20 +17087,18,18 +17088,19,19 +17089,18,18 +17090,18,18 +17091,18,18 +17092,20,20 +17093,20,20 +17094,18,18 +17095,18,18 +17096,20,20 +17097,18,18 +17098,18,18 +17099,19,18 +17100,18,18 +17101,20,20 +17102,20,20 +17103,19,19 +17104,20,20 +17105,20,20 +17106,18,18 +17107,20,20 +17108,19,19 +17109,20,20 +17110,18,18 +17111,19,18 +17112,20,20 +17113,19,19 +17114,19,19 +17115,19,19 +17116,18,18 +17117,20,20 +17118,18,18 +17119,18,18 +17120,20,20 +17121,18,18 +17122,18,18 +17123,18,18 +17124,20,20 +17125,18,18 +17126,20,20 +17127,19,19 +17128,18,18 +17129,20,20 +17130,20,20 +17131,18,18 +17132,18,18 +17133,20,20 +17134,20,20 +17135,20,19 +17136,19,19 +17137,18,18 +17138,20,20 +17139,20,20 +17140,18,18 +17141,20,20 +17142,20,20 +17143,20,20 +17144,20,19 +17145,20,20 +17146,18,18 +17147,18,18 +17148,20,20 +17149,18,18 +17150,18,18 +17151,18,18 +17152,18,18 +17153,20,20 +17154,20,20 +17155,19,18 +17156,20,20 +17157,19,19 +17158,20,20 +17159,20,20 +17160,18,18 +17161,19,19 +17162,19,19 +17163,18,18 +17164,18,18 +17165,20,20 +17166,18,18 +17167,20,20 +17168,18,20 +17169,19,19 +17170,20,20 +17171,18,18 +17172,20,20 +17173,18,18 +17174,20,20 +17175,20,20 +17176,18,18 +17177,20,18 +17178,20,20 +17179,18,18 +17180,18,18 +17181,20,20 +17182,18,18 +17183,18,18 +17184,19,19 +17185,20,20 +17186,18,18 +17187,18,18 +17188,18,18 +17189,20,20 +17190,20,20 +17191,18,18 +17192,18,18 +17193,20,20 +17194,18,18 +17195,20,20 +17196,18,18 +17197,18,18 +17198,19,19 +17199,18,18 +17200,19,19 +17201,20,20 +17202,20,20 +17203,18,18 +17204,18,18 +17205,18,18 +17206,20,20 +17207,20,20 +17208,20,20 +17209,19,19 +17210,20,20 +17211,18,18 +17212,18,18 +17213,19,18 +17214,18,18 +17215,18,18 +17216,20,20 +17217,18,18 +17218,18,18 +17219,18,18 +17220,20,20 +17221,18,18 +17222,19,18 +17223,20,20 +17224,20,20 +17225,20,20 +17226,20,20 +17227,20,20 +17228,20,20 +17229,18,18 +17230,20,20 +17231,20,20 +17232,20,20 +17233,20,20 +17234,20,20 +17235,18,18 +17236,20,20 +17237,20,20 +17238,20,20 +17239,20,20 +17240,20,20 +17241,20,20 +17242,18,18 +17243,20,20 +17244,18,18 +17245,20,20 +17246,20,20 +17247,20,20 +17248,18,18 +17249,18,18 +17250,20,20 +17251,18,18 +17252,18,18 +17253,20,20 +17254,20,20 +17255,18,18 +17256,20,20 +17257,20,20 +17258,20,20 +17259,18,18 +17260,18,18 +17261,20,20 +17262,20,20 +17263,20,20 +17264,20,20 +17265,18,18 +17266,20,20 +17267,20,20 +17268,20,20 +17269,20,20 +17270,18,18 +17271,20,20 +17272,20,20 +17273,18,18 +17274,20,20 +17275,20,20 +17276,18,18 +17277,19,20 +17278,20,20 +17279,20,20 +17280,18,18 +17281,20,20 +17282,18,18 +17283,20,20 +17284,20,20 +17285,20,20 +17286,20,20 +17287,18,18 +17288,18,18 +17289,20,20 +17290,20,20 +17291,20,20 +17292,19,19 +17293,20,20 +17294,19,19 +17295,18,18 +17296,18,18 +17297,20,20 +17298,20,20 +17299,19,19 +17300,18,18 +17301,20,20 +17302,20,20 +17303,20,20 +17304,20,20 +17305,18,18 +17306,20,20 +17307,20,20 +17308,20,20 +17309,20,20 +17310,20,20 +17311,20,20 +17312,18,18 +17313,18,18 +17314,18,18 +17315,20,20 +17316,20,20 +17317,20,20 +17318,18,18 +17319,18,18 +17320,20,20 +17321,18,18 +17322,20,20 +17323,20,20 +17324,18,18 +17325,20,20 +17326,20,20 +17327,20,20 +17328,20,20 +17329,20,20 +17330,20,20 +17331,20,20 +17332,20,20 +17333,20,20 +17334,18,18 +17335,18,18 +17336,20,20 +17337,20,20 +17338,20,20 +17339,19,19 +17340,20,20 +17341,18,18 +17342,20,20 +17343,20,20 +17344,18,18 +17345,20,20 +17346,19,19 +17347,20,20 +17348,20,20 +17349,18,18 +17350,20,20 diff --git a/BertToSimple/BiGRU/metrics/__init__.py b/BertToSimple/BiGRU/metrics/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/BertToSimple/BiGRU/metrics/custom_metrics.py b/BertToSimple/BiGRU/metrics/custom_metrics.py new file mode 100644 index 0000000..c2354d6 --- /dev/null +++ b/BertToSimple/BiGRU/metrics/custom_metrics.py @@ -0,0 +1,332 @@ +#encoding:utf-8 +import torch +from tqdm import tqdm +import numpy as np +from collections import Counter +from sklearn.metrics import roc_auc_score +from sklearn.metrics import f1_score, classification_report + +__call__ = ['Accuracy','AUC','F1Score','EntityScore','ClassReport','MultiLabelReport','AccuracyThresh'] + +class Metric: + def __init__(self): + pass + + def __call__(self, outputs, target): + raise NotImplementedError + + def reset(self): + raise NotImplementedError + + def value(self): + raise NotImplementedError + + def name(self): + raise NotImplementedError + +class Accuracy(Metric): + ''' + 计算准确度 + 可以使用topK参数设定计算K准确度 + Example: + >>> metrics = Accuracy(**) + >>> for epoch in range(epochs): + >>> metrics.reset() + >>> for batch in batchs: + >>> logits = model() + >>> metrics(logits,target) + >>> print(metrics.name(),metrics.value()) + ''' + def __init__(self,topK): + super(Accuracy,self).__init__() + self.topK = topK + self.reset() + + def __call__(self, logits, target): + _, pred = logits.topk(self.topK, 1, True, True) + pred = pred.t() + correct = pred.eq(target.view(1, -1).expand_as(pred)) + self.correct_k = correct[:self.topK].view(-1).float().sum(0) + self.total = target.size(0) + + def reset(self): + self.correct_k = 0 + self.total = 0 + + def value(self): + return float(self.correct_k) / self.total + + def name(self): + return 'accuracy' + + +class AccuracyThresh(Metric): + ''' + 计算准确度 + 可以使用topK参数设定计算K准确度 + Example: + >>> metrics = AccuracyThresh(**) + >>> for epoch in range(epochs): + >>> metrics.reset() + >>> for batch in batchs: + >>> logits = model() + >>> metrics(logits,target) + >>> print(metrics.name(),metrics.value()) + ''' + def __init__(self,thresh = 0.5): + super(AccuracyThresh,self).__init__() + self.thresh = thresh + self.reset() + + def __call__(self, logits, target): + self.y_pred = logits.sigmoid() + self.y_true = target + + def reset(self): + self.correct_k = 0 + self.total = 0 + + def value(self): + data_size = self.y_pred.size(0) + acc = np.mean(((self.y_pred>self.thresh)==self.y_true.byte()).float().cpu().numpy(), axis=1).sum() + return acc / data_size + + def name(self): + return 'accuracy' + + +class AUC(Metric): + ''' + AUC score + micro: + Calculate metrics globally by considering each element of the label + indicator matrix as a label. + macro: + Calculate metrics for each label, and find their unweighted + mean. This does not take label imbalance into account. + weighted: + Calculate metrics for each label, and find their average, weighted + by support (the number of true instances for each label). + samples: + Calculate metrics for each instance, and find their average. + Example: + >>> metrics = AUC(**) + >>> for epoch in range(epochs): + >>> metrics.reset() + >>> for batch in batchs: + >>> logits = model() + >>> metrics(logits,target) + >>> print(metrics.name(),metrics.value()) + ''' + + def __init__(self,task_type = 'binary',average = 'binary'): + super(AUC, self).__init__() + + assert task_type in ['binary','multiclass'] + assert average in ['binary','micro', 'macro', 'samples', 'weighted'] + + self.task_type = task_type + self.average = average + + def __call__(self,logits,target): + ''' + 计算整个结果 + ''' + if self.task_type == 'binary': + self.y_prob = logits.sigmoid().data.cpu().numpy() + else: + self.y_prob = logits.softmax(-1).data.cpu().detach().numpy() + self.y_true = target.cpu().numpy() + + def reset(self): + self.y_prob = 0 + self.y_true = 0 + + def value(self): + ''' + 计算指标得分 + ''' + auc = roc_auc_score(y_score=self.y_prob, y_true=self.y_true, average=self.average) + return auc + + def name(self): + return 'auc' + +class F1Score(Metric): + ''' + F1 Score + binary: + Only report results for the class specified by ``pos_label``. + This is applicable only if targets (``y_{true,pred}``) are binary. + micro: + Calculate metrics globally by considering each element of the label + indicator matrix as a label. + macro: + Calculate metrics for each label, and find their unweighted + mean. This does not take label imbalance into account. + weighted: + Calculate metrics for each label, and find their average, weighted + by support (the number of true instances for each label). + samples: + Calculate metrics for each instance, and find their average. + Example: + >>> metrics = F1Score(**) + >>> for epoch in range(epochs): + >>> metrics.reset() + >>> for batch in batchs: + >>> logits = model() + >>> metrics(logits,target) + >>> print(metrics.name(),metrics.value()) + ''' + def __init__(self,thresh = 0.5, normalizate = True,task_type = 'binary',average = 'binary',search_thresh = False): + super(F1Score).__init__() + assert task_type in ['binary','multiclass'] + assert average in ['binary','micro', 'macro', 'samples', 'weighted'] + + self.thresh = thresh + self.task_type = task_type + self.normalizate = normalizate + self.search_thresh = search_thresh + self.average = average + + def thresh_search(self,y_prob): + ''' + 对于f1评分的指标，一般我们需要对阈值进行调整，一般不会使用默认的0.5值，因此 + 这里我们队Thresh进行优化 + :return: + ''' + best_threshold = 0 + best_score = 0 + for threshold in tqdm([i * 0.01 for i in range(100)], disable=True): + self.y_pred = y_prob > threshold + score = self.value() + if score > best_score: + best_threshold = threshold + best_score = score + return best_threshold,best_score + + def __call__(self,logits,target): + ''' + 计算整个结果 + :return: + ''' + self.y_true = target.cpu().numpy() + if self.normalizate and self.task_type == 'binary': + y_prob = logits.sigmoid().data.cpu().numpy() + elif self.normalizate and self.task_type == 'multiclass': + y_prob = logits.softmax(-1).data.cpu().detach().numpy() + else: + y_prob = logits.cpu().detach().numpy() + + if self.task_type == 'binary': + if self.thresh and self.search_thresh == False: + self.y_pred = (y_prob > self.thresh ).astype(int) + self.value() + else: + thresh,f1 = self.thresh_search(y_prob = y_prob) + print(f"Best thresh: {thresh:.4f} - F1 Score: {f1:.4f}") + + if self.task_type == 'multiclass': + self.y_pred = np.argmax(self.y_pred, 1) + + def reset(self): + self.y_pred = 0 + self.y_true = 0 + + def value(self): + ''' + 计算指标得分 + ''' + if self.task_type == 'binary': + f1 = f1_score(y_true=self.y_true, y_pred=self.y_pred, average=self.average) + return f1 + if self.task_type == 'multiclass': + f1 = f1_score(y_true=self.y_true, y_pred=self.y_pred, average=self.average) + return f1 + + def name(self): + return 'f1' + +class ClassReport(Metric): + ''' + class report + ''' + def __init__(self,target_names = None): + super(ClassReport).__init__() + self.target_names = target_names + + def reset(self): + self.y_pred = 0 + self.y_true = 0 + + def value(self): + ''' + 计算指标得分 + ''' + score = classification_report(y_true = self.y_true, y_pred = self.y_pred, target_names=self.target_names) + print(f"\n\n classification report: {score}") + + def __call__(self,logits,target): + _, y_pred = torch.max(logits.data, 1) + self.y_pred = y_pred.cpu().numpy() + self.y_true = target.cpu().numpy() + + def name(self): + return "class_report" + +class MultiLabelReport(Metric): + ''' + multi label report + ''' + def __init__(self,id2label = None): + super(MultiLabelReport).__init__() + self.id2label = id2label + + def reset(self): + self.y_prob = 0 + self.y_true = 0 + + def __call__(self,logits,target): + + self.y_prob = logits.sigmoid().data.cpu().detach().numpy() + self.y_true = target.cpu().numpy() + + def value(self): + ''' + 计算指标得分 + ''' + for i, label in self.id2label.items(): + auc = roc_auc_score(y_score=self.y_prob[:, i], y_true=self.y_true[:, i]) + print(f"label:{label} - auc: {auc:.4f}") + + def name(self): + return "multilabel_report" + + +class LMAccuracy(Metric): + def __init__(self,topK =1): + super(LMAccuracy).__init__() + self.topK = topK + self.reset() + + def __call__(self,logits,target): + pred = torch.argmax(logits, 1) + active_acc = target.view(-1) != -1 + active_pred = pred[active_acc] + active_labels = target[active_acc] + + correct = active_pred.eq(active_labels) + self.correct_k = correct.float().sum(0) + self.total = active_labels.size(0) + + def reset(self): + self.correct_k = 0 + self.total = 0 + + def value(self): + return float(self.correct_k) / self.total + + def name(self): + return 'accuracy' + + diff --git a/BertToSimple/BiGRU/metrics/glue_compute_metrics.py b/BertToSimple/BiGRU/metrics/glue_compute_metrics.py new file mode 100644 index 0000000..747d82a --- /dev/null +++ b/BertToSimple/BiGRU/metrics/glue_compute_metrics.py @@ -0,0 +1,83 @@ +# coding=utf-8 +# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import csv +import sys +import logging + +logger = logging.getLogger(__name__) +try: + from scipy.stats import pearsonr, spearmanr + from sklearn.metrics import matthews_corrcoef, f1_score + _has_sklearn = True +except (AttributeError, ImportError) as e: + logger.warning("To use data.metrics please install scikit-learn. See https://scikit-learn.org/stable/index.html") + _has_sklearn = False + +def simple_accuracy(preds, labels): + return (preds == labels).mean() + + +def acc_and_f1(preds, labels): + acc = simple_accuracy(preds, labels) + f1 = f1_score(y_true=labels, y_pred=preds) + return { + "acc": acc, + "f1": f1, + "acc_and_f1": (acc + f1) / 2, + } + +def only_acc(preds, labels): + acc = simple_accuracy(preds, labels) + return { + "acc": acc, + } + +def pearson_and_spearman(preds, labels): + pearson_corr = pearsonr(preds, labels)[0] + spearman_corr = spearmanr(preds, labels)[0] + return { + "pearson": pearson_corr, + "spearmanr": spearman_corr, + "corr": (pearson_corr + spearman_corr) / 2, + } + +def compute_metrics(task_name, preds, labels): + assert len(preds) == len(labels) + if task_name == "cola": + return {"mcc": matthews_corrcoef(labels, preds)} + elif task_name == "sst-2": + return {"acc": simple_accuracy(preds, labels)} + elif task_name == "mrpc": + return only_acc(preds, labels) + elif task_name == "sts-b": + return pearson_and_spearman(preds, labels) + elif task_name == "qqp": + return acc_and_f1(preds, labels) + elif task_name == "mnli": + return {"acc": simple_accuracy(preds, labels)} + elif task_name == "mnli-mm": + return {"acc": simple_accuracy(preds, labels)} + elif task_name == "qnli": + return {"acc": simple_accuracy(preds, labels)} + elif task_name == "rte": + return {"acc": simple_accuracy(preds, labels)} + elif task_name == "wnli": + return {"acc": simple_accuracy(preds, labels)} + elif task_name == "lcqmc": + return {"acc": simple_accuracy(preds, labels)} + else: + raise KeyError(task_name) diff --git a/BertToSimple/BiGRU/model/__init__.py b/BertToSimple/BiGRU/model/__init__.py new file mode 100644 index 0000000..661b40d --- /dev/null +++ b/BertToSimple/BiGRU/model/__init__.py @@ -0,0 +1 @@ +#encoding:utf-8 \ No newline at end of file diff --git a/BertToSimple/BiGRU/model/configuration_albert.py b/BertToSimple/BiGRU/model/configuration_albert.py new file mode 100644 index 0000000..0d48a02 --- /dev/null +++ b/BertToSimple/BiGRU/model/configuration_albert.py @@ -0,0 +1,80 @@ +""" BERT model configuration """ +from __future__ import absolute_import, division, print_function, unicode_literals + +import json +import logging +import sys +from io import open + +from .configuration_utils import PretrainedConfig +logger = logging.getLogger(__name__) + +class AlbertConfig(PretrainedConfig): + r""" + Arguments: + vocab_size_or_config_json_file: Vocabulary size of `inputs_ids` in `BertModel`. + hidden_size: Size of the encoder layers and the pooler layer. + num_hidden_layers: Number of hidden layers in the Transformer encoder. + num_attention_heads: Number of attention heads for each attention layer in + the Transformer encoder. + intermediate_size: The size of the "intermediate" (i.e., feed-forward) + layer in the Transformer encoder. + hidden_act: The non-linear activation function (function or string) in the + encoder and pooler. If string, "gelu", "relu" and "swish" are supported. + hidden_dropout_prob: The dropout probabilitiy for all fully connected + layers in the embeddings, encoder, and pooler. + attention_probs_dropout_prob: The dropout ratio for the attention + probabilities. + max_position_embeddings: The maximum sequence length that this model might + ever be used with. Typically set this to something large just in case + (e.g., 512 or 1024 or 2048). + type_vocab_size: The vocabulary size of the `token_type_ids` passed into + `BertModel`. + initializer_range: The sttdev of the truncated_normal_initializer for + initializing all weight matrices. + layer_norm_eps: The epsilon used by LayerNorm. + """ + def __init__(self, + vocab_size_or_config_json_file=30000, + embedding_size=128, + hidden_size=4096, + num_hidden_layers=12, + num_hidden_groups=1, + num_attention_heads=64, + intermediate_size=16384, + inner_group_num=1, + hidden_act="gelu_new", + hidden_dropout_prob=0, + attention_probs_dropout_prob=0, + max_position_embeddings=512, + type_vocab_size=2, + initializer_range=0.02, + layer_norm_eps=1e-12, + **kwargs): + super(AlbertConfig, self).__init__(**kwargs) + print(vocab_size_or_config_json_file) + if isinstance(vocab_size_or_config_json_file, str) or (sys.version_info[0] == 2 + and isinstance(vocab_size_or_config_json_file, unicode)): + with open(vocab_size_or_config_json_file, "r", encoding='utf-8') as reader: + json_config = json.loads(reader.read()) + for key, value in json_config.items(): + self.__dict__[key] = value + elif isinstance(vocab_size_or_config_json_file, int): + self.vocab_size = vocab_size_or_config_json_file + self.hidden_size = hidden_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.hidden_act = hidden_act + self.intermediate_size = intermediate_size + self.hidden_dropout_prob = hidden_dropout_prob + self.attention_probs_dropout_prob = attention_probs_dropout_prob + self.max_position_embeddings = max_position_embeddings + self.type_vocab_size = type_vocab_size + self.initializer_range = initializer_range + self.layer_norm_eps = layer_norm_eps + self.embedding_size = embedding_size + self.inner_group_num = inner_group_num + self.num_hidden_groups = num_hidden_groups + else: + raise ValueError("First argument must be either a vocabulary size (int)" + " or the path to a pretrained model config file (str)") diff --git a/BertToSimple/BiGRU/model/configuration_bert.py b/BertToSimple/BiGRU/model/configuration_bert.py new file mode 100644 index 0000000..1524a4b --- /dev/null +++ b/BertToSimple/BiGRU/model/configuration_bert.py @@ -0,0 +1,83 @@ + +""" BERT model configuration """ + +from __future__ import absolute_import, division, print_function, unicode_literals + +import json +import logging +import sys +from io import open + +from .configuration_utils import PretrainedConfig + +logger = logging.getLogger(__name__) + +BERT_PRETRAINED_CONFIG_ARCHIVE_MAP = {} +class BertConfig(PretrainedConfig): + r""" + :class:`~pytorch_transformers.BertConfig` is the configuration class to store the configuration of a + `BertModel`. + + + Arguments: + vocab_size_or_config_json_file: Vocabulary size of `inputs_ids` in `BertModel`. + hidden_size: Size of the encoder layers and the pooler layer. + num_hidden_layers: Number of hidden layers in the Transformer encoder. + num_attention_heads: Number of attention heads for each attention layer in + the Transformer encoder. + intermediate_size: The size of the "intermediate" (i.e., feed-forward) + layer in the Transformer encoder. + hidden_act: The non-linear activation function (function or string) in the + encoder and pooler. If string, "gelu", "relu" and "swish" are supported. + hidden_dropout_prob: The dropout probabilitiy for all fully connected + layers in the embeddings, encoder, and pooler. + attention_probs_dropout_prob: The dropout ratio for the attention + probabilities. + max_position_embeddings: The maximum sequence length that this model might + ever be used with. Typically set this to something large just in case + (e.g., 512 or 1024 or 2048). + type_vocab_size: The vocabulary size of the `token_type_ids` passed into + `BertModel`. + initializer_range: The sttdev of the truncated_normal_initializer for + initializing all weight matrices. + layer_norm_eps: The epsilon used by LayerNorm. + """ + pretrained_config_archive_map = BERT_PRETRAINED_CONFIG_ARCHIVE_MAP + + def __init__(self, + vocab_size_or_config_json_file=30522, + hidden_size=768, + num_hidden_layers=12, + num_attention_heads=12, + intermediate_size=3072, + hidden_act="gelu", + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + max_position_embeddings=512, + type_vocab_size=2, + initializer_range=0.02, + layer_norm_eps=1e-12, + **kwargs): + super(BertConfig, self).__init__(**kwargs) + if isinstance(vocab_size_or_config_json_file, str) or (sys.version_info[0] == 2 + and isinstance(vocab_size_or_config_json_file, unicode)): + with open(vocab_size_or_config_json_file, "r", encoding='utf-8') as reader: + json_config = json.loads(reader.read()) + for key, value in json_config.items(): + self.__dict__[key] = value + elif isinstance(vocab_size_or_config_json_file, int): + self.vocab_size = vocab_size_or_config_json_file + self.hidden_size = hidden_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.hidden_act = hidden_act + self.intermediate_size = intermediate_size + self.hidden_dropout_prob = hidden_dropout_prob + self.attention_probs_dropout_prob = attention_probs_dropout_prob + self.max_position_embeddings = max_position_embeddings + self.type_vocab_size = type_vocab_size + self.initializer_range = initializer_range + self.layer_norm_eps = layer_norm_eps + else: + raise ValueError("First argument must be either a vocabulary size (int)" + " or the path to a pretrained model config file (str)") diff --git a/BertToSimple/BiGRU/model/configuration_utils.py b/BertToSimple/BiGRU/model/configuration_utils.py new file mode 100644 index 0000000..d7a9b50 --- /dev/null +++ b/BertToSimple/BiGRU/model/configuration_utils.py @@ -0,0 +1,207 @@ +# coding=utf-8 +# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" Configuration base class and utilities.""" + +from __future__ import (absolute_import, division, print_function, + unicode_literals) + +import copy +import json +import logging +import os +from io import open + +from model.file_utils import cached_path, CONFIG_NAME + +logger = logging.getLogger(__name__) + +class PretrainedConfig(object): + r""" Base class for all configuration classes. + Handles a few parameters tools to all models' configurations as well as methods for loading/downloading/saving configurations. + + Note: + A configuration file can be loaded and saved to disk. Loading the configuration file and using this file to initialize a model does **not** load the model weights. + It only affects the model's configuration. + + Class attributes (overridden by derived classes): + - ``pretrained_config_archive_map``: a python ``dict`` of with `short-cut-names` (string) as keys and `url` (string) of associated pretrained model configurations as values. + + Parameters: + ``finetuning_task``: string, default `None`. Name of the task used to fine-tune the model. This can be used when converting from an original (TensorFlow or PyTorch) checkpoint. + ``num_labels``: integer, default `2`. Number of classes to use when the model is a classification model (sequences/tokens) + ``output_attentions``: boolean, default `False`. Should the model returns attentions weights. + ``output_hidden_states``: string, default `False`. Should the model returns all hidden-states. + ``torchscript``: string, default `False`. Is the model used with Torchscript. + """ + pretrained_config_archive_map = {} + + def __init__(self, **kwargs): + self.finetuning_task = kwargs.pop('finetuning_task', None) + self.num_labels = kwargs.pop('num_labels', 2) + self.output_attentions = kwargs.pop('output_attentions', False) + self.output_hidden_states = kwargs.pop('output_hidden_states', False) + self.torchscript = kwargs.pop('torchscript', False) + self.pruned_heads = kwargs.pop('pruned_heads', {}) + + def save_pretrained(self, save_directory): + """ Save a configuration object to the directory `save_directory`, so that it + can be re-loaded using the :func:`~pytorch_transformers.PretrainedConfig.from_pretrained` class method. + """ + assert os.path.isdir(save_directory), "Saving path should be a directory where the model and configuration can be saved" + + # If we save using the predefined names, we can load using `from_pretrained` + output_config_file = os.path.join(save_directory, CONFIG_NAME) + + self.to_json_file(output_config_file) + + @classmethod + def from_pretrained(cls, pretrained_model_name_or_path, **kwargs): + r""" Instantiate a :class:`~pytorch_transformers.PretrainedConfig` (or a derived class) from a pre-trained model configuration. + + Parameters: + pretrained_model_name_or_path: either: + + - a string with the `shortcut name` of a pre-trained model configuration to load from cache or download, e.g.: ``bert-base-uncased``. + - a path to a `directory` containing a configuration file saved using the :func:`~pytorch_transformers.PretrainedConfig.save_pretrained` method, e.g.: ``./my_model_directory/``. + - a path or url to a saved configuration JSON `file`, e.g.: ``./my_model_directory/configuration.json``. + + cache_dir: (`optional`) string: + Path to a directory in which a downloaded pre-trained model + configuration should be cached if the standard cache should not be used. + + kwargs: (`optional`) dict: key/value pairs with which to update the configuration object after loading. + + - The values in kwargs of any keys which are configuration attributes will be used to override the loaded values. + - Behavior concerning key/value pairs whose keys are *not* configuration attributes is controlled by the `return_unused_kwargs` keyword parameter. + + force_download: (`optional`) boolean, default False: + Force to (re-)download the model weights and configuration files and override the cached versions if they exists. + + proxies: (`optional`) dict, default None: + A dictionary of proxy servers to use by protocol or endpoint, e.g.: {'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}. + The proxies are used on each request. + + return_unused_kwargs: (`optional`) bool: + + - If False, then this function returns just the final configuration object. + - If True, then this functions returns a tuple `(config, unused_kwargs)` where `unused_kwargs` is a dictionary consisting of the key/value pairs whose keys are not configuration attributes: ie the part of kwargs which has not been used to update `config` and is otherwise ignored. + + Examples:: + + # We can't instantiate directly the base class `PretrainedConfig` so let's show the examples on a + # derived class: BertConfig + config = BertConfig.from_pretrained('bert-base-uncased') # Download configuration from S3 and cache. + config = BertConfig.from_pretrained('./test/saved_model/') # E.g. config (or model) was saved using `save_pretrained('./test/saved_model/')` + config = BertConfig.from_pretrained('./test/saved_model/my_configuration.json') + config = BertConfig.from_pretrained('bert-base-uncased', output_attention=True, foo=False) + assert config.output_attention == True + config, unused_kwargs = BertConfig.from_pretrained('bert-base-uncased', output_attention=True, + foo=False, return_unused_kwargs=True) + assert config.output_attention == True + assert unused_kwargs == {'foo': False} + + """ + cache_dir = kwargs.pop('cache_dir', None) + force_download = kwargs.pop('force_download', False) + proxies = kwargs.pop('proxies', None) + return_unused_kwargs = kwargs.pop('return_unused_kwargs', False) + + if pretrained_model_name_or_path in cls.pretrained_config_archive_map: + config_file = cls.pretrained_config_archive_map[pretrained_model_name_or_path] + elif os.path.isdir(pretrained_model_name_or_path): + config_file = os.path.join(pretrained_model_name_or_path, CONFIG_NAME) + else: + config_file = pretrained_model_name_or_path + # redirect to the cache, if necessary + try: + resolved_config_file = cached_path(config_file, cache_dir=cache_dir, force_download=force_download, proxies=proxies) + except EnvironmentError as e: + if pretrained_model_name_or_path in cls.pretrained_config_archive_map: + logger.error( + "Couldn't reach server at '{}' to download pretrained model configuration file.".format( + config_file)) + else: + logger.error( + "Model name '{}' was not found in model name list ({}). " + "We assumed '{}' was a path or url but couldn't find any file " + "associated to this path or url.".format( + pretrained_model_name_or_path, + ', '.join(cls.pretrained_config_archive_map.keys()), + config_file)) + raise e + if resolved_config_file == config_file: + logger.info("loading configuration file {}".format(config_file)) + else: + logger.info("loading configuration file {} from cache at {}".format( + config_file, resolved_config_file)) + + # Load config + config = cls.from_json_file(resolved_config_file) + + if hasattr(config, 'pruned_heads'): + config.pruned_heads = dict((int(key), set(value)) for key, value in config.pruned_heads.items()) + + # Update config with kwargs if needed + to_remove = [] + for key, value in kwargs.items(): + if hasattr(config, key): + setattr(config, key, value) + to_remove.append(key) + else: + setattr(config,key,value) + for key in to_remove: + kwargs.pop(key, None) + + logger.info("Model config %s", config) + if return_unused_kwargs: + return config, kwargs + else: + return config + + @classmethod + def from_dict(cls, json_object): + """Constructs a `Config` from a Python dictionary of parameters.""" + config = cls(vocab_size_or_config_json_file=-1) + for key, value in json_object.items(): + config.__dict__[key] = value + return config + # @classmethod不需要实例化 + @classmethod + def from_json_file(cls, json_file): + """Constructs a `BertConfig` from a json file of parameters.""" + with open(json_file, "r", encoding='utf-8') as reader: + text = reader.read() + return cls.from_dict(json.loads(text)) + + def __eq__(self, other): + return self.__dict__ == other.__dict__ + + def __repr__(self): + return str(self.to_json_string()) + + def to_dict(self): + """Serializes this instance to a Python dictionary.""" + output = copy.deepcopy(self.__dict__) + return output + + def to_json_string(self): + """Serializes this instance to a JSON string.""" + return json.dumps(self.to_dict(), indent=2, sort_keys=True) + "\n" + + def to_json_file(self, json_file_path): + """ Save this instance to a json file.""" + with open(json_file_path, "w", encoding='utf-8') as writer: + writer.write(self.to_json_string()) diff --git a/BertToSimple/BiGRU/model/file_utils.py b/BertToSimple/BiGRU/model/file_utils.py new file mode 100644 index 0000000..3fe7fa8 --- /dev/null +++ b/BertToSimple/BiGRU/model/file_utils.py @@ -0,0 +1,294 @@ +""" +Utilities for working with the local dataset cache. +This file is adapted from the AllenNLP library at https://github.com/allenai/allennlp +Copyright by the AllenNLP authors. +""" +from __future__ import (absolute_import, division, print_function, unicode_literals) + +import sys +import json +import logging +import os +import six +import shutil +import tempfile +import fnmatch +from functools import wraps +from hashlib import sha256 +from io import open + +import boto3 +from botocore.config import Config +from botocore.exceptions import ClientError +import requests +from tqdm import tqdm + +try: + from torch.hub import _get_torch_home + torch_cache_home = _get_torch_home() +except ImportError: + torch_cache_home = os.path.expanduser( + os.getenv('TORCH_HOME', os.path.join( + os.getenv('XDG_CACHE_HOME', '~/.cache'), 'torch'))) +default_cache_path = os.path.join(torch_cache_home, 'pytorch_transformers') + +try: + from urllib.parse import urlparse +except ImportError: + from urlparse import urlparse + +try: + from pathlib import Path + PYTORCH_PRETRAINED_BERT_CACHE = Path( + os.getenv('PYTORCH_TRANSFORMERS_CACHE', os.getenv('PYTORCH_PRETRAINED_BERT_CACHE', default_cache_path))) +except (AttributeError, ImportError): + PYTORCH_PRETRAINED_BERT_CACHE = os.getenv('PYTORCH_TRANSFORMERS_CACHE', + os.getenv('PYTORCH_PRETRAINED_BERT_CACHE', + default_cache_path)) + +PYTORCH_TRANSFORMERS_CACHE = PYTORCH_PRETRAINED_BERT_CACHE # Kept for backward compatibility + +WEIGHTS_NAME = "pytorch_model.bin" +TF_WEIGHTS_NAME = 'model.ckpt' +CONFIG_NAME = "config.json" + +logger = logging.getLogger(__name__) # pylint: disable=invalid-name + +if not six.PY2: + def add_start_docstrings(*docstr): + def docstring_decorator(fn): + fn.__doc__ = ''.join(docstr) + fn.__doc__ + return fn + return docstring_decorator + + def add_end_docstrings(*docstr): + def docstring_decorator(fn): + fn.__doc__ = fn.__doc__ + ''.join(docstr) + return fn + return docstring_decorator +else: + # Not possible to update class docstrings on python2 + def add_start_docstrings(*docstr): + def docstring_decorator(fn): + return fn + return docstring_decorator + + def add_end_docstrings(*docstr): + def docstring_decorator(fn): + return fn + return docstring_decorator + +def url_to_filename(url, etag=None): + """ + Convert `url` into a hashed filename in a repeatable way. + If `etag` is specified, append its hash to the url's, delimited + by a period. + """ + url_bytes = url.encode('utf-8') + url_hash = sha256(url_bytes) + filename = url_hash.hexdigest() + + if etag: + etag_bytes = etag.encode('utf-8') + etag_hash = sha256(etag_bytes) + filename += '.' + etag_hash.hexdigest() + + return filename + + +def filename_to_url(filename, cache_dir=None): + """ + Return the url and etag (which may be ``None``) stored for `filename`. + Raise ``EnvironmentError`` if `filename` or its stored metadata do not exist. + """ + if cache_dir is None: + cache_dir = PYTORCH_TRANSFORMERS_CACHE + if sys.version_info[0] == 3 and isinstance(cache_dir, Path): + cache_dir = str(cache_dir) + + cache_path = os.path.join(cache_dir, filename) + if not os.path.exists(cache_path): + raise EnvironmentError("file {} not found".format(cache_path)) + + meta_path = cache_path + '.json' + if not os.path.exists(meta_path): + raise EnvironmentError("file {} not found".format(meta_path)) + + with open(meta_path, encoding="utf-8") as meta_file: + metadata = json.load(meta_file) + url = metadata['url'] + etag = metadata['etag'] + + return url, etag + + +def cached_path(url_or_filename, cache_dir=None, force_download=False, proxies=None): + """ + Given something that might be a URL (or might be a local path), + determine which. If it's a URL, download the file and cache it, and + return the path to the cached file. If it's already a local path, + make sure the file exists and then return the path. + Args: + cache_dir: specify a cache directory to save the file to (overwrite the default cache dir). + force_download: if True, re-dowload the file even if it's already cached in the cache dir. + """ + if cache_dir is None: + cache_dir = PYTORCH_TRANSFORMERS_CACHE + if sys.version_info[0] == 3 and isinstance(url_or_filename, Path): + url_or_filename = str(url_or_filename) + if sys.version_info[0] == 3 and isinstance(cache_dir, Path): + cache_dir = str(cache_dir) + + parsed = urlparse(url_or_filename) + + if parsed.scheme in ('http', 'https', 's3'): + # URL, so get it from the cache (downloading if necessary) + return get_from_cache(url_or_filename, cache_dir=cache_dir, force_download=force_download, proxies=proxies) + elif os.path.exists(url_or_filename): + # File, and it exists. + return url_or_filename + elif parsed.scheme == '': + # File, but it doesn't exist. + raise EnvironmentError("file {} not found".format(url_or_filename)) + else: + # Something unknown + raise ValueError("unable to parse {} as a URL or as a local path".format(url_or_filename)) + + +def split_s3_path(url): + """Split a full s3 path into the bucket name and path.""" + parsed = urlparse(url) + if not parsed.netloc or not parsed.path: + raise ValueError("bad s3 path {}".format(url)) + bucket_name = parsed.netloc + s3_path = parsed.path + # Remove '/' at beginning of path. + if s3_path.startswith("/"): + s3_path = s3_path[1:] + return bucket_name, s3_path + + +def s3_request(func): + """ + Wrapper function for s3 requests in order to create more helpful error + messages. + """ + + @wraps(func) + def wrapper(url, *args, **kwargs): + try: + return func(url, *args, **kwargs) + except ClientError as exc: + if int(exc.response["Error"]["Code"]) == 404: + raise EnvironmentError("file {} not found".format(url)) + else: + raise + + return wrapper + + +@s3_request +def s3_etag(url, proxies=None): + """Check ETag on S3 object.""" + s3_resource = boto3.resource("s3", config=Config(proxies=proxies)) + bucket_name, s3_path = split_s3_path(url) + s3_object = s3_resource.Object(bucket_name, s3_path) + return s3_object.e_tag + + +@s3_request +def s3_get(url, temp_file, proxies=None): + """Pull a file directly from S3.""" + s3_resource = boto3.resource("s3", config=Config(proxies=proxies)) + bucket_name, s3_path = split_s3_path(url) + s3_resource.Bucket(bucket_name).download_fileobj(s3_path, temp_file) + + +def http_get(url, temp_file, proxies=None): + req = requests.get(url, stream=True, proxies=proxies) + content_length = req.headers.get('Content-Length') + total = int(content_length) if content_length is not None else None + progress = tqdm(unit="B", total=total) + for chunk in req.iter_content(chunk_size=1024): + if chunk: # filter out keep-alive new chunks + progress.update(len(chunk)) + temp_file.write(chunk) + progress.close() + + +def get_from_cache(url, cache_dir=None, force_download=False, proxies=None): + """ + Given a URL, look for the corresponding dataset in the local cache. + If it's not there, download it. Then return the path to the cached file. + """ + if cache_dir is None: + cache_dir = PYTORCH_TRANSFORMERS_CACHE + if sys.version_info[0] == 3 and isinstance(cache_dir, Path): + cache_dir = str(cache_dir) + if sys.version_info[0] == 2 and not isinstance(cache_dir, str): + cache_dir = str(cache_dir) + + if not os.path.exists(cache_dir): + os.makedirs(cache_dir) + + # Get eTag to add to filename, if it exists. + if url.startswith("s3://"): + etag = s3_etag(url, proxies=proxies) + else: + try: + response = requests.head(url, allow_redirects=True, proxies=proxies) + if response.status_code != 200: + etag = None + else: + etag = response.headers.get("ETag") + except EnvironmentError: + etag = None + + if sys.version_info[0] == 2 and etag is not None: + etag = etag.decode('utf-8') + filename = url_to_filename(url, etag) + + # get cache path to put the file + cache_path = os.path.join(cache_dir, filename) + + # If we don't have a connection (etag is None) and can't identify the file + # try to get the last downloaded one + if not os.path.exists(cache_path) and etag is None: + matching_files = fnmatch.filter(os.listdir(cache_dir), filename + '.*') + matching_files = list(filter(lambda s: not s.endswith('.json'), matching_files)) + if matching_files: + cache_path = os.path.join(cache_dir, matching_files[-1]) + + if not os.path.exists(cache_path) or force_download: + # Download to temporary file, then copy to cache dir once finished. + # Otherwise you get corrupt cache entries if the download gets interrupted. + with tempfile.NamedTemporaryFile() as temp_file: + logger.info("%s not found in cache or force_download set to True, downloading to %s", url, temp_file.name) + + # GET file object + if url.startswith("s3://"): + s3_get(url, temp_file, proxies=proxies) + else: + http_get(url, temp_file, proxies=proxies) + + # we are copying the file before closing it, so flush to avoid truncation + temp_file.flush() + # shutil.copyfileobj() starts at the current position, so go to the start + temp_file.seek(0) + + logger.info("copying %s to cache at %s", temp_file.name, cache_path) + with open(cache_path, 'wb') as cache_file: + shutil.copyfileobj(temp_file, cache_file) + + logger.info("creating metadata file for %s", cache_path) + meta = {'url': url, 'etag': etag} + meta_path = cache_path + '.json' + with open(meta_path, 'w') as meta_file: + output_string = json.dumps(meta) + if sys.version_info[0] == 2 and isinstance(output_string, str): + output_string = unicode(output_string, 'utf-8') # The beauty of python 2 + meta_file.write(output_string) + + logger.info("removing temp file %s", temp_file.name) + + return cache_path diff --git a/BertToSimple/BiGRU/model/modeling_albert.py b/BertToSimple/BiGRU/model/modeling_albert.py new file mode 100644 index 0000000..98dba51 --- /dev/null +++ b/BertToSimple/BiGRU/model/modeling_albert.py @@ -0,0 +1,1087 @@ +"""PyTorch ALBERT model. """ +from __future__ import absolute_import, division, print_function, unicode_literals +import logging +import math +import os +import sys +import torch +from torch import nn +from torch.nn import CrossEntropyLoss, MSELoss +from .modeling_utils import PreTrainedModel, prune_linear_layer +from .configuration_albert import AlbertConfig +from .file_utils import add_start_docstrings +logger = logging.getLogger(__name__) + +ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP = { + 'albert-base': "", + 'albert-large': "", + 'albert-xlarge': "", + 'albert-xxlarge': "", +} +def load_tf_weights_in_albert(model, config, tf_checkpoint_path): + """ Load tf checkpoints in a pytorch model. + """ + try: + import re + import numpy as np + import tensorflow as tf + except ImportError: + logger.error("Loading a TensorFlow model in PyTorch, requires TensorFlow to be installed. Please see " + "https://www.tensorflow.org/install/ for installation instructions.") + raise + tf_path = os.path.abspath(tf_checkpoint_path) + logger.info("Converting TensorFlow checkpoint from {}".format(tf_path)) + if not os.path.exists(tf_path+'/checkpoint'): + tf_path = tf_path + "/variables/variables" + # Load weights from TF model + init_vars = tf.train.list_variables(tf_path) + names = [] + arrays = [] + for name, shape in init_vars: + logger.info("Loading TF weight {} with shape {}".format(name, shape)) + array = tf.train.load_variable(tf_path, name) + names.append(name) + arrays.append(array) + for name, array in zip(names, arrays): + name = name.replace("attention_1","attention") + name = name.replace("ffn_1","ffn") + name = name.split('/') + # adam_v and adam_m are variables used in AdamWeightDecayOptimizer to calculated m and v + # which are not required for using pretrained model + if any(n in ["adam_v", "adam_m", "global_step"] for n in name): + logger.info("Skipping {}".format("/".join(name))) + continue + pointer = model + for m_name in name: + if re.fullmatch(r'[A-Za-z]+_\d+', m_name): + l = re.split(r'_(\d+)', m_name) + elif re.fullmatch(r'[A-Za-z]+_+[A-Za-z]+_\d+', m_name): + l = re.split(r'_(\d+)', m_name) + else: + l = [m_name] + if l[0] in ['LayerNorm', 'attention', 'ffn'] and len(l) >= 2: + l = ["_".join(l[:-1])] + if l[0] == 'kernel' or l[0] == 'gamma': + pointer = getattr(pointer, 'weight') + elif l[0] == 'output_bias' or l[0] == 'beta': + pointer = getattr(pointer, 'bias') + elif l[0] == 'output_weights': + pointer = getattr(pointer, 'weight') + elif l[0] == 'squad': + pointer = getattr(pointer, 'classifier') + else: + try: + pointer = getattr(pointer, l[0]) + except AttributeError: + logger.info("Skipping {}".format("/".join(name))) + continue + if len(l) >= 2: + num = int(l[1]) + pointer = pointer[num] + + if m_name[-11:] == '_embeddings': + pointer = getattr(pointer, 'weight') + elif m_name == 'kernel': + array = np.transpose(array) + try: + assert pointer.shape == array.shape + except AssertionError as e: + e.args += (pointer.shape, array.shape) + raise + logger.info("Initialize PyTorch weight {}".format(name)) + pointer.data = torch.from_numpy(array) + return model + +def gelu(x): + """ Original Implementation of the gelu activation function in Google Bert repo when initially created. + For information: OpenAI GPT's gelu is slightly different (and gives slightly different results): + 0.5 * x * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3)))) + Also see https://arxiv.org/abs/1606.08415 + """ + return x * 0.5 * (1.0 + torch.erf(x / math.sqrt(2.0))) + +def gelu_new(x): + """ Implementation of the gelu activation function currently in Google Bert repo (identical to OpenAI GPT). + Also see https://arxiv.org/abs/1606.08415 + """ + return 0.5 * x * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3)))) + +def swish(x): + return x * torch.sigmoid(x) + +ACT2FN = {"gelu": gelu, "relu": torch.nn.functional.relu, "swish": swish, "gelu_new": gelu_new} +AlbertLayerNorm = torch.nn.LayerNorm + +class AlbertEmbeddings(nn.Module): + """Construct the embeddings from word, position and token_type embeddings. + """ + def __init__(self, config): + super(AlbertEmbeddings, self).__init__() + self.word_embeddings = nn.Embedding(config.vocab_size, config.embedding_size, padding_idx=0) + self.position_embeddings = nn.Embedding(config.max_position_embeddings, config.embedding_size) + self.token_type_embeddings = nn.Embedding(config.type_vocab_size, config.embedding_size) + # self.LayerNorm is not snake-cased to stick with TensorFlow model variable name and be able to load + self.LayerNorm = AlbertLayerNorm(config.embedding_size, eps=config.layer_norm_eps) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + + def forward(self, input_ids, token_type_ids=None, position_ids=None): + seq_length = input_ids.size(1) + if position_ids is None: + position_ids = torch.arange(seq_length, dtype=torch.long, device=input_ids.device) + position_ids = position_ids.unsqueeze(0).expand_as(input_ids) + if token_type_ids is None: + token_type_ids = torch.zeros_like(input_ids) + words_embeddings = self.word_embeddings(input_ids) + position_embeddings = self.position_embeddings(position_ids) + token_type_embeddings = self.token_type_embeddings(token_type_ids) + embeddings = words_embeddings + position_embeddings + token_type_embeddings + embeddings = self.LayerNorm(embeddings) + embeddings = self.dropout(embeddings) + return embeddings + +class AlbertSelfAttention(nn.Module): + def __init__(self, config): + super(AlbertSelfAttention, self).__init__() + if config.hidden_size % config.num_attention_heads != 0: + raise ValueError( + "The hidden size (%d) is not a multiple of the number of attention " + "heads (%d)" % (config.hidden_size, config.num_attention_heads)) + self.output_attentions = config.output_attentions + self.num_attention_heads = config.num_attention_heads + self.attention_head_size = int(config.hidden_size / config.num_attention_heads) + self.all_head_size = self.num_attention_heads * self.attention_head_size + self.query = nn.Linear(config.hidden_size, self.all_head_size) + self.key = nn.Linear(config.hidden_size, self.all_head_size) + self.value = nn.Linear(config.hidden_size, self.all_head_size) + self.dropout = nn.Dropout(config.attention_probs_dropout_prob) + + def transpose_for_scores(self, x): + new_x_shape = x.size()[:-1] + (self.num_attention_heads, self.attention_head_size) + x = x.view(*new_x_shape) + return x.permute(0, 2, 1, 3) + + def forward(self, hidden_states, attention_mask=None, head_mask=None): + mixed_query_layer = self.query(hidden_states) + mixed_key_layer = self.key(hidden_states) + mixed_value_layer = self.value(hidden_states) + + query_layer = self.transpose_for_scores(mixed_query_layer) + key_layer = self.transpose_for_scores(mixed_key_layer) + value_layer = self.transpose_for_scores(mixed_value_layer) + + # Take the dot product between "query" and "key" to get the raw attention scores. + attention_scores = torch.matmul(query_layer, key_layer.transpose(-1, -2)) + attention_scores = attention_scores / math.sqrt(self.attention_head_size) + if attention_mask is not None: + # Apply the attention mask is (precomputed for all layers in BertModel forward() function) + attention_scores = attention_scores + attention_mask + + # Normalize the attention scores to probabilities. + attention_probs = nn.Softmax(dim=-1)(attention_scores) + + # This is actually dropping out entire tokens to attend to, which might + # seem a bit unusual, but is taken from the original Transformer paper. + attention_probs = self.dropout(attention_probs) + + # Mask heads if we want to + if head_mask is not None: + attention_probs = attention_probs * head_mask + + context_layer = torch.matmul(attention_probs, value_layer) + + context_layer = context_layer.permute(0, 2, 1, 3).contiguous() + new_context_layer_shape = context_layer.size()[:-2] + (self.all_head_size,) + context_layer = context_layer.view(*new_context_layer_shape) + outputs = (context_layer, attention_probs) if self.output_attentions else (context_layer,) + return outputs + +class AlbertSelfOutput(nn.Module): + def __init__(self, config): + super(AlbertSelfOutput, self).__init__() + self.dense = nn.Linear(config.hidden_size, config.hidden_size) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + def forward(self, hidden_states, input_tensor): + hidden_states = self.dense(hidden_states) + hidden_states = self.dropout(hidden_states) + return hidden_states + +class AlbertAttention(nn.Module): + def __init__(self, config): + super(AlbertAttention, self).__init__() + self.self = AlbertSelfAttention(config) + self.output = AlbertSelfOutput(config) + self.pruned_heads = set() + + def prune_heads(self, heads): + if len(heads) == 0: + return + mask = torch.ones(self.self.num_attention_heads, self.self.attention_head_size) + heads = set(heads) - self.pruned_heads # Convert to set and emove already pruned heads + for head in heads: + # Compute how many pruned heads are before the head and move the index accordingly + head = head - sum(1 if h < head else 0 for h in self.pruned_heads) + mask[head] = 0 + mask = mask.view(-1).contiguous().eq(1) + index = torch.arange(len(mask))[mask].long() + + # Prune linear layers + self.self.query = prune_linear_layer(self.self.query, index) + self.self.key = prune_linear_layer(self.self.key, index) + self.self.value = prune_linear_layer(self.self.value, index) + self.output.dense = prune_linear_layer(self.output.dense, index, dim=1) + + # Update hyper params and store pruned heads + self.self.num_attention_heads = self.self.num_attention_heads - len(heads) + self.self.all_head_size = self.self.attention_head_size * self.self.num_attention_heads + self.pruned_heads = self.pruned_heads.union(heads) + + def forward(self, input_tensor, attention_mask=None, head_mask=None): + self_outputs = self.self(input_tensor, attention_mask, head_mask) + attention_output = self.output(self_outputs[0], input_tensor) + outputs = (attention_output,self_outputs) + return outputs + +class AlbertOutput(nn.Module): + def __init__(self, config): + super(AlbertOutput, self).__init__() + self.dense = nn.Linear(config.intermediate_size, config.hidden_size) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + + def forward(self, hidden_states): + hidden_states = self.dense(hidden_states) + hidden_states = self.dropout(hidden_states) + return hidden_states + +class AlbertIntermediate(nn.Module): + def __init__(self, config): + super(AlbertIntermediate, self).__init__() + self.dense = nn.Linear(config.hidden_size, config.intermediate_size) + self.output = AlbertOutput(config) + if isinstance(config.hidden_act, str) or (sys.version_info[0] == 2 and isinstance(config.hidden_act, unicode)): + self.intermediate_act_fn = ACT2FN[config.hidden_act] + else: + self.intermediate_act_fn = config.hidden_act + + def forward(self, hidden_states): + intermediate_output = self.dense(hidden_states) + intermediate_output = self.intermediate_act_fn(intermediate_output) + output = self.output(intermediate_output) + return output + +class AlbertFFN(nn.Module): + def __init__(self, config): + super(AlbertFFN, self).__init__() + self.intermediate = AlbertIntermediate(config) + + def forward(self, attention_output): + output = self.intermediate(attention_output) + return output + +class AlbertLayer(nn.Module): + def __init__(self, config): + super(AlbertLayer, self).__init__() + self.attention = AlbertAttention(config) + self.ffn = AlbertFFN(config) + self.LayerNorm = AlbertLayerNorm(config.hidden_size, eps=config.layer_norm_eps) + self.LayerNorm_1 = AlbertLayerNorm(config.hidden_size, eps=config.layer_norm_eps) + + def forward(self, hidden_states, attention_mask=None, head_mask=None): + attention_outputs = self.attention(hidden_states, attention_mask, head_mask) + attention_output = self.LayerNorm(attention_outputs[0] + hidden_states) + ffn_output = self.ffn(attention_output) + ffn_output = self.LayerNorm_1(ffn_output+attention_output) + outputs = (ffn_output,) + attention_outputs[1:] # add attentions if we output them + return outputs + +class AlbertGroup(nn.Module): + def __init__(self, config): + super(AlbertGroup, self).__init__() + self.inner_group_num = config.inner_group_num + self.inner_group = nn.ModuleList([AlbertLayer(config) for _ in range(config.inner_group_num)]) + + def forward(self, hidden_states, attention_mask, head_mask): + layer_attentions = () + layer_hidden_states = () + for inner_group_idx in range(self.inner_group_num): + layer_module = self.inner_group[inner_group_idx] + layer_outputs = layer_module(hidden_states, attention_mask, head_mask) + hidden_states = layer_outputs[0] + layer_attentions = layer_attentions + (layer_outputs[1],) + layer_hidden_states = layer_hidden_states + (hidden_states,) + return (layer_hidden_states, layer_attentions) + +class AlbertTransformer(nn.Module): + def __init__(self, config): + super(AlbertTransformer, self).__init__() + self.output_attentions = config.output_attentions + self.output_hidden_states = config.output_hidden_states + self.num_hidden_layers = config.num_hidden_layers + self.num_hidden_groups = config.num_hidden_groups + self.group = nn.ModuleList([AlbertGroup(config) for _ in range(config.num_hidden_groups)]) + + def forward(self, hidden_states, attention_mask, head_mask): + all_hidden_states = () + all_attentions = () + for layer_idx in range(self.num_hidden_layers): + if self.output_hidden_states and layer_idx == 0: + all_hidden_states = all_hidden_states + (hidden_states,) + group_idx = int(layer_idx / self.num_hidden_layers * self.num_hidden_groups) + layer_module = self.group[group_idx] + layer_outputs = layer_module(hidden_states, attention_mask, head_mask[layer_idx]) + hidden_states = layer_outputs[0][-1] + if self.output_attentions: + all_attentions = all_attentions + layer_outputs[1] + if self.output_hidden_states: + all_hidden_states = all_hidden_states + layer_outputs[0] + outputs = (hidden_states,) + if self.output_hidden_states: + outputs = outputs + (all_hidden_states,) + if self.output_attentions: + outputs = outputs + (all_attentions,) + return outputs # last-layer hidden state, (all hidden states), (all attentions) + +class AlbertEncoder(nn.Module): + def __init__(self, config): + super(AlbertEncoder, self).__init__() + self.hidden_size = config.hidden_size + self.embedding_size = config.embedding_size + self.embedding_hidden_mapping_in = nn.Linear(self.embedding_size, self.hidden_size) + self.transformer = AlbertTransformer(config) + + def forward(self, hidden_states, attention_mask=None, head_mask=None): + if self.embedding_size != self.hidden_size: + prev_output = self.embedding_hidden_mapping_in(hidden_states) + else: + prev_output = hidden_states + outputs = self.transformer(prev_output, attention_mask, head_mask) + return outputs # last-layer hidden state, (all hidden states), (all attentions) + +class AlbertPooler(nn.Module): + def __init__(self, config): + super(AlbertPooler, self).__init__() + self.dense = nn.Linear(config.hidden_size, config.hidden_size) + self.activation = nn.Tanh() + + def forward(self, hidden_states): + # We "pool" the model by simply taking the hidden state corresponding + # to the first token. + first_token_tensor = hidden_states[:, 0] + pooled_output = self.dense(first_token_tensor) + pooled_output = self.activation(pooled_output) + return pooled_output + +class AlbertPredictionHeadTransform(nn.Module): + def __init__(self, config): + super(AlbertPredictionHeadTransform, self).__init__() + self.dense = nn.Linear(config.hidden_size, config.embedding_size) + if isinstance(config.hidden_act, str) or (sys.version_info[0] == 2 and isinstance(config.hidden_act, unicode)): + self.transform_act_fn = ACT2FN[config.hidden_act] + else: + self.transform_act_fn = config.hidden_act + self.LayerNorm = AlbertLayerNorm(config.embedding_size, eps=config.layer_norm_eps) + + def forward(self, hidden_states): + hidden_states = self.dense(hidden_states) + hidden_states = self.transform_act_fn(hidden_states) + hidden_states = self.LayerNorm(hidden_states) + return hidden_states + +class AlbertLMPredictionHead(nn.Module): + def __init__(self, config): + super(AlbertLMPredictionHead, self).__init__() + self.transform = AlbertPredictionHeadTransform(config) + # The output weights are the same as the input embeddings, but there is + # an output-only bias for each token. + self.decoder = nn.Linear(config.embedding_size,config.vocab_size,bias=False) + self.bias = nn.Parameter(torch.zeros(config.vocab_size)) + + def forward(self, hidden_states): + hidden_states = self.transform(hidden_states) + hidden_states = self.decoder(hidden_states) + self.bias + return hidden_states + +class AlbertOnlyMLMHead(nn.Module): + def __init__(self, config): + super(AlbertOnlyMLMHead, self).__init__() + self.predictions = AlbertLMPredictionHead(config) + + def forward(self, sequence_output): + prediction_scores = self.predictions(sequence_output) + return prediction_scores + +class AlbertOnlyNSPHead(nn.Module): + def __init__(self, config): + super(AlbertOnlyNSPHead, self).__init__() + self.seq_relationship = nn.Linear(config.hidden_size, 2) + + def forward(self, pooled_output): + seq_relationship_score = self.seq_relationship(pooled_output) + return seq_relationship_score + +class AlbertPreTrainingHeads(nn.Module): + def __init__(self, config): + super(AlbertPreTrainingHeads, self).__init__() + self.predictions = AlbertLMPredictionHead(config) + self.seq_relationship = nn.Linear(config.hidden_size, 2) + + def forward(self, sequence_output, pooled_output): + prediction_scores = self.predictions(sequence_output) + seq_relationship_score = self.seq_relationship(pooled_output) + return prediction_scores, seq_relationship_score + +class AlbertPreTrainedModel(PreTrainedModel): + """ An abstract class to handle weights initialization and + a simple interface for dowloading and loading pretrained models. + """ + config_class = AlbertConfig + pretrained_model_archive_map = ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP + load_tf_weights = load_tf_weights_in_albert + base_model_prefix = "bert" + + def _init_weights(self, module): + """ Initialize the weights """ + if isinstance(module, (nn.Linear, nn.Embedding)): + # Slightly different from the TF version which uses truncated_normal for initialization + # cf https://github.com/pytorch/pytorch/pull/5617 + module.weight.data.normal_(mean=0.0, std=self.config.initializer_range) + elif isinstance(module, AlbertLayerNorm): + module.bias.data.zero_() + module.weight.data.fill_(1.0) + if isinstance(module, nn.Linear) and module.bias is not None: + module.bias.data.zero_() + + +ALBERT_START_DOCSTRING = r""" The ALBERT model was proposed in + `ALBERT: A Lite BERT for Self-supervised Learning of Language Representations`_ + by Zhenzhong Lan, Mingda Chen, Sebastian Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut. + This model is a PyTorch `torch.nn.Module`_ sub-class. Use it as a regular PyTorch Module and + refer to the PyTorch documentation for all matter related to general usage and behavior. + .. _`BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding`: + https://arxiv.org/abs/1909.11942 + .. _`torch.nn.Module`: + https://pytorch.org/docs/stable/nn.html#module + Parameters: + config (:class:`~transformers.ALbertConfig`): Model configuration class with all the parameters of the model. + Initializing with a config file does not load the weights associated with the model, only the configuration. + Check out the :meth:`~transformers.PreTrainedModel.from_pretrained` method to load the model weights. +""" + +ALBERT_INPUTS_DOCSTRING = r""" + Inputs: + **input_ids**: ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Indices of input sequence tokens in the vocabulary. + To match pre-training, ALBERT input sequence should be formatted with [CLS] and [SEP] tokens as follows: + (a) For sequence pairs: + ``tokens: [CLS] is this jack ##son ##ville ? [SEP] no it is not . [SEP]`` + ``token_type_ids: 0 0 0 0 0 0 0 0 1 1 1 1 1 1`` + (b) For single sequences: + ``tokens: [CLS] the dog is hairy . [SEP]`` + ``token_type_ids: 0 0 0 0 0 0 0`` + ALBert is a model with absolute position embeddings so it's usually advised to pad the inputs on + the right rather than the left. + Indices can be obtained using :class:`transformers.BertTokenizer`. + See :func:`transformers.PreTrainedTokenizer.encode` and + :func:`transformers.PreTrainedTokenizer.convert_tokens_to_ids` for details. + **attention_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, sequence_length)``: + Mask to avoid performing attention on padding token indices. + Mask values selected in ``[0, 1]``: + ``1`` for tokens that are NOT MASKED, ``0`` for MASKED tokens. + **token_type_ids**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Segment token indices to indicate first and second portions of the inputs. + Indices are selected in ``[0, 1]``: ``0`` corresponds to a `sentence A` token, ``1`` + corresponds to a `sentence B` token + (see `BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding`_ for more details). + **position_ids**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Indices of positions of each input sequence tokens in the position embeddings. + Selected in the range ``[0, config.max_position_embeddings - 1]``. + **head_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(num_heads,)`` or ``(num_layers, num_heads)``: + Mask to nullify selected heads of the self-attention modules. + Mask values selected in ``[0, 1]``: + ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**. +""" + +@add_start_docstrings("The bare Albert Model transformer outputting raw hidden-states without any specific head on top.", + ALBERT_START_DOCSTRING, ALBERT_INPUTS_DOCSTRING) +class AlbertModel(AlbertPreTrainedModel): + r""" + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **last_hidden_state**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, hidden_size)`` + Sequence of hidden-states at the output of the last layer of the model. + **pooler_output**: ``torch.FloatTensor`` of shape ``(batch_size, hidden_size)`` + Last layer hidden-state of the first token of the sequence (classification token) + further processed by a Linear layer and a Tanh activation function. The Linear + layer weights are trained from the next sentence prediction (classification) + objective during Bert pretraining. This output is usually *not* a good summary + of the semantic content of the input, you're often better with averaging or pooling + the sequence of hidden-states for the whole input sequence. + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + Examples:: + tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') + model = BertModel.from_pretrained('bert-base-uncased') + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + outputs = model(input_ids) + last_hidden_states = outputs[0] # The last hidden-state is the first element of the output tuple + """ + + def __init__(self, config): + super(AlbertModel, self).__init__(config) + + self.embeddings = AlbertEmbeddings(config) + self.encoder = AlbertEncoder(config) + self.pooler = AlbertPooler(config) + + self.init_weights() + + def _resize_token_embeddings(self, new_num_tokens): + old_embeddings = self.embeddings.word_embeddings + new_embeddings = self._get_resized_embeddings(old_embeddings, new_num_tokens) + self.embeddings.word_embeddings = new_embeddings + return self.embeddings.word_embeddings + + def _prune_heads(self, heads_to_prune): + """ Prunes heads of the model. + heads_to_prune: dict of {layer_num: list of heads to prune in this layer} + See base class PreTrainedModel + """ + for layer, heads in heads_to_prune.items(): + self.encoder.layer[layer].attention.prune_heads(heads) + + def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None): + if attention_mask is None: + attention_mask = torch.ones_like(input_ids) + if token_type_ids is None: + token_type_ids = torch.zeros_like(input_ids) + + # We create a 3D attention mask from a 2D tensor mask. + # Sizes are [batch_size, 1, 1, to_seq_length] + # So we can broadcast to [batch_size, num_heads, from_seq_length, to_seq_length] + # this attention mask is more simple than the triangular masking of causal attention + # used in OpenAI GPT, we just need to prepare the broadcast dimension here. + extended_attention_mask = attention_mask.unsqueeze(1).unsqueeze(2) + + # Since attention_mask is 1.0 for positions we want to attend and 0.0 for + # masked positions, this operation will create a tensor which is 0.0 for + # positions we want to attend and -10000.0 for masked positions. + # Since we are adding it to the raw scores before the softmax, this is + # effectively the same as removing these entirely. + extended_attention_mask = extended_attention_mask.to(dtype=next(self.parameters()).dtype) # fp16 compatibility + extended_attention_mask = (1.0 - extended_attention_mask) * -10000.0 + + # Prepare head mask if needed + # 1.0 in head_mask indicate we keep the head + # attention_probs has shape bsz x n_heads x N x N + # input head_mask has shape [num_heads] or [num_hidden_layers x num_heads] + # and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length] + if head_mask is not None: + if head_mask.dim() == 1: + head_mask = head_mask.unsqueeze(0).unsqueeze(0).unsqueeze(-1).unsqueeze(-1) + head_mask = head_mask.expand(self.config.num_hidden_layers, -1, -1, -1, -1) + elif head_mask.dim() == 2: + head_mask = head_mask.unsqueeze(1).unsqueeze(-1).unsqueeze( + -1) # We can specify head_mask for each layer + head_mask = head_mask.to( + dtype=next(self.parameters()).dtype) # switch to fload if need + fp16 compatibility + else: + head_mask = [None] * self.config.num_hidden_layers + + embedding_output = self.embeddings(input_ids, position_ids=position_ids, token_type_ids=token_type_ids) + encoder_outputs = self.encoder(embedding_output, + extended_attention_mask, + head_mask=head_mask) + sequence_output = encoder_outputs[0] + pooled_output = self.pooler(sequence_output) + + outputs = (sequence_output, pooled_output,) + encoder_outputs[ + 1:] # add hidden_states and attentions if they are here + return outputs # sequence_output, pooled_output, (hidden_states), (attentions) + +@add_start_docstrings("""Bert Model with two heads on top as done during the pre-training: + a `masked language modeling` head and a `next sentence prediction (classification)` head. """, + ALBERT_START_DOCSTRING, ALBERT_INPUTS_DOCSTRING) +class AlbertForPreTraining(AlbertPreTrainedModel): + r""" + **masked_lm_labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Labels for computing the masked language modeling loss. + Indices should be in ``[-1, 0, ..., config.vocab_size]`` (see ``input_ids`` docstring) + Tokens with indices set to ``-1`` are ignored (masked), the loss is only computed for the tokens with labels + in ``[0, ..., config.vocab_size]`` + **next_sentence_label**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``: + Labels for computing the next sequence prediction (classification) loss. Input should be a sequence pair (see ``input_ids`` docstring) + Indices should be in ``[0, 1]``. + ``0`` indicates sequence B is a continuation of sequence A, + ``1`` indicates sequence B is a random sequence. + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **loss**: (`optional`, returned when both ``masked_lm_labels`` and ``next_sentence_label`` are provided) ``torch.FloatTensor`` of shape ``(1,)``: + Total loss as the sum of the masked language modeling loss and the next sequence prediction (classification) loss. + **prediction_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, config.vocab_size)`` + Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). + **seq_relationship_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, 2)`` + Prediction scores of the next sequence prediction (classification) head (scores of True/False continuation before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + Examples:: + tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') + model = BertForPreTraining.from_pretrained('bert-base-uncased') + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + outputs = model(input_ids) + prediction_scores, seq_relationship_scores = outputs[:2] + """ + + def __init__(self, config): + super(AlbertForPreTraining, self).__init__(config) + self.bert = AlbertModel(config) + self.cls = AlbertPreTrainingHeads(config) + + self.init_weights() + self.tie_weights() + + def tie_weights(self): + """ Make sure we are sharing the input and output embeddings. + Export to TorchScript can't handle parameter sharing so we are cloning them instead. + """ + self._tie_or_clone_weights(self.cls.predictions.decoder, + self.bert.embeddings.word_embeddings) + + def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, + masked_lm_labels=None, next_sentence_label=None): + outputs = self.bert(input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + position_ids=position_ids, + head_mask=head_mask) + + sequence_output, pooled_output = outputs[:2] + prediction_scores, seq_relationship_score = self.cls(sequence_output, pooled_output) + + outputs = (prediction_scores, seq_relationship_score,) + outputs[ + 2:] # add hidden states and attention if they are here + + if masked_lm_labels is not None and next_sentence_label is not None: + loss_fct = CrossEntropyLoss(ignore_index=-1) + masked_lm_loss = loss_fct(prediction_scores.view(-1, self.config.vocab_size), masked_lm_labels.view(-1)) + next_sentence_loss = loss_fct(seq_relationship_score.view(-1, 2), next_sentence_label.view(-1)) + total_loss = masked_lm_loss + next_sentence_loss + outputs = (total_loss,) + outputs + return outputs # (loss), prediction_scores, seq_relationship_score, (hidden_states), (attentions) + +@add_start_docstrings("""Bert Model with a `language modeling` head on top. """, + ALBERT_START_DOCSTRING, ALBERT_INPUTS_DOCSTRING) +class AlbertForMaskedLM(AlbertPreTrainedModel): + r""" + **masked_lm_labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Labels for computing the masked language modeling loss. + Indices should be in ``[-1, 0, ..., config.vocab_size]`` (see ``input_ids`` docstring) + Tokens with indices set to ``-1`` are ignored (masked), the loss is only computed for the tokens with labels + in ``[0, ..., config.vocab_size]`` + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **loss**: (`optional`, returned when ``masked_lm_labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: + Masked language modeling loss. + **prediction_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, config.vocab_size)`` + Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + Examples:: + tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') + model = BertForMaskedLM.from_pretrained('bert-base-uncased') + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + outputs = model(input_ids, masked_lm_labels=input_ids) + loss, prediction_scores = outputs[:2] + """ + + def __init__(self, config): + super(AlbertForMaskedLM, self).__init__(config) + + self.bert = AlbertModel(config) + self.cls = AlbertOnlyMLMHead(config) + + self.init_weights() + self.tie_weights() + + def tie_weights(self): + """ Make sure we are sharing the input and output embeddings. + Export to TorchScript can't handle parameter sharing so we are cloning them instead. + """ + self._tie_or_clone_weights(self.cls.predictions.decoder, + self.bert.embeddings.word_embeddings) + + def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, + masked_lm_labels=None): + outputs = self.bert(input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + position_ids=position_ids, + head_mask=head_mask) + + sequence_output = outputs[0] + prediction_scores = self.cls(sequence_output) + + outputs = (prediction_scores,) + outputs[2:] # Add hidden states and attention if they are here + if masked_lm_labels is not None: + loss_fct = CrossEntropyLoss(ignore_index=-1) + masked_lm_loss = loss_fct(prediction_scores.view(-1, self.config.vocab_size), masked_lm_labels.view(-1)) + outputs = (masked_lm_loss,) + outputs + + return outputs # (masked_lm_loss), prediction_scores, (hidden_states), (attentions) + + +@add_start_docstrings("""Bert Model with a `next sentence prediction (classification)` head on top. """, + ALBERT_START_DOCSTRING, ALBERT_INPUTS_DOCSTRING) +class AlbertForNextSentencePrediction(AlbertPreTrainedModel): + r""" + **next_sentence_label**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``: + Labels for computing the next sequence prediction (classification) loss. Input should be a sequence pair (see ``input_ids`` docstring) + Indices should be in ``[0, 1]``. + ``0`` indicates sequence B is a continuation of sequence A, + ``1`` indicates sequence B is a random sequence. + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **loss**: (`optional`, returned when ``next_sentence_label`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: + Next sequence prediction (classification) loss. + **seq_relationship_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, 2)`` + Prediction scores of the next sequence prediction (classification) head (scores of True/False continuation before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + Examples:: + tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') + model = BertForNextSentencePrediction.from_pretrained('bert-base-uncased') + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + outputs = model(input_ids) + seq_relationship_scores = outputs[0] + """ + + def __init__(self, config): + super(AlbertForNextSentencePrediction, self).__init__(config) + + self.bert = AlbertModel(config) + self.cls = AlbertOnlyNSPHead(config) + + self.init_weights() + + def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, + next_sentence_label=None): + outputs = self.bert(input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + position_ids=position_ids, + head_mask=head_mask) + + pooled_output = outputs[1] + + seq_relationship_score = self.cls(pooled_output) + + outputs = (seq_relationship_score,) + outputs[2:] # add hidden states and attention if they are here + if next_sentence_label is not None: + loss_fct = CrossEntropyLoss(ignore_index=-1) + next_sentence_loss = loss_fct(seq_relationship_score.view(-1, 2), next_sentence_label.view(-1)) + outputs = (next_sentence_loss,) + outputs + + return outputs # (next_sentence_loss), seq_relationship_score, (hidden_states), (attentions) + + +@add_start_docstrings("""Bert Model transformer with a sequence classification/regression head on top (a linear layer on top of + the pooled output) e.g. for GLUE tasks. """, + ALBERT_START_DOCSTRING, ALBERT_INPUTS_DOCSTRING) +class AlbertForSequenceClassification(AlbertPreTrainedModel): + r""" + **labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``: + Labels for computing the sequence classification/regression loss. + Indices should be in ``[0, ..., config.num_labels - 1]``. + If ``config.num_labels == 1`` a regression loss is computed (Mean-Square loss), + If ``config.num_labels > 1`` a classification loss is computed (Cross-Entropy). + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **loss**: (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: + Classification (or regression if config.num_labels==1) loss. + **logits**: ``torch.FloatTensor`` of shape ``(batch_size, config.num_labels)`` + Classification (or regression if config.num_labels==1) scores (before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + Examples:: + tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') + model = BertForSequenceClassification.from_pretrained('bert-base-uncased') + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + labels = torch.tensor([1]).unsqueeze(0) # Batch size 1 + outputs = model(input_ids, labels=labels) + loss, logits = outputs[:2] + """ + + def __init__(self, config): + super(AlbertForSequenceClassification, self).__init__(config) + self.num_labels = config.num_labels + # 根据config定义一个和预训练模型一致的albert模型 + self.bert = AlbertModel(config) + self.dropout = nn.Dropout(0.1 if config.hidden_dropout_prob == 0 else config.hidden_dropout_prob) + self.classifier = nn.Linear(config.hidden_size, self.config.num_labels) + + self.init_weights() + + def forward(self, input_ids, attention_mask=None, token_type_ids=None, + position_ids=None, head_mask=None, labels=None): + + outputs = self.bert(input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + position_ids=position_ids, + head_mask=head_mask) + # output[0]->(48,60,1024) + # output[1]->(48,1024)->（batch,feature） + pooled_output = outputs[1] + pooled_output = self.dropout(pooled_output+0.1) + # logits未进入softmax的 + logits = self.classifier(pooled_output) + #实际上这里idx2之后没东西 啥都没加 + outputs = (logits,) + outputs[2:] # add hidden states and attention if they are here + if labels is not None: + if self.num_labels == 1: + # We are doing regression + loss_fct = MSELoss() + loss = loss_fct(logits.view(-1), labels.view(-1)) + else: + loss_fct = CrossEntropyLoss() + loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1)) + outputs = (loss,) + outputs + return outputs # (loss), logits, (hidden_states), (attentions) + + +@add_start_docstrings("""Bert Model with a multiple choice classification head on top (a linear layer on top of + the pooled output and a softmax) e.g. for RocStories/SWAG tasks. """, + ALBERT_START_DOCSTRING, ALBERT_INPUTS_DOCSTRING) +class AlbertForMultipleChoice(AlbertPreTrainedModel): + r""" + **labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``: + Labels for computing the multiple choice classification loss. + Indices should be in ``[0, ..., num_choices]`` where `num_choices` is the size of the second dimension + of the input tensors. (see `input_ids` above) + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **loss**: (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: + Classification loss. + **classification_scores**: ``torch.FloatTensor`` of shape ``(batch_size, num_choices)`` where `num_choices` is the size of the second dimension + of the input tensors. (see `input_ids` above). + Classification scores (before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + Examples:: + tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') + model = BertForMultipleChoice.from_pretrained('bert-base-uncased') + choices = ["Hello, my dog is cute", "Hello, my cat is amazing"] + input_ids = torch.tensor([tokenizer.encode(s) for s in choices]).unsqueeze(0) # Batch size 1, 2 choices + labels = torch.tensor(1).unsqueeze(0) # Batch size 1 + outputs = model(input_ids, labels=labels) + loss, classification_scores = outputs[:2] + """ + + def __init__(self, config): + super(AlbertForMultipleChoice, self).__init__(config) + + self.bert = AlbertModel(config) + self.dropout = nn.Dropout(0.1 if config.hidden_dropout_prob == 0 else config.hidden_dropout_prob) + self.classifier = nn.Linear(config.hidden_size, 1) + + self.init_weights() + + def forward(self, input_ids, attention_mask=None, token_type_ids=None, + position_ids=None, head_mask=None, labels=None): + num_choices = input_ids.shape[1] + + input_ids = input_ids.view(-1, input_ids.size(-1)) + attention_mask = attention_mask.view(-1, attention_mask.size(-1)) if attention_mask is not None else None + token_type_ids = token_type_ids.view(-1, token_type_ids.size(-1)) if token_type_ids is not None else None + position_ids = position_ids.view(-1, position_ids.size(-1)) if position_ids is not None else None + outputs = self.bert(input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + position_ids=position_ids, + head_mask=head_mask) + pooled_output = outputs[1] + pooled_output = self.dropout(pooled_output) + logits = self.classifier(pooled_output) + reshaped_logits = logits.view(-1, num_choices) + outputs = (reshaped_logits,) + outputs[2:] # add hidden states and attention if they are here + if labels is not None: + loss_fct = CrossEntropyLoss() + loss = loss_fct(reshaped_logits, labels) + outputs = (loss,) + outputs + + return outputs # (loss), reshaped_logits, (hidden_states), (attentions) + + +@add_start_docstrings("""Bert Model with a token classification head on top (a linear layer on top of + the hidden-states output) e.g. for Named-Entity-Recognition (NER) tasks. """, + ALBERT_START_DOCSTRING, ALBERT_INPUTS_DOCSTRING) + +class AlbertForTokenClassification(AlbertPreTrainedModel): + r""" + **labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Labels for computing the token classification loss. + Indices should be in ``[0, ..., config.num_labels - 1]``. + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **loss**: (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: + Classification loss. + **scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, config.num_labels)`` + Classification scores (before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + Examples:: + tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') + model = BertForTokenClassification.from_pretrained('bert-base-uncased') + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + labels = torch.tensor([1] * input_ids.size(1)).unsqueeze(0) # Batch size 1 + outputs = model(input_ids, labels=labels) + loss, scores = outputs[:2] + """ + + def __init__(self, config): + super(AlbertForTokenClassification, self).__init__(config) + self.num_labels = config.num_labels + + self.bert = AlbertModel(config) + self.dropout = nn.Dropout(0.1 if config.hidden_dropout_prob == 0 else config.hidden_dropout_prob) + self.classifier = nn.Linear(config.hidden_size, config.num_labels) + + self.init_weights() + + def forward(self, input_ids, attention_mask=None, token_type_ids=None, + position_ids=None, head_mask=None, labels=None): + + outputs = self.bert(input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + position_ids=position_ids, + head_mask=head_mask) + + sequence_output = outputs[0] + + sequence_output = self.dropout(sequence_output) + logits = self.classifier(sequence_output) + + outputs = (logits,) + outputs[2:] # add hidden states and attention if they are here + if labels is not None: + loss_fct = CrossEntropyLoss() + # Only keep active parts of the loss + if attention_mask is not None: + active_loss = attention_mask.view(-1) == 1 + active_logits = logits.view(-1, self.num_labels)[active_loss] + active_labels = labels.view(-1)[active_loss] + loss = loss_fct(active_logits, active_labels) + else: + loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1)) + outputs = (loss,) + outputs + + return outputs # (loss), scores, (hidden_states), (attentions) + + +@add_start_docstrings("""Bert Model with a span classification head on top for extractive question-answering tasks like SQuAD (a linear layers on top of + the hidden-states output to compute `span start logits` and `span end logits`). """, + ALBERT_START_DOCSTRING, ALBERT_INPUTS_DOCSTRING) +class AlbertForQuestionAnswering(AlbertPreTrainedModel): + r""" + **start_positions**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``: + Labels for position (index) of the start of the labelled span for computing the token classification loss. + Positions are clamped to the length of the sequence (`sequence_length`). + Position outside of the sequence are not taken into account for computing the loss. + **end_positions**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``: + Labels for position (index) of the end of the labelled span for computing the token classification loss. + Positions are clamped to the length of the sequence (`sequence_length`). + Position outside of the sequence are not taken into account for computing the loss. + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **loss**: (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: + Total span extraction loss is the sum of a Cross-Entropy for the start and end positions. + **start_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length,)`` + Span-start scores (before SoftMax). + **end_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length,)`` + Span-end scores (before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + Examples:: + tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') + model = BertForQuestionAnswering.from_pretrained('bert-base-uncased') + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + start_positions = torch.tensor([1]) + end_positions = torch.tensor([3]) + outputs = model(input_ids, start_positions=start_positions, end_positions=end_positions) + loss, start_scores, end_scores = outputs[:2] + """ + + def __init__(self, config): + super(AlbertForQuestionAnswering, self).__init__(config) + self.num_labels = config.num_labels + + self.bert = AlbertModel(config) + self.qa_outputs = nn.Linear(config.hidden_size, config.num_labels) + + self.init_weights() + + def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, + start_positions=None, end_positions=None): + + outputs = self.bert(input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + position_ids=position_ids, + head_mask=head_mask) + + sequence_output = outputs[0] + + logits = self.qa_outputs(sequence_output) + start_logits, end_logits = logits.split(1, dim=-1) + start_logits = start_logits.squeeze(-1) + end_logits = end_logits.squeeze(-1) + + outputs = (start_logits, end_logits,) + outputs[2:] + if start_positions is not None and end_positions is not None: + # If we are on multi-GPU, split add a dimension + if len(start_positions.size()) > 1: + start_positions = start_positions.squeeze(-1) + if len(end_positions.size()) > 1: + end_positions = end_positions.squeeze(-1) + # sometimes the start/end positions are outside our model inputs, we ignore these terms + ignored_index = start_logits.size(1) + start_positions.clamp_(0, ignored_index) + end_positions.clamp_(0, ignored_index) + + loss_fct = CrossEntropyLoss(ignore_index=ignored_index) + start_loss = loss_fct(start_logits, start_positions) + end_loss = loss_fct(end_logits, end_positions) + total_loss = (start_loss + end_loss) / 2 + outputs = (total_loss,) + outputs + + return outputs # (loss), start_logits, end_logits, (hidden_states), (attentions) diff --git a/BertToSimple/BiGRU/model/modeling_albert_bright.py b/BertToSimple/BiGRU/model/modeling_albert_bright.py new file mode 100644 index 0000000..f12a4c9 --- /dev/null +++ b/BertToSimple/BiGRU/model/modeling_albert_bright.py @@ -0,0 +1,1002 @@ +"""PyTorch brightmart version ALBERT model. """ +from __future__ import absolute_import, division, print_function, unicode_literals + +import logging +import os + +import torch +from torch import nn +from torch.nn import CrossEntropyLoss, MSELoss + +from .modeling_utils import PreTrainedModel, prune_linear_layer +from .configuration_albert import AlbertConfig +from .file_utils import add_start_docstrings +from .modeling_bert import (ACT2FN, + BertSelfAttention, + BertIntermediate, + BertPooler, + BertPredictionHeadTransform) + +logger = logging.getLogger(__name__) + +ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP = { + 'albert-base': "", + 'albert-large': "", + 'albert-xlarge': "", + 'albert-xxlarge': "", +} +def load_tf_weights_in_albert(model, config, tf_checkpoint_path): + """ Load tf checkpoints in a pytorch model. + """ + try: + import re + import numpy as np + import tensorflow as tf + except ImportError: + logger.error("Loading a TensorFlow model in PyTorch, requires TensorFlow to be installed. Please see " + "https://www.tensorflow.org/install/ for installation instructions.") + raise + tf_path = os.path.abspath(tf_checkpoint_path) + logger.info("Converting TensorFlow checkpoint from {}".format(tf_path)) + # Load weights from TF model + init_vars = tf.train.list_variables(tf_path) + names = [] + arrays = [] + for name, shape in init_vars: + logger.info("Loading TF weight {} with shape {}".format(name, shape)) + array = tf.train.load_variable(tf_path, name) + names.append(name) + arrays.append(array) + for name, array in zip(names, arrays): + name = name.split('/') + # adam_v and adam_m are variables used in AdamWeightDecayOptimizer to calculated m and v + # which are not required for using pretrained model + if any(n in ["adam_v", "adam_m", "global_step"] for n in name): + logger.info("Skipping {}".format("/".join(name))) + continue + pointer = model + for m_name in name: + if re.fullmatch(r'[A-Za-z]+_\d+', m_name): + l = re.split(r'_(\d+)', m_name) + else: + l = [m_name] + if l[0] == 'kernel' or l[0] == 'gamma': + pointer = getattr(pointer, 'weight') + elif l[0] == 'output_bias' or l[0] == 'beta': + pointer = getattr(pointer, 'bias') + elif l[0] == 'output_weights': + pointer = getattr(pointer, 'weight') + elif l[0] == 'squad': + pointer = getattr(pointer, 'classifier') + else: + try: + pointer = getattr(pointer, l[0]) + except AttributeError: + logger.info("Skipping {}".format("/".join(name))) + continue + if len(l) >= 2: + num = int(l[1]) + pointer = pointer[num] + if m_name[-11:] == '_embeddings': + pointer = getattr(pointer, 'weight') + elif m_name[-13:] == '_embeddings_2': + pointer = getattr(pointer, 'weight') + array = np.transpose(array) + elif m_name == 'kernel': + array = np.transpose(array) + try: + assert pointer.shape == array.shape + except AssertionError as e: + e.args += (pointer.shape, array.shape) + raise + logger.info("Initialize PyTorch weight {}".format(name)) + pointer.data = torch.from_numpy(array) + return model + +AlbertLayerNorm = torch.nn.LayerNorm +class AlbertEmbeddings(nn.Module): + """Construct the embeddings from word, position and token_type embeddings. + """ + def __init__(self, config): + super(AlbertEmbeddings, self).__init__() + self.word_embeddings = nn.Embedding(config.vocab_size, config.embedding_size, padding_idx=0) + # project layer + self.word_embeddings_2 = nn.Linear(config.embedding_size, config.hidden_size, bias=False) + + self.position_embeddings = nn.Embedding(config.max_position_embeddings, config.hidden_size) + self.token_type_embeddings = nn.Embedding(config.type_vocab_size, config.hidden_size) + + # self.LayerNorm is not snake-cased to stick with TensorFlow model variable name and be able to load + # any TensorFlow checkpoint file + self.LayerNorm =AlbertLayerNorm(config.hidden_size, eps=config.layer_norm_eps) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + + def forward(self, input_ids, token_type_ids=None, position_ids=None): + seq_length = input_ids.size(1) + if position_ids is None: + position_ids = torch.arange(seq_length, dtype=torch.long, device=input_ids.device) + position_ids = position_ids.unsqueeze(0).expand_as(input_ids) + if token_type_ids is None: + token_type_ids = torch.zeros_like(input_ids) + + words_embeddings = self.word_embeddings(input_ids) + # project transform + words_embeddings = self.word_embeddings_2(words_embeddings) + position_embeddings = self.position_embeddings(position_ids) + token_type_embeddings = self.token_type_embeddings(token_type_ids) + + embeddings = words_embeddings + position_embeddings + token_type_embeddings + embeddings = self.LayerNorm(embeddings) + embeddings = self.dropout(embeddings) + return embeddings + +class AlbertSelfOutput(nn.Module): + def __init__(self, config): + super(AlbertSelfOutput, self).__init__() + self.dense = nn.Linear(config.hidden_size, config.hidden_size) + self.LayerNorm = AlbertLayerNorm(config.hidden_size, eps=config.layer_norm_eps) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + + def forward(self, hidden_states, input_tensor): + hidden_states = self.dense(hidden_states) + hidden_states = self.dropout(hidden_states) + # postln + hidden_states = self.LayerNorm(hidden_states + input_tensor) + return hidden_states + + +class AlbertAttention(nn.Module): + def __init__(self, config): + super(AlbertAttention, self).__init__() + self.self = BertSelfAttention(config) + self.output = AlbertSelfOutput(config) + self.pruned_heads = set() + + def prune_heads(self, heads): + if len(heads) == 0: + return + mask = torch.ones(self.self.num_attention_heads, self.self.attention_head_size) + heads = set(heads) - self.pruned_heads # Convert to set and emove already pruned heads + for head in heads: + # Compute how many pruned heads are before the head and move the index accordingly + head = head - sum(1 if h < head else 0 for h in self.pruned_heads) + mask[head] = 0 + mask = mask.view(-1).contiguous().eq(1) + index = torch.arange(len(mask))[mask].long() + + # Prune linear layers + self.self.query = prune_linear_layer(self.self.query, index) + self.self.key = prune_linear_layer(self.self.key, index) + self.self.value = prune_linear_layer(self.self.value, index) + self.output.dense = prune_linear_layer(self.output.dense, index, dim=1) + + # Update hyper params and store pruned heads + self.self.num_attention_heads = self.self.num_attention_heads - len(heads) + self.self.all_head_size = self.self.attention_head_size * self.self.num_attention_heads + self.pruned_heads = self.pruned_heads.union(heads) + + def forward(self, input_tensor, attention_mask=None, head_mask=None): + # postln + self_outputs = self.self(input_tensor, attention_mask, head_mask) + attention_output = self.output(self_outputs[0], input_tensor) + outputs = (attention_output,) + self_outputs[1:] # add attentions if we output them + return outputs + +class AlbertOutput(nn.Module): + def __init__(self, config): + super(AlbertOutput, self).__init__() + self.dense = nn.Linear(config.intermediate_size, config.hidden_size) + self.LayerNorm = AlbertLayerNorm(config.hidden_size, eps=config.layer_norm_eps) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + + def forward(self, hidden_states, input_tensor): + hidden_states = self.dense(hidden_states) + hidden_states = self.dropout(hidden_states) + # postln + hidden_states = self.LayerNorm(hidden_states + input_tensor) + return hidden_states + +class BertLayer(nn.Module): + def __init__(self, config): + super(BertLayer, self).__init__() + self.attention = AlbertAttention(config) + self.intermediate = BertIntermediate(config) + self.output = AlbertOutput(config) + + def forward(self, hidden_states, attention_mask=None, head_mask=None): + attention_outputs = self.attention(hidden_states, attention_mask, head_mask) + attention_output = attention_outputs[0] + # postln + attention_output_pre = attention_output + intermediate_output = self.intermediate(attention_output_pre) + layer_output = self.output(intermediate_output, attention_output) + outputs = (layer_output,) + attention_outputs[1:] # add attentions if we output them + return outputs + +class AlbertEncoder(nn.Module): + def __init__(self, config): + super(AlbertEncoder, self).__init__() + self.output_attentions = config.output_attentions + self.output_hidden_states = config.output_hidden_states + self.num_hidden_layers = config.num_hidden_layers + self.layer_shared = BertLayer(config) + + def forward(self, hidden_states, attention_mask=None, head_mask=None): + all_hidden_states = () + all_attentions = () + for i in range(self.num_hidden_layers): + layer_module = self.layer_shared + if self.output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states,) + layer_outputs = layer_module(hidden_states, attention_mask, head_mask[i]) + hidden_states = layer_outputs[0] + + if self.output_attentions: + all_attentions = all_attentions + (layer_outputs[1],) + # Add last layer + if self.output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states,) + outputs = (hidden_states,) + if self.output_hidden_states: + outputs = outputs + (all_hidden_states,) + if self.output_attentions: + outputs = outputs + (all_attentions,) + return outputs # last-layer hidden state, (all hidden states), (all attentions) + +class AlbertLMPredictionHead(nn.Module): + def __init__(self, config): + super(AlbertLMPredictionHead, self).__init__() + self.transform = BertPredictionHeadTransform(config) + # The output weights are the same as the input embeddings, but there is + # an output-only bias for each token. + self.project_layer = nn.Linear(config.hidden_size, config.embedding_size, bias=False) + self.decoder = nn.Linear(config.embedding_size, + config.vocab_size, + bias=False) + self.bias = nn.Parameter(torch.zeros(config.vocab_size)) + + def forward(self, hidden_states): + hidden_states = self.transform(hidden_states) + hidden_states = self.project_layer(hidden_states) + hidden_states = self.decoder(hidden_states) + self.bias + return hidden_states + +class AlbertOnlyMLMHead(nn.Module): + def __init__(self, config): + super(AlbertOnlyMLMHead, self).__init__() + self.predictions = AlbertLMPredictionHead(config) + + def forward(self, sequence_output): + prediction_scores = self.predictions(sequence_output) + return prediction_scores + +class AlbertOnlyNSPHead(nn.Module): + def __init__(self, config): + super(AlbertOnlyNSPHead, self).__init__() + self.seq_relationship = nn.Linear(config.hidden_size, 2) + + def forward(self, pooled_output): + seq_relationship_score = self.seq_relationship(pooled_output) + return seq_relationship_score + +class AlbertPreTrainingHeads(nn.Module): + def __init__(self, config): + super(AlbertPreTrainingHeads, self).__init__() + self.predictions = AlbertLMPredictionHead(config) + self.seq_relationship = nn.Linear(config.hidden_size, 2) + + def forward(self, sequence_output, pooled_output): + prediction_scores = self.predictions(sequence_output) + seq_relationship_score = self.seq_relationship(pooled_output) + return prediction_scores, seq_relationship_score + +class AlbertPreTrainedModel(PreTrainedModel): + """ An abstract class to handle weights initialization and + a simple interface for dowloading and loading pretrained models. + """ + config_class = AlbertConfig + pretrained_model_archive_map = ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP + load_tf_weights = load_tf_weights_in_albert + base_model_prefix = "bert" + + def _init_weights(self, module): + """ Initialize the weights """ + if isinstance(module, (nn.Linear, nn.Embedding)): + # Slightly different from the TF version which uses truncated_normal for initialization + # cf https://github.com/pytorch/pytorch/pull/5617 + module.weight.data.normal_(mean=0.0, std=self.config.initializer_range) + elif isinstance(module, AlbertLayerNorm): + module.bias.data.zero_() + module.weight.data.fill_(1.0) + if isinstance(module, nn.Linear) and module.bias is not None: + module.bias.data.zero_() + +BERT_START_DOCSTRING = r""" The BERT model was proposed in + `BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding`_ + by Jacob Devlin, Ming-Wei Chang, Kenton Lee and Kristina Toutanova. It's a bidirectional transformer + pre-trained using a combination of masked language modeling objective and next sentence prediction + on a large corpus comprising the Toronto Book Corpus and Wikipedia. + + This model is a PyTorch `torch.nn.Module`_ sub-class. Use it as a regular PyTorch Module and + refer to the PyTorch documentation for all matter related to general usage and behavior. + + .. _`BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding`: + https://arxiv.org/abs/1810.04805 + + .. _`torch.nn.Module`: + https://pytorch.org/docs/stable/nn.html#module + + Parameters: + config (:class:`~transformers.BertConfig`): Model configuration class with all the parameters of the model. + Initializing with a config file does not load the weights associated with the model, only the configuration. + Check out the :meth:`~transformers.PreTrainedModel.from_pretrained` method to load the model weights. +""" + +BERT_INPUTS_DOCSTRING = r""" + Inputs: + **input_ids**: ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Indices of input sequence tokens in the vocabulary. + To match pre-training, BERT input sequence should be formatted with [CLS] and [SEP] tokens as follows: + + (a) For sequence pairs: + + ``tokens: [CLS] is this jack ##son ##ville ? [SEP] no it is not . [SEP]`` + + ``token_type_ids: 0 0 0 0 0 0 0 0 1 1 1 1 1 1`` + + (b) For single sequences: + + ``tokens: [CLS] the dog is hairy . [SEP]`` + + ``token_type_ids: 0 0 0 0 0 0 0`` + + Bert is a model with absolute position embeddings so it's usually advised to pad the inputs on + the right rather than the left. + + Indices can be obtained using :class:`transformers.BertTokenizer`. + See :func:`transformers.PreTrainedTokenizer.encode` and + :func:`transformers.PreTrainedTokenizer.convert_tokens_to_ids` for details. + **attention_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, sequence_length)``: + Mask to avoid performing attention on padding token indices. + Mask values selected in ``[0, 1]``: + ``1`` for tokens that are NOT MASKED, ``0`` for MASKED tokens. + **token_type_ids**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Segment token indices to indicate first and second portions of the inputs. + Indices are selected in ``[0, 1]``: ``0`` corresponds to a `sentence A` token, ``1`` + corresponds to a `sentence B` token + (see `BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding`_ for more details). + **position_ids**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Indices of positions of each input sequence tokens in the position embeddings. + Selected in the range ``[0, config.max_position_embeddings - 1]``. + **head_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(num_heads,)`` or ``(num_layers, num_heads)``: + Mask to nullify selected heads of the self-attention modules. + Mask values selected in ``[0, 1]``: + ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**. +""" + + +@add_start_docstrings("The bare Bert Model transformer outputting raw hidden-states without any specific head on top.", + BERT_START_DOCSTRING, BERT_INPUTS_DOCSTRING) +class AlbertModel(AlbertPreTrainedModel): + r""" + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **last_hidden_state**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, hidden_size)`` + Sequence of hidden-states at the output of the last layer of the model. + **pooler_output**: ``torch.FloatTensor`` of shape ``(batch_size, hidden_size)`` + Last layer hidden-state of the first token of the sequence (classification token) + further processed by a Linear layer and a Tanh activation function. The Linear + layer weights are trained from the next sentence prediction (classification) + objective during Bert pretraining. This output is usually *not* a good summary + of the semantic content of the input, you're often better with averaging or pooling + the sequence of hidden-states for the whole input sequence. + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') + model = BertModel.from_pretrained('bert-base-uncased') + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + outputs = model(input_ids) + last_hidden_states = outputs[0] # The last hidden-state is the first element of the output tuple + + """ + + def __init__(self, config): + super(AlbertModel, self).__init__(config) + + self.embeddings = AlbertEmbeddings(config) + self.encoder = AlbertEncoder(config) + self.pooler = BertPooler(config) + + self.init_weights() + + def _resize_token_embeddings(self, new_num_tokens): + old_embeddings = self.embeddings.word_embeddings + new_embeddings = self._get_resized_embeddings(old_embeddings, new_num_tokens) + self.embeddings.word_embeddings = new_embeddings + return self.embeddings.word_embeddings + + def _prune_heads(self, heads_to_prune): + """ Prunes heads of the model. + heads_to_prune: dict of {layer_num: list of heads to prune in this layer} + See base class PreTrainedModel + """ + for layer, heads in heads_to_prune.items(): + self.encoder.layer[layer].attention.prune_heads(heads) + + def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None): + if attention_mask is None: + attention_mask = torch.ones_like(input_ids) + if token_type_ids is None: + token_type_ids = torch.zeros_like(input_ids) + + # We create a 3D attention mask from a 2D tensor mask. + # Sizes are [batch_size, 1, 1, to_seq_length] + # So we can broadcast to [batch_size, num_heads, from_seq_length, to_seq_length] + # this attention mask is more simple than the triangular masking of causal attention + # used in OpenAI GPT, we just need to prepare the broadcast dimension here. + extended_attention_mask = attention_mask.unsqueeze(1).unsqueeze(2) + + # Since attention_mask is 1.0 for positions we want to attend and 0.0 for + # masked positions, this operation will create a tensor which is 0.0 for + # positions we want to attend and -10000.0 for masked positions. + # Since we are adding it to the raw scores before the softmax, this is + # effectively the same as removing these entirely. + extended_attention_mask = extended_attention_mask.to(dtype=next(self.parameters()).dtype) # fp16 compatibility + extended_attention_mask = (1.0 - extended_attention_mask) * -10000.0 + + # Prepare head mask if needed + # 1.0 in head_mask indicate we keep the head + # attention_probs has shape bsz x n_heads x N x N + # input head_mask has shape [num_heads] or [num_hidden_layers x num_heads] + # and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length] + if head_mask is not None: + if head_mask.dim() == 1: + head_mask = head_mask.unsqueeze(0).unsqueeze(0).unsqueeze(-1).unsqueeze(-1) + head_mask = head_mask.expand(self.config.num_hidden_layers, -1, -1, -1, -1) + elif head_mask.dim() == 2: + head_mask = head_mask.unsqueeze(1).unsqueeze(-1).unsqueeze( + -1) # We can specify head_mask for each layer + head_mask = head_mask.to( + dtype=next(self.parameters()).dtype) # switch to fload if need + fp16 compatibility + else: + head_mask = [None] * self.config.num_hidden_layers + + embedding_output = self.embeddings(input_ids, position_ids=position_ids, token_type_ids=token_type_ids) + encoder_outputs = self.encoder(embedding_output, + extended_attention_mask, + head_mask=head_mask) + sequence_output = encoder_outputs[0] + pooled_output = self.pooler(sequence_output) + + outputs = (sequence_output, pooled_output,) + encoder_outputs[ + 1:] # add hidden_states and attentions if they are here + return outputs # sequence_output, pooled_output, (hidden_states), (attentions) + + +@add_start_docstrings("""Bert Model with two heads on top as done during the pre-training: + a `masked language modeling` head and a `next sentence prediction (classification)` head. """, + BERT_START_DOCSTRING, BERT_INPUTS_DOCSTRING) +class AlbertForPreTraining(AlbertPreTrainedModel): + r""" + **masked_lm_labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Labels for computing the masked language modeling loss. + Indices should be in ``[-1, 0, ..., config.vocab_size]`` (see ``input_ids`` docstring) + Tokens with indices set to ``-1`` are ignored (masked), the loss is only computed for the tokens with labels + in ``[0, ..., config.vocab_size]`` + **next_sentence_label**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``: + Labels for computing the next sequence prediction (classification) loss. Input should be a sequence pair (see ``input_ids`` docstring) + Indices should be in ``[0, 1]``. + ``0`` indicates sequence B is a continuation of sequence A, + ``1`` indicates sequence B is a random sequence. + + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **loss**: (`optional`, returned when both ``masked_lm_labels`` and ``next_sentence_label`` are provided) ``torch.FloatTensor`` of shape ``(1,)``: + Total loss as the sum of the masked language modeling loss and the next sequence prediction (classification) loss. + **prediction_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, config.vocab_size)`` + Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). + **seq_relationship_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, 2)`` + Prediction scores of the next sequence prediction (classification) head (scores of True/False continuation before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') + model = BertForPreTraining.from_pretrained('bert-base-uncased') + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + outputs = model(input_ids) + prediction_scores, seq_relationship_scores = outputs[:2] + + """ + + def __init__(self, config): + super(AlbertForPreTraining, self).__init__(config) + + self.bert = AlbertModel(config) + self.cls = AlbertPreTrainingHeads(config) + + self.init_weights() + self.tie_weights() + + def tie_weights(self): + """ Make sure we are sharing the input and output embeddings. + Export to TorchScript can't handle parameter sharing so we are cloning them instead. + """ + self._tie_or_clone_weights(self.cls.predictions.decoder, + self.bert.embeddings.word_embeddings) + + def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, + masked_lm_labels=None, next_sentence_label=None): + outputs = self.bert(input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + position_ids=position_ids, + head_mask=head_mask) + + sequence_output, pooled_output = outputs[:2] + prediction_scores, seq_relationship_score = self.cls(sequence_output, pooled_output) + + outputs = (prediction_scores, seq_relationship_score,) + outputs[ + 2:] # add hidden states and attention if they are here + + if masked_lm_labels is not None and next_sentence_label is not None: + loss_fct = CrossEntropyLoss(ignore_index=-1) + masked_lm_loss = loss_fct(prediction_scores.view(-1, self.config.vocab_size), masked_lm_labels.view(-1)) + next_sentence_loss = loss_fct(seq_relationship_score.view(-1, 2), next_sentence_label.view(-1)) + total_loss = masked_lm_loss + next_sentence_loss + outputs = (total_loss,) + outputs + + return outputs # (loss), prediction_scores, seq_relationship_score, (hidden_states), (attentions) + + +@add_start_docstrings("""Bert Model with a `language modeling` head on top. """, + BERT_START_DOCSTRING, BERT_INPUTS_DOCSTRING) +class AlbertForMaskedLM(AlbertPreTrainedModel): + r""" + **masked_lm_labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Labels for computing the masked language modeling loss. + Indices should be in ``[-1, 0, ..., config.vocab_size]`` (see ``input_ids`` docstring) + Tokens with indices set to ``-1`` are ignored (masked), the loss is only computed for the tokens with labels + in ``[0, ..., config.vocab_size]`` + + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **loss**: (`optional`, returned when ``masked_lm_labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: + Masked language modeling loss. + **prediction_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, config.vocab_size)`` + Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') + model = BertForMaskedLM.from_pretrained('bert-base-uncased') + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + outputs = model(input_ids, masked_lm_labels=input_ids) + loss, prediction_scores = outputs[:2] + + """ + + def __init__(self, config): + super(AlbertForMaskedLM, self).__init__(config) + + self.bert = AlbertModel(config) + self.cls = AlbertOnlyMLMHead(config) + + self.init_weights() + self.tie_weights() + + def tie_weights(self): + """ Make sure we are sharing the input and output embeddings. + Export to TorchScript can't handle parameter sharing so we are cloning them instead. + """ + self._tie_or_clone_weights(self.cls.predictions.decoder, + self.bert.embeddings.word_embeddings) + + def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, + masked_lm_labels=None): + outputs = self.bert(input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + position_ids=position_ids, + head_mask=head_mask) + + sequence_output = outputs[0] + prediction_scores = self.cls(sequence_output) + + outputs = (prediction_scores,) + outputs[2:] # Add hidden states and attention if they are here + if masked_lm_labels is not None: + loss_fct = CrossEntropyLoss(ignore_index=-1) + masked_lm_loss = loss_fct(prediction_scores.view(-1, self.config.vocab_size), masked_lm_labels.view(-1)) + outputs = (masked_lm_loss,) + outputs + + return outputs # (masked_lm_loss), prediction_scores, (hidden_states), (attentions) + + +@add_start_docstrings("""Bert Model with a `next sentence prediction (classification)` head on top. """, + BERT_START_DOCSTRING, BERT_INPUTS_DOCSTRING) +class AlbertForNextSentencePrediction(AlbertPreTrainedModel): + r""" + **next_sentence_label**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``: + Labels for computing the next sequence prediction (classification) loss. Input should be a sequence pair (see ``input_ids`` docstring) + Indices should be in ``[0, 1]``. + ``0`` indicates sequence B is a continuation of sequence A, + ``1`` indicates sequence B is a random sequence. + + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **loss**: (`optional`, returned when ``next_sentence_label`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: + Next sequence prediction (classification) loss. + **seq_relationship_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, 2)`` + Prediction scores of the next sequence prediction (classification) head (scores of True/False continuation before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') + model = BertForNextSentencePrediction.from_pretrained('bert-base-uncased') + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + outputs = model(input_ids) + seq_relationship_scores = outputs[0] + + """ + + def __init__(self, config): + super(AlbertForNextSentencePrediction, self).__init__(config) + + self.bert = AlbertModel(config) + self.cls = AlbertOnlyNSPHead(config) + + self.init_weights() + + def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, + next_sentence_label=None): + outputs = self.bert(input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + position_ids=position_ids, + head_mask=head_mask) + + pooled_output = outputs[1] + + seq_relationship_score = self.cls(pooled_output) + + outputs = (seq_relationship_score,) + outputs[2:] # add hidden states and attention if they are here + if next_sentence_label is not None: + loss_fct = CrossEntropyLoss(ignore_index=-1) + next_sentence_loss = loss_fct(seq_relationship_score.view(-1, 2), next_sentence_label.view(-1)) + outputs = (next_sentence_loss,) + outputs + + return outputs # (next_sentence_loss), seq_relationship_score, (hidden_states), (attentions) + + +@add_start_docstrings("""Bert Model transformer with a sequence classification/regression head on top (a linear layer on top of + the pooled output) e.g. for GLUE tasks. """, + BERT_START_DOCSTRING, BERT_INPUTS_DOCSTRING) +class AlbertForSequenceClassification(AlbertPreTrainedModel): + r""" + **labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``: + Labels for computing the sequence classification/regression loss. + Indices should be in ``[0, ..., config.num_labels - 1]``. + If ``config.num_labels == 1`` a regression loss is computed (Mean-Square loss), + If ``config.num_labels > 1`` a classification loss is computed (Cross-Entropy). + + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **loss**: (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: + Classification (or regression if config.num_labels==1) loss. + **logits**: ``torch.FloatTensor`` of shape ``(batch_size, config.num_labels)`` + Classification (or regression if config.num_labels==1) scores (before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') + model = BertForSequenceClassification.from_pretrained('bert-base-uncased') + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + labels = torch.tensor([1]).unsqueeze(0) # Batch size 1 + outputs = model(input_ids, labels=labels) + loss, logits = outputs[:2] + + """ + + def __init__(self, config): + super(AlbertForSequenceClassification, self).__init__(config) + self.num_labels = config.num_labels + + self.bert = AlbertModel(config) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + self.classifier = nn.Linear(config.hidden_size, self.config.num_labels) + + self.init_weights() + + def forward(self, input_ids, attention_mask=None, token_type_ids=None, + position_ids=None, head_mask=None, labels=None): + + outputs = self.bert(input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + position_ids=position_ids, + head_mask=head_mask) + + pooled_output = outputs[1] + + pooled_output = self.dropout(pooled_output) + logits = self.classifier(pooled_output) + + outputs = (logits,) + outputs[2:] # add hidden states and attention if they are here + + if labels is not None: + if self.num_labels == 1: + # We are doing regression + loss_fct = MSELoss() + loss = loss_fct(logits.view(-1), labels.view(-1)) + else: + loss_fct = CrossEntropyLoss() + loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1)) + outputs = (loss,) + outputs + + return outputs # (loss), logits, (hidden_states), (attentions) + + +@add_start_docstrings("""Bert Model with a multiple choice classification head on top (a linear layer on top of + the pooled output and a softmax) e.g. for RocStories/SWAG tasks. """, + BERT_START_DOCSTRING, BERT_INPUTS_DOCSTRING) +class AlbertForMultipleChoice(AlbertPreTrainedModel): + r""" + **labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``: + Labels for computing the multiple choice classification loss. + Indices should be in ``[0, ..., num_choices]`` where `num_choices` is the size of the second dimension + of the input tensors. (see `input_ids` above) + + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **loss**: (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: + Classification loss. + **classification_scores**: ``torch.FloatTensor`` of shape ``(batch_size, num_choices)`` where `num_choices` is the size of the second dimension + of the input tensors. (see `input_ids` above). + Classification scores (before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') + model = BertForMultipleChoice.from_pretrained('bert-base-uncased') + choices = ["Hello, my dog is cute", "Hello, my cat is amazing"] + input_ids = torch.tensor([tokenizer.encode(s) for s in choices]).unsqueeze(0) # Batch size 1, 2 choices + labels = torch.tensor(1).unsqueeze(0) # Batch size 1 + outputs = model(input_ids, labels=labels) + loss, classification_scores = outputs[:2] + + """ + + def __init__(self, config): + super(AlbertForMultipleChoice, self).__init__(config) + + self.bert = AlbertModel(config) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + self.classifier = nn.Linear(config.hidden_size, 1) + + self.init_weights() + + def forward(self, input_ids, attention_mask=None, token_type_ids=None, + position_ids=None, head_mask=None, labels=None): + num_choices = input_ids.shape[1] + + input_ids = input_ids.view(-1, input_ids.size(-1)) + attention_mask = attention_mask.view(-1, attention_mask.size(-1)) if attention_mask is not None else None + token_type_ids = token_type_ids.view(-1, token_type_ids.size(-1)) if token_type_ids is not None else None + position_ids = position_ids.view(-1, position_ids.size(-1)) if position_ids is not None else None + + outputs = self.bert(input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + position_ids=position_ids, + head_mask=head_mask) + + pooled_output = outputs[1] + + pooled_output = self.dropout(pooled_output) + logits = self.classifier(pooled_output) + reshaped_logits = logits.view(-1, num_choices) + + outputs = (reshaped_logits,) + outputs[2:] # add hidden states and attention if they are here + + if labels is not None: + loss_fct = CrossEntropyLoss() + loss = loss_fct(reshaped_logits, labels) + outputs = (loss,) + outputs + + return outputs # (loss), reshaped_logits, (hidden_states), (attentions) + + +@add_start_docstrings("""Bert Model with a token classification head on top (a linear layer on top of + the hidden-states output) e.g. for Named-Entity-Recognition (NER) tasks. """, + BERT_START_DOCSTRING, BERT_INPUTS_DOCSTRING) +class AlbertForTokenClassification(AlbertPreTrainedModel): + r""" + **labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Labels for computing the token classification loss. + Indices should be in ``[0, ..., config.num_labels - 1]``. + + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **loss**: (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: + Classification loss. + **scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, config.num_labels)`` + Classification scores (before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') + model = BertForTokenClassification.from_pretrained('bert-base-uncased') + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + labels = torch.tensor([1] * input_ids.size(1)).unsqueeze(0) # Batch size 1 + outputs = model(input_ids, labels=labels) + loss, scores = outputs[:2] + + """ + + def __init__(self, config): + super(AlbertForTokenClassification, self).__init__(config) + self.num_labels = config.num_labels + + self.bert = AlbertModel(config) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + self.classifier = nn.Linear(config.hidden_size, config.num_labels) + + self.init_weights() + + def forward(self, input_ids, attention_mask=None, token_type_ids=None, + position_ids=None, head_mask=None, labels=None): + + outputs = self.bert(input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + position_ids=position_ids, + head_mask=head_mask) + + sequence_output = outputs[0] + + sequence_output = self.dropout(sequence_output) + logits = self.classifier(sequence_output) + + outputs = (logits,) + outputs[2:] # add hidden states and attention if they are here + if labels is not None: + loss_fct = CrossEntropyLoss() + # Only keep active parts of the loss + if attention_mask is not None: + active_loss = attention_mask.view(-1) == 1 + active_logits = logits.view(-1, self.num_labels)[active_loss] + active_labels = labels.view(-1)[active_loss] + loss = loss_fct(active_logits, active_labels) + else: + loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1)) + outputs = (loss,) + outputs + + return outputs # (loss), scores, (hidden_states), (attentions) + + +@add_start_docstrings("""Bert Model with a span classification head on top for extractive question-answering tasks like SQuAD (a linear layers on top of + the hidden-states output to compute `span start logits` and `span end logits`). """, + BERT_START_DOCSTRING, BERT_INPUTS_DOCSTRING) +class AlbertForQuestionAnswering(AlbertPreTrainedModel): + r""" + **start_positions**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``: + Labels for position (index) of the start of the labelled span for computing the token classification loss. + Positions are clamped to the length of the sequence (`sequence_length`). + Position outside of the sequence are not taken into account for computing the loss. + **end_positions**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``: + Labels for position (index) of the end of the labelled span for computing the token classification loss. + Positions are clamped to the length of the sequence (`sequence_length`). + Position outside of the sequence are not taken into account for computing the loss. + + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **loss**: (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: + Total span extraction loss is the sum of a Cross-Entropy for the start and end positions. + **start_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length,)`` + Span-start scores (before SoftMax). + **end_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length,)`` + Span-end scores (before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') + model = BertForQuestionAnswering.from_pretrained('bert-base-uncased') + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + start_positions = torch.tensor([1]) + end_positions = torch.tensor([3]) + outputs = model(input_ids, start_positions=start_positions, end_positions=end_positions) + loss, start_scores, end_scores = outputs[:2] + + """ + + def __init__(self, config): + super(AlbertForQuestionAnswering, self).__init__(config) + self.num_labels = config.num_labels + + self.bert = AlbertModel(config) + self.qa_outputs = nn.Linear(config.hidden_size, config.num_labels) + + self.init_weights() + + def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, + start_positions=None, end_positions=None): + + outputs = self.bert(input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + position_ids=position_ids, + head_mask=head_mask) + + sequence_output = outputs[0] + + logits = self.qa_outputs(sequence_output) + start_logits, end_logits = logits.split(1, dim=-1) + start_logits = start_logits.squeeze(-1) + end_logits = end_logits.squeeze(-1) + + outputs = (start_logits, end_logits,) + outputs[2:] + if start_positions is not None and end_positions is not None: + # If we are on multi-GPU, split add a dimension + if len(start_positions.size()) > 1: + start_positions = start_positions.squeeze(-1) + if len(end_positions.size()) > 1: + end_positions = end_positions.squeeze(-1) + # sometimes the start/end positions are outside our model inputs, we ignore these terms + ignored_index = start_logits.size(1) + start_positions.clamp_(0, ignored_index) + end_positions.clamp_(0, ignored_index) + + loss_fct = CrossEntropyLoss(ignore_index=ignored_index) + start_loss = loss_fct(start_logits, start_positions) + end_loss = loss_fct(end_logits, end_positions) + total_loss = (start_loss + end_loss) / 2 + outputs = (total_loss,) + outputs + + return outputs # (loss), start_logits, end_logits, (hidden_states), (attentions) diff --git a/BertToSimple/BiGRU/model/modeling_bert.py b/BertToSimple/BiGRU/model/modeling_bert.py new file mode 100644 index 0000000..fecf1e4 --- /dev/null +++ b/BertToSimple/BiGRU/model/modeling_bert.py @@ -0,0 +1,1149 @@ +# coding=utf-8 +# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""PyTorch BERT model. """ + +from __future__ import absolute_import, division, print_function, unicode_literals + +import json +import logging +import math +import os +import sys +from io import open + +import torch +from torch import nn +from torch.nn import CrossEntropyLoss, MSELoss + +from .modeling_utils import PreTrainedModel, prune_linear_layer +from .configuration_bert import BertConfig +from .file_utils import add_start_docstrings + +logger = logging.getLogger(__name__) + +BERT_PRETRAINED_MODEL_ARCHIVE_MAP = { + 'bert-base-uncased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-pytorch_model.bin", + 'bert-large-uncased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-uncased-pytorch_model.bin", + 'bert-base-cased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-cased-pytorch_model.bin", + 'bert-large-cased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-cased-pytorch_model.bin", + 'bert-base-multilingual-uncased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-multilingual-uncased-pytorch_model.bin", + 'bert-base-multilingual-cased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-multilingual-cased-pytorch_model.bin", + 'bert-base-chinese': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-chinese-pytorch_model.bin", + 'bert-base-german-cased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-german-cased-pytorch_model.bin", + 'bert-large-uncased-whole-word-masking': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-uncased-whole-word-masking-pytorch_model.bin", + 'bert-large-cased-whole-word-masking': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-cased-whole-word-masking-pytorch_model.bin", + 'bert-large-uncased-whole-word-masking-finetuned-squad': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-uncased-whole-word-masking-finetuned-squad-pytorch_model.bin", + 'bert-large-cased-whole-word-masking-finetuned-squad': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-cased-whole-word-masking-finetuned-squad-pytorch_model.bin", + 'bert-base-cased-finetuned-mrpc': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-cased-finetuned-mrpc-pytorch_model.bin", + 'bert-base-german-dbmdz-cased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-german-dbmdz-cased-pytorch_model.bin", + 'bert-base-german-dbmdz-uncased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-german-dbmdz-uncased-pytorch_model.bin", +} + +def load_tf_weights_in_bert(model, config, tf_checkpoint_path): + """ Load tf checkpoints in a pytorch model. + """ + try: + import re + import numpy as np + import tensorflow as tf + except ImportError: + logger.error("Loading a TensorFlow model in PyTorch, requires TensorFlow to be installed. Please see " + "https://www.tensorflow.org/install/ for installation instructions.") + raise + tf_path = os.path.abspath(tf_checkpoint_path) + logger.info("Converting TensorFlow checkpoint from {}".format(tf_path)) + # Load weights from TF model + init_vars = tf.train.list_variables(tf_path) + names = [] + arrays = [] + for name, shape in init_vars: + logger.info("Loading TF weight {} with shape {}".format(name, shape)) + array = tf.train.load_variable(tf_path, name) + names.append(name) + arrays.append(array) + + for name, array in zip(names, arrays): + name = name.split('/') + # adam_v and adam_m are variables used in AdamWeightDecayOptimizer to calculated m and v + # which are not required for using pretrained model + if any(n in ["adam_v", "adam_m", "global_step"] for n in name): + logger.info("Skipping {}".format("/".join(name))) + continue + pointer = model + for m_name in name: + if re.fullmatch(r'[A-Za-z]+_\d+', m_name): + l = re.split(r'_(\d+)', m_name) + else: + l = [m_name] + if l[0] == 'kernel' or l[0] == 'gamma': + pointer = getattr(pointer, 'weight') + elif l[0] == 'output_bias' or l[0] == 'beta': + pointer = getattr(pointer, 'bias') + elif l[0] == 'output_weights': + pointer = getattr(pointer, 'weight') + elif l[0] == 'squad': + pointer = getattr(pointer, 'classifier') + else: + try: + pointer = getattr(pointer, l[0]) + except AttributeError: + logger.info("Skipping {}".format("/".join(name))) + continue + if len(l) >= 2: + num = int(l[1]) + pointer = pointer[num] + if m_name[-11:] == '_embeddings': + pointer = getattr(pointer, 'weight') + elif m_name == 'kernel': + array = np.transpose(array) + try: + assert pointer.shape == array.shape + except AssertionError as e: + e.args += (pointer.shape, array.shape) + raise + logger.info("Initialize PyTorch weight {}".format(name)) + pointer.data = torch.from_numpy(array) + return model + + +def gelu(x): + """ Original Implementation of the gelu activation function in Google Bert repo when initially created. + For information: OpenAI GPT's gelu is slightly different (and gives slightly different results): + 0.5 * x * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3)))) + Also see https://arxiv.org/abs/1606.08415 + """ + return x * 0.5 * (1.0 + torch.erf(x / math.sqrt(2.0))) + +def gelu_new(x): + """ Implementation of the gelu activation function currently in Google Bert repo (identical to OpenAI GPT). + Also see https://arxiv.org/abs/1606.08415 + """ + return 0.5 * x * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3)))) + +def swish(x): + return x * torch.sigmoid(x) + + +ACT2FN = {"gelu": gelu, "relu": torch.nn.functional.relu, "swish": swish, "gelu_new": gelu_new} + + +BertLayerNorm = torch.nn.LayerNorm + +class BertEmbeddings(nn.Module): + """Construct the embeddings from word, position and token_type embeddings. + """ + def __init__(self, config): + super(BertEmbeddings, self).__init__() + self.word_embeddings = nn.Embedding(config.vocab_size, config.hidden_size, padding_idx=0) + self.position_embeddings = nn.Embedding(config.max_position_embeddings, config.hidden_size) + self.token_type_embeddings = nn.Embedding(config.type_vocab_size, config.hidden_size) + + # self.LayerNorm is not snake-cased to stick with TensorFlow model variable name and be able to load + # any TensorFlow checkpoint file + self.LayerNorm = BertLayerNorm(config.hidden_size, eps=config.layer_norm_eps) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + + def forward(self, input_ids, token_type_ids=None, position_ids=None): + seq_length = input_ids.size(1) + if position_ids is None: + position_ids = torch.arange(seq_length, dtype=torch.long, device=input_ids.device) + position_ids = position_ids.unsqueeze(0).expand_as(input_ids) + if token_type_ids is None: + token_type_ids = torch.zeros_like(input_ids) + + words_embeddings = self.word_embeddings(input_ids) + position_embeddings = self.position_embeddings(position_ids) + token_type_embeddings = self.token_type_embeddings(token_type_ids) + + embeddings = words_embeddings + position_embeddings + token_type_embeddings + embeddings = self.LayerNorm(embeddings) + embeddings = self.dropout(embeddings) + return embeddings + + +class BertSelfAttention(nn.Module): + def __init__(self, config): + super(BertSelfAttention, self).__init__() + if config.hidden_size % config.num_attention_heads != 0: + raise ValueError( + "The hidden size (%d) is not a multiple of the number of attention " + "heads (%d)" % (config.hidden_size, config.num_attention_heads)) + self.output_attentions = config.output_attentions + + self.num_attention_heads = config.num_attention_heads + self.attention_head_size = int(config.hidden_size / config.num_attention_heads) + self.all_head_size = self.num_attention_heads * self.attention_head_size + + self.query = nn.Linear(config.hidden_size, self.all_head_size) + self.key = nn.Linear(config.hidden_size, self.all_head_size) + self.value = nn.Linear(config.hidden_size, self.all_head_size) + + self.dropout = nn.Dropout(config.attention_probs_dropout_prob) + + def transpose_for_scores(self, x): + new_x_shape = x.size()[:-1] + (self.num_attention_heads, self.attention_head_size) + x = x.view(*new_x_shape) + return x.permute(0, 2, 1, 3) + + def forward(self, hidden_states, attention_mask=None, head_mask=None): + mixed_query_layer = self.query(hidden_states) + mixed_key_layer = self.key(hidden_states) + mixed_value_layer = self.value(hidden_states) + + query_layer = self.transpose_for_scores(mixed_query_layer) + key_layer = self.transpose_for_scores(mixed_key_layer) + value_layer = self.transpose_for_scores(mixed_value_layer) + + # Take the dot product between "query" and "key" to get the raw attention scores. + attention_scores = torch.matmul(query_layer, key_layer.transpose(-1, -2)) + attention_scores = attention_scores / math.sqrt(self.attention_head_size) + if attention_mask is not None: + # Apply the attention mask is (precomputed for all layers in BertModel forward() function) + attention_scores = attention_scores + attention_mask + + # Normalize the attention scores to probabilities. + attention_probs = nn.Softmax(dim=-1)(attention_scores) + + # This is actually dropping out entire tokens to attend to, which might + # seem a bit unusual, but is taken from the original Transformer paper. + attention_probs = self.dropout(attention_probs) + + # Mask heads if we want to + if head_mask is not None: + attention_probs = attention_probs * head_mask + + context_layer = torch.matmul(attention_probs, value_layer) + + context_layer = context_layer.permute(0, 2, 1, 3).contiguous() + new_context_layer_shape = context_layer.size()[:-2] + (self.all_head_size,) + context_layer = context_layer.view(*new_context_layer_shape) + + outputs = (context_layer, attention_probs) if self.output_attentions else (context_layer,) + return outputs + + +class BertSelfOutput(nn.Module): + def __init__(self, config): + super(BertSelfOutput, self).__init__() + self.dense = nn.Linear(config.hidden_size, config.hidden_size) + self.LayerNorm = BertLayerNorm(config.hidden_size, eps=config.layer_norm_eps) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + + def forward(self, hidden_states, input_tensor): + hidden_states = self.dense(hidden_states) + hidden_states = self.dropout(hidden_states) + hidden_states = self.LayerNorm(hidden_states + input_tensor) + return hidden_states + + +class BertAttention(nn.Module): + def __init__(self, config): + super(BertAttention, self).__init__() + self.self = BertSelfAttention(config) + self.output = BertSelfOutput(config) + self.pruned_heads = set() + + def prune_heads(self, heads): + if len(heads) == 0: + return + mask = torch.ones(self.self.num_attention_heads, self.self.attention_head_size) + heads = set(heads) - self.pruned_heads # Convert to set and emove already pruned heads + for head in heads: + # Compute how many pruned heads are before the head and move the index accordingly + head = head - sum(1 if h < head else 0 for h in self.pruned_heads) + mask[head] = 0 + mask = mask.view(-1).contiguous().eq(1) + index = torch.arange(len(mask))[mask].long() + + # Prune linear layers + self.self.query = prune_linear_layer(self.self.query, index) + self.self.key = prune_linear_layer(self.self.key, index) + self.self.value = prune_linear_layer(self.self.value, index) + self.output.dense = prune_linear_layer(self.output.dense, index, dim=1) + + # Update hyper params and store pruned heads + self.self.num_attention_heads = self.self.num_attention_heads - len(heads) + self.self.all_head_size = self.self.attention_head_size * self.self.num_attention_heads + self.pruned_heads = self.pruned_heads.union(heads) + + def forward(self, input_tensor, attention_mask=None, head_mask=None): + self_outputs = self.self(input_tensor, attention_mask, head_mask) + attention_output = self.output(self_outputs[0], input_tensor) + outputs = (attention_output,) + self_outputs[1:] # add attentions if we output them + return outputs + + +class BertIntermediate(nn.Module): + def __init__(self, config): + super(BertIntermediate, self).__init__() + self.dense = nn.Linear(config.hidden_size, config.intermediate_size) + if isinstance(config.hidden_act, str) or (sys.version_info[0] == 2 and isinstance(config.hidden_act, unicode)): + self.intermediate_act_fn = ACT2FN[config.hidden_act] + else: + self.intermediate_act_fn = config.hidden_act + + def forward(self, hidden_states): + hidden_states = self.dense(hidden_states) + hidden_states = self.intermediate_act_fn(hidden_states) + return hidden_states + + +class BertOutput(nn.Module): + def __init__(self, config): + super(BertOutput, self).__init__() + self.dense = nn.Linear(config.intermediate_size, config.hidden_size) + self.LayerNorm = BertLayerNorm(config.hidden_size, eps=config.layer_norm_eps) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + + def forward(self, hidden_states, input_tensor): + hidden_states = self.dense(hidden_states) + hidden_states = self.dropout(hidden_states) + hidden_states = self.LayerNorm(hidden_states + input_tensor) + return hidden_states + + +class BertLayer(nn.Module): + def __init__(self, config): + super(BertLayer, self).__init__() + self.attention = BertAttention(config) + self.intermediate = BertIntermediate(config) + self.output = BertOutput(config) + + def forward(self, hidden_states, attention_mask=None, head_mask=None): + attention_outputs = self.attention(hidden_states, attention_mask, head_mask) + attention_output = attention_outputs[0] + intermediate_output = self.intermediate(attention_output) + layer_output = self.output(intermediate_output, attention_output) + outputs = (layer_output,) + attention_outputs[1:] # add attentions if we output them + return outputs + + +class BertEncoder(nn.Module): + def __init__(self, config): + super(BertEncoder, self).__init__() + self.output_attentions = config.output_attentions + self.output_hidden_states = config.output_hidden_states + self.layer = nn.ModuleList([BertLayer(config) for _ in range(config.num_hidden_layers)]) + + def forward(self, hidden_states, attention_mask=None, head_mask=None): + all_hidden_states = () + all_attentions = () + for i, layer_module in enumerate(self.layer): + if self.output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states,) + + layer_outputs = layer_module(hidden_states, attention_mask, head_mask[i]) + hidden_states = layer_outputs[0] + + if self.output_attentions: + all_attentions = all_attentions + (layer_outputs[1],) + + # Add last layer + if self.output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states,) + + outputs = (hidden_states,) + if self.output_hidden_states: + outputs = outputs + (all_hidden_states,) + if self.output_attentions: + outputs = outputs + (all_attentions,) + return outputs # last-layer hidden state, (all hidden states), (all attentions) + + +class BertPooler(nn.Module): + def __init__(self, config): + super(BertPooler, self).__init__() + self.dense = nn.Linear(config.hidden_size, config.hidden_size) + self.activation = nn.Tanh() + + def forward(self, hidden_states): + # We "pool" the model by simply taking the hidden state corresponding + # to the first token. + first_token_tensor = hidden_states[:, 0] + pooled_output = self.dense(first_token_tensor) + pooled_output = self.activation(pooled_output) + return pooled_output + + +class BertPredictionHeadTransform(nn.Module): + def __init__(self, config): + super(BertPredictionHeadTransform, self).__init__() + self.dense = nn.Linear(config.hidden_size, config.hidden_size) + if isinstance(config.hidden_act, str) or (sys.version_info[0] == 2 and isinstance(config.hidden_act, unicode)): + self.transform_act_fn = ACT2FN[config.hidden_act] + else: + self.transform_act_fn = config.hidden_act + self.LayerNorm = BertLayerNorm(config.hidden_size, eps=config.layer_norm_eps) + + def forward(self, hidden_states): + hidden_states = self.dense(hidden_states) + hidden_states = self.transform_act_fn(hidden_states) + hidden_states = self.LayerNorm(hidden_states) + return hidden_states + + +class BertLMPredictionHead(nn.Module): + def __init__(self, config): + super(BertLMPredictionHead, self).__init__() + self.transform = BertPredictionHeadTransform(config) + + # The output weights are the same as the input embeddings, but there is + # an output-only bias for each token. + self.decoder = nn.Linear(config.hidden_size, + config.vocab_size, + bias=False) + + self.bias = nn.Parameter(torch.zeros(config.vocab_size)) + + def forward(self, hidden_states): + hidden_states = self.transform(hidden_states) + hidden_states = self.decoder(hidden_states) + self.bias + return hidden_states + + +class BertOnlyMLMHead(nn.Module): + def __init__(self, config): + super(BertOnlyMLMHead, self).__init__() + self.predictions = BertLMPredictionHead(config) + + def forward(self, sequence_output): + prediction_scores = self.predictions(sequence_output) + return prediction_scores + + +class BertOnlyNSPHead(nn.Module): + def __init__(self, config): + super(BertOnlyNSPHead, self).__init__() + self.seq_relationship = nn.Linear(config.hidden_size, 2) + + def forward(self, pooled_output): + seq_relationship_score = self.seq_relationship(pooled_output) + return seq_relationship_score + + +class BertPreTrainingHeads(nn.Module): + def __init__(self, config): + super(BertPreTrainingHeads, self).__init__() + self.predictions = BertLMPredictionHead(config) + self.seq_relationship = nn.Linear(config.hidden_size, 2) + + def forward(self, sequence_output, pooled_output): + prediction_scores = self.predictions(sequence_output) + seq_relationship_score = self.seq_relationship(pooled_output) + return prediction_scores, seq_relationship_score + + +class BertPreTrainedModel(PreTrainedModel): + """ An abstract class to handle weights initialization and + a simple interface for dowloading and loading pretrained models. + """ + config_class = BertConfig + pretrained_model_archive_map = BERT_PRETRAINED_MODEL_ARCHIVE_MAP + load_tf_weights = load_tf_weights_in_bert + base_model_prefix = "bert" + + def _init_weights(self, module): + """ Initialize the weights """ + if isinstance(module, (nn.Linear, nn.Embedding)): + # Slightly different from the TF version which uses truncated_normal for initialization + # cf https://github.com/pytorch/pytorch/pull/5617 + module.weight.data.normal_(mean=0.0, std=self.config.initializer_range) + elif isinstance(module, BertLayerNorm): + module.bias.data.zero_() + module.weight.data.fill_(1.0) + if isinstance(module, nn.Linear) and module.bias is not None: + module.bias.data.zero_() + + +BERT_START_DOCSTRING = r""" The BERT model was proposed in + `BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding`_ + by Jacob Devlin, Ming-Wei Chang, Kenton Lee and Kristina Toutanova. It's a bidirectional transformer + pre-trained using a combination of masked language modeling objective and next sentence prediction + on a large corpus comprising the Toronto Book Corpus and Wikipedia. + + This model is a PyTorch `torch.nn.Module`_ sub-class. Use it as a regular PyTorch Module and + refer to the PyTorch documentation for all matter related to general usage and behavior. + + .. _`BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding`: + https://arxiv.org/abs/1810.04805 + + .. _`torch.nn.Module`: + https://pytorch.org/docs/stable/nn.html#module + + Parameters: + config (:class:`~transformers.BertConfig`): Model configuration class with all the parameters of the model. + Initializing with a config file does not load the weights associated with the model, only the configuration. + Check out the :meth:`~transformers.PreTrainedModel.from_pretrained` method to load the model weights. +""" + +BERT_INPUTS_DOCSTRING = r""" + Inputs: + **input_ids**: ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Indices of input sequence tokens in the vocabulary. + To match pre-training, BERT input sequence should be formatted with [CLS] and [SEP] tokens as follows: + + (a) For sequence pairs: + + ``tokens: [CLS] is this jack ##son ##ville ? [SEP] no it is not . [SEP]`` + + ``token_type_ids: 0 0 0 0 0 0 0 0 1 1 1 1 1 1`` + + (b) For single sequences: + + ``tokens: [CLS] the dog is hairy . [SEP]`` + + ``token_type_ids: 0 0 0 0 0 0 0`` + + Bert is a model with absolute position embeddings so it's usually advised to pad the inputs on + the right rather than the left. + + Indices can be obtained using :class:`transformers.BertTokenizer`. + See :func:`transformers.PreTrainedTokenizer.encode` and + :func:`transformers.PreTrainedTokenizer.convert_tokens_to_ids` for details. + **attention_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, sequence_length)``: + Mask to avoid performing attention on padding token indices. + Mask values selected in ``[0, 1]``: + ``1`` for tokens that are NOT MASKED, ``0`` for MASKED tokens. + **token_type_ids**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Segment token indices to indicate first and second portions of the inputs. + Indices are selected in ``[0, 1]``: ``0`` corresponds to a `sentence A` token, ``1`` + corresponds to a `sentence B` token + (see `BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding`_ for more details). + **position_ids**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Indices of positions of each input sequence tokens in the position embeddings. + Selected in the range ``[0, config.max_position_embeddings - 1]``. + **head_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(num_heads,)`` or ``(num_layers, num_heads)``: + Mask to nullify selected heads of the self-attention modules. + Mask values selected in ``[0, 1]``: + ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**. +""" + +@add_start_docstrings("The bare Bert Model transformer outputting raw hidden-states without any specific head on top.", + BERT_START_DOCSTRING, BERT_INPUTS_DOCSTRING) +class BertModel(BertPreTrainedModel): + r""" + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **last_hidden_state**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, hidden_size)`` + Sequence of hidden-states at the output of the last layer of the model. + **pooler_output**: ``torch.FloatTensor`` of shape ``(batch_size, hidden_size)`` + Last layer hidden-state of the first token of the sequence (classification token) + further processed by a Linear layer and a Tanh activation function. The Linear + layer weights are trained from the next sentence prediction (classification) + objective during Bert pretraining. This output is usually *not* a good summary + of the semantic content of the input, you're often better with averaging or pooling + the sequence of hidden-states for the whole input sequence. + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') + model = BertModel.from_pretrained('bert-base-uncased') + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + outputs = model(input_ids) + last_hidden_states = outputs[0] # The last hidden-state is the first element of the output tuple + + """ + def __init__(self, config): + super(BertModel, self).__init__(config) + + self.embeddings = BertEmbeddings(config) + self.encoder = BertEncoder(config) + self.pooler = BertPooler(config) + + self.init_weights() + + def _resize_token_embeddings(self, new_num_tokens): + old_embeddings = self.embeddings.word_embeddings + new_embeddings = self._get_resized_embeddings(old_embeddings, new_num_tokens) + self.embeddings.word_embeddings = new_embeddings + return self.embeddings.word_embeddings + + def _prune_heads(self, heads_to_prune): + """ Prunes heads of the model. + heads_to_prune: dict of {layer_num: list of heads to prune in this layer} + See base class PreTrainedModel + """ + for layer, heads in heads_to_prune.items(): + self.encoder.layer[layer].attention.prune_heads(heads) + + def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None): + if attention_mask is None: + attention_mask = torch.ones_like(input_ids) + if token_type_ids is None: + token_type_ids = torch.zeros_like(input_ids) + + # We create a 3D attention mask from a 2D tensor mask. + # Sizes are [batch_size, 1, 1, to_seq_length] + # So we can broadcast to [batch_size, num_heads, from_seq_length, to_seq_length] + # this attention mask is more simple than the triangular masking of causal attention + # used in OpenAI GPT, we just need to prepare the broadcast dimension here. + extended_attention_mask = attention_mask.unsqueeze(1).unsqueeze(2) + + # Since attention_mask is 1.0 for positions we want to attend and 0.0 for + # masked positions, this operation will create a tensor which is 0.0 for + # positions we want to attend and -10000.0 for masked positions. + # Since we are adding it to the raw scores before the softmax, this is + # effectively the same as removing these entirely. + extended_attention_mask = extended_attention_mask.to(dtype=next(self.parameters()).dtype) # fp16 compatibility + extended_attention_mask = (1.0 - extended_attention_mask) * -10000.0 + + # Prepare head mask if needed + # 1.0 in head_mask indicate we keep the head + # attention_probs has shape bsz x n_heads x N x N + # input head_mask has shape [num_heads] or [num_hidden_layers x num_heads] + # and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length] + if head_mask is not None: + if head_mask.dim() == 1: + head_mask = head_mask.unsqueeze(0).unsqueeze(0).unsqueeze(-1).unsqueeze(-1) + head_mask = head_mask.expand(self.config.num_hidden_layers, -1, -1, -1, -1) + elif head_mask.dim() == 2: + head_mask = head_mask.unsqueeze(1).unsqueeze(-1).unsqueeze(-1) # We can specify head_mask for each layer + head_mask = head_mask.to(dtype=next(self.parameters()).dtype) # switch to fload if need + fp16 compatibility + else: + head_mask = [None] * self.config.num_hidden_layers + + embedding_output = self.embeddings(input_ids, position_ids=position_ids, token_type_ids=token_type_ids) + encoder_outputs = self.encoder(embedding_output, + extended_attention_mask, + head_mask=head_mask) + sequence_output = encoder_outputs[0] + pooled_output = self.pooler(sequence_output) + + outputs = (sequence_output, pooled_output,) + encoder_outputs[1:] # add hidden_states and attentions if they are here + return outputs # sequence_output, pooled_output, (hidden_states), (attentions) + + +@add_start_docstrings("""Bert Model with two heads on top as done during the pre-training: + a `masked language modeling` head and a `next sentence prediction (classification)` head. """, + BERT_START_DOCSTRING, BERT_INPUTS_DOCSTRING) +class BertForPreTraining(BertPreTrainedModel): + r""" + **masked_lm_labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Labels for computing the masked language modeling loss. + Indices should be in ``[-1, 0, ..., config.vocab_size]`` (see ``input_ids`` docstring) + Tokens with indices set to ``-1`` are ignored (masked), the loss is only computed for the tokens with labels + in ``[0, ..., config.vocab_size]`` + **next_sentence_label**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``: + Labels for computing the next sequence prediction (classification) loss. Input should be a sequence pair (see ``input_ids`` docstring) + Indices should be in ``[0, 1]``. + ``0`` indicates sequence B is a continuation of sequence A, + ``1`` indicates sequence B is a random sequence. + + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **loss**: (`optional`, returned when both ``masked_lm_labels`` and ``next_sentence_label`` are provided) ``torch.FloatTensor`` of shape ``(1,)``: + Total loss as the sum of the masked language modeling loss and the next sequence prediction (classification) loss. + **prediction_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, config.vocab_size)`` + Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). + **seq_relationship_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, 2)`` + Prediction scores of the next sequence prediction (classification) head (scores of True/False continuation before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') + model = BertForPreTraining.from_pretrained('bert-base-uncased') + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + outputs = model(input_ids) + prediction_scores, seq_relationship_scores = outputs[:2] + + """ + def __init__(self, config): + super(BertForPreTraining, self).__init__(config) + + self.bert = BertModel(config) + self.cls = BertPreTrainingHeads(config) + + self.init_weights() + self.tie_weights() + + def tie_weights(self): + """ Make sure we are sharing the input and output embeddings. + Export to TorchScript can't handle parameter sharing so we are cloning them instead. + """ + self._tie_or_clone_weights(self.cls.predictions.decoder, + self.bert.embeddings.word_embeddings) + + def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, + masked_lm_labels=None, next_sentence_label=None): + + outputs = self.bert(input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + position_ids=position_ids, + head_mask=head_mask) + + sequence_output, pooled_output = outputs[:2] + prediction_scores, seq_relationship_score = self.cls(sequence_output, pooled_output) + + outputs = (prediction_scores, seq_relationship_score,) + outputs[2:] # add hidden states and attention if they are here + + if masked_lm_labels is not None and next_sentence_label is not None: + loss_fct = CrossEntropyLoss(ignore_index=-1) + masked_lm_loss = loss_fct(prediction_scores.view(-1, self.config.vocab_size), masked_lm_labels.view(-1)) + next_sentence_loss = loss_fct(seq_relationship_score.view(-1, 2), next_sentence_label.view(-1)) + total_loss = masked_lm_loss + next_sentence_loss + outputs = (total_loss,) + outputs + + return outputs # (loss), prediction_scores, seq_relationship_score, (hidden_states), (attentions) + + +@add_start_docstrings("""Bert Model with a `language modeling` head on top. """, + BERT_START_DOCSTRING, BERT_INPUTS_DOCSTRING) +class BertForMaskedLM(BertPreTrainedModel): + r""" + **masked_lm_labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Labels for computing the masked language modeling loss. + Indices should be in ``[-1, 0, ..., config.vocab_size]`` (see ``input_ids`` docstring) + Tokens with indices set to ``-1`` are ignored (masked), the loss is only computed for the tokens with labels + in ``[0, ..., config.vocab_size]`` + + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **loss**: (`optional`, returned when ``masked_lm_labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: + Masked language modeling loss. + **prediction_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, config.vocab_size)`` + Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') + model = BertForMaskedLM.from_pretrained('bert-base-uncased') + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + outputs = model(input_ids, masked_lm_labels=input_ids) + loss, prediction_scores = outputs[:2] + + """ + def __init__(self, config): + super(BertForMaskedLM, self).__init__(config) + + self.bert = BertModel(config) + self.cls = BertOnlyMLMHead(config) + + self.init_weights() + self.tie_weights() + + def tie_weights(self): + """ Make sure we are sharing the input and output embeddings. + Export to TorchScript can't handle parameter sharing so we are cloning them instead. + """ + self._tie_or_clone_weights(self.cls.predictions.decoder, + self.bert.embeddings.word_embeddings) + + def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, + masked_lm_labels=None): + + outputs = self.bert(input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + position_ids=position_ids, + head_mask=head_mask) + + sequence_output = outputs[0] + prediction_scores = self.cls(sequence_output) + + outputs = (prediction_scores,) + outputs[2:] # Add hidden states and attention if they are here + if masked_lm_labels is not None: + loss_fct = CrossEntropyLoss(ignore_index=-1) + masked_lm_loss = loss_fct(prediction_scores.view(-1, self.config.vocab_size), masked_lm_labels.view(-1)) + outputs = (masked_lm_loss,) + outputs + + return outputs # (masked_lm_loss), prediction_scores, (hidden_states), (attentions) + + +@add_start_docstrings("""Bert Model with a `next sentence prediction (classification)` head on top. """, + BERT_START_DOCSTRING, BERT_INPUTS_DOCSTRING) +class BertForNextSentencePrediction(BertPreTrainedModel): + r""" + **next_sentence_label**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``: + Labels for computing the next sequence prediction (classification) loss. Input should be a sequence pair (see ``input_ids`` docstring) + Indices should be in ``[0, 1]``. + ``0`` indicates sequence B is a continuation of sequence A, + ``1`` indicates sequence B is a random sequence. + + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **loss**: (`optional`, returned when ``next_sentence_label`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: + Next sequence prediction (classification) loss. + **seq_relationship_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, 2)`` + Prediction scores of the next sequence prediction (classification) head (scores of True/False continuation before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') + model = BertForNextSentencePrediction.from_pretrained('bert-base-uncased') + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + outputs = model(input_ids) + seq_relationship_scores = outputs[0] + + """ + def __init__(self, config): + super(BertForNextSentencePrediction, self).__init__(config) + + self.bert = BertModel(config) + self.cls = BertOnlyNSPHead(config) + + self.init_weights() + + def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, + next_sentence_label=None): + + outputs = self.bert(input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + position_ids=position_ids, + head_mask=head_mask) + + pooled_output = outputs[1] + + seq_relationship_score = self.cls(pooled_output) + + outputs = (seq_relationship_score,) + outputs[2:] # add hidden states and attention if they are here + if next_sentence_label is not None: + loss_fct = CrossEntropyLoss(ignore_index=-1) + next_sentence_loss = loss_fct(seq_relationship_score.view(-1, 2), next_sentence_label.view(-1)) + outputs = (next_sentence_loss,) + outputs + + return outputs # (next_sentence_loss), seq_relationship_score, (hidden_states), (attentions) + + +@add_start_docstrings("""Bert Model transformer with a sequence classification/regression head on top (a linear layer on top of + the pooled output) e.g. for GLUE tasks. """, + BERT_START_DOCSTRING, BERT_INPUTS_DOCSTRING) +class BertForSequenceClassification(BertPreTrainedModel): + r""" + **labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``: + Labels for computing the sequence classification/regression loss. + Indices should be in ``[0, ..., config.num_labels - 1]``. + If ``config.num_labels == 1`` a regression loss is computed (Mean-Square loss), + If ``config.num_labels > 1`` a classification loss is computed (Cross-Entropy). + + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **loss**: (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: + Classification (or regression if config.num_labels==1) loss. + **logits**: ``torch.FloatTensor`` of shape ``(batch_size, config.num_labels)`` + Classification (or regression if config.num_labels==1) scores (before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') + model = BertForSequenceClassification.from_pretrained('bert-base-uncased') + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + labels = torch.tensor([1]).unsqueeze(0) # Batch size 1 + outputs = model(input_ids, labels=labels) + loss, logits = outputs[:2] + + """ + def __init__(self, config): + super(BertForSequenceClassification, self).__init__(config) + self.num_labels = config.num_labels + + self.bert = BertModel(config) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + self.classifier = nn.Linear(config.hidden_size, self.config.num_labels) + + self.init_weights() + + def forward(self, input_ids, attention_mask=None, token_type_ids=None, + position_ids=None, head_mask=None, labels=None): + + outputs = self.bert(input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + position_ids=position_ids, + head_mask=head_mask) + + pooled_output = outputs[1] + + pooled_output = self.dropout(pooled_output) + logits = self.classifier(pooled_output) + + outputs = (logits,) + outputs[2:] # add hidden states and attention if they are here + + if labels is not None: + if self.num_labels == 1: + # We are doing regression + loss_fct = MSELoss() + loss = loss_fct(logits.view(-1), labels.view(-1)) + else: + loss_fct = CrossEntropyLoss() + loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1)) + outputs = (loss,) + outputs + + return outputs # (loss), logits, (hidden_states), (attentions) + + +@add_start_docstrings("""Bert Model with a multiple choice classification head on top (a linear layer on top of + the pooled output and a softmax) e.g. for RocStories/SWAG tasks. """, + BERT_START_DOCSTRING, BERT_INPUTS_DOCSTRING) +class BertForMultipleChoice(BertPreTrainedModel): + r""" + **labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``: + Labels for computing the multiple choice classification loss. + Indices should be in ``[0, ..., num_choices]`` where `num_choices` is the size of the second dimension + of the input tensors. (see `input_ids` above) + + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **loss**: (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: + Classification loss. + **classification_scores**: ``torch.FloatTensor`` of shape ``(batch_size, num_choices)`` where `num_choices` is the size of the second dimension + of the input tensors. (see `input_ids` above). + Classification scores (before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') + model = BertForMultipleChoice.from_pretrained('bert-base-uncased') + choices = ["Hello, my dog is cute", "Hello, my cat is amazing"] + input_ids = torch.tensor([tokenizer.encode(s) for s in choices]).unsqueeze(0) # Batch size 1, 2 choices + labels = torch.tensor(1).unsqueeze(0) # Batch size 1 + outputs = model(input_ids, labels=labels) + loss, classification_scores = outputs[:2] + + """ + def __init__(self, config): + super(BertForMultipleChoice, self).__init__(config) + + self.bert = BertModel(config) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + self.classifier = nn.Linear(config.hidden_size, 1) + + self.init_weights() + + def forward(self, input_ids, attention_mask=None, token_type_ids=None, + position_ids=None, head_mask=None, labels=None): + num_choices = input_ids.shape[1] + + input_ids = input_ids.view(-1, input_ids.size(-1)) + attention_mask = attention_mask.view(-1, attention_mask.size(-1)) if attention_mask is not None else None + token_type_ids = token_type_ids.view(-1, token_type_ids.size(-1)) if token_type_ids is not None else None + position_ids = position_ids.view(-1, position_ids.size(-1)) if position_ids is not None else None + + outputs = self.bert(input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + position_ids=position_ids, + head_mask=head_mask) + + pooled_output = outputs[1] + + pooled_output = self.dropout(pooled_output) + logits = self.classifier(pooled_output) + reshaped_logits = logits.view(-1, num_choices) + + outputs = (reshaped_logits,) + outputs[2:] # add hidden states and attention if they are here + + if labels is not None: + loss_fct = CrossEntropyLoss() + loss = loss_fct(reshaped_logits, labels) + outputs = (loss,) + outputs + + return outputs # (loss), reshaped_logits, (hidden_states), (attentions) + + +@add_start_docstrings("""Bert Model with a token classification head on top (a linear layer on top of + the hidden-states output) e.g. for Named-Entity-Recognition (NER) tasks. """, + BERT_START_DOCSTRING, BERT_INPUTS_DOCSTRING) +class BertForTokenClassification(BertPreTrainedModel): + r""" + **labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Labels for computing the token classification loss. + Indices should be in ``[0, ..., config.num_labels - 1]``. + + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **loss**: (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: + Classification loss. + **scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, config.num_labels)`` + Classification scores (before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') + model = BertForTokenClassification.from_pretrained('bert-base-uncased') + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + labels = torch.tensor([1] * input_ids.size(1)).unsqueeze(0) # Batch size 1 + outputs = model(input_ids, labels=labels) + loss, scores = outputs[:2] + + """ + def __init__(self, config): + super(BertForTokenClassification, self).__init__(config) + self.num_labels = config.num_labels + + self.bert = BertModel(config) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + self.classifier = nn.Linear(config.hidden_size, config.num_labels) + + self.init_weights() + + def forward(self, input_ids, attention_mask=None, token_type_ids=None, + position_ids=None, head_mask=None, labels=None): + + outputs = self.bert(input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + position_ids=position_ids, + head_mask=head_mask) + + sequence_output = outputs[0] + + sequence_output = self.dropout(sequence_output) + logits = self.classifier(sequence_output) + + outputs = (logits,) + outputs[2:] # add hidden states and attention if they are here + if labels is not None: + loss_fct = CrossEntropyLoss() + # Only keep active parts of the loss + if attention_mask is not None: + active_loss = attention_mask.view(-1) == 1 + active_logits = logits.view(-1, self.num_labels)[active_loss] + active_labels = labels.view(-1)[active_loss] + loss = loss_fct(active_logits, active_labels) + else: + loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1)) + outputs = (loss,) + outputs + + return outputs # (loss), scores, (hidden_states), (attentions) + + +@add_start_docstrings("""Bert Model with a span classification head on top for extractive question-answering tasks like SQuAD (a linear layers on top of + the hidden-states output to compute `span start logits` and `span end logits`). """, + BERT_START_DOCSTRING, BERT_INPUTS_DOCSTRING) +class BertForQuestionAnswering(BertPreTrainedModel): + r""" + **start_positions**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``: + Labels for position (index) of the start of the labelled span for computing the token classification loss. + Positions are clamped to the length of the sequence (`sequence_length`). + Position outside of the sequence are not taken into account for computing the loss. + **end_positions**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``: + Labels for position (index) of the end of the labelled span for computing the token classification loss. + Positions are clamped to the length of the sequence (`sequence_length`). + Position outside of the sequence are not taken into account for computing the loss. + + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **loss**: (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: + Total span extraction loss is the sum of a Cross-Entropy for the start and end positions. + **start_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length,)`` + Span-start scores (before SoftMax). + **end_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length,)`` + Span-end scores (before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') + model = BertForQuestionAnswering.from_pretrained('bert-base-uncased') + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + start_positions = torch.tensor([1]) + end_positions = torch.tensor([3]) + outputs = model(input_ids, start_positions=start_positions, end_positions=end_positions) + loss, start_scores, end_scores = outputs[:2] + + """ + def __init__(self, config): + super(BertForQuestionAnswering, self).__init__(config) + self.num_labels = config.num_labels + + self.bert = BertModel(config) + self.qa_outputs = nn.Linear(config.hidden_size, config.num_labels) + + self.init_weights() + + def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, + start_positions=None, end_positions=None): + + outputs = self.bert(input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + position_ids=position_ids, + head_mask=head_mask) + + sequence_output = outputs[0] + + logits = self.qa_outputs(sequence_output) + start_logits, end_logits = logits.split(1, dim=-1) + start_logits = start_logits.squeeze(-1) + end_logits = end_logits.squeeze(-1) + + outputs = (start_logits, end_logits,) + outputs[2:] + if start_positions is not None and end_positions is not None: + # If we are on multi-GPU, split add a dimension + if len(start_positions.size()) > 1: + start_positions = start_positions.squeeze(-1) + if len(end_positions.size()) > 1: + end_positions = end_positions.squeeze(-1) + # sometimes the start/end positions are outside our model inputs, we ignore these terms + ignored_index = start_logits.size(1) + start_positions.clamp_(0, ignored_index) + end_positions.clamp_(0, ignored_index) + + loss_fct = CrossEntropyLoss(ignore_index=ignored_index) + start_loss = loss_fct(start_logits, start_positions) + end_loss = loss_fct(end_logits, end_positions) + total_loss = (start_loss + end_loss) / 2 + outputs = (total_loss,) + outputs + + return outputs # (loss), start_logits, end_logits, (hidden_states), (attentions) diff --git a/BertToSimple/BiGRU/model/modeling_utils.py b/BertToSimple/BiGRU/model/modeling_utils.py new file mode 100644 index 0000000..5294716 --- /dev/null +++ b/BertToSimple/BiGRU/model/modeling_utils.py @@ -0,0 +1,756 @@ +"""PyTorch BERT model.""" + +from __future__ import (absolute_import, division, print_function, + unicode_literals) + +import logging +import os + +import torch +from torch import nn +from torch.nn import CrossEntropyLoss +from torch.nn import functional as F + +from model.configuration_utils import PretrainedConfig +from model.file_utils import cached_path, WEIGHTS_NAME, TF_WEIGHTS_NAME + +logger = logging.getLogger(__name__) + + +try: + from torch.nn import Identity +except ImportError: + # Older PyTorch compatibility + class Identity(nn.Module): + r"""A placeholder identity operator that is argument-insensitive. + """ + def __init__(self, *args, **kwargs): + super(Identity, self).__init__() + + def forward(self, input): + return input + +class PreTrainedModel(nn.Module): + r""" Base class for all models. + + :class:`~pytorch_transformers.PreTrainedModel` takes care of storing the configuration of the models and handles methods for loading/downloading/saving models + as well as a few methods commons to all models to (i) resize the input embeddings and (ii) prune heads in the self-attention heads. + + Class attributes (overridden by derived classes): + - ``config_class``: a class derived from :class:`~pytorch_transformers.PretrainedConfig` to use as configuration class for this model architecture. + - ``pretrained_model_archive_map``: a python ``dict`` of with `short-cut-names` (string) as keys and `url` (string) of associated pretrained weights as values. + - ``load_tf_weights``: a python ``method`` for loading a TensorFlow checkpoint in a PyTorch model, taking as arguments: + + - ``model``: an instance of the relevant subclass of :class:`~pytorch_transformers.PreTrainedModel`, + - ``config``: an instance of the relevant subclass of :class:`~pytorch_transformers.PretrainedConfig`, + - ``path``: a path (string) to the TensorFlow checkpoint. + + - ``base_model_prefix``: a string indicating the attribute associated to the base model in derived classes of the same architecture adding modules on top of the base model. + """ + config_class = None + pretrained_model_archive_map = {} + load_tf_weights = lambda model, config, path: None + base_model_prefix = "" + + def __init__(self, config, *inputs, **kwargs): + super(PreTrainedModel, self).__init__() + if not isinstance(config, PretrainedConfig): + raise ValueError( + "Parameter config in `{}(config)` should be an instance of class `PretrainedConfig`. " + "To create a model from a pretrained model use " + "`model = {}.from_pretrained(PRETRAINED_MODEL_NAME)`".format( + self.__class__.__name__, self.__class__.__name__ + )) + # Save config in model + self.config = config + + def _get_resized_embeddings(self, old_embeddings, new_num_tokens=None): + """ Build a resized Embedding Module from a provided token Embedding Module. + Increasing the size will add newly initialized vectors at the end + Reducing the size will remove vectors from the end + + Args: + new_num_tokens: (`optional`) int + New number of tokens in the embedding matrix. + Increasing the size will add newly initialized vectors at the end + Reducing the size will remove vectors from the end + If not provided or None: return the provided token Embedding Module. + Return: ``torch.nn.Embeddings`` + Pointer to the resized Embedding Module or the old Embedding Module if new_num_tokens is None + """ + if new_num_tokens is None: + return old_embeddings + + old_num_tokens, old_embedding_dim = old_embeddings.weight.size() + if old_num_tokens == new_num_tokens: + return old_embeddings + + # Build new embeddings + new_embeddings = nn.Embedding(new_num_tokens, old_embedding_dim) + new_embeddings.to(old_embeddings.weight.device) + + # initialize all new embeddings (in particular added tokens) + self._init_weights(new_embeddings) + + # Copy word embeddings from the previous weights + num_tokens_to_copy = min(old_num_tokens, new_num_tokens) + new_embeddings.weight.data[:num_tokens_to_copy, :] = old_embeddings.weight.data[:num_tokens_to_copy, :] + + return new_embeddings + + def _tie_or_clone_weights(self, first_module, second_module): + """ Tie or clone module weights depending of weither we are using TorchScript or not + """ + + if self.config.torchscript: + first_module.weight = nn.Parameter(second_module.weight.clone()) + else: + first_module.weight = second_module.weight + + + if hasattr(first_module, 'bias') and first_module.bias is not None: + first_module.bias.data = torch.nn.functional.pad( + first_module.bias.data, + (0, first_module.weight.shape[0] - first_module.bias.shape[0]), + 'constant', + 0 + ) + + def resize_token_embeddings(self, new_num_tokens=None): + """ Resize input token embeddings matrix of the model if new_num_tokens != config.vocab_size. + Take care of tying weights embeddings afterwards if the model class has a `tie_weights()` method. + + Arguments: + + new_num_tokens: (`optional`) int: + New number of tokens in the embedding matrix. Increasing the size will add newly initialized vectors at the end. Reducing the size will remove vectors from the end. + If not provided or None: does nothing and just returns a pointer to the input tokens ``torch.nn.Embeddings`` Module of the model. + + Return: ``torch.nn.Embeddings`` + Pointer to the input tokens Embeddings Module of the model + """ + base_model = getattr(self, self.base_model_prefix, self) # get the base model if needed + model_embeds = base_model._resize_token_embeddings(new_num_tokens) + if new_num_tokens is None: + return model_embeds + + # Update base model and current model config + self.config.vocab_size = new_num_tokens + base_model.vocab_size = new_num_tokens + + # Tie weights again if needed + if hasattr(self, 'tie_weights'): + self.tie_weights() + + return model_embeds + + def init_weights(self): + """ Initialize and prunes weights if needed. """ + # Initialize weights + self.apply(self._init_weights) + + # Prune heads if needed + if self.config.pruned_heads: + self.prune_heads(self.config.pruned_heads) + + def prune_heads(self, heads_to_prune): + """ Prunes heads of the base model. + + Arguments: + + heads_to_prune: dict with keys being selected layer indices (`int`) and associated values being the list of heads to prune in said layer (list of `int`). + E.g. {1: [0, 2], 2: [2, 3]} will prune heads 0 and 2 on layer 1 and heads 2 and 3 on layer 2. + """ + base_model = getattr(self, self.base_model_prefix, self) # get the base model if needed + + # save new sets of pruned heads as union of previously stored pruned heads and newly pruned heads + for layer, heads in heads_to_prune.items(): + union_heads = set(self.config.pruned_heads.get(layer, [])) | set(heads) + self.config.pruned_heads[layer] = list(union_heads) # Unfortunately we have to store it as list for JSON + + base_model._prune_heads(heads_to_prune) + + def save_pretrained(self, save_directory): + """ Save a model and its configuration file to a directory, so that it + can be re-loaded using the `:func:`~pytorch_transformers.PreTrainedModel.from_pretrained`` class method. + """ + assert os.path.isdir(save_directory), "Saving path should be a directory where the model and configuration can be saved" + + # Only save the model it-self if we are using distributed training + model_to_save = self.module if hasattr(self, 'module') else self + + # Save configuration file + model_to_save.config.save_pretrained(save_directory) + + # If we save using the predefined names, we can load using `from_pretrained` + output_model_file = os.path.join(save_directory, WEIGHTS_NAME) + + torch.save(model_to_save.state_dict(), output_model_file) + + @classmethod + def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs): + r"""Instantiate a pretrained pytorch model from a pre-trained model configuration. + + The model is set in evaluation mode by default using ``model.eval()`` (Dropout modules are deactivated) + To train the model, you should first set it back in training mode with ``model.train()`` + + The warning ``Weights from XXX not initialized from pretrained model`` means that the weights of XXX do not come pre-trained with the rest of the model. + It is up to you to train those weights with a downstream fine-tuning task. + + The warning ``Weights from XXX not used in YYY`` means that the layer XXX is not used by YYY, therefore those weights are discarded. + + Parameters: + pretrained_model_name_or_path: either: + + - a string with the `shortcut name` of a pre-trained model to load from cache or download, e.g.: ``bert-base-uncased``. + - a path to a `directory` containing model weights saved using :func:`~pytorch_transformers.PreTrainedModel.save_pretrained`, e.g.: ``./my_model_directory/``. + - a path or url to a `tensorflow index checkpoint file` (e.g. `./tf_model/model.ckpt.index`). In this case, ``from_tf`` should be set to True and a configuration object should be provided as ``config`` argument. This loading path is slower than converting the TensorFlow checkpoint in a PyTorch model using the provided conversion scripts and loading the PyTorch model afterwards. + + model_args: (`optional`) Sequence of positional arguments: + All remaning positional arguments will be passed to the underlying model's ``__init__`` method + + config: (`optional`) instance of a class derived from :class:`~pytorch_transformers.PretrainedConfig`: + Configuration for the model to use instead of an automatically loaded configuation. Configuration can be automatically loaded when: + + - the model is a model provided by the library (loaded with the ``shortcut-name`` string of a pretrained model), or + - the model was saved using :func:`~pytorch_transformers.PreTrainedModel.save_pretrained` and is reloaded by suppling the save directory. + - the model is loaded by suppling a local directory as ``pretrained_model_name_or_path`` and a configuration JSON file named `config.json` is found in the directory. + + state_dict: (`optional`) dict: + an optional state dictionnary for the model to use instead of a state dictionary loaded from saved weights file. + This option can be used if you want to create a model from a pretrained configuration but load your own weights. + In this case though, you should check if using :func:`~pytorch_transformers.PreTrainedModel.save_pretrained` and :func:`~pytorch_transformers.PreTrainedModel.from_pretrained` is not a simpler option. + + cache_dir: (`optional`) string: + Path to a directory in which a downloaded pre-trained model + configuration should be cached if the standard cache should not be used. + + force_download: (`optional`) boolean, default False: + Force to (re-)download the model weights and configuration files and override the cached versions if they exists. + + proxies: (`optional`) dict, default None: + A dictionary of proxy servers to use by protocol or endpoint, e.g.: {'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}. + The proxies are used on each request. + + output_loading_info: (`optional`) boolean: + Set to ``True`` to also return a dictionnary containing missing keys, unexpected keys and error messages. + + kwargs: (`optional`) Remaining dictionary of keyword arguments: + Can be used to update the configuration object (after it being loaded) and initiate the model. (e.g. ``output_attention=True``). Behave differently depending on whether a `config` is provided or automatically loaded: + + - If a configuration is provided with ``config``, ``**kwargs`` will be directly passed to the underlying model's ``__init__`` method (we assume all relevant updates to the configuration have already been done) + - If a configuration is not provided, ``kwargs`` will be first passed to the configuration class initialization function (:func:`~pytorch_transformers.PretrainedConfig.from_pretrained`). Each key of ``kwargs`` that corresponds to a configuration attribute will be used to override said attribute with the supplied ``kwargs`` value. Remaining keys that do not correspond to any configuration attribute will be passed to the underlying model's ``__init__`` function. + + Examples:: + + model = BertModel.from_pretrained('bert-base-uncased') # Download model and configuration from S3 and cache. + model = BertModel.from_pretrained('./test/saved_model/') # E.g. model was saved using `save_pretrained('./test/saved_model/')` + model = BertModel.from_pretrained('bert-base-uncased', output_attention=True) # Update configuration during loading + assert model.config.output_attention == True + # Loading from a TF checkpoint file instead of a PyTorch model (slower) + config = BertConfig.from_json_file('./tf_model/my_tf_model_config.json') + model = BertModel.from_pretrained('./tf_model/my_tf_checkpoint.ckpt.index', from_tf=True, config=config) + + """ + config = kwargs.pop('config', None) + state_dict = kwargs.pop('state_dict', None) + cache_dir = kwargs.pop('cache_dir', None) + from_tf = kwargs.pop('from_tf', False) + force_download = kwargs.pop('force_download', False) + proxies = kwargs.pop('proxies', None) + output_loading_info = kwargs.pop('output_loading_info', False) + + # Load config + if config is None: + config, model_kwargs = cls.config_class.from_pretrained( + pretrained_model_name_or_path, *model_args, + cache_dir=cache_dir, return_unused_kwargs=True, + force_download=force_download, + **kwargs + ) + else: + model_kwargs = kwargs + print(model_kwargs) + + # Load model + if pretrained_model_name_or_path in cls.pretrained_model_archive_map: + archive_file = cls.pretrained_model_archive_map[pretrained_model_name_or_path] + elif os.path.isdir(pretrained_model_name_or_path): + if from_tf: + # Directly load from a TensorFlow checkpoint + archive_file = os.path.join(pretrained_model_name_or_path, TF_WEIGHTS_NAME + ".index") + else: + archive_file = os.path.join(pretrained_model_name_or_path, WEIGHTS_NAME) + else: + if from_tf: + # Directly load from a TensorFlow checkpoint + archive_file = pretrained_model_name_or_path + ".index" + else: + archive_file = pretrained_model_name_or_path + # redirect to the cache, if necessary + try: + resolved_archive_file = cached_path(archive_file, cache_dir=cache_dir, force_download=force_download, proxies=proxies) + except EnvironmentError as e: + if pretrained_model_name_or_path in cls.pretrained_model_archive_map: + logger.error( + "Couldn't reach server at '{}' to download pretrained weights.".format( + archive_file)) + else: + logger.error( + "Model name '{}' was not found in model name list ({}). " + "We assumed '{}' was a path or url but couldn't find any file " + "associated to this path or url.".format( + pretrained_model_name_or_path, + ', '.join(cls.pretrained_model_archive_map.keys()), + archive_file)) + raise e + if resolved_archive_file == archive_file: + logger.info("loading weights file {}".format(archive_file)) + else: + logger.info("loading weights file {} from cache at {}".format( + archive_file, resolved_archive_file)) + + # Instantiate model. + model = cls(config, *model_args, **model_kwargs) + + if state_dict is None and not from_tf: + state_dict = torch.load(resolved_archive_file, map_location='cpu') + if from_tf: + # Directly load from a TensorFlow checkpoint + return cls.load_tf_weights(model, config, resolved_archive_file[:-6]) # Remove the '.index' + + # Convert old format to new format if needed from a PyTorch state_dict + old_keys = [] + new_keys = [] + for key in state_dict.keys(): + new_key = None + if 'gamma' in key: + new_key = key.replace('gamma', 'weight') + if 'beta' in key: + new_key = key.replace('beta', 'bias') + if new_key: + old_keys.append(key) + new_keys.append(new_key) + for old_key, new_key in zip(old_keys, new_keys): + state_dict[new_key] = state_dict.pop(old_key) + + # Load from a PyTorch state_dict + missing_keys = [] + unexpected_keys = [] + error_msgs = [] + # copy state_dict so _load_from_state_dict can modify it + metadata = getattr(state_dict, '_metadata', None) + state_dict = state_dict.copy() + if metadata is not None: + state_dict._metadata = metadata + + def load(module, prefix=''): + local_metadata = {} if metadata is None else metadata.get(prefix[:-1], {}) + module._load_from_state_dict( + state_dict, prefix, local_metadata, True, missing_keys, unexpected_keys, error_msgs) + for name, child in module._modules.items(): + if child is not None: + load(child, prefix + name + '.') + + # Make sure we are able to load base models as well as derived models (with heads) + start_prefix = '' + model_to_load = model + if not hasattr(model, cls.base_model_prefix) and any(s.startswith(cls.base_model_prefix) for s in state_dict.keys()): + start_prefix = cls.base_model_prefix + '.' + if hasattr(model, cls.base_model_prefix) and not any(s.startswith(cls.base_model_prefix) for s in state_dict.keys()): + model_to_load = getattr(model, cls.base_model_prefix) + + load(model_to_load, prefix=start_prefix) + if len(missing_keys) > 0: + logger.info("Weights of {} not initialized from pretrained model: {}".format( + model.__class__.__name__, missing_keys)) + if len(unexpected_keys) > 0: + logger.info("Weights from pretrained model not used in {}: {}".format( + model.__class__.__name__, unexpected_keys)) + if len(error_msgs) > 0: + raise RuntimeError('Error(s) in loading state_dict for {}:\n\t{}'.format( + model.__class__.__name__, "\n\t".join(error_msgs))) + + if hasattr(model, 'tie_weights'): + model.tie_weights() # make sure word embedding weights are still tied + + # Set model in evaluation mode to desactivate DropOut modules by default + model.eval() + + if output_loading_info: + loading_info = {"missing_keys": missing_keys, "unexpected_keys": unexpected_keys, "error_msgs": error_msgs} + return model, loading_info + + return model + + +class Conv1D(nn.Module): + def __init__(self, nf, nx): + """ Conv1D layer as defined by Radford et al. for OpenAI GPT (and also used in GPT-2) + Basically works like a Linear layer but the weights are transposed + """ + super(Conv1D, self).__init__() + self.nf = nf + w = torch.empty(nx, nf) + nn.init.normal_(w, std=0.02) + self.weight = nn.Parameter(w) + self.bias = nn.Parameter(torch.zeros(nf)) + + def forward(self, x): + size_out = x.size()[:-1] + (self.nf,) + x = torch.addmm(self.bias, x.view(-1, x.size(-1)), self.weight) + x = x.view(*size_out) + return x + + +class PoolerStartLogits(nn.Module): + """ Compute SQuAD start_logits from sequence hidden states. """ + def __init__(self, config): + super(PoolerStartLogits, self).__init__() + self.dense = nn.Linear(config.hidden_size, 1) + + def forward(self, hidden_states, p_mask=None): + """ Args: + **p_mask**: (`optional`) ``torch.FloatTensor`` of shape `(batch_size, seq_len)` + invalid position mask such as query and special symbols (PAD, SEP, CLS) + 1.0 means token should be masked. + """ + x = self.dense(hidden_states).squeeze(-1) + + if p_mask is not None: + x = x * (1 - p_mask) - 1e30 * p_mask + + return x + + +class PoolerEndLogits(nn.Module): + """ Compute SQuAD end_logits from sequence hidden states and start token hidden state. + """ + def __init__(self, config): + super(PoolerEndLogits, self).__init__() + self.dense_0 = nn.Linear(config.hidden_size * 2, config.hidden_size) + self.activation = nn.Tanh() + self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) + self.dense_1 = nn.Linear(config.hidden_size, 1) + + def forward(self, hidden_states, start_states=None, start_positions=None, p_mask=None): + """ Args: + One of ``start_states``, ``start_positions`` should be not None. + If both are set, ``start_positions`` overrides ``start_states``. + + **start_states**: ``torch.LongTensor`` of shape identical to hidden_states + hidden states of the first tokens for the labeled span. + **start_positions**: ``torch.LongTensor`` of shape ``(batch_size,)`` + position of the first token for the labeled span: + **p_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, seq_len)`` + Mask of invalid position such as query and special symbols (PAD, SEP, CLS) + 1.0 means token should be masked. + """ + assert start_states is not None or start_positions is not None, "One of start_states, start_positions should be not None" + if start_positions is not None: + slen, hsz = hidden_states.shape[-2:] + start_positions = start_positions[:, None, None].expand(-1, -1, hsz) # shape (bsz, 1, hsz) + start_states = hidden_states.gather(-2, start_positions) # shape (bsz, 1, hsz) + start_states = start_states.expand(-1, slen, -1) # shape (bsz, slen, hsz) + + x = self.dense_0(torch.cat([hidden_states, start_states], dim=-1)) + x = self.activation(x) + x = self.LayerNorm(x) + x = self.dense_1(x).squeeze(-1) + + if p_mask is not None: + x = x * (1 - p_mask) - 1e30 * p_mask + + return x + + +class PoolerAnswerClass(nn.Module): + """ Compute SQuAD 2.0 answer class from classification and start tokens hidden states. """ + def __init__(self, config): + super(PoolerAnswerClass, self).__init__() + self.dense_0 = nn.Linear(config.hidden_size * 2, config.hidden_size) + self.activation = nn.Tanh() + self.dense_1 = nn.Linear(config.hidden_size, 1, bias=False) + + def forward(self, hidden_states, start_states=None, start_positions=None, cls_index=None): + """ + Args: + One of ``start_states``, ``start_positions`` should be not None. + If both are set, ``start_positions`` overrides ``start_states``. + + **start_states**: ``torch.LongTensor`` of shape identical to ``hidden_states``. + hidden states of the first tokens for the labeled span. + **start_positions**: ``torch.LongTensor`` of shape ``(batch_size,)`` + position of the first token for the labeled span. + **cls_index**: torch.LongTensor of shape ``(batch_size,)`` + position of the CLS token. If None, take the last token. + + note(Original repo): + no dependency on end_feature so that we can obtain one single `cls_logits` + for each sample + """ + hsz = hidden_states.shape[-1] + assert start_states is not None or start_positions is not None, "One of start_states, start_positions should be not None" + if start_positions is not None: + start_positions = start_positions[:, None, None].expand(-1, -1, hsz) # shape (bsz, 1, hsz) + start_states = hidden_states.gather(-2, start_positions).squeeze(-2) # shape (bsz, hsz) + + if cls_index is not None: + cls_index = cls_index[:, None, None].expand(-1, -1, hsz) # shape (bsz, 1, hsz) + cls_token_state = hidden_states.gather(-2, cls_index).squeeze(-2) # shape (bsz, hsz) + else: + cls_token_state = hidden_states[:, -1, :] # shape (bsz, hsz) + + x = self.dense_0(torch.cat([start_states, cls_token_state], dim=-1)) + x = self.activation(x) + x = self.dense_1(x).squeeze(-1) + + return x + + +class SQuADHead(nn.Module): + r""" A SQuAD head inspired by XLNet. + + Parameters: + config (:class:`~pytorch_transformers.XLNetConfig`): Model configuration class with all the parameters of the model. + + Inputs: + **hidden_states**: ``torch.FloatTensor`` of shape ``(batch_size, seq_len, hidden_size)`` + hidden states of sequence tokens + **start_positions**: ``torch.LongTensor`` of shape ``(batch_size,)`` + position of the first token for the labeled span. + **end_positions**: ``torch.LongTensor`` of shape ``(batch_size,)`` + position of the last token for the labeled span. + **cls_index**: torch.LongTensor of shape ``(batch_size,)`` + position of the CLS token. If None, take the last token. + **is_impossible**: ``torch.LongTensor`` of shape ``(batch_size,)`` + Whether the question has a possible answer in the paragraph or not. + **p_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, seq_len)`` + Mask of invalid position such as query and special symbols (PAD, SEP, CLS) + 1.0 means token should be masked. + + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **loss**: (`optional`, returned if both ``start_positions`` and ``end_positions`` are provided) ``torch.FloatTensor`` of shape ``(1,)``: + Classification loss as the sum of start token, end token (and is_impossible if provided) classification losses. + **start_top_log_probs**: (`optional`, returned if ``start_positions`` or ``end_positions`` is not provided) + ``torch.FloatTensor`` of shape ``(batch_size, config.start_n_top)`` + Log probabilities for the top config.start_n_top start token possibilities (beam-search). + **start_top_index**: (`optional`, returned if ``start_positions`` or ``end_positions`` is not provided) + ``torch.LongTensor`` of shape ``(batch_size, config.start_n_top)`` + Indices for the top config.start_n_top start token possibilities (beam-search). + **end_top_log_probs**: (`optional`, returned if ``start_positions`` or ``end_positions`` is not provided) + ``torch.FloatTensor`` of shape ``(batch_size, config.start_n_top * config.end_n_top)`` + Log probabilities for the top ``config.start_n_top * config.end_n_top`` end token possibilities (beam-search). + **end_top_index**: (`optional`, returned if ``start_positions`` or ``end_positions`` is not provided) + ``torch.LongTensor`` of shape ``(batch_size, config.start_n_top * config.end_n_top)`` + Indices for the top ``config.start_n_top * config.end_n_top`` end token possibilities (beam-search). + **cls_logits**: (`optional`, returned if ``start_positions`` or ``end_positions`` is not provided) + ``torch.FloatTensor`` of shape ``(batch_size,)`` + Log probabilities for the ``is_impossible`` label of the answers. + """ + def __init__(self, config): + super(SQuADHead, self).__init__() + self.start_n_top = config.start_n_top + self.end_n_top = config.end_n_top + + self.start_logits = PoolerStartLogits(config) + self.end_logits = PoolerEndLogits(config) + self.answer_class = PoolerAnswerClass(config) + + def forward(self, hidden_states, start_positions=None, end_positions=None, + cls_index=None, is_impossible=None, p_mask=None): + outputs = () + + start_logits = self.start_logits(hidden_states, p_mask=p_mask) + + if start_positions is not None and end_positions is not None: + # If we are on multi-GPU, let's remove the dimension added by batch splitting + for x in (start_positions, end_positions, cls_index, is_impossible): + if x is not None and x.dim() > 1: + x.squeeze_(-1) + + # during training, compute the end logits based on the ground truth of the start position + end_logits = self.end_logits(hidden_states, start_positions=start_positions, p_mask=p_mask) + + loss_fct = CrossEntropyLoss() + start_loss = loss_fct(start_logits, start_positions) + end_loss = loss_fct(end_logits, end_positions) + total_loss = (start_loss + end_loss) / 2 + + if cls_index is not None and is_impossible is not None: + # Predict answerability from the representation of CLS and START + cls_logits = self.answer_class(hidden_states, start_positions=start_positions, cls_index=cls_index) + loss_fct_cls = nn.BCEWithLogitsLoss() + cls_loss = loss_fct_cls(cls_logits, is_impossible) + + # note(zhiliny): by default multiply the loss by 0.5 so that the scale is comparable to start_loss and end_loss + total_loss += cls_loss * 0.5 + + outputs = (total_loss,) + outputs + + else: + # during inference, compute the end logits based on beam search + bsz, slen, hsz = hidden_states.size() + start_log_probs = F.softmax(start_logits, dim=-1) # shape (bsz, slen) + + start_top_log_probs, start_top_index = torch.topk(start_log_probs, self.start_n_top, dim=-1) # shape (bsz, start_n_top) + start_top_index_exp = start_top_index.unsqueeze(-1).expand(-1, -1, hsz) # shape (bsz, start_n_top, hsz) + start_states = torch.gather(hidden_states, -2, start_top_index_exp) # shape (bsz, start_n_top, hsz) + start_states = start_states.unsqueeze(1).expand(-1, slen, -1, -1) # shape (bsz, slen, start_n_top, hsz) + + hidden_states_expanded = hidden_states.unsqueeze(2).expand_as(start_states) # shape (bsz, slen, start_n_top, hsz) + p_mask = p_mask.unsqueeze(-1) if p_mask is not None else None + end_logits = self.end_logits(hidden_states_expanded, start_states=start_states, p_mask=p_mask) + end_log_probs = F.softmax(end_logits, dim=1) # shape (bsz, slen, start_n_top) + + end_top_log_probs, end_top_index = torch.topk(end_log_probs, self.end_n_top, dim=1) # shape (bsz, end_n_top, start_n_top) + end_top_log_probs = end_top_log_probs.view(-1, self.start_n_top * self.end_n_top) + end_top_index = end_top_index.view(-1, self.start_n_top * self.end_n_top) + + start_states = torch.einsum("blh,bl->bh", hidden_states, start_log_probs) + cls_logits = self.answer_class(hidden_states, start_states=start_states, cls_index=cls_index) + + outputs = (start_top_log_probs, start_top_index, end_top_log_probs, end_top_index, cls_logits) + outputs + + # return start_top_log_probs, start_top_index, end_top_log_probs, end_top_index, cls_logits + # or (if labels are provided) (total_loss,) + return outputs + + +class SequenceSummary(nn.Module): + r""" Compute a single vector summary of a sequence hidden states according to various possibilities: + Args of the config class: + summary_type: + - 'last' => [default] take the last token hidden state (like XLNet) + - 'first' => take the first token hidden state (like Bert) + - 'mean' => take the mean of all tokens hidden states + - 'cls_index' => supply a Tensor of classification token position (GPT/GPT-2) + - 'attn' => Not implemented now, use multi-head attention + summary_use_proj: Add a projection after the vector extraction + summary_proj_to_labels: If True, the projection outputs to config.num_labels classes (otherwise to hidden_size). Default: False. + summary_activation: 'tanh' => add a tanh activation to the output, Other => no activation. Default + summary_first_dropout: Add a dropout before the projection and activation + summary_last_dropout: Add a dropout after the projection and activation + """ + def __init__(self, config): + super(SequenceSummary, self).__init__() + + self.summary_type = config.summary_type if hasattr(config, 'summary_use_proj') else 'last' + if self.summary_type == 'attn': + # We should use a standard multi-head attention module with absolute positional embedding for that. + # Cf. https://github.com/zihangdai/xlnet/blob/master/modeling.py#L253-L276 + # We can probably just use the multi-head attention module of PyTorch >=1.1.0 + raise NotImplementedError + + self.summary = Identity() + if hasattr(config, 'summary_use_proj') and config.summary_use_proj: + if hasattr(config, 'summary_proj_to_labels') and config.summary_proj_to_labels and config.num_labels > 0: + num_classes = config.num_labels + else: + num_classes = config.hidden_size + self.summary = nn.Linear(config.hidden_size, num_classes) + + self.activation = Identity() + if hasattr(config, 'summary_activation') and config.summary_activation == 'tanh': + self.activation = nn.Tanh() + + self.first_dropout = Identity() + if hasattr(config, 'summary_first_dropout') and config.summary_first_dropout > 0: + self.first_dropout = nn.Dropout(config.summary_first_dropout) + + self.last_dropout = Identity() + if hasattr(config, 'summary_last_dropout') and config.summary_last_dropout > 0: + self.last_dropout = nn.Dropout(config.summary_last_dropout) + + def forward(self, hidden_states, cls_index=None): + """ hidden_states: float Tensor in shape [bsz, seq_len, hidden_size], the hidden-states of the last layer. + cls_index: [optional] position of the classification token if summary_type == 'cls_index', + shape (bsz,) or more generally (bsz, ...) where ... are optional leading dimensions of hidden_states. + if summary_type == 'cls_index' and cls_index is None: + we take the last token of the sequence as classification token + """ + if self.summary_type == 'last': + output = hidden_states[:, -1] + elif self.summary_type == 'first': + output = hidden_states[:, 0] + elif self.summary_type == 'mean': + output = hidden_states.mean(dim=1) + elif self.summary_type == 'cls_index': + if cls_index is None: + cls_index = torch.full_like(hidden_states[..., :1, :], hidden_states.shape[-2]-1, dtype=torch.long) + else: + cls_index = cls_index.unsqueeze(-1).unsqueeze(-1) + cls_index = cls_index.expand((-1,) * (cls_index.dim()-1) + (hidden_states.size(-1),)) + # shape of cls_index: (bsz, XX, 1, hidden_size) where XX are optional leading dim of hidden_states + output = hidden_states.gather(-2, cls_index).squeeze(-2) # shape (bsz, XX, hidden_size) + elif self.summary_type == 'attn': + raise NotImplementedError + + output = self.first_dropout(output) + output = self.summary(output) + output = self.activation(output) + output = self.last_dropout(output) + + return output + + +def prune_linear_layer(layer, index, dim=0): + """ Prune a linear layer (a model parameters) to keep only entries in index. + Return the pruned layer as a new layer with requires_grad=True. + Used to remove heads. + """ + index = index.to(layer.weight.device) + W = layer.weight.index_select(dim, index).clone().detach() + if layer.bias is not None: + if dim == 1: + b = layer.bias.clone().detach() + else: + b = layer.bias[index].clone().detach() + new_size = list(layer.weight.size()) + new_size[dim] = len(index) + new_layer = nn.Linear(new_size[1], new_size[0], bias=layer.bias is not None).to(layer.weight.device) + new_layer.weight.requires_grad = False + new_layer.weight.copy_(W.contiguous()) + new_layer.weight.requires_grad = True + if layer.bias is not None: + new_layer.bias.requires_grad = False + new_layer.bias.copy_(b.contiguous()) + new_layer.bias.requires_grad = True + return new_layer + + +def prune_conv1d_layer(layer, index, dim=1): + """ Prune a Conv1D layer (a model parameters) to keep only entries in index. + A Conv1D work as a Linear layer (see e.g. BERT) but the weights are transposed. + Return the pruned layer as a new layer with requires_grad=True. + Used to remove heads. + """ + index = index.to(layer.weight.device) + W = layer.weight.index_select(dim, index).clone().detach() + if dim == 0: + b = layer.bias.clone().detach() + else: + b = layer.bias[index].clone().detach() + new_size = list(layer.weight.size()) + new_size[dim] = len(index) + new_layer = Conv1D(new_size[1], new_size[0]).to(layer.weight.device) + new_layer.weight.requires_grad = False + new_layer.weight.copy_(W.contiguous()) + new_layer.weight.requires_grad = True + new_layer.bias.requires_grad = False + new_layer.bias.copy_(b.contiguous()) + new_layer.bias.requires_grad = True + return new_layer + + +def prune_layer(layer, index, dim=None): + """ Prune a Conv1D or nn.Linear layer (a model parameters) to keep only entries in index. + Return the pruned layer as a new layer with requires_grad=True. + Used to remove heads. + """ + if isinstance(layer, nn.Linear): + return prune_linear_layer(layer, index, dim=0 if dim is None else dim) + elif isinstance(layer, Conv1D): + return prune_conv1d_layer(layer, index, dim=1 if dim is None else dim) + else: + raise ValueError("Can't prune layer of class {}".format(layer.__class__)) diff --git a/BertToSimple/BiGRU/model/tokenization_albert.py b/BertToSimple/BiGRU/model/tokenization_albert.py new file mode 100644 index 0000000..4e8a147 --- /dev/null +++ b/BertToSimple/BiGRU/model/tokenization_albert.py @@ -0,0 +1,359 @@ +"""Tokenization classes.""" + +from __future__ import (absolute_import, division, print_function, + unicode_literals) +import collections +import unicodedata +import six +import logging +# 分词算法 +import sentencepiece as spm + +logger = logging.getLogger(__name__) +SPIECE_UNDERLINE = u"▁" + +def preprocess_text(inputs,remove_space=True,do_lower_case=True): + if remove_space: + outputs = ' '.join(inputs.strip().split()) + else: + outputs = inputs + outputs = outputs.replace("``", '"').replace("''", '"') + if six.PY2 and isinstance(outputs, str): + outputs = outputs.decode('utf-8') + outputs = unicodedata.normalize("NFKD", outputs) + outputs = "".join([c for c in outputs if not unicodedata.combining(c)]) + if do_lower_case: + outputs = outputs.lower() + return outputs + +def encode_pieces(sp_model, text, return_unicode=True, sample=False): + """turn sentences into word pieces.""" + text = preprocess_text(text,) + if six.PY2 and isinstance(text, unicode): + text = text.encode('utf-8') + if not sample: + pieces = sp_model.EncodeAsPieces(text) + else: + pieces = sp_model.SampleEncodeAsPieces(text, 64, 0.1) + new_pieces = [] + for piece in pieces: + if len(piece) > 1 and piece[-1] == ',' and piece[-2].isdigit(): + cur_pieces = sp_model.EncodeAsPieces( + piece[:-1].replace(SPIECE_UNDERLINE, '')) + if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE: + if len(cur_pieces[0]) == 1: + cur_pieces = cur_pieces[1:] + else: + cur_pieces[0] = cur_pieces[0][1:] + cur_pieces.append(piece[-1]) + new_pieces.extend(cur_pieces) + else: + new_pieces.append(piece) + + # note(zhiliny): convert back to unicode for py2 + if six.PY2 and return_unicode: + ret_pieces = [] + for piece in new_pieces: + if isinstance(piece, str): + piece = piece.decode(piece, "utf-8") + ret_pieces.append(piece) + new_pieces = ret_pieces + + return new_pieces + +def encode_ids(sp_model, text, sample=False): + pieces = encode_pieces(sp_model, text, return_unicode=False, sample=sample) + ids = [sp_model.PieceToId(piece) for piece in pieces] + return ids + + +def load_vocab(vocab_file): + """Loads a vocabulary file into a dictionary.""" + vocab = collections.OrderedDict() + with open(vocab_file, "r", encoding="utf-8") as reader: + tokens = reader.readlines() + for index, token in enumerate(tokens): + # 删除字符 + token = token.rstrip('\n') + vocab[token] = index + return vocab + +def convert_by_vocab(vocab, items): + """Converts a sequence of [tokens|ids] using the vocab.""" + output = [] + for item in items: + output.append(vocab[item]) + return output + +def convert_tokens_to_ids(vocab, tokens): + return convert_by_vocab(vocab, tokens) + +def convert_ids_to_tokens(inv_vocab, ids): + return convert_by_vocab(inv_vocab, ids) + +def whitespace_tokenize(text): + """Runs basic whitespace cleaning and splitting on a piece of text.""" + text = text.strip() + if not text: + return [] + tokens = text.split() + return tokens + +class FullTokenizer(object): + """Runs end-to-end tokenziation.""" + + def __init__(self, vocab_file, do_lower_case=True, spm_model_file=None): + self.vocab = None + self.sp_model = None + if spm_model_file: + self.sp_model = spm.SentencePieceProcessor() + logger.info("loading sentence piece model") + self.sp_model.Load(spm_model_file) + # # Note(mingdachen): For the purpose of consisent API, we are + # # generating a vocabulary for the sentence piece tokenizer. + self.vocab = {self.sp_model.IdToPiece(i): i for i + in range(self.sp_model.GetPieceSize())} + else: + print("load vocab") + # 返回字->index的字典 + self.vocab = load_vocab(vocab_file) + print("load token") + # 对象里包含了各种各样处理中文的方法 + self.basic_tokenizer = BasicTokenizer(do_lower_case=do_lower_case) + # 这个对象的方法处理单词，数据需要先经过basic_tokenizer方法的处理 + self.wordpiece_tokenizer = WordpieceTokenizer(vocab=self.vocab,unk_token="[UNK]", max_input_chars_per_word=100) + # index->字的字典 + self.inv_vocab = {v: k for k, v in self.vocab.items()} + + def tokenize(self, text): + if self.sp_model: + split_tokens = encode_pieces(self.sp_model, text, return_unicode=False) + else: + split_tokens = [] + # 对每句的每个词进行处理 + for token in self.basic_tokenizer.tokenize(text): + for sub_token in self.wordpiece_tokenizer.tokenize(token): + split_tokens.append(sub_token) + + return split_tokens + + def convert_tokens_to_ids(self, tokens): + if self.sp_model: + return [self.sp_model.PieceToId(token) for token in tokens] + else: + return convert_by_vocab(self.vocab, tokens) + + def convert_ids_to_tokens(self, ids): + if self.sp_model: + logger.info("using sentence piece tokenzier.") + return [self.sp_model.IdToPiece(id_) for id_ in ids] + else: + return convert_by_vocab(self.inv_vocab, ids) + +class BasicTokenizer(object): + """Runs basic tokenization (punctuation splitting, lower casing, etc.).""" + + def __init__(self, do_lower_case=True): + """Constructs a BasicTokenizer. + + Args: + do_lower_case: Whether to lower case the input. + """ + self.do_lower_case = do_lower_case + + def tokenize(self, text): + """Tokenizes a piece of text.""" + text = self._clean_text(text) + + # This was added on November 1st, 2018 for the multilingual and Chinese + # models. This is also applied to the English models now, but it doesn't + # matter since the English models were not trained on any Chinese data + # and generally don't have any Chinese data in them (there are Chinese + # characters in the vocabulary because Wikipedia does have some Chinese + # words in the English Wikipedia.). + text = self._tokenize_chinese_chars(text) + orig_tokens = whitespace_tokenize(text) + split_tokens = [] + for token in orig_tokens: + if self.do_lower_case: + token = token.lower() + token = self._run_strip_accents(token) + split_tokens.extend(self._run_split_on_punc(token)) + output_tokens = whitespace_tokenize(" ".join(split_tokens)) + return output_tokens + + def _run_strip_accents(self, text): + """Strips accents from a piece of text.""" + text = unicodedata.normalize("NFD", text) + output = [] + for char in text: + cat = unicodedata.category(char) + if cat == "Mn": + continue + output.append(char) + return "".join(output) + + def _run_split_on_punc(self, text): + """Splits punctuation on a piece of text.""" + chars = list(text) + i = 0 + start_new_word = True + output = [] + while i < len(chars): + char = chars[i] + if _is_punctuation(char): + output.append([char]) + start_new_word = True + else: + if start_new_word: + output.append([]) + start_new_word = False + output[-1].append(char) + i += 1 + + return ["".join(x) for x in output] + + def _tokenize_chinese_chars(self, text): + """Adds whitespace around any CJK character.""" + output = [] + for char in text: + cp = ord(char) + if self._is_chinese_char(cp): + output.append(" ") + output.append(char) + output.append(" ") + else: + output.append(char) + return "".join(output) + + def _is_chinese_char(self, cp): + """Checks whether CP is the codepoint of a CJK character.""" + # This defines a "chinese character" as anything in the CJK Unicode block: + # https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block) + # + # Note that the CJK Unicode block is NOT all Japanese and Korean characters, + # despite its name. The modern Korean Hangul alphabet is a different block, + # as is Japanese Hiragana and Katakana. Those alphabets are used to write + # space-separated words, so they are not treated specially and handled + # like the all of the other languages. + if ((cp >= 0x4E00 and cp <= 0x9FFF) or # + (cp >= 0x3400 and cp <= 0x4DBF) or # + (cp >= 0x20000 and cp <= 0x2A6DF) or # + (cp >= 0x2A700 and cp <= 0x2B73F) or # + (cp >= 0x2B740 and cp <= 0x2B81F) or # + (cp >= 0x2B820 and cp <= 0x2CEAF) or + (cp >= 0xF900 and cp <= 0xFAFF) or # + (cp >= 0x2F800 and cp <= 0x2FA1F)): # + return True + + return False + + def _clean_text(self, text): + """Performs invalid character removal and whitespace cleanup on text.""" + output = [] + for char in text: + cp = ord(char) + if cp == 0 or cp == 0xfffd or _is_control(char): + continue + if _is_whitespace(char): + output.append(" ") + else: + output.append(char) + return "".join(output) + +class WordpieceTokenizer(object): + """Runs WordPiece tokenization.""" + + def __init__(self, vocab, unk_token, max_input_chars_per_word=100): + self.vocab = vocab + self.unk_token = unk_token + self.max_input_chars_per_word = max_input_chars_per_word + + def tokenize(self, text): + """Tokenizes a piece of text into its word pieces. + + This uses a greedy longest-match-first algorithm to perform tokenization + using the given vocabulary. + + For example: + input = "unaffable" + output = ["un", "##aff", "##able"] + + Args: + text: A single token or whitespace separated tokens. This should have + already been passed through `BasicTokenizer`. + + Returns: + A list of wordpiece tokens. + """ + + output_tokens = [] + for token in whitespace_tokenize(text): + chars = list(token) + if len(chars) > self.max_input_chars_per_word: + output_tokens.append(self.unk_token) + continue + + is_bad = False + start = 0 + sub_tokens = [] + while start < len(chars): + end = len(chars) + cur_substr = None + while start < end: + substr = "".join(chars[start:end]) + if start > 0: + substr = "##" + substr + if substr in self.vocab: + cur_substr = substr + break + end -= 1 + if cur_substr is None: + is_bad = True + break + sub_tokens.append(cur_substr) + start = end + + if is_bad: + output_tokens.append(self.unk_token) + else: + output_tokens.extend(sub_tokens) + return output_tokens + +def _is_whitespace(char): + """Checks whether `chars` is a whitespace character.""" + # \t, \n, and \r are technically control characters but we treat them + # as whitespace since they are generally considered as such. + if char == " " or char == "\t" or char == "\n" or char == "\r": + return True + cat = unicodedata.category(char) + if cat == "Zs": + return True + return False + + +def _is_control(char): + """Checks whether `chars` is a control character.""" + # These are technically control characters but we count them as whitespace + # characters. + if char == "\t" or char == "\n" or char == "\r": + return False + cat = unicodedata.category(char) + if cat in ("Cc", "Cf"): + return True + return False + +def _is_punctuation(char): + """Checks whether `chars` is a punctuation character.""" + cp = ord(char) + # We treat all non-letter/number ASCII as punctuation. + # Characters such as "^", "$", and "`" are not in the Unicode + # Punctuation class but we treat them as punctuation anyways, for + # consistency. + if ((cp >= 33 and cp <= 47) or (cp >= 58 and cp <= 64) or + (cp >= 91 and cp <= 96) or (cp >= 123 and cp <= 126)): + return True + cat = unicodedata.category(char) + if cat.startswith("P"): + return True + return False diff --git a/BertToSimple/BiGRU/model/tokenization_bert.py b/BertToSimple/BiGRU/model/tokenization_bert.py new file mode 100644 index 0000000..ad3d480 --- /dev/null +++ b/BertToSimple/BiGRU/model/tokenization_bert.py @@ -0,0 +1,441 @@ +# coding=utf-8 +# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Tokenization classes.""" + +from __future__ import absolute_import, division, print_function, unicode_literals + +import collections +import logging +import os +import unicodedata +from io import open + +from .tokenization_utils import PreTrainedTokenizer + +logger = logging.getLogger(__name__) + +VOCAB_FILES_NAMES = {'vocab_file': 'vocab.txt'} + +def load_vocab(vocab_file): + """Loads a vocabulary file into a dictionary.""" + vocab = collections.OrderedDict() + with open(vocab_file, "r", encoding="utf-8") as reader: + tokens = reader.readlines() + for index, token in enumerate(tokens): + token = token.rstrip('\n') + vocab[token] = index + return vocab + + +def whitespace_tokenize(text): + """Runs basic whitespace cleaning and splitting on a piece of text.""" + text = text.strip() + if not text: + return [] + tokens = text.split() + return tokens + + +class BertTokenizer(PreTrainedTokenizer): + r""" + Constructs a BertTokenizer. + :class:`~transformers.BertTokenizer` runs end-to-end tokenization: punctuation splitting + wordpiece + + Args: + vocab_file: Path to a one-wordpiece-per-line vocabulary file + do_lower_case: Whether to lower case the input. Only has an effect when do_wordpiece_only=False + do_basic_tokenize: Whether to do basic tokenization before wordpiece. + max_len: An artificial maximum length to truncate tokenized sequences to; Effective maximum length is always the + minimum of this value (if specified) and the underlying BERT model's sequence length. + never_split: List of tokens which will never be split during tokenization. Only has an effect when + do_wordpiece_only=False + """ + + vocab_files_names = VOCAB_FILES_NAMES + + def __init__(self, vocab_file, do_lower_case=True, do_basic_tokenize=True, never_split=None, + unk_token="[UNK]", sep_token="[SEP]", pad_token="[PAD]", cls_token="[CLS]", + mask_token="[MASK]", tokenize_chinese_chars=True, **kwargs): + """Constructs a BertTokenizer. + + Args: + **vocab_file**: Path to a one-wordpiece-per-line vocabulary file + **do_lower_case**: (`optional`) boolean (default True) + Whether to lower case the input + Only has an effect when do_basic_tokenize=True + **do_basic_tokenize**: (`optional`) boolean (default True) + Whether to do basic tokenization before wordpiece. + **never_split**: (`optional`) list of string + List of tokens which will never be split during tokenization. + Only has an effect when do_basic_tokenize=True + **tokenize_chinese_chars**: (`optional`) boolean (default True) + Whether to tokenize Chinese characters. + This should likely be deactivated for Japanese: + see: https://github.com/huggingface/pytorch-pretrained-BERT/issues/328 + """ + super(BertTokenizer, self).__init__(unk_token=unk_token, sep_token=sep_token, + pad_token=pad_token, cls_token=cls_token, + mask_token=mask_token, **kwargs) + self.max_len_single_sentence = self.max_len - 2 # take into account special tokens + self.max_len_sentences_pair = self.max_len - 3 # take into account special tokens + + if not os.path.isfile(vocab_file): + raise ValueError( + "Can't find a vocabulary file at path '{}'. To load the vocabulary from a Google pretrained " + "model use `tokenizer = BertTokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`".format(vocab_file)) + self.vocab = load_vocab(vocab_file) + self.ids_to_tokens = collections.OrderedDict( + [(ids, tok) for tok, ids in self.vocab.items()]) + self.do_basic_tokenize = do_basic_tokenize + if do_basic_tokenize: + self.basic_tokenizer = BasicTokenizer(do_lower_case=do_lower_case, + never_split=never_split, + tokenize_chinese_chars=tokenize_chinese_chars) + self.wordpiece_tokenizer = WordpieceTokenizer(vocab=self.vocab, unk_token=self.unk_token) + + @property + def vocab_size(self): + return len(self.vocab) + + def _tokenize(self, text): + split_tokens = [] + if self.do_basic_tokenize: + for token in self.basic_tokenizer.tokenize(text, never_split=self.all_special_tokens): + for sub_token in self.wordpiece_tokenizer.tokenize(token): + split_tokens.append(sub_token) + else: + split_tokens = self.wordpiece_tokenizer.tokenize(text) + return split_tokens + + def _convert_token_to_id(self, token): + """ Converts a token (str/unicode) in an id using the vocab. """ + return self.vocab.get(token, self.vocab.get(self.unk_token)) + + def _convert_id_to_token(self, index): + """Converts an index (integer) in a token (string/unicode) using the vocab.""" + return self.ids_to_tokens.get(index, self.unk_token) + + def convert_tokens_to_string(self, tokens): + """ Converts a sequence of tokens (string) in a single string. """ + out_string = ' '.join(tokens).replace(' ##', '').strip() + return out_string + + def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None): + """ + Build model inputs from a sequence or a pair of sequence for sequence classification tasks + by concatenating and adding special tokens. + A BERT sequence has the following format: + single sequence: [CLS] X [SEP] + pair of sequences: [CLS] A [SEP] B [SEP] + """ + if token_ids_1 is None: + return [self.cls_token_id] + token_ids_0 + [self.sep_token_id] + cls = [self.cls_token_id] + sep = [self.sep_token_id] + return cls + token_ids_0 + sep + token_ids_1 + sep + + def get_special_tokens_mask(self, token_ids_0, token_ids_1=None, already_has_special_tokens=False): + """ + Retrieves sequence ids from a token list that has no special tokens added. This method is called when adding + special tokens using the tokenizer ``prepare_for_model`` or ``encode_plus`` methods. + + Args: + token_ids_0: list of ids (must not contain special tokens) + token_ids_1: Optional list of ids (must not contain special tokens), necessary when fetching sequence ids + for sequence pairs + already_has_special_tokens: (default False) Set to True if the token list is already formated with + special tokens for the model + + Returns: + A list of integers in the range [0, 1]: 0 for a special token, 1 for a sequence token. + """ + + if already_has_special_tokens: + if token_ids_1 is not None: + raise ValueError("You should not supply a second sequence if the provided sequence of " + "ids is already formated with special tokens for the model.") + return list(map(lambda x: 1 if x in [self.sep_token_id, self.cls_token_id] else 0, token_ids_0)) + + if token_ids_1 is not None: + return [1] + ([0] * len(token_ids_0)) + [1] + ([0] * len(token_ids_1)) + [1] + return [1] + ([0] * len(token_ids_0)) + [1] + + def create_token_type_ids_from_sequences(self, token_ids_0, token_ids_1=None): + """ + Creates a mask from the two sequences passed to be used in a sequence-pair classification task. + A BERT sequence pair mask has the following format: + 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 + | first sequence | second sequence + + if token_ids_1 is None, only returns the first portion of the mask (0's). + """ + sep = [self.sep_token_id] + cls = [self.cls_token_id] + if token_ids_1 is None: + return len(cls + token_ids_0 + sep) * [0] + return len(cls + token_ids_0 + sep) * [0] + len(token_ids_1 + sep) * [1] + + def save_vocabulary(self, vocab_path): + """Save the tokenizer vocabulary to a directory or file.""" + index = 0 + if os.path.isdir(vocab_path): + vocab_file = os.path.join(vocab_path, VOCAB_FILES_NAMES['vocab_file']) + else: + vocab_file = vocab_path + with open(vocab_file, "w", encoding="utf-8") as writer: + for token, token_index in sorted(self.vocab.items(), key=lambda kv: kv[1]): + if index != token_index: + logger.warning("Saving vocabulary to {}: vocabulary indices are not consecutive." + " Please check that the vocabulary is not corrupted!".format(vocab_file)) + index = token_index + writer.write(token + u'\n') + index += 1 + return (vocab_file,) + + +class BasicTokenizer(object): + """Runs basic tokenization (punctuation splitting, lower casing, etc.).""" + + def __init__(self, do_lower_case=True, never_split=None, tokenize_chinese_chars=True): + """ Constructs a BasicTokenizer. + + Args: + **do_lower_case**: Whether to lower case the input. + **never_split**: (`optional`) list of str + Kept for backward compatibility purposes. + Now implemented directly at the base class level (see :func:`PreTrainedTokenizer.tokenize`) + List of token not to split. + **tokenize_chinese_chars**: (`optional`) boolean (default True) + Whether to tokenize Chinese characters. + This should likely be deactivated for Japanese: + see: https://github.com/huggingface/pytorch-pretrained-BERT/issues/328 + """ + if never_split is None: + never_split = [] + self.do_lower_case = do_lower_case + self.never_split = never_split + self.tokenize_chinese_chars = tokenize_chinese_chars + + def tokenize(self, text, never_split=None): + """ Basic Tokenization of a piece of text. + Split on "white spaces" only, for sub-word tokenization, see WordPieceTokenizer. + + Args: + **never_split**: (`optional`) list of str + Kept for backward compatibility purposes. + Now implemented directly at the base class level (see :func:`PreTrainedTokenizer.tokenize`) + List of token not to split. + """ + never_split = self.never_split + (never_split if never_split is not None else []) + text = self._clean_text(text) + # This was added on November 1st, 2018 for the multilingual and Chinese + # models. This is also applied to the English models now, but it doesn't + # matter since the English models were not trained on any Chinese data + # and generally don't have any Chinese data in them (there are Chinese + # characters in the vocabulary because Wikipedia does have some Chinese + # words in the English Wikipedia.). + if self.tokenize_chinese_chars: + text = self._tokenize_chinese_chars(text) + orig_tokens = whitespace_tokenize(text) + split_tokens = [] + for token in orig_tokens: + if self.do_lower_case and token not in never_split: + token = token.lower() + token = self._run_strip_accents(token) + split_tokens.extend(self._run_split_on_punc(token)) + + output_tokens = whitespace_tokenize(" ".join(split_tokens)) + return output_tokens + + def _run_strip_accents(self, text): + """Strips accents from a piece of text.""" + text = unicodedata.normalize("NFD", text) + output = [] + for char in text: + cat = unicodedata.category(char) + if cat == "Mn": + continue + output.append(char) + return "".join(output) + + def _run_split_on_punc(self, text, never_split=None): + """Splits punctuation on a piece of text.""" + if never_split is not None and text in never_split: + return [text] + chars = list(text) + i = 0 + start_new_word = True + output = [] + while i < len(chars): + char = chars[i] + if _is_punctuation(char): + output.append([char]) + start_new_word = True + else: + if start_new_word: + output.append([]) + start_new_word = False + output[-1].append(char) + i += 1 + + return ["".join(x) for x in output] + + def _tokenize_chinese_chars(self, text): + """Adds whitespace around any CJK character.""" + output = [] + for char in text: + cp = ord(char) + if self._is_chinese_char(cp): + output.append(" ") + output.append(char) + output.append(" ") + else: + output.append(char) + return "".join(output) + + def _is_chinese_char(self, cp): + """Checks whether CP is the codepoint of a CJK character.""" + # This defines a "chinese character" as anything in the CJK Unicode block: + # https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block) + # + # Note that the CJK Unicode block is NOT all Japanese and Korean characters, + # despite its name. The modern Korean Hangul alphabet is a different block, + # as is Japanese Hiragana and Katakana. Those alphabets are used to write + # space-separated words, so they are not treated specially and handled + # like the all of the other languages. + if ((cp >= 0x4E00 and cp <= 0x9FFF) or # + (cp >= 0x3400 and cp <= 0x4DBF) or # + (cp >= 0x20000 and cp <= 0x2A6DF) or # + (cp >= 0x2A700 and cp <= 0x2B73F) or # + (cp >= 0x2B740 and cp <= 0x2B81F) or # + (cp >= 0x2B820 and cp <= 0x2CEAF) or + (cp >= 0xF900 and cp <= 0xFAFF) or # + (cp >= 0x2F800 and cp <= 0x2FA1F)): # + return True + + return False + + def _clean_text(self, text): + """Performs invalid character removal and whitespace cleanup on text.""" + output = [] + for char in text: + cp = ord(char) + if cp == 0 or cp == 0xfffd or _is_control(char): + continue + if _is_whitespace(char): + output.append(" ") + else: + output.append(char) + return "".join(output) + + +class WordpieceTokenizer(object): + """Runs WordPiece tokenization.""" + + def __init__(self, vocab, unk_token, max_input_chars_per_word=100): + self.vocab = vocab + self.unk_token = unk_token + self.max_input_chars_per_word = max_input_chars_per_word + + def tokenize(self, text): + """Tokenizes a piece of text into its word pieces. + + This uses a greedy longest-match-first algorithm to perform tokenization + using the given vocabulary. + + For example: + input = "unaffable" + output = ["un", "##aff", "##able"] + + Args: + text: A single token or whitespace separated tokens. This should have + already been passed through `BasicTokenizer`. + + Returns: + A list of wordpiece tokens. + """ + + output_tokens = [] + for token in whitespace_tokenize(text): + chars = list(token) + if len(chars) > self.max_input_chars_per_word: + output_tokens.append(self.unk_token) + continue + + is_bad = False + start = 0 + sub_tokens = [] + while start < len(chars): + end = len(chars) + cur_substr = None + while start < end: + substr = "".join(chars[start:end]) + if start > 0: + substr = "##" + substr + if substr in self.vocab: + cur_substr = substr + break + end -= 1 + if cur_substr is None: + is_bad = True + break + sub_tokens.append(cur_substr) + start = end + + if is_bad: + output_tokens.append(self.unk_token) + else: + output_tokens.extend(sub_tokens) + return output_tokens + + +def _is_whitespace(char): + """Checks whether `chars` is a whitespace character.""" + # \t, \n, and \r are technically contorl characters but we treat them + # as whitespace since they are generally considered as such. + if char == " " or char == "\t" or char == "\n" or char == "\r": + return True + cat = unicodedata.category(char) + if cat == "Zs": + return True + return False + + +def _is_control(char): + """Checks whether `chars` is a control character.""" + # These are technically control characters but we count them as whitespace + # characters. + if char == "\t" or char == "\n" or char == "\r": + return False + cat = unicodedata.category(char) + if cat.startswith("C"): + return True + return False + + +def _is_punctuation(char): + """Checks whether `chars` is a punctuation character.""" + cp = ord(char) + # We treat all non-letter/number ASCII as punctuation. + # Characters such as "^", "$", and "`" are not in the Unicode + # Punctuation class but we treat them as punctuation anyways, for + # consistency. + if ((cp >= 33 and cp <= 47) or (cp >= 58 and cp <= 64) or + (cp >= 91 and cp <= 96) or (cp >= 123 and cp <= 126)): + return True + cat = unicodedata.category(char) + if cat.startswith("P"): + return True + return False diff --git a/BertToSimple/BiGRU/model/tokenization_utils.py b/BertToSimple/BiGRU/model/tokenization_utils.py new file mode 100644 index 0000000..c80a764 --- /dev/null +++ b/BertToSimple/BiGRU/model/tokenization_utils.py @@ -0,0 +1,1065 @@ +# coding=utf-8 +# Copyright 2018 The Open AI Team Authors and The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Tokenization classes for OpenAI GPT.""" +from __future__ import (absolute_import, division, print_function, + unicode_literals) + +import logging +import os +import json +import six +import copy +from io import open + +from .file_utils import cached_path + +import torch + +logger = logging.getLogger(__name__) + +SPECIAL_TOKENS_MAP_FILE = 'special_tokens_map.json' +ADDED_TOKENS_FILE = 'added_tokens.json' +TOKENIZER_CONFIG_FILE = 'tokenizer_config.json' + +class PreTrainedTokenizer(object): + """ Base class for all tokenizers. + Handle all the shared methods for tokenization and special tokens as well as methods dowloading/caching/loading pretrained tokenizers as well as adding tokens to the vocabulary. + + This class also contain the added tokens in a unified way on top of all tokenizers so we don't have to handle the specific vocabulary augmentation methods of the various underlying dictionary structures (BPE, sentencepiece...). + + Class attributes (overridden by derived classes): + + - ``vocab_files_names``: a python ``dict`` with, as keys, the ``__init__`` keyword name of each vocabulary file required by the model, and as associated values, the filename for saving the associated file (string). + - ``pretrained_vocab_files_map``: a python ``dict of dict`` the high-level keys being the ``__init__`` keyword name of each vocabulary file required by the model, the low-level being the `short-cut-names` (string) of the pretrained models with, as associated values, the `url` (string) to the associated pretrained vocabulary file. + - ``max_model_input_sizes``: a python ``dict`` with, as keys, the `short-cut-names` (string) of the pretrained models, and as associated values, the maximum length of the sequence inputs of this model, or None if the model has no maximum input size. + - ``pretrained_init_configuration``: a python ``dict`` with, as keys, the `short-cut-names` (string) of the pretrained models, and as associated values, a dictionnary of specific arguments to pass to the ``__init__``method of the tokenizer class for this pretrained model when loading the tokenizer with the ``from_pretrained()`` method. + + Parameters: + + - ``bos_token``: (`Optional`) string: a beginning of sentence token. Will be associated to ``self.bos_token`` and ``self.bos_token_id`` + + - ``eos_token``: (`Optional`) string: an end of sentence token. Will be associated to ``self.eos_token`` and ``self.eos_token_id`` + + - ``unk_token``: (`Optional`) string: an unknown token. Will be associated to ``self.unk_token`` and ``self.unk_token_id`` + + - ``sep_token``: (`Optional`) string: a separation token (e.g. to separate context and query in an input sequence). Will be associated to ``self.sep_token`` and ``self.sep_token_id`` + + - ``pad_token``: (`Optional`) string: a padding token. Will be associated to ``self.pad_token`` and ``self.pad_token_id`` + + - ``cls_token``: (`Optional`) string: a classification token (e.g. to extract a summary of an input sequence leveraging self-attention along the full depth of the model). Will be associated to ``self.cls_token`` and ``self.cls_token_id`` + + - ``mask_token``: (`Optional`) string: a masking token (e.g. when training a model with masked-language modeling). Will be associated to ``self.mask_token`` and ``self.mask_token_id`` + + - ``additional_special_tokens``: (`Optional`) list: a list of additional special tokens. Adding all special tokens here ensure they won't be split by the tokenization process. Will be associated to ``self.additional_special_tokens`` and ``self.additional_special_tokens_ids`` + """ + vocab_files_names = {} + pretrained_vocab_files_map = {} + pretrained_init_configuration = {} + max_model_input_sizes = {} + + SPECIAL_TOKENS_ATTRIBUTES = ["bos_token", "eos_token", "unk_token", "sep_token", + "pad_token", "cls_token", "mask_token", + "additional_special_tokens"] + + @property + def bos_token(self): + """ Beginning of sentence token (string). Log an error if used while not having been set. """ + if self._bos_token is None: + logger.error("Using bos_token, but it is not set yet.") + return self._bos_token + + @property + def eos_token(self): + """ End of sentence token (string). Log an error if used while not having been set. """ + if self._eos_token is None: + logger.error("Using eos_token, but it is not set yet.") + return self._eos_token + + @property + def unk_token(self): + """ Unknown token (string). Log an error if used while not having been set. """ + if self._unk_token is None: + logger.error("Using unk_token, but it is not set yet.") + return self._unk_token + + @property + def sep_token(self): + """ Separation token (string). E.g. separate context and query in an input sequence. Log an error if used while not having been set. """ + if self._sep_token is None: + logger.error("Using sep_token, but it is not set yet.") + return self._sep_token + + @property + def pad_token(self): + """ Padding token (string). Log an error if used while not having been set. """ + if self._pad_token is None: + logger.error("Using pad_token, but it is not set yet.") + return self._pad_token + + @property + def cls_token(self): + """ Classification token (string). E.g. to extract a summary of an input sequence leveraging self-attention along the full depth of the model. Log an error if used while not having been set. """ + if self._cls_token is None: + logger.error("Using cls_token, but it is not set yet.") + return self._cls_token + + @property + def mask_token(self): + """ Mask token (string). E.g. when training a model with masked-language modeling. Log an error if used while not having been set. """ + if self._mask_token is None: + logger.error("Using mask_token, but it is not set yet.") + return self._mask_token + + @property + def additional_special_tokens(self): + """ All the additional special tokens you may want to use (list of strings). Log an error if used while not having been set. """ + if self._additional_special_tokens is None: + logger.error("Using additional_special_tokens, but it is not set yet.") + return self._additional_special_tokens + + @bos_token.setter + def bos_token(self, value): + self._bos_token = value + + @eos_token.setter + def eos_token(self, value): + self._eos_token = value + + @unk_token.setter + def unk_token(self, value): + self._unk_token = value + + @sep_token.setter + def sep_token(self, value): + self._sep_token = value + + @pad_token.setter + def pad_token(self, value): + self._pad_token = value + + @cls_token.setter + def cls_token(self, value): + self._cls_token = value + + @mask_token.setter + def mask_token(self, value): + self._mask_token = value + + @additional_special_tokens.setter + def additional_special_tokens(self, value): + self._additional_special_tokens = value + + @property + def bos_token_id(self): + """ Id of the beginning of sentence token in the vocabulary. Log an error if used while not having been set. """ + return self.convert_tokens_to_ids(self.bos_token) + + @property + def eos_token_id(self): + """ Id of the end of sentence token in the vocabulary. Log an error if used while not having been set. """ + return self.convert_tokens_to_ids(self.eos_token) + + @property + def unk_token_id(self): + """ Id of the unknown token in the vocabulary. Log an error if used while not having been set. """ + return self.convert_tokens_to_ids(self.unk_token) + + @property + def sep_token_id(self): + """ Id of the separation token in the vocabulary. E.g. separate context and query in an input sequence. Log an error if used while not having been set. """ + return self.convert_tokens_to_ids(self.sep_token) + + @property + def pad_token_id(self): + """ Id of the padding token in the vocabulary. Log an error if used while not having been set. """ + return self.convert_tokens_to_ids(self.pad_token) + + @property + def cls_token_id(self): + """ Id of the classification token in the vocabulary. E.g. to extract a summary of an input sequence leveraging self-attention along the full depth of the model. Log an error if used while not having been set. """ + return self.convert_tokens_to_ids(self.cls_token) + + @property + def mask_token_id(self): + """ Id of the mask token in the vocabulary. E.g. when training a model with masked-language modeling. Log an error if used while not having been set. """ + return self.convert_tokens_to_ids(self.mask_token) + + @property + def additional_special_tokens_ids(self): + """ Ids of all the additional special tokens in the vocabulary (list of integers). Log an error if used while not having been set. """ + return self.convert_tokens_to_ids(self.additional_special_tokens) + + def __init__(self, max_len=None, **kwargs): + self._bos_token = None + self._eos_token = None + self._unk_token = None + self._sep_token = None + self._pad_token = None + self._cls_token = None + self._mask_token = None + self._additional_special_tokens = [] + + self.max_len = max_len if max_len is not None else int(1e12) + + # Added tokens + self.added_tokens_encoder = {} + self.added_tokens_decoder = {} + + # inputs and kwargs for saving and re-loading (see ``from_pretrained`` and ``save_pretrained``) + self.init_inputs = () + self.init_kwargs = {} + + for key, value in kwargs.items(): + if key in self.SPECIAL_TOKENS_ATTRIBUTES: + if key == 'additional_special_tokens': + assert isinstance(value, (list, tuple)) and all(isinstance(t, str) or (six.PY2 and isinstance(t, unicode)) for t in value) + else: + assert isinstance(value, str) or (six.PY2 and isinstance(value, unicode)) + setattr(self, key, value) + + + @classmethod + def from_pretrained(cls, *inputs, **kwargs): + r""" + Instantiate a :class:`~transformers.PreTrainedTokenizer` (or a derived class) from a predefined tokenizer. + + Args: + pretrained_model_name_or_path: either: + + - a string with the `shortcut name` of a predefined tokenizer to load from cache or download, e.g.: ``bert-base-uncased``. + - a path to a `directory` containing vocabulary files required by the tokenizer, for instance saved using the :func:`~transformers.PreTrainedTokenizer.save_pretrained` method, e.g.: ``./my_model_directory/``. + - (not applicable to all derived classes) a path or url to a single saved vocabulary file if and only if the tokenizer only requires a single vocabulary file (e.g. Bert, XLNet), e.g.: ``./my_model_directory/vocab.txt``. + + cache_dir: (`optional`) string: + Path to a directory in which a downloaded predefined tokenizer vocabulary files should be cached if the standard cache should not be used. + + force_download: (`optional`) boolean, default False: + Force to (re-)download the vocabulary files and override the cached versions if they exists. + + proxies: (`optional`) dict, default None: + A dictionary of proxy servers to use by protocol or endpoint, e.g.: {'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}. + The proxies are used on each request. + + inputs: (`optional`) positional arguments: will be passed to the Tokenizer ``__init__`` method. + + kwargs: (`optional`) keyword arguments: will be passed to the Tokenizer ``__init__`` method. Can be used to set special tokens like ``bos_token``, ``eos_token``, ``unk_token``, ``sep_token``, ``pad_token``, ``cls_token``, ``mask_token``, ``additional_special_tokens``. See parameters in the doc string of :class:`~transformers.PreTrainedTokenizer` for details. + + Examples:: + + # We can't instantiate directly the base class `PreTrainedTokenizer` so let's show our examples on a derived class: BertTokenizer + + # Download vocabulary from S3 and cache. + tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') + + # If vocabulary files are in a directory (e.g. tokenizer was saved using `save_pretrained('./test/saved_model/')`) + tokenizer = BertTokenizer.from_pretrained('./test/saved_model/') + + # If the tokenizer uses a single vocabulary file, you can point directly to this file + tokenizer = BertTokenizer.from_pretrained('./test/saved_model/my_vocab.txt') + + # You can link tokens to special vocabulary when instantiating + tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', unk_token='') + # You should be sure '' is in the vocabulary when doing that. + # Otherwise use tokenizer.add_special_tokens({'unk_token': ''}) instead) + assert tokenizer.unk_token == '' + + """ + return cls._from_pretrained(*inputs, **kwargs) + + + @classmethod + def _from_pretrained(cls, pretrained_model_name_or_path, *init_inputs, **kwargs): + cache_dir = kwargs.pop('cache_dir', None) + force_download = kwargs.pop('force_download', False) + proxies = kwargs.pop('proxies', None) + + s3_models = list(cls.max_model_input_sizes.keys()) + vocab_files = {} + init_configuration = {} + if pretrained_model_name_or_path in s3_models: + # Get the vocabulary from AWS S3 bucket + for file_id, map_list in cls.pretrained_vocab_files_map.items(): + vocab_files[file_id] = map_list[pretrained_model_name_or_path] + if cls.pretrained_init_configuration and pretrained_model_name_or_path in cls.pretrained_init_configuration: + init_configuration = cls.pretrained_init_configuration[pretrained_model_name_or_path] + else: + # Get the vocabulary from local files + logger.info( + "Model name '{}' not found in model shortcut name list ({}). " + "Assuming '{}' is a path or url to a directory containing tokenizer files.".format( + pretrained_model_name_or_path, ', '.join(s3_models), + pretrained_model_name_or_path)) + + # Look for the tokenizer main vocabulary files + for file_id, file_name in cls.vocab_files_names.items(): + if os.path.isdir(pretrained_model_name_or_path): + # If a directory is provided we look for the standard filenames + full_file_name = os.path.join(pretrained_model_name_or_path, file_name) + else: + # If a path to a file is provided we use it (will only work for non-BPE tokenizer using a single vocabulary file) + full_file_name = pretrained_model_name_or_path + if not os.path.exists(full_file_name): + logger.info("Didn't find file {}. We won't load it.".format(full_file_name)) + full_file_name = None + vocab_files[file_id] = full_file_name + + # Look for the additional tokens files + additional_files_names = {'added_tokens_file': ADDED_TOKENS_FILE, + 'special_tokens_map_file': SPECIAL_TOKENS_MAP_FILE, + 'tokenizer_config_file': TOKENIZER_CONFIG_FILE, + } + + # If a path to a file was provided, get the parent directory + saved_directory = pretrained_model_name_or_path + if os.path.exists(saved_directory) and not os.path.isdir(saved_directory): + saved_directory = os.path.dirname(saved_directory) + + for file_id, file_name in additional_files_names.items(): + full_file_name = os.path.join(saved_directory, file_name) + if not os.path.exists(full_file_name): + logger.info("Didn't find file {}. We won't load it.".format(full_file_name)) + full_file_name = None + vocab_files[file_id] = full_file_name + + if all(full_file_name is None for full_file_name in vocab_files.values()): + raise EnvironmentError( + "Model name '{}' was not found in tokenizers model name list ({}). " + "We assumed '{}' was a path or url to a directory containing vocabulary files " + "named {} but couldn't find such vocabulary files at this path or url.".format( + pretrained_model_name_or_path, ', '.join(s3_models), + pretrained_model_name_or_path, + list(cls.vocab_files_names.values()))) + + # Get files from url, cache, or disk depending on the case + try: + resolved_vocab_files = {} + for file_id, file_path in vocab_files.items(): + if file_path is None: + resolved_vocab_files[file_id] = None + else: + resolved_vocab_files[file_id] = cached_path(file_path, cache_dir=cache_dir, force_download=force_download, proxies=proxies) + except EnvironmentError: + if pretrained_model_name_or_path in s3_models: + msg = "Couldn't reach server at '{}' to download vocabulary files." + else: + msg = "Model name '{}' was not found in tokenizers model name list ({}). " \ + "We assumed '{}' was a path or url to a directory containing vocabulary files " \ + "named {}, but couldn't find such vocabulary files at this path or url.".format( + pretrained_model_name_or_path, ', '.join(s3_models), + pretrained_model_name_or_path, + list(cls.vocab_files_names.values())) + + raise EnvironmentError(msg) + + for file_id, file_path in vocab_files.items(): + if file_path == resolved_vocab_files[file_id]: + logger.info("loading file {}".format(file_path)) + else: + logger.info("loading file {} from cache at {}".format( + file_path, resolved_vocab_files[file_id])) + + # Prepare tokenizer initialization kwargs + # Did we saved some inputs and kwargs to reload ? + tokenizer_config_file = resolved_vocab_files.pop('tokenizer_config_file', None) + if tokenizer_config_file is not None: + init_kwargs = json.load(open(tokenizer_config_file, encoding="utf-8")) + saved_init_inputs = init_kwargs.pop('init_inputs', ()) + if not init_inputs: + init_inputs = saved_init_inputs + else: + init_kwargs = init_configuration + + # Update with newly provided kwargs + init_kwargs.update(kwargs) + + # Set max length if needed + if pretrained_model_name_or_path in cls.max_model_input_sizes: + # if we're using a pretrained model, ensure the tokenizer + # wont index sequences longer than the number of positional embeddings + max_len = cls.max_model_input_sizes[pretrained_model_name_or_path] + if max_len is not None and isinstance(max_len, (int, float)): + init_kwargs['max_len'] = min(init_kwargs.get('max_len', int(1e12)), max_len) + + # Merge resolved_vocab_files arguments in init_kwargs. + added_tokens_file = resolved_vocab_files.pop('added_tokens_file', None) + special_tokens_map_file = resolved_vocab_files.pop('special_tokens_map_file', None) + for args_name, file_path in resolved_vocab_files.items(): + if args_name not in init_kwargs: + init_kwargs[args_name] = file_path + if special_tokens_map_file is not None: + special_tokens_map = json.load(open(special_tokens_map_file, encoding="utf-8")) + for key, value in special_tokens_map.items(): + if key not in init_kwargs: + init_kwargs[key] = value + + # Instantiate tokenizer. + tokenizer = cls(*init_inputs, **init_kwargs) + + # Save inputs and kwargs for saving and re-loading with ``save_pretrained`` + tokenizer.init_inputs = init_inputs + tokenizer.init_kwargs = init_kwargs + + # Add supplementary tokens. + if added_tokens_file is not None: + added_tok_encoder = json.load(open(added_tokens_file, encoding="utf-8")) + added_tok_decoder = {v:k for k, v in added_tok_encoder.items()} + tokenizer.added_tokens_encoder.update(added_tok_encoder) + tokenizer.added_tokens_decoder.update(added_tok_decoder) + + return tokenizer + + + def save_pretrained(self, save_directory): + """ Save the tokenizer vocabulary files together with: + - added tokens, + - special-tokens-to-class-attributes-mapping, + - tokenizer instantiation positional and keywords inputs (e.g. do_lower_case for Bert). + + This won't save modifications other than (added tokens and special token mapping) you may have + applied to the tokenizer after the instantiation (e.g. modifying tokenizer.do_lower_case after creation). + + This method make sure the full tokenizer can then be re-loaded using the :func:`~transformers.PreTrainedTokenizer.from_pretrained` class method. + """ + if not os.path.isdir(save_directory): + logger.error("Saving directory ({}) should be a directory".format(save_directory)) + return + + special_tokens_map_file = os.path.join(save_directory, SPECIAL_TOKENS_MAP_FILE) + added_tokens_file = os.path.join(save_directory, ADDED_TOKENS_FILE) + tokenizer_config_file = os.path.join(save_directory, TOKENIZER_CONFIG_FILE) + + tokenizer_config = copy.deepcopy(self.init_kwargs) + tokenizer_config['init_inputs'] = copy.deepcopy(self.init_inputs) + for file_id in self.vocab_files_names.keys(): + tokenizer_config.pop(file_id, None) + + with open(tokenizer_config_file, 'w', encoding='utf-8') as f: + f.write(json.dumps(tokenizer_config, ensure_ascii=False)) + + with open(special_tokens_map_file, 'w', encoding='utf-8') as f: + f.write(json.dumps(self.special_tokens_map, ensure_ascii=False)) + + with open(added_tokens_file, 'w', encoding='utf-8') as f: + if self.added_tokens_encoder: + out_str = json.dumps(self.added_tokens_encoder, ensure_ascii=False) + else: + out_str = u"{}" + f.write(out_str) + + vocab_files = self.save_vocabulary(save_directory) + + return vocab_files + (special_tokens_map_file, added_tokens_file) + + + def save_vocabulary(self, save_directory): + """ Save the tokenizer vocabulary to a directory. This method does *NOT* save added tokens + and special token mappings. + + Please use :func:`~transformers.PreTrainedTokenizer.save_pretrained` `()` to save the full Tokenizer state if you want to reload it using the :func:`~transformers.PreTrainedTokenizer.from_pretrained` class method. + """ + raise NotImplementedError + + + def vocab_size(self): + """ Size of the base vocabulary (without the added tokens) """ + raise NotImplementedError + + + def __len__(self): + """ Size of the full vocabulary with the added tokens """ + return self.vocab_size + len(self.added_tokens_encoder) + + + def add_tokens(self, new_tokens): + """ + Add a list of new tokens to the tokenizer class. If the new tokens are not in the + vocabulary, they are added to it with indices starting from length of the current vocabulary. + + Args: + new_tokens: list of string. Each string is a token to add. Tokens are only added if they are not already in the vocabulary (tested by checking if the tokenizer assign the index of the ``unk_token`` to them). + + Returns: + Number of tokens added to the vocabulary. + + Examples:: + + # Let's see how to increase the vocabulary of Bert model and tokenizer + tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') + model = BertModel.from_pretrained('bert-base-uncased') + + num_added_toks = tokenizer.add_tokens(['new_tok1', 'my_new-tok2']) + print('We have added', num_added_toks, 'tokens') + model.resize_token_embeddings(len(tokenizer)) # Notice: resize_token_embeddings expect to receive the full size of the new vocabulary, i.e. the length of the tokenizer. + """ + if not new_tokens: + return 0 + + to_add_tokens = [] + for token in new_tokens: + assert isinstance(token, str) or (six.PY2 and isinstance(token, unicode)) + if token != self.unk_token and \ + self.convert_tokens_to_ids(token) == self.convert_tokens_to_ids(self.unk_token) and \ + token not in to_add_tokens: + to_add_tokens.append(token) + logger.info("Adding %s to the vocabulary", token) + + added_tok_encoder = dict((tok, len(self) + i) for i, tok in enumerate(to_add_tokens)) + added_tok_decoder = {v:k for k, v in added_tok_encoder.items()} + self.added_tokens_encoder.update(added_tok_encoder) + self.added_tokens_decoder.update(added_tok_decoder) + + return len(to_add_tokens) + + def num_added_tokens(self, pair=False): + """ + Returns the number of added tokens when encoding a sequence with special tokens. + + Note: + This encodes inputs and checks the number of added tokens, and is therefore not efficient. Do not put this + inside your training loop. + + Args: + pair: Returns the number of added tokens in the case of a sequence pair if set to True, returns the + number of added tokens in the case of a single sequence if set to False. + + Returns: + Number of tokens added to sequences + """ + token_ids_0 = [] + token_ids_1 = [] + return len(self.build_inputs_with_special_tokens(token_ids_0, token_ids_1 if pair else None)) + + def add_special_tokens(self, special_tokens_dict): + """ + Add a dictionary of special tokens (eos, pad, cls...) to the encoder and link them + to class attributes. If special tokens are NOT in the vocabulary, they are added + to it (indexed starting from the last index of the current vocabulary). + + Using `add_special_tokens` will ensure your special tokens can be used in several ways: + + - special tokens are carefully handled by the tokenizer (they are never split) + - you can easily refer to special tokens using tokenizer class attributes like `tokenizer.cls_token`. This makes it easy to develop model-agnostic training and fine-tuning scripts. + + When possible, special tokens are already registered for provided pretrained models (ex: BertTokenizer cls_token is already registered to be '[CLS]' and XLM's one is also registered to be '') + + Args: + special_tokens_dict: dict of string. Keys should be in the list of predefined special attributes: + [``bos_token``, ``eos_token``, ``unk_token``, ``sep_token``, ``pad_token``, ``cls_token``, ``mask_token``, + ``additional_special_tokens``]. + + Tokens are only added if they are not already in the vocabulary (tested by checking if the tokenizer assign the index of the ``unk_token`` to them). + + Returns: + Number of tokens added to the vocabulary. + + Examples:: + + # Let's see how to add a new classification token to GPT-2 + tokenizer = GPT2Tokenizer.from_pretrained('gpt2') + model = GPT2Model.from_pretrained('gpt2') + + special_tokens_dict = {'cls_token': ''} + + num_added_toks = tokenizer.add_special_tokens(special_tokens_dict) + print('We have added', num_added_toks, 'tokens') + model.resize_token_embeddings(len(tokenizer)) # Notice: resize_token_embeddings expect to receive the full size of the new vocabulary, i.e. the length of the tokenizer. + + assert tokenizer.cls_token == '' + """ + if not special_tokens_dict: + return 0 + + added_tokens = 0 + for key, value in special_tokens_dict.items(): + assert key in self.SPECIAL_TOKENS_ATTRIBUTES + if key == 'additional_special_tokens': + assert isinstance(value, (list, tuple)) and all(isinstance(t, str) or (six.PY2 and isinstance(t, unicode)) for t in value) + added_tokens += self.add_tokens(value) + else: + assert isinstance(value, str) or (six.PY2 and isinstance(value, unicode)) + added_tokens += self.add_tokens([value]) + logger.info("Assigning %s to the %s key of the tokenizer", value, key) + setattr(self, key, value) + + return added_tokens + + def tokenize(self, text, **kwargs): + """ Converts a string in a sequence of tokens (string), using the tokenizer. + Split in words for word-based vocabulary or sub-words for sub-word-based + vocabularies (BPE/SentencePieces/WordPieces). + + Take care of added tokens. + """ + def split_on_token(tok, text): + result = [] + split_text = text.split(tok) + for i, sub_text in enumerate(split_text): + sub_text = sub_text.strip() + if i == 0 and not sub_text: + result += [tok] + elif i == len(split_text) - 1: + if sub_text: + result += [sub_text] + else: + pass + else: + if sub_text: + result += [sub_text] + result += [tok] + return result + + def split_on_tokens(tok_list, text): + if not text: + return [] + if not tok_list: + return self._tokenize(text, **kwargs) + + tokenized_text = [] + text_list = [text] + for tok in tok_list: + tokenized_text = [] + for sub_text in text_list: + if sub_text not in self.added_tokens_encoder \ + and sub_text not in self.all_special_tokens: + tokenized_text += split_on_token(tok, sub_text) + else: + tokenized_text += [sub_text] + text_list = tokenized_text + + return sum((self._tokenize(token, **kwargs) if token not \ + in self.added_tokens_encoder and token not in self.all_special_tokens \ + else [token] for token in tokenized_text), []) + + added_tokens = list(self.added_tokens_encoder.keys()) + self.all_special_tokens + tokenized_text = split_on_tokens(added_tokens, text) + return tokenized_text + + def _tokenize(self, text, **kwargs): + """ Converts a string in a sequence of tokens (string), using the tokenizer. + Split in words for word-based vocabulary or sub-words for sub-word-based + vocabularies (BPE/SentencePieces/WordPieces). + + Do NOT take care of added tokens. + """ + raise NotImplementedError + + def convert_tokens_to_ids(self, tokens): + """ Converts a single token, or a sequence of tokens, (str/unicode) in a single integer id + (resp. a sequence of ids), using the vocabulary. + """ + if tokens is None: + return None + + if isinstance(tokens, str) or (six.PY2 and isinstance(tokens, unicode)): + return self._convert_token_to_id_with_added_voc(tokens) + + ids = [] + for token in tokens: + ids.append(self._convert_token_to_id_with_added_voc(token)) + if len(ids) > self.max_len: + logger.warning("Token indices sequence length is longer than the specified maximum sequence length " + "for this model ({} > {}). Running this sequence through the model will result in " + "indexing errors".format(len(ids), self.max_len)) + return ids + + def _convert_token_to_id_with_added_voc(self, token): + if token is None: + return None + + if token in self.added_tokens_encoder: + return self.added_tokens_encoder[token] + return self._convert_token_to_id(token) + + def _convert_token_to_id(self, token): + raise NotImplementedError + + def encode(self, + text, + text_pair=None, + add_special_tokens=False, + max_length=None, + stride=0, + truncation_strategy='longest_first', + return_tensors=None, + **kwargs): + """ + Converts a string in a sequence of ids (integer), using the tokenizer and vocabulary. + + Same as doing ``self.convert_tokens_to_ids(self.tokenize(text))``. + + Args: + text: The first sequence to be encoded. This can be a string, a list of strings (tokenized string using + the `tokenize` method) or a list of integers (tokenized string ids using the `convert_tokens_to_ids` + method) + text_pair: Optional second sequence to be encoded. This can be a string, a list of strings (tokenized + string using the `tokenize` method) or a list of integers (tokenized string ids using the + `convert_tokens_to_ids` method) + add_special_tokens: if set to ``True``, the sequences will be encoded with the special tokens relative + to their model. + max_length: if set to a number, will limit the total sequence returned so that it has a maximum length. + If there are overflowing tokens, those will be added to the returned dictionary + stride: if set to a number along with max_length, the overflowing tokens returned will contain some tokens + from the main sequence returned. The value of this argument defines the number of additional tokens. + truncation_strategy: string selected in the following options: + - 'longest_first' (default) Iteratively reduce the inputs sequence until the input is under max_length + starting from the longest one at each token (when there is a pair of input sequences) + - 'only_first': Only truncate the first sequence + - 'only_second': Only truncate the second sequence + - 'do_not_truncate': Does not truncate (raise an error if the input sequence is longer than max_length) + return_tensors: (optional) can be set to 'tf' or 'pt' to return respectively TensorFlow tf.constant + or PyTorch torch.Tensor instead of a list of python integers. + **kwargs: passed to the `self.tokenize()` method + """ + encoded_inputs = self.encode_plus(text, + text_pair=text_pair, + max_length=max_length, + add_special_tokens=add_special_tokens, + stride=stride, + truncation_strategy=truncation_strategy, + return_tensors=return_tensors, + **kwargs) + + return encoded_inputs["input_ids"] + + def encode_plus(self, + text, + text_pair=None, + add_special_tokens=False, + max_length=None, + stride=0, + truncation_strategy='longest_first', + return_tensors=None, + **kwargs): + """ + Returns a dictionary containing the encoded sequence or sequence pair and additional informations: + the mask for sequence classification and the overflowing elements if a ``max_length`` is specified. + + Args: + text: The first sequence to be encoded. This can be a string, a list of strings (tokenized string using + the `tokenize` method) or a list of integers (tokenized string ids using the `convert_tokens_to_ids` + method) + text_pair: Optional second sequence to be encoded. This can be a string, a list of strings (tokenized + string using the `tokenize` method) or a list of integers (tokenized string ids using the + `convert_tokens_to_ids` method) + add_special_tokens: if set to ``True``, the sequences will be encoded with the special tokens relative + to their model. + max_length: if set to a number, will limit the total sequence returned so that it has a maximum length. + If there are overflowing tokens, those will be added to the returned dictionary + stride: if set to a number along with max_length, the overflowing tokens returned will contain some tokens + from the main sequence returned. The value of this argument defines the number of additional tokens. + truncation_strategy: string selected in the following options: + - 'longest_first' (default) Iteratively reduce the inputs sequence until the input is under max_length + starting from the longest one at each token (when there is a pair of input sequences) + - 'only_first': Only truncate the first sequence + - 'only_second': Only truncate the second sequence + - 'do_not_truncate': Does not truncate (raise an error if the input sequence is longer than max_length) + return_tensors: (optional) can be set to 'tf' or 'pt' to return respectively TensorFlow tf.constant + or PyTorch torch.Tensor instead of a list of python integers. + **kwargs: passed to the `self.tokenize()` method + """ + + def get_input_ids(text): + if isinstance(text, six.string_types): + return self.convert_tokens_to_ids(self.tokenize(text, **kwargs)) + elif isinstance(text, (list, tuple)) and len(text) > 0 and isinstance(text[0], six.string_types): + return self.convert_tokens_to_ids(text) + elif isinstance(text, (list, tuple)) and len(text) > 0 and isinstance(text[0], int): + return text + else: + raise ValueError("Input is not valid. Should be a string, a list/tuple of strings or a list/tuple of integers.") + + first_ids = get_input_ids(text) + second_ids = get_input_ids(text_pair) if text_pair is not None else None + + return self.prepare_for_model(first_ids, + pair_ids=second_ids, + max_length=max_length, + add_special_tokens=add_special_tokens, + stride=stride, + truncation_strategy=truncation_strategy, + return_tensors=return_tensors) + + def prepare_for_model(self, ids, pair_ids=None, max_length=None, add_special_tokens=False, stride=0, + truncation_strategy='longest_first', return_tensors=None): + """ + Prepares a sequence of input id, or a pair of sequences of inputs ids so that it can be used by the model. + It adds special tokens, truncates + sequences if overflowing while taking into account the special tokens and manages a window stride for + overflowing tokens + + Args: + ids: list of tokenized input ids. Can be obtained from a string by chaining the + `tokenize` and `convert_tokens_to_ids` methods. + pair_ids: Optional second list of input ids. Can be obtained from a string by chaining the + `tokenize` and `convert_tokens_to_ids` methods. + max_length: maximum length of the returned list. Will truncate by taking into account the special tokens. + add_special_tokens: if set to ``True``, the sequences will be encoded with the special tokens relative + to their model. + stride: window stride for overflowing tokens. Can be useful for edge effect removal when using sequential + list of inputs. + truncation_strategy: string selected in the following options: + - 'longest_first' (default) Iteratively reduce the inputs sequence until the input is under max_length + starting from the longest one at each token (when there is a pair of input sequences) + - 'only_first': Only truncate the first sequence + - 'only_second': Only truncate the second sequence + - 'do_not_truncate': Does not truncate (raise an error if the input sequence is longer than max_length) + return_tensors: (optional) can be set to 'tf' or 'pt' to return respectively TensorFlow tf.constant + or PyTorch torch.Tensor instead of a list of python integers. + + Return: + A Dictionary of shape:: + + { + input_ids: list[int], + overflowing_tokens: list[int] if a ``max_length`` is specified, else None + special_tokens_mask: list[int] if ``add_special_tokens`` if set to ``True`` + } + + With the fields: + ``input_ids``: list of tokens to be fed to a model + + ``overflowing_tokens``: list of overflowing tokens if a max length is specified. + + ``special_tokens_mask``: if adding special tokens, this is a list of [0, 1], with 0 specifying special added + tokens and 1 specifying sequence tokens. + """ + pair = bool(pair_ids is not None) + len_ids = len(ids) + len_pair_ids = len(pair_ids) if pair else 0 + + encoded_inputs = {} + total_len = len_ids + len_pair_ids + (self.num_added_tokens(pair=pair) if add_special_tokens else 0) + if max_length and total_len > max_length: + ids, pair_ids, overflowing_tokens = self.truncate_sequences(ids, pair_ids=pair_ids, + num_tokens_to_remove=total_len-max_length, + truncation_strategy=truncation_strategy, + stride=stride) + encoded_inputs["overflowing_tokens"] = overflowing_tokens + encoded_inputs["num_truncated_tokens"] = total_len - max_length + + if add_special_tokens: + sequence = self.build_inputs_with_special_tokens(ids, pair_ids) + token_type_ids = self.create_token_type_ids_from_sequences(ids, pair_ids) + encoded_inputs["special_tokens_mask"] = self.get_special_tokens_mask(ids, pair_ids) + else: + sequence = ids + pair_ids if pair else ids + token_type_ids = [0] * len(ids) + ([1] * len(pair_ids) if pair else []) + + if return_tensors == 'tf' and is_tf_available(): + sequence = tf.constant([sequence]) + token_type_ids = tf.constant([token_type_ids]) + elif return_tensors == 'pt' and is_torch_available(): + sequence = torch.tensor([sequence]) + token_type_ids = torch.tensor([token_type_ids]) + elif return_tensors is not None: + logger.warning("Unable to convert output to tensors format {}, PyTorch or TensorFlow is not available.".format(return_tensors)) + + encoded_inputs["input_ids"] = sequence + encoded_inputs["token_type_ids"] = token_type_ids + + if max_length and len(encoded_inputs["input_ids"]) > max_length: + encoded_inputs["input_ids"] = encoded_inputs["input_ids"][:max_length] + encoded_inputs["token_type_ids"] = encoded_inputs["token_type_ids"][:max_length] + encoded_inputs["special_tokens_mask"] = encoded_inputs["special_tokens_mask"][:max_length] + + return encoded_inputs + + def truncate_sequences(self, ids, pair_ids=None, num_tokens_to_remove=0, truncation_strategy='longest_first', stride=0): + """Truncates a sequence pair in place to the maximum length. + truncation_strategy: string selected in the following options: + - 'longest_first' (default) Iteratively reduce the inputs sequence until the input is under max_length + starting from the longest one at each token (when there is a pair of input sequences). + Overflowing tokens only contains overflow from the first sequence. + - 'only_first': Only truncate the first sequence. raise an error if the first sequence is shorter or equal to than num_tokens_to_remove. + - 'only_second': Only truncate the second sequence + - 'do_not_truncate': Does not truncate (raise an error if the input sequence is longer than max_length) + """ + if num_tokens_to_remove <= 0: + return ids, pair_ids, [] + + if truncation_strategy == 'longest_first': + overflowing_tokens = [] + for _ in range(num_tokens_to_remove): + if pair_ids is None or len(ids) > len(pair_ids): + overflowing_tokens = [ids[-1]] + overflowing_tokens + ids = ids[:-1] + else: + pair_ids = pair_ids[:-1] + window_len = min(len(ids), stride) + if window_len > 0: + overflowing_tokens = ids[-window_len:] + overflowing_tokens + elif truncation_strategy == 'only_first': + assert len(ids) > num_tokens_to_remove + window_len = min(len(ids), stride + num_tokens_to_remove) + overflowing_tokens = ids[-window_len:] + ids = ids[:-num_tokens_to_remove] + elif truncation_strategy == 'only_second': + assert pair_ids is not None and len(pair_ids) > num_tokens_to_remove + window_len = min(len(pair_ids), stride + num_tokens_to_remove) + overflowing_tokens = pair_ids[-window_len:] + pair_ids = pair_ids[:-num_tokens_to_remove] + elif truncation_strategy == 'do_not_truncate': + raise ValueError("Input sequence are too long for max_length. Please select a truncation strategy.") + else: + raise ValueError("Truncation_strategy should be selected in ['longest_first', 'only_first', 'only_second', 'do_not_truncate']") + return (ids, pair_ids, overflowing_tokens) + + def create_token_type_ids_from_sequences(self, token_ids_0, token_ids_1=None): + logger.warning("This tokenizer does not make use of special tokens.") + if token_ids_1 is None: + return len(token_ids_0) * [0] + return [0] * len(token_ids_0) + [1] * len(token_ids_1) + + def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None): + """ + Build model inputs from a sequence or a pair of sequence for sequence classification tasks + by concatenating and adding special tokens. + A RoBERTa sequence has the following format: + single sequence: X + pair of sequences: A B + """ + logger.warning("This tokenizer does not make use of special tokens. Input is returned with no modification.") + if token_ids_1 is None: + return token_ids_0 + return token_ids_0 + token_ids_1 + + def get_special_tokens_mask(self, token_ids_0, token_ids_1=None, already_has_special_tokens=False): + """ + Retrieves sequence ids from a token list that has no special tokens added. This method is called when adding + special tokens using the tokenizer ``prepare_for_model`` or ``encode_plus`` methods. + + Args: + token_ids_0: list of ids (must not contain special tokens) + token_ids_1: Optional list of ids (must not contain special tokens), necessary when fetching sequence ids + for sequence pairs + already_has_special_tokens: (default False) Set to True if the token list is already formated with + special tokens for the model + + Returns: + A list of integers in the range [0, 1]: 0 for a special token, 1 for a sequence token. + """ + return [0] * ((len(token_ids_1) if token_ids_1 else 0) + len(token_ids_0)) + + def convert_ids_to_tokens(self, ids, skip_special_tokens=False): + """ Converts a single index or a sequence of indices (integers) in a token " + (resp.) a sequence of tokens (str/unicode), using the vocabulary and added tokens. + + Args: + skip_special_tokens: Don't decode special tokens (self.all_special_tokens). Default: False + """ + if isinstance(ids, int): + if ids in self.added_tokens_decoder: + return self.added_tokens_decoder[ids] + else: + return self._convert_id_to_token(ids) + tokens = [] + for index in ids: + if skip_special_tokens and index in self.all_special_ids: + continue + if index in self.added_tokens_decoder: + tokens.append(self.added_tokens_decoder[index]) + else: + tokens.append(self._convert_id_to_token(index)) + return tokens + + def _convert_id_to_token(self, index): + raise NotImplementedError + + def convert_tokens_to_string(self, tokens): + """ Converts a sequence of tokens (string) in a single string. + The most simple way to do it is ' '.join(self.convert_ids_to_tokens(token_ids)) + but we often want to remove sub-word tokenization artifacts at the same time. + """ + return ' '.join(self.convert_ids_to_tokens(tokens)) + + def decode(self, token_ids, skip_special_tokens=False, clean_up_tokenization_spaces=True): + """ + Converts a sequence of ids (integer) in a string, using the tokenizer and vocabulary + with options to remove special tokens and clean up tokenization spaces. + Similar to doing ``self.convert_tokens_to_string(self.convert_ids_to_tokens(token_ids))``. + + Args: + token_ids: list of tokenized input ids. Can be obtained using the `encode` or `encode_plus` methods. + skip_special_tokens: if set to True, will replace special tokens. + clean_up_tokenization_spaces: if set to True, will clean up the tokenization spaces. + """ + filtered_tokens = self.convert_ids_to_tokens(token_ids, skip_special_tokens=skip_special_tokens) + + # To avoid mixing byte-level and unicode for byte-level BPT + # we need to build string separatly for added tokens and byte-level tokens + # cf. https://github.com/huggingface/transformers/issues/1133 + sub_texts = [] + current_sub_text = [] + for token in filtered_tokens: + if skip_special_tokens and token in self.all_special_ids: + continue + if token in self.added_tokens_encoder: + if current_sub_text: + sub_texts.append(self.convert_tokens_to_string(current_sub_text)) + current_sub_text = [] + sub_texts.append(" " + token) + else: + current_sub_text.append(token) + if current_sub_text: + sub_texts.append(self.convert_tokens_to_string(current_sub_text)) + text = ''.join(sub_texts) + + if clean_up_tokenization_spaces: + clean_text = self.clean_up_tokenization(text) + return clean_text + else: + return text + + @property + def special_tokens_map(self): + """ A dictionary mapping special token class attribute (cls_token, unk_token...) to their + values ('', ''...) + """ + set_attr = {} + for attr in self.SPECIAL_TOKENS_ATTRIBUTES: + attr_value = getattr(self, "_" + attr) + if attr_value: + set_attr[attr] = attr_value + return set_attr + + @property + def all_special_tokens(self): + """ List all the special tokens ('', ''...) mapped to class attributes + (cls_token, unk_token...). + """ + all_toks = [] + set_attr = self.special_tokens_map + for attr_value in set_attr.values(): + all_toks = all_toks + (list(attr_value) if isinstance(attr_value, (list, tuple)) else [attr_value]) + all_toks = list(set(all_toks)) + return all_toks + + @property + def all_special_ids(self): + """ List the vocabulary indices of the special tokens ('', ''...) mapped to + class attributes (cls_token, unk_token...). + """ + all_toks = self.all_special_tokens + all_ids = list(self._convert_token_to_id(t) for t in all_toks) + return all_ids + + @staticmethod + def clean_up_tokenization(out_string): + """ Clean up a list of simple English tokenization artifacts like spaces before punctuations and abreviated forms. + """ + out_string = out_string.replace(' .', '.').replace(' ?', '?').replace(' !', '!').replace(' ,', ',' + ).replace(" ' ", "'").replace(" n't", "n't").replace(" 'm", "'m").replace(" do not", " don't" + ).replace(" 's", "'s").replace(" 've", "'ve").replace(" 're", "'re") + return out_string diff --git a/BertToSimple/BiGRU/my_model.py b/BertToSimple/BiGRU/my_model.py new file mode 100644 index 0000000..a25556c --- /dev/null +++ b/BertToSimple/BiGRU/my_model.py @@ -0,0 +1,152 @@ +import torch +# coding: utf-8 +import torch.nn as nn +import torch.nn.functional as F +import numpy as np +import time +from torch.nn import CrossEntropyLoss, MSELoss +from torch.nn.utils.rnn import pack_padded_sequence,pad_packed_sequence +from model.modeling_albert import AlbertConfig, AlbertForSequenceClassification +class DistillModel(nn.Module): + def __init__(self,args,config): + super(DistillModel, self).__init__() + raw_model = AlbertForSequenceClassification.from_pretrained(args.model_name_or_path, + from_tf=bool('.ckpt' in args.model_name_or_path), + config=config) + print(raw_model) + self.albert_embedding = list(raw_model.children())[0] + self.albert_embedding = list(self.albert_embedding.children())[0] + self.albert_embedding = list(self.albert_embedding.children())[0] + print(config) + self.drop = nn.Dropout(0.1) + self.gru = nn.GRU(input_size=128, hidden_size=1024, + num_layers=1, batch_first=True, bidirectional=True) + # self.output_layer = nn.Sequential(nn.Linear()) + # 只要Embedding结构 + #print(self.albert_embedding) + self.output_layer = nn.Sequential(nn.Linear(2*1024,2*1024), + nn.ReLU(True), + nn.Linear(2*1024,21)) + + + + def init_weights(self): + init_uniform = 0.1 + # baseline initial=========================================== + self.encoder.weight.data.uniform_(-init_uniform, init_uniform) + self.decoder.bias.data.zero_() + self.decoder.weight.data.uniform_(-init_uniform, init_uniform) + + + def forward(self, input_ids, attention_mask=None, token_type_ids=None, + position_ids=None, head_mask=None, labels=None): + # input_idx->[batch_size,64] + # attention_mask->[batch_size,64] + len_list = [] + device = input_ids.device + for i in range(attention_mask.size(0)): + L = torch.sum(attention_mask[i]==1).item() + len_list.append(L) + embeddings = self.albert_embedding(input_ids) + # embeddings->[batch_size,64,128] + len_list = torch.Tensor(len_list).to(device) + X = pack_padded_sequence(embeddings,len_list,batch_first=True,enforce_sorted=False) + output_and_len,hid = self.gru(X) + output,len_list = pad_packed_sequence(output_and_len,batch_first=True) + # print("t5-t4",t5-t4) + # output-> [batch_ize,seq_len,2*1024] + output_list = [] + for i in range(output.size(0)): + output_list.append(output[i,len_list[i]-1,:].unsqueeze(0)) + output = torch.cat(output_list,0) + logits = self.output_layer(output) + if labels != None: + mse_loss = MSELoss() + cross_loss = CrossEntropyLoss() + loss = 0.2*cross_loss(logits,labels[:,0].long())+0.8*mse_loss(logits,labels[:,1:]) + return (loss,logits) + else: + return (None,logits) + + def init_hidden(self,batch_size,use_gpu): + weight = next(self.parameters()).data + if use_gpu == True: + hidden_and_C = (weight.new(self.nlayers,batch_size,self.nhid).zero_().cuda(), + weight.new(self.nlayers,batch_size,self.nhid).zero_().cuda()) + else: + hidden_and_C = (weight.new(self.nlayers, batch_size, self.nhid).zero_(), + weight.new(self.nlayers, batch_size, self.nhid).zero_()) + return hidden_and_C + + + +class DistillModel1(nn.Module): + def __init__(self,args,config): + super(DistillModel1, self).__init__() + raw_model = AlbertForSequenceClassification.from_pretrained(args.model_name_or_path, + from_tf=bool('.ckpt' in args.model_name_or_path), + config=config) + self.albert_embedding = list(raw_model.children())[0] + self.albert_embedding = list(self.albert_embedding.children())[0] + self.albert_embedding = list(self.albert_embedding.children())[0] + print(config) + self.drop = nn.Dropout(0.1) + self.gru = nn.GRU(input_size=128, hidden_size=1024, + num_layers=1, batch_first=True, bidirectional=True) + # self.output_layer = nn.Sequential(nn.Linear()) + # 只要Embedding结构 + #print(self.albert_embedding) + self.output_layer = nn.Sequential(nn.Linear(2*1024,2*1024), + nn.ReLU(True), + nn.Linear(2*1024,21)) + + + + def init_weights(self): + init_uniform = 0.1 + # baseline initial=========================================== + self.encoder.weight.data.uniform_(-init_uniform, init_uniform) + self.decoder.bias.data.zero_() + self.decoder.weight.data.uniform_(-init_uniform, init_uniform) + + + def forward(self, input_ids, attention_mask=None, token_type_ids=None, + position_ids=None, head_mask=None, labels=None): + # input_idx->[batch_size,64] + # attention_mask->[batch_size,64] + len_list = [] + device = input_ids.device + for i in range(attention_mask.size(0)): + L = torch.sum(attention_mask[i]==1).item() + len_list.append(L) + embeddings = self.albert_embedding(input_ids) + # embeddings->[batch_size,64,128] + len_list = torch.Tensor(len_list).to(device) + X = pack_padded_sequence(embeddings,len_list,batch_first=True,enforce_sorted=False) + output_and_len,hid = self.gru(X) + output,len_list = pad_packed_sequence(output_and_len,batch_first=True) + # print("t5-t4",t5-t4) + # output-> [batch_ize,seq_len,2*1024] + output = self.output_layer(output) + # 取有效序列最后一位？ + output_list = [] + for i in range(output.size(0)): + output_list.append(output[i,len_list[i]-1,:].unsqueeze(0)) + logits = torch.cat(output_list,0).to(device) + if labels != None: + mse_loss = MSELoss() + cross_loss = CrossEntropyLoss() + loss = 0.2*cross_loss(logits,labels[:,0].long())+0.8*mse_loss(logits,labels[:,1:]) + return (loss,logits) + else: + return (None,logits) + + def init_hidden(self,batch_size,use_gpu): + weight = next(self.parameters()).data + if use_gpu == True: + hidden_and_C = (weight.new(self.nlayers,batch_size,self.nhid).zero_().cuda(), + weight.new(self.nlayers,batch_size,self.nhid).zero_().cuda()) + else: + hidden_and_C = (weight.new(self.nlayers, batch_size, self.nhid).zero_(), + weight.new(self.nlayers, batch_size, self.nhid).zero_()) + return hidden_and_C \ No newline at end of file diff --git a/BertToSimple/BiGRU/prepare_lm_data_mask.py b/BertToSimple/BiGRU/prepare_lm_data_mask.py new file mode 100644 index 0000000..05b7f8d --- /dev/null +++ b/BertToSimple/BiGRU/prepare_lm_data_mask.py @@ -0,0 +1,289 @@ +import os +import json +import random +import collections +from tools.common import logger, init_logger +from argparse import ArgumentParser +from tools.common import seed_everything +from model.tokenization_bert import BertTokenizer +from callback.progressbar import ProgressBar +from pathlib import Path + +MaskedLmInstance = collections.namedtuple("MaskedLmInstance", ["index", "label"]) + +def truncate_seq_pair(tokens_a, tokens_b, max_num_tokens): + """Truncates a pair of sequences to a maximum sequence length.""" + while True: + total_length = len(tokens_a) + len(tokens_b) + if total_length <= max_num_tokens: + break + trunc_tokens = tokens_a if len(tokens_a) > len(tokens_b) else tokens_b + assert len(trunc_tokens) >= 1 + # We want to sometimes truncate from the front and sometimes from the + # back to add more randomness and avoid biases. + if random.random() < 0.5: + del trunc_tokens[0] + else: + trunc_tokens.pop() + +def create_instances_from_document(all_documents, document_index, max_seq_length, short_seq_prob, + masked_lm_prob, max_predictions_per_seq, vocab_words): + """Creates `TrainingInstance`s for a single document. + This method is changed to create sentence-order prediction (SOP) followed by idea from paper of ALBERT, 2019-08-28, brightmart + """ + document = all_documents[document_index] # 得到一个文档 + + # Account for [CLS], [SEP], [SEP] + max_num_tokens = max_seq_length - 3 + + # We *usually* want to fill up the entire sequence since we are padding + # to `max_seq_length` anyways, so short sequences are generally wasted + # computation. However, we *sometimes* + # (i.e., short_seq_prob == 0.1 == 10% of the time) want to use shorter + # sequences to minimize the mismatch between pre-training and fine-tuning. + # The `target_seq_length` is just a rough target however, whereas + # `max_seq_length` is a hard limit. + target_seq_length = max_num_tokens + if random.random() < short_seq_prob: # 有一定的比例，如10%的概率，我们使用比较短的序列长度，以缓解预训练的长序列和调优阶段（可能的）短序列的不一致情况 + target_seq_length = random.randint(2, max_num_tokens) + + # We DON'T just concatenate all of the tokens from a document into a long + # sequence and choose an arbitrary split point because this would make the + # next sentence prediction task too easy. Instead, we split the input into + # segments "A" and "B" based on the actual "sentences" provided by the user + # input. + # 设法使用实际的句子，而不是任意的截断句子，从而更好的构造句子连贯性预测的任务 + instances = [] + current_chunk = [] # 当前处理的文本段，包含多个句子 + current_length = 0 + i = 0 + # print("###document:",document) # 一个document可以是一整篇文章、新闻、词条等. document:[['是', '爷', '们', '，', '就', '得', '给', '媳', '妇', '幸', '福'], ['关', '注', '【', '晨', '曦', '教', '育', '】', '，', '获', '取', '育', '儿', '的', '智', '慧', '，', '与', '孩', '子', '一', '同', '成', '长', '！'], ['方', '法', ':', '打', '开', '微', '信', '→', '添', '加', '朋', '友', '→', '搜', '号', '→', '##he', '##bc', '##x', '##jy', '##→', '关', '注', '!', '我', '是', '一', '个', '爷', '们', '，', '孝', '顺', '是', '做', '人', '的', '第', '一', '准', '则', '。'], ['甭', '管', '小', '时', '候', '怎', '么', '跟', '家', '长', '犯', '混', '蛋', '，', '长', '大', '了', '，', '就', '底', '报', '答', '父', '母', '，', '以', '后', '我', '媳', '妇', '也', '必', '须', '孝', '顺', '。'], ['我', '是', '一', '个', '爷', '们', '，', '可', '以', '花', '心', '，', '可', '以', '好', '玩', '。'], ['但', '我', '一', '定', '会', '找', '一', '个', '管', '的', '住', '我', '的', '女', '人', '，', '和', '我', '一', '起', '生', '活', '。'], ['28', '岁', '以', '前', '在', '怎', '么', '玩', '都', '行', '，', '但', '我', '最', '后', '一', '定', '会', '找', '一', '个', '勤', '俭', '持', '家', '的', '女', '人', '。'], ['我', '是', '一', '爷', '们', '，', '我', '不', '会', '让', '自', '己', '的', '女', '人', '受', '一', '点', '委', '屈', '，', '每', '次', '把', '她', '抱', '在', '怀', '里', '，', '看', '她', '洋', '溢', '着', '幸', '福', '的', '脸', '，', '我', '都', '会', '引', '以', '为', '傲', '，', '这', '特', '么', '就', '是', '我', '的', '女', '人', '。'], ['我', '是', '一', '爷', '们', '，', '干', '什', '么', '也', '不', '能', '忘', '了', '自', '己', '媳', '妇', '，', '就', '算', '和', '哥', '们', '一', '起', '喝', '酒', '，', '喝', '到', '很', '晚', '，', '也', '要', '提', '前', '打', '电', '话', '告', '诉', '她', '，', '让', '她', '早', '点', '休', '息', '。'], ['我', '是', '一', '爷', '们', '，', '我', '媳', '妇', '绝', '对', '不', '能', '抽', '烟', '，', '喝', '酒', '还', '勉', '强', '过', '得', '去', '，', '不', '过', '该', '喝', '的', '时', '候', '喝', '，', '不', '该', '喝', '的', '时', '候', '，', '少', '扯', '纳', '极', '薄', '蛋', '。'], ['我', '是', '一', '爷', '们', '，', '我', '媳', '妇', '必', '须', '听', '我', '话', '，', '在', '人', '前', '一', '定', '要', '给', '我', '面', '子', '，', '回', '家', '了', '咱', '什', '么', '都', '好', '说', '。'], ['我', '是', '一', '爷', '们', '，', '就', '算', '难', '的', '吃', '不', '上', '饭', '了', '，', '都', '不', '张', '口', '跟', '媳', '妇', '要', '一', '分', '钱', '。'], ['我', '是', '一', '爷', '们', '，', '不', '管', '上', '学', '还', '是', '上', '班', '，', '我', '都', '会', '送', '媳', '妇', '回', '家', '。'], ['我', '是', '一', '爷', '们', '，', '交', '往', '不', '到', '1', '年', '，', '绝', '对', '不', '会', '和', '媳', '妇', '提', '过', '分', '的', '要', '求', '，', '我', '会', '尊', '重', '她', '。'], ['我', '是', '一', '爷', '们', '，', '游', '戏', '永', '远', '比', '不', '上', '我', '媳', '妇', '重', '要', '，', '只', '要', '媳', '妇', '发', '话', '，', '我', '绝', '对', '唯', '命', '是', '从', '。'], ['我', '是', '一', '爷', '们', '，', '上', 'q', '绝', '对', '是', '为', '了', '等', '媳', '妇', '，', '所', '有', '暧', '昧', '的', '心', '情', '只', '为', '她', '一', '个', '女', '人', '而', '写', '，', '我', '不', '一', '定', '会', '经', '常', '写', '日', '志', '，', '可', '是', '我', '会', '告', '诉', '全', '世', '界', '，', '我', '很', '爱', '她', '。'], ['我', '是', '一', '爷', '们', '，', '不', '一', '定', '要', '经', '常', '制', '造', '浪', '漫', '、', '偶', '尔', '过', '个', '节', '日', '也', '要', '送', '束', '玫', '瑰', '花', '给', '媳', '妇', '抱', '回', '家', '。'], ['我', '是', '一', '爷', '们', '，', '手', '机', '会', '24', '小', '时', '为', '她', '开', '机', '，', '让', '她', '半', '夜', '痛', '经', '的', '时', '候', '，', '做', '恶', '梦', '的', '时', '候', '，', '随', '时', '可', '以', '联', '系', '到', '我', '。'], ['我', '是', '一', '爷', '们', '，', '我', '会', '经', '常', '带', '媳', '妇', '出', '去', '玩', '，', '她', '不', '一', '定', '要', '和', '我', '所', '有', '的', '哥', '们', '都', '认', '识', '，', '但', '见', '面', '能', '说', '的', '上', '话', '就', '行', '。'], ['我', '是', '一', '爷', '们', '，', '我', '会', '和', '媳', '妇', '的', '姐', '妹', '哥', '们', '搞', '好', '关', '系', '，', '让', '她', '们', '相', '信', '我', '一', '定', '可', '以', '给', '我', '媳', '妇', '幸', '福', '。'], ['我', '是', '一', '爷', '们', '，', '吵', '架', '后', '、', '也', '要', '主', '动', '打', '电', '话', '关', '心', '她', '，', '咱', '是', '一', '爷', '们', '，', '给', '媳', '妇', '服', '个', '软', '，', '道', '个', '歉', '怎', '么', '了', '？'], ['我', '是', '一', '爷', '们', '，', '绝', '对', '不', '会', '嫌', '弃', '自', '己', '媳', '妇', '，', '拿', '她', '和', '别', '人', '比', '，', '说', '她', '这', '不', '如', '人', '家', '，', '纳', '不', '如', '人', '家', '的', '。'], ['我', '是', '一', '爷', '们', '，', '陪', '媳', '妇', '逛', '街', '时', '，', '碰', '见', '熟', '人', '，', '无', '论', '我', '媳', '妇', '长', '的', '好', '看', '与', '否', '，', '我', '都', '会', '大', '方', '的', '介', '绍', '。'], ['谁', '让', '咱', '爷', '们', '就', '好', '这', '口', '呢', '。'], ['我', '是', '一', '爷', '们', '，', '我', '想', '我', '会', '给', '我', '媳', '妇', '最', '好', '的', '幸', '福', '。'], ['【', '我', '们', '重', '在', '分', '享', '。'], ['所', '有', '文', '字', '和', '美', '图', '，', '来', '自', '网', '络', '，', '晨', '欣', '教', '育', '整', '理', '。'], ['对', '原', '文', '作', '者', '，', '表', '示', '敬', '意', '。'], ['】', '关', '注', '晨', '曦', '教', '育', '[UNK]', '[UNK]', '晨', '曦', '教', '育', '（', '微', '信', '号', '：', 'he', '##bc', '##x', '##jy', '）', '。'], ['打', '开', '微', '信', '，', '扫', '描', '二', '维', '码', '，', '关', '注', '[UNK]', '晨', '曦', '教', '育', '[UNK]', '，', '获', '取', '更', '多', '育', '儿', '资', '源', '。'], ['点', '击', '下', '面', '订', '阅', '按', '钮', '订', '阅', '，', '会', '有', '更', '多', '惊', '喜', '哦', '！']] + while i < len(document): # 从文档的第一个位置开始，按个往下看 + segment = document[ + i] # segment是列表，代表的是按字分开的一个完整句子，如 segment=['我', '是', '一', '爷', '们', '，', '我', '想', '我', '会', '给', '我', '媳', '妇', '最', '好', '的', '幸', '福', '。'] + # segment = get_new_segment(segment) # whole word mask for chinese: 结合分词的中文的whole mask设置即在需要的地方加上“##” + current_chunk.append(segment) # 将一个独立的句子加入到当前的文本块中 + current_length += len(segment) # 累计到为止位置接触到句子的总长度 + if i == len(document) - 1 or current_length >= target_seq_length: + # 如果累计的序列长度达到了目标的长度，或当前走到了文档结尾==>构造并添加到“A[SEP]B“中的A和B中； + if current_chunk: # 如果当前块不为空 + # `a_end` is how many segments from `current_chunk` go into the `A` + # (first) sentence. + a_end = 1 + if len(current_chunk) >= 2: # 当前块，如果包含超过两个句子，取当前块的一部分作为“A[SEP]B“中的A部分 + a_end = random.randint(1, len(current_chunk) - 1) + # 将当前文本段中选取出来的前半部分，赋值给A即tokens_a + tokens_a = [] + for j in range(a_end): + tokens_a.extend(current_chunk[j]) + + # 构造“A[SEP]B“中的B部分(有一部分是正常的当前文档中的后半部;在原BERT的实现中一部分是随机的从另一个文档中选取的，） + tokens_b = [] + for j in range(a_end, len(current_chunk)): + tokens_b.extend(current_chunk[j]) + + # 有百分之50%的概率交换一下tokens_a和tokens_b的位置 + # print("tokens_a length1:",len(tokens_a)) + # print("tokens_b length1:",len(tokens_b)) # len(tokens_b) = 0 + if len(tokens_a) == 0 or len(tokens_b) == 0: continue + if random.random() < 0.5: # 交换一下tokens_a和tokens_b + is_random_next = True + temp = tokens_a + tokens_a = tokens_b + tokens_b = temp + else: + is_random_next = False + truncate_seq_pair(tokens_a, tokens_b, max_num_tokens) + assert len(tokens_a) >= 1 + assert len(tokens_b) >= 1 + + # 把tokens_a & tokens_b加入到按照bert的风格，即以[CLS]tokens_a[SEP]tokens_b[SEP]的形式，结合到一起，作为最终的tokens; 也带上segment_ids，前面部分segment_ids的值是0，后面部分的值是1. + tokens = ["[CLS]"] + tokens_a + ["[SEP]"] + tokens_b + ["[SEP]"] + # The segment IDs are 0 for the [CLS] token, the A tokens and the first [SEP] + # They are 1 for the B tokens and the final [SEP] + segment_ids = [0 for _ in range(len(tokens_a) + 2)] + [1 for _ in range(len(tokens_b) + 1)] + + # 创建masked LM的任务的数据 Creates the predictions for the masked LM objective + tokens, masked_lm_positions, masked_lm_labels = create_masked_lm_predictions( + tokens, masked_lm_prob, max_predictions_per_seq, vocab_words) + instance = { + "tokens": tokens, + "segment_ids": segment_ids, + "is_random_next": is_random_next, + "masked_lm_positions": masked_lm_positions, + "masked_lm_labels": masked_lm_labels} + instances.append(instance) + current_chunk = [] # 清空当前块 + current_length = 0 # 重置当前文本块的长度 + i += 1 # 接着文档中的内容往后看 + return instances + + +def create_masked_lm_predictions(tokens, masked_lm_prob, max_predictions_per_seq, vocab_list): + """Creates the predictions for the masked LM objective. This is mostly copied from the Google BERT repo, but + with several refactors to clean it up and remove a lot of unnecessary variables.""" + cand_indices = [] + for (i, token) in enumerate(tokens): + if token == "[CLS]" or token == "[SEP]": + continue + # Whole Word Masking means that if we mask all of the wordpieces + # corresponding to an original word. When a word has been split into + # WordPieces, the first token does not have any marker and any subsequence + # tokens are prefixed with ##. So whenever we see the ## token, we + # append it to the previous set of word indexes. + # + # Note that Whole Word Masking does *not* change the training code + # at all -- we still predict each WordPiece independently, softmaxed + # over the entire vocabulary. + cand_indices.append(i) + + num_to_mask = min(max_predictions_per_seq, max(1, int(round(len(tokens) * masked_lm_prob)))) + random.shuffle(cand_indices) + mask_indices = sorted(random.sample(cand_indices, num_to_mask)) + masked_token_labels = [] + for index in mask_indices: + # 80% of the time, replace with [MASK] + if random.random() < 0.8: + masked_token = "[MASK]" + else: + # 10% of the time, keep original + if random.random() < 0.5: + masked_token = tokens[index] + # 10% of the time, replace with random word + else: + masked_token = random.choice(vocab_list) + masked_token_labels.append(MaskedLmInstance(index=index, label=tokens[index])) + tokens[index] = masked_token + assert len(masked_token_labels) <= num_to_mask + masked_token_labels = sorted(masked_token_labels, key=lambda x: x.index) + mask_indices = [p.index for p in masked_token_labels] + masked_labels = [p.label for p in masked_token_labels] + return tokens, mask_indices, masked_labels + + +def create_training_instances(input_file, tokenizer, max_seq_len, short_seq_prob, + masked_lm_prob, max_predictions_per_seq): + """Create `TrainingInstance`s from raw text.""" + all_documents = [[]] + # Input file format: + # (1) One sentence per line. These should ideally be actual sentences, not + # entire paragraphs or arbitrary spans of text. (Because we use the + # sentence boundaries for the "next sentence prediction" task). + # (2) Blank lines between documents. Document boundaries are needed so + # that the "next sentence prediction" task doesn't span between documents. + f = open(input_file, 'r') + lines = f.readlines() + pbar = ProgressBar(n_total=len(lines), desc='read data') + for line_cnt, line in enumerate(lines): + line = line.strip() + # Empty lines are used as document delimiters + if not line: + all_documents.append([]) + tokens = tokenizer.tokenize(line) + if tokens: + all_documents[-1].append(tokens) + pbar(step=line_cnt) + print(' ') + # Remove empty documents + all_documents = [x for x in all_documents if x] + random.shuffle(all_documents) + + vocab_words = list(tokenizer.vocab.keys()) + instances = [] + pbar = ProgressBar(n_total=len(all_documents), desc='create instances') + for document_index in range(len(all_documents)): + instances.extend( + create_instances_from_document( + all_documents, document_index, max_seq_len, short_seq_prob, + masked_lm_prob, max_predictions_per_seq, vocab_words)) + pbar(step=document_index) + print(' ') + ex_idx = 0 + while ex_idx < 5: + instance = instances[ex_idx] + logger.info("-------------------------Example-----------------------") + logger.info(f"id: {ex_idx}") + logger.info(f"tokens: {' '.join([str(x) for x in instance['tokens']])}") + logger.info(f"masked_lm_labels: {' '.join([str(x) for x in instance['masked_lm_labels']])}") + logger.info(f"segment_ids: {' '.join([str(x) for x in instance['segment_ids']])}") + logger.info(f"masked_lm_positions: {' '.join([str(x) for x in instance['masked_lm_positions']])}") + logger.info(f"is_random_next : {instance['is_random_next']}") + ex_idx += 1 + random.shuffle(instances) + return instances + + +def main(): + parser = ArgumentParser() + ## Required parameters + parser.add_argument("--data_dir", default=None, type=str, required=True) + parser.add_argument("--vocab_path", default=None, type=str, required=True) + parser.add_argument("--output_dir", default=None, type=str, required=True) + + parser.add_argument('--data_name', default='albert', type=str) + parser.add_argument("--do_data", default=False, action='store_true') + parser.add_argument("--do_split", default=False, action='store_true') + parser.add_argument("--do_lower_case", default=False, action='store_true') + parser.add_argument('--seed', default=42, type=int) + parser.add_argument("--line_per_file", default=1000000000, type=int) + parser.add_argument("--file_num", type=int, default=10, + help="Number of dynamic masking to pregenerate (with different masks)") + parser.add_argument("--max_seq_len", type=int, default=128) + parser.add_argument("--short_seq_prob", type=float, default=0.1, + help="Probability of making a short sentence as a training example") + parser.add_argument("--masked_lm_prob", type=float, default=0.15, + help="Probability of masking each token for the LM task") + parser.add_argument("--max_predictions_per_seq", type=int, default=20, # 128 * 0.15 + help="Maximum number of tokens to mask in each sequence") + args = parser.parse_args() + seed_everything(args.seed) + args.data_dir = Path(args.data_dir) + if not os.path.exists(args.output_dir): + os.mkdir(args.output_dir) + init_logger(log_file=args.output_dir +"pregenerate_training_data.log") + logger.info("pregenerate training data parameters:\n %s", args) + tokenizer = BertTokenizer(vocab_file=args.vocab_path, do_lower_case=args.do_lower_case) + + # split big file + if args.do_split: + corpus_path = args.data_dir / "corpus/corpus.txt" + split_save_path = args.data_dir / "/corpus/train" + if not split_save_path.exists(): + split_save_path.mkdir(exist_ok=True) + line_per_file = args.line_per_file + command = f'split -a 4 -l {line_per_file} -d {corpus_path} {split_save_path}/shard_' + os.system(f"{command}") + + # generator train data + if args.do_data: + data_path = args.data_dir / "corpus/train" + files = sorted([f for f in data_path.parent.iterdir() if f.exists() and '.txt' in str(f)]) + for idx in range(args.file_num): + logger.info(f"pregenetate {args.data_name}_file_{idx}.json") + save_filename = data_path / f"{args.data_name}_file_{idx}.json" + num_instances = 0 + with save_filename.open('w') as fw: + for file_idx in range(len(files)): + file_path = files[file_idx] + file_examples = create_training_instances(input_file=file_path, + tokenizer=tokenizer, + max_seq_len=args.max_seq_len, + short_seq_prob=args.short_seq_prob, + masked_lm_prob=args.masked_lm_prob, + max_predictions_per_seq=args.max_predictions_per_seq) + file_examples = [json.dumps(instance) for instance in file_examples] + for instance in file_examples: + fw.write(instance + '\n') + num_instances += 1 + metrics_file = data_path / f"{args.data_name}_file_{idx}_metrics.json" + print(f"num_instances: {num_instances}") + with metrics_file.open('w') as metrics_file: + metrics = { + "num_training_examples": num_instances, + "max_seq_len": args.max_seq_len + } + metrics_file.write(json.dumps(metrics)) + + +if __name__ == '__main__': + main() diff --git a/BertToSimple/BiGRU/prepare_lm_data_ngram.py b/BertToSimple/BiGRU/prepare_lm_data_ngram.py new file mode 100644 index 0000000..7079e5f --- /dev/null +++ b/BertToSimple/BiGRU/prepare_lm_data_ngram.py @@ -0,0 +1,316 @@ +import os +import json +import random +import numpy as np +import collections +from pathlib import Path +from tools.common import logger, init_logger +from argparse import ArgumentParser +from tools.common import seed_everything +from model.tokenization_bert import BertTokenizer +from callback.progressbar import ProgressBar + +MaskedLmInstance = collections.namedtuple("MaskedLmInstance", ["index", "label"]) + +def truncate_seq_pair(tokens_a, tokens_b, max_num_tokens): + """Truncates a pair of sequences to a maximum sequence length.""" + while True: + total_length = len(tokens_a) + len(tokens_b) + if total_length <= max_num_tokens: + break + trunc_tokens = tokens_a if len(tokens_a) > len(tokens_b) else tokens_b + assert len(trunc_tokens) >= 1 + # We want to sometimes truncate from the front and sometimes from the + # back to add more randomness and avoid biases. + if random.random() < 0.5: + del trunc_tokens[0] + else: + trunc_tokens.pop() + +def create_instances_from_document(all_documents, document_index, max_seq_length, short_seq_prob, + max_ngram, masked_lm_prob, max_predictions_per_seq, vocab_words): + """Creates `TrainingInstance`s for a single document. + This method is changed to create sentence-order prediction (SOP) followed by idea from paper of ALBERT, 2019-08-28, brightmart + """ + document = all_documents[document_index] # 得到一个文档 + + # Account for [CLS], [SEP], [SEP] + max_num_tokens = max_seq_length - 3 + + # We *usually* want to fill up the entire sequence since we are padding + # to `max_seq_length` anyways, so short sequences are generally wasted + # computation. However, we *sometimes* + # (i.e., short_seq_prob == 0.1 == 10% of the time) want to use shorter + # sequences to minimize the mismatch between pre-training and fine-tuning. + # The `target_seq_length` is just a rough target however, whereas + # `max_seq_length` is a hard limit. + target_seq_length = max_num_tokens + if random.random() < short_seq_prob: # 有一定的比例，如10%的概率，我们使用比较短的序列长度，以缓解预训练的长序列和调优阶段（可能的）短序列的不一致情况 + target_seq_length = random.randint(2, max_num_tokens) + + # We DON'T just concatenate all of the tokens from a document into a long + # sequence and choose an arbitrary split point because this would make the + # next sentence prediction task too easy. Instead, we split the input into + # segments "A" and "B" based on the actual "sentences" provided by the user + # input. + # 设法使用实际的句子，而不是任意的截断句子，从而更好的构造句子连贯性预测的任务 + instances = [] + current_chunk = [] # 当前处理的文本段，包含多个句子 + current_length = 0 + i = 0 + # print("###document:",document) # 一个document可以是一整篇文章、新闻、词条等. document:[['是', '爷', '们', '，', '就', '得', '给', '媳', '妇', '幸', '福'], ['关', '注', '【', '晨', '曦', '教', '育', '】', '，', '获', '取', '育', '儿', '的', '智', '慧', '，', '与', '孩', '子', '一', '同', '成', '长', '！'], ['方', '法', ':', '打', '开', '微', '信', '→', '添', '加', '朋', '友', '→', '搜', '号', '→', '##he', '##bc', '##x', '##jy', '##→', '关', '注', '!', '我', '是', '一', '个', '爷', '们', '，', '孝', '顺', '是', '做', '人', '的', '第', '一', '准', '则', '。'], ['甭', '管', '小', '时', '候', '怎', '么', '跟', '家', '长', '犯', '混', '蛋', '，', '长', '大', '了', '，', '就', '底', '报', '答', '父', '母', '，', '以', '后', '我', '媳', '妇', '也', '必', '须', '孝', '顺', '。'], ['我', '是', '一', '个', '爷', '们', '，', '可', '以', '花', '心', '，', '可', '以', '好', '玩', '。'], ['但', '我', '一', '定', '会', '找', '一', '个', '管', '的', '住', '我', '的', '女', '人', '，', '和', '我', '一', '起', '生', '活', '。'], ['28', '岁', '以', '前', '在', '怎', '么', '玩', '都', '行', '，', '但', '我', '最', '后', '一', '定', '会', '找', '一', '个', '勤', '俭', '持', '家', '的', '女', '人', '。'], ['我', '是', '一', '爷', '们', '，', '我', '不', '会', '让', '自', '己', '的', '女', '人', '受', '一', '点', '委', '屈', '，', '每', '次', '把', '她', '抱', '在', '怀', '里', '，', '看', '她', '洋', '溢', '着', '幸', '福', '的', '脸', '，', '我', '都', '会', '引', '以', '为', '傲', '，', '这', '特', '么', '就', '是', '我', '的', '女', '人', '。'], ['我', '是', '一', '爷', '们', '，', '干', '什', '么', '也', '不', '能', '忘', '了', '自', '己', '媳', '妇', '，', '就', '算', '和', '哥', '们', '一', '起', '喝', '酒', '，', '喝', '到', '很', '晚', '，', '也', '要', '提', '前', '打', '电', '话', '告', '诉', '她', '，', '让', '她', '早', '点', '休', '息', '。'], ['我', '是', '一', '爷', '们', '，', '我', '媳', '妇', '绝', '对', '不', '能', '抽', '烟', '，', '喝', '酒', '还', '勉', '强', '过', '得', '去', '，', '不', '过', '该', '喝', '的', '时', '候', '喝', '，', '不', '该', '喝', '的', '时', '候', '，', '少', '扯', '纳', '极', '薄', '蛋', '。'], ['我', '是', '一', '爷', '们', '，', '我', '媳', '妇', '必', '须', '听', '我', '话', '，', '在', '人', '前', '一', '定', '要', '给', '我', '面', '子', '，', '回', '家', '了', '咱', '什', '么', '都', '好', '说', '。'], ['我', '是', '一', '爷', '们', '，', '就', '算', '难', '的', '吃', '不', '上', '饭', '了', '，', '都', '不', '张', '口', '跟', '媳', '妇', '要', '一', '分', '钱', '。'], ['我', '是', '一', '爷', '们', '，', '不', '管', '上', '学', '还', '是', '上', '班', '，', '我', '都', '会', '送', '媳', '妇', '回', '家', '。'], ['我', '是', '一', '爷', '们', '，', '交', '往', '不', '到', '1', '年', '，', '绝', '对', '不', '会', '和', '媳', '妇', '提', '过', '分', '的', '要', '求', '，', '我', '会', '尊', '重', '她', '。'], ['我', '是', '一', '爷', '们', '，', '游', '戏', '永', '远', '比', '不', '上', '我', '媳', '妇', '重', '要', '，', '只', '要', '媳', '妇', '发', '话', '，', '我', '绝', '对', '唯', '命', '是', '从', '。'], ['我', '是', '一', '爷', '们', '，', '上', 'q', '绝', '对', '是', '为', '了', '等', '媳', '妇', '，', '所', '有', '暧', '昧', '的', '心', '情', '只', '为', '她', '一', '个', '女', '人', '而', '写', '，', '我', '不', '一', '定', '会', '经', '常', '写', '日', '志', '，', '可', '是', '我', '会', '告', '诉', '全', '世', '界', '，', '我', '很', '爱', '她', '。'], ['我', '是', '一', '爷', '们', '，', '不', '一', '定', '要', '经', '常', '制', '造', '浪', '漫', '、', '偶', '尔', '过', '个', '节', '日', '也', '要', '送', '束', '玫', '瑰', '花', '给', '媳', '妇', '抱', '回', '家', '。'], ['我', '是', '一', '爷', '们', '，', '手', '机', '会', '24', '小', '时', '为', '她', '开', '机', '，', '让', '她', '半', '夜', '痛', '经', '的', '时', '候', '，', '做', '恶', '梦', '的', '时', '候', '，', '随', '时', '可', '以', '联', '系', '到', '我', '。'], ['我', '是', '一', '爷', '们', '，', '我', '会', '经', '常', '带', '媳', '妇', '出', '去', '玩', '，', '她', '不', '一', '定', '要', '和', '我', '所', '有', '的', '哥', '们', '都', '认', '识', '，', '但', '见', '面', '能', '说', '的', '上', '话', '就', '行', '。'], ['我', '是', '一', '爷', '们', '，', '我', '会', '和', '媳', '妇', '的', '姐', '妹', '哥', '们', '搞', '好', '关', '系', '，', '让', '她', '们', '相', '信', '我', '一', '定', '可', '以', '给', '我', '媳', '妇', '幸', '福', '。'], ['我', '是', '一', '爷', '们', '，', '吵', '架', '后', '、', '也', '要', '主', '动', '打', '电', '话', '关', '心', '她', '，', '咱', '是', '一', '爷', '们', '，', '给', '媳', '妇', '服', '个', '软', '，', '道', '个', '歉', '怎', '么', '了', '？'], ['我', '是', '一', '爷', '们', '，', '绝', '对', '不', '会', '嫌', '弃', '自', '己', '媳', '妇', '，', '拿', '她', '和', '别', '人', '比', '，', '说', '她', '这', '不', '如', '人', '家', '，', '纳', '不', '如', '人', '家', '的', '。'], ['我', '是', '一', '爷', '们', '，', '陪', '媳', '妇', '逛', '街', '时', '，', '碰', '见', '熟', '人', '，', '无', '论', '我', '媳', '妇', '长', '的', '好', '看', '与', '否', '，', '我', '都', '会', '大', '方', '的', '介', '绍', '。'], ['谁', '让', '咱', '爷', '们', '就', '好', '这', '口', '呢', '。'], ['我', '是', '一', '爷', '们', '，', '我', '想', '我', '会', '给', '我', '媳', '妇', '最', '好', '的', '幸', '福', '。'], ['【', '我', '们', '重', '在', '分', '享', '。'], ['所', '有', '文', '字', '和', '美', '图', '，', '来', '自', '网', '络', '，', '晨', '欣', '教', '育', '整', '理', '。'], ['对', '原', '文', '作', '者', '，', '表', '示', '敬', '意', '。'], ['】', '关', '注', '晨', '曦', '教', '育', '[UNK]', '[UNK]', '晨', '曦', '教', '育', '（', '微', '信', '号', '：', 'he', '##bc', '##x', '##jy', '）', '。'], ['打', '开', '微', '信', '，', '扫', '描', '二', '维', '码', '，', '关', '注', '[UNK]', '晨', '曦', '教', '育', '[UNK]', '，', '获', '取', '更', '多', '育', '儿', '资', '源', '。'], ['点', '击', '下', '面', '订', '阅', '按', '钮', '订', '阅', '，', '会', '有', '更', '多', '惊', '喜', '哦', '！']] + while i < len(document): # 从文档的第一个位置开始，按个往下看 + segment = document[ + i] # segment是列表，代表的是按字分开的一个完整句子，如 segment=['我', '是', '一', '爷', '们', '，', '我', '想', '我', '会', '给', '我', '媳', '妇', '最', '好', '的', '幸', '福', '。'] + # segment = get_new_segment(segment) # whole word mask for chinese: 结合分词的中文的whole mask设置即在需要的地方加上“##” + current_chunk.append(segment) # 将一个独立的句子加入到当前的文本块中 + current_length += len(segment) # 累计到为止位置接触到句子的总长度 + if i == len(document) - 1 or current_length >= target_seq_length: + # 如果累计的序列长度达到了目标的长度，或当前走到了文档结尾==>构造并添加到“A[SEP]B“中的A和B中； + if current_chunk: # 如果当前块不为空 + # `a_end` is how many segments from `current_chunk` go into the `A` + # (first) sentence. + a_end = 1 + if len(current_chunk) >= 2: # 当前块，如果包含超过两个句子，取当前块的一部分作为“A[SEP]B“中的A部分 + a_end = random.randint(1, len(current_chunk) - 1) + # 将当前文本段中选取出来的前半部分，赋值给A即tokens_a + tokens_a = [] + for j in range(a_end): + tokens_a.extend(current_chunk[j]) + + # 构造“A[SEP]B“中的B部分(有一部分是正常的当前文档中的后半部;在原BERT的实现中一部分是随机的从另一个文档中选取的，） + tokens_b = [] + for j in range(a_end, len(current_chunk)): + tokens_b.extend(current_chunk[j]) + + # 有百分之50%的概率交换一下tokens_a和tokens_b的位置 + # print("tokens_a length1:",len(tokens_a)) + # print("tokens_b length1:",len(tokens_b)) # len(tokens_b) = 0 + if len(tokens_a) == 0 or len(tokens_b) == 0: continue + if random.random() < 0.5: # 交换一下tokens_a和tokens_b + is_random_next = True + temp = tokens_a + tokens_a = tokens_b + tokens_b = temp + else: + is_random_next = False + truncate_seq_pair(tokens_a, tokens_b, max_num_tokens) + assert len(tokens_a) >= 1 + assert len(tokens_b) >= 1 + + # 把tokens_a & tokens_b加入到按照bert的风格，即以[CLS]tokens_a[SEP]tokens_b[SEP]的形式，结合到一起，作为最终的tokens; 也带上segment_ids，前面部分segment_ids的值是0，后面部分的值是1. + tokens = ["[CLS]"] + tokens_a + ["[SEP]"] + tokens_b + ["[SEP]"] + # The segment IDs are 0 for the [CLS] token, the A tokens and the first [SEP] + # They are 1 for the B tokens and the final [SEP] + segment_ids = [0 for _ in range(len(tokens_a) + 2)] + [1 for _ in range(len(tokens_b) + 1)] + + # 创建masked LM的任务的数据 Creates the predictions for the masked LM objective + tokens, masked_lm_positions, masked_lm_labels = create_masked_lm_predictions( + tokens, max_ngram, masked_lm_prob, max_predictions_per_seq, vocab_words) + instance = { + "tokens": tokens, + "segment_ids": segment_ids, + "is_random_next": is_random_next, + "masked_lm_positions": masked_lm_positions, + "masked_lm_labels": masked_lm_labels} + instances.append(instance) + current_chunk = [] # 清空当前块 + current_length = 0 # 重置当前文本块的长度 + i += 1 # 接着文档中的内容往后看 + return instances + + +def create_masked_lm_predictions(tokens, max_ngram, masked_lm_prob, max_predictions_per_seq, vocab_list): + """Creates the predictions for the masked LM objective. This is mostly copied from the Google BERT repo, but + with several refactors to clean it up and remove a lot of unnecessary variables.""" + + # n-gram masking Albert + ngrams = np.arange(1, max_ngram + 1, dtype=np.int64) + pvals = 1. / np.arange(1, max_ngram + 1) + pvals /= pvals.sum(keepdims=True) # p(n) = 1/n / sigma(1/k) + cand_indices = [] + for (i, token) in enumerate(tokens): + if token == "[CLS]" or token == "[SEP]": + continue + # Whole Word Masking means that if we mask all of the wordpieces + # corresponding to an original word. When a word has been split into + # WordPieces, the first token does not have any marker and any subsequence + # tokens are prefixed with ##. So whenever we see the ## token, we + # append it to the previous set of word indexes. + # + # Note that Whole Word Masking does *not* change the training code + # at all -- we still predict each WordPiece independently, softmaxed + # over the entire vocabulary. + cand_indices.append(i) + num_to_mask = min(max_predictions_per_seq, max(1, int(round(len(tokens) * masked_lm_prob)))) + random.shuffle(cand_indices) + masked_token_labels = [] + covered_indices = set() + for index in cand_indices: + n = np.random.choice(ngrams, p=pvals) + if len(masked_token_labels) >= num_to_mask: + break + if index in covered_indices: + continue + if index < len(cand_indices) - (n - 1): + for i in range(n): + ind = index + i + if ind in covered_indices: + continue + covered_indices.add(ind) + # 80% of the time, replace with [MASK] + if random.random() < 0.8: + masked_token = "[MASK]" + else: + # 10% of the time, keep original + if random.random() < 0.5: + masked_token = tokens[ind] + # 10% of the time, replace with random word + else: + masked_token = random.choice(vocab_list) + masked_token_labels.append(MaskedLmInstance(index=ind, label=tokens[ind])) + tokens[ind] = masked_token + + #assert len(masked_token_labels) <= num_to_mask + masked_token_labels = sorted(masked_token_labels, key=lambda x: x.index) + mask_indices = [p.index for p in masked_token_labels] + masked_labels = [p.label for p in masked_token_labels] + return tokens, mask_indices, masked_labels + +def create_training_instances(input_file, tokenizer, max_seq_len, short_seq_prob, + max_ngram, masked_lm_prob, max_predictions_per_seq): + """Create `TrainingInstance`s from raw text.""" + all_documents = [[]] + # Input file format: + # (1) One sentence per line. These should ideally be actual sentences, not + # entire paragraphs or arbitrary spans of text. (Because we use the + # sentence boundaries for the "next sentence prediction" task). + # (2) Blank lines between documents. Document boundaries are needed so + # that the "next sentence prediction" task doesn't span between documents. + f = open(input_file, 'r') + lines = f.readlines() + pbar = ProgressBar(n_total=len(lines), desc='read data') + for line_cnt, line in enumerate(lines): + line = line.strip() + # Empty lines are used as document delimiters + if not line: + all_documents.append([]) + tokens = tokenizer.tokenize(line) + if tokens: + all_documents[-1].append(tokens) + pbar(step=line_cnt) + print(' ') + # Remove empty documents + all_documents = [x for x in all_documents if x] + random.shuffle(all_documents) + + vocab_words = list(tokenizer.vocab.keys()) + instances = [] + pbar = ProgressBar(n_total=len(all_documents), desc='create instances') + for document_index in range(len(all_documents)): + instances.extend( + create_instances_from_document( + all_documents, document_index, max_seq_len, short_seq_prob, + max_ngram, masked_lm_prob, max_predictions_per_seq, vocab_words)) + pbar(step=document_index) + print(' ') + ex_idx = 0 + while ex_idx < 5: + instance = instances[ex_idx] + logger.info("-------------------------Example-----------------------") + logger.info(f"id: {ex_idx}") + logger.info(f"tokens: {' '.join([str(x) for x in instance['tokens']])}") + logger.info(f"masked_lm_labels: {' '.join([str(x) for x in instance['masked_lm_labels']])}") + logger.info(f"segment_ids: {' '.join([str(x) for x in instance['segment_ids']])}") + logger.info(f"masked_lm_positions: {' '.join([str(x) for x in instance['masked_lm_positions']])}") + logger.info(f"is_random_next : {instance['is_random_next']}") + ex_idx += 1 + random.shuffle(instances) + return instances + + +def main(): + parser = ArgumentParser() + ## Required parameters + parser.add_argument("--data_dir", default=None, type=str, required=True) + parser.add_argument("--vocab_path", default=None, type=str, required=True) + parser.add_argument("--output_dir", default=None, type=str, required=True) + + parser.add_argument('--data_name', default='albert', type=str) + parser.add_argument('--max_ngram', default=3, type=int) + parser.add_argument("--do_data", default=False, action='store_true') + parser.add_argument("--do_split", default=False, action='store_true') + parser.add_argument("--do_lower_case", default=False, action='store_true') + parser.add_argument('--seed', default=42, type=int) + parser.add_argument("--line_per_file", default=1000000000, type=int) + parser.add_argument("--file_num", type=int, default=10, + help="Number of dynamic masking to pregenerate (with different masks)") + parser.add_argument("--max_seq_len", type=int, default=128) + parser.add_argument("--short_seq_prob", type=float, default=0.1, + help="Probability of making a short sentence as a training example") + parser.add_argument("--masked_lm_prob", type=float, default=0.15, + help="Probability of masking each token for the LM task") + parser.add_argument("--max_predictions_per_seq", type=int, default=20, # 128 * 0.15 + help="Maximum number of tokens to mask in each sequence") + args = parser.parse_args() + seed_everything(args.seed) + args.data_dir = Path(args.data_dir) + if not os.path.exists(args.output_dir): + os.mkdir(args.output_dir) + init_logger(log_file=args.output_dir +"pregenerate_training_data_ngram.log") + logger.info("pregenerate training data parameters:\n %s", args) + tokenizer = BertTokenizer(vocab_file=args.vocab_path, do_lower_case=args.do_lower_case) + + # split big file + if args.do_split: + corpus_path =args.data_dir / "corpus/corpus.txt" + split_save_path = args.data_dir / "/corpus/train" + if not split_save_path.exists(): + split_save_path.mkdir(exist_ok=True) + line_per_file = args.line_per_file + command = f'split -a 4 -l {line_per_file} -d {corpus_path} {split_save_path}/shard_' + os.system(f"{command}") + + # generator train data + if args.do_data: + data_path = args.data_dir / "corpus/train" + files = sorted([f for f in data_path.parent.iterdir() if f.exists() and '.txt' in str(f)]) + for idx in range(args.file_num): + logger.info(f"pregenetate {args.data_name}_file_{idx}.json") + save_filename = data_path / f"{args.data_name}_file_{idx}.json" + num_instances = 0 + with save_filename.open('w') as fw: + for file_idx in range(len(files)): + file_path = files[file_idx] + file_examples = create_training_instances(input_file=file_path, + tokenizer=tokenizer, + max_seq_len=args.max_seq_len, + max_ngram=args.max_ngram, + short_seq_prob=args.short_seq_prob, + masked_lm_prob=args.masked_lm_prob, + max_predictions_per_seq=args.max_predictions_per_seq) + file_examples = [json.dumps(instance) for instance in file_examples] + for instance in file_examples: + fw.write(instance + '\n') + num_instances += 1 + metrics_file = data_path / f"{args.data_name}_file_{idx}_metrics.json" + print(f"num_instances: {num_instances}") + with metrics_file.open('w') as metrics_file: + metrics = { + "num_training_examples": num_instances, + "max_seq_len": args.max_seq_len + } + metrics_file.write(json.dumps(metrics)) + +if __name__ == '__main__': + main() + +''' +python prepare_lm_data_ngram.py \ + --data_dir=dataset/ \ + --vocab_path=vocab.txt \ + --output_dir=outputs/ \ + --data_name=albert \ + --max_ngram=3 \ + --do_data +''' diff --git a/BertToSimple/BiGRU/prev_trained_model/albert_21_93/config.json b/BertToSimple/BiGRU/prev_trained_model/albert_21_93/config.json new file mode 100644 index 0000000..242a15c --- /dev/null +++ b/BertToSimple/BiGRU/prev_trained_model/albert_21_93/config.json @@ -0,0 +1,28 @@ +{ + "attention_probs_dropout_prob": 0, + "down_scale_factor": 1, + "embedding_size": 128, + "finetuning_task": "mrpc", + "gap_size": 0, + "hidden_act": "relu", + "hidden_dropout_prob": 0, + "hidden_size": 1024, + "initializer_range": 0.02, + "inner_group_num": 1, + "intermediate_size": 4096, + "layer_norm_eps": 1e-12, + "layers_to_keep": [], + "max_position_embeddings": 512, + "net_structure_type": 0, + "num_attention_heads": 16, + "num_hidden_groups": 1, + "num_hidden_layers": 24, + "num_labels": 21, + "num_memory_blocks": 0, + "output_attentions": false, + "output_hidden_states": false, + "pruned_heads": {}, + "torchscript": false, + "type_vocab_size": 2, + "vocab_size": 21128 +} diff --git a/BertToSimple/BiGRU/prev_trained_model/albert_large_zh/config.json b/BertToSimple/BiGRU/prev_trained_model/albert_large_zh/config.json new file mode 100644 index 0000000..e9608ab --- /dev/null +++ b/BertToSimple/BiGRU/prev_trained_model/albert_large_zh/config.json @@ -0,0 +1,21 @@ +{ + "attention_probs_dropout_prob": 0, + "hidden_act": "relu", + "hidden_dropout_prob": 0, + "embedding_size": 128, + "hidden_size": 1024, + "initializer_range": 0.02, + "intermediate_size": 4096, + "max_position_embeddings": 512, + "num_attention_heads": 16, + "num_hidden_layers": 24, + "num_hidden_groups": 1, + "net_structure_type": 0, + "layers_to_keep": [], + "gap_size": 0, + "num_memory_blocks": 0, + "inner_group_num": 1, + "down_scale_factor": 1, + "type_vocab_size": 2, + "vocab_size": 21128 +} diff --git a/BertToSimple/BiGRU/prev_trained_model/albert_large_zh/vocab.txt b/BertToSimple/BiGRU/prev_trained_model/albert_large_zh/vocab.txt new file mode 100644 index 0000000..ca4f978 --- /dev/null +++ b/BertToSimple/BiGRU/prev_trained_model/albert_large_zh/vocab.txt @@ -0,0 +1,21128 @@ +[PAD] +[unused1] +[unused2] +[unused3] +[unused4] +[unused5] +[unused6] +[unused7] +[unused8] +[unused9] +[unused10] +[unused11] +[unused12] +[unused13] +[unused14] +[unused15] +[unused16] +[unused17] +[unused18] +[unused19] +[unused20] +[unused21] +[unused22] +[unused23] +[unused24] +[unused25] +[unused26] +[unused27] +[unused28] +[unused29] +[unused30] +[unused31] +[unused32] +[unused33] +[unused34] +[unused35] +[unused36] +[unused37] +[unused38] +[unused39] +[unused40] +[unused41] +[unused42] +[unused43] +[unused44] +[unused45] +[unused46] +[unused47] +[unused48] +[unused49] +[unused50] +[unused51] +[unused52] +[unused53] +[unused54] +[unused55] +[unused56] +[unused57] +[unused58] +[unused59] +[unused60] +[unused61] +[unused62] +[unused63] +[unused64] +[unused65] +[unused66] +[unused67] +[unused68] +[unused69] +[unused70] +[unused71] +[unused72] +[unused73] +[unused74] +[unused75] +[unused76] +[unused77] +[unused78] +[unused79] +[unused80] +[unused81] +[unused82] +[unused83] +[unused84] +[unused85] +[unused86] +[unused87] +[unused88] +[unused89] +[unused90] +[unused91] +[unused92] +[unused93] +[unused94] +[unused95] +[unused96] +[unused97] +[unused98] +[unused99] +[UNK] +[CLS] +[SEP] +[MASK] + + +! +" +# +$ +% +& +' +( +) +* ++ +, +- +. +/ +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +: +; +< += +> +? +@ +[ +\ +] +^ +_ +a +b +c +d +e +f +g +h +i +j +k +l +m +n +o +p +q +r +s +t +u +v +w +x +y +z +{ +| +} +~ +£ +¤ +¥ +§ +© +« +® +° +± +² +³ +µ +· +¹ +º +» +¼ +× +ß +æ +÷ +ø +đ +ŋ +ɔ +ə +ɡ +ʰ +ˇ +ˈ +ˊ +ˋ +ˍ +ː +˙ +˚ +ˢ +α +β +γ +δ +ε +η +θ +ι +κ +λ +μ +ν +ο +π +ρ +ς +σ +τ +υ +φ +χ +ψ +ω +а +б +в +г +д +е +ж +з +и +к +л +м +н +о +п +р +с +т +у +ф +х +ц +ч +ш +ы +ь +я +і +ا +ب +ة +ت +د +ر +س +ع +ل +م +ن +ه +و +ي +۩ +ก +ง +น +ม +ย +ร +อ +า +เ +๑ +་ +ღ +ᄀ +ᄁ +ᄂ +ᄃ +ᄅ +ᄆ +ᄇ +ᄈ +ᄉ +ᄋ +ᄌ +ᄎ +ᄏ +ᄐ +ᄑ +ᄒ +ᅡ +ᅢ +ᅣ +ᅥ +ᅦ +ᅧ +ᅨ +ᅩ +ᅪ +ᅬ +ᅭ +ᅮ +ᅯ +ᅲ +ᅳ +ᅴ +ᅵ +ᆨ +ᆫ +ᆯ +ᆷ +ᆸ +ᆺ +ᆻ +ᆼ +ᗜ +ᵃ +ᵉ +ᵍ +ᵏ +ᵐ +ᵒ +ᵘ +‖ +„ +† +• +‥ +‧ +
 +‰ +′ +″ +‹ +› +※ +‿ +⁄ +ⁱ +⁺ +ⁿ +₁ +₂ +₃ +₄ +€ +℃ +№ +™ +ⅰ +ⅱ +ⅲ +ⅳ +ⅴ +← +↑ +→ +↓ +↔ +↗ +↘ +⇒ +∀ +− +∕ +∙ +√ +∞ +∟ +∠ +∣ +∥ +∩ +∮ +∶ +∼ +∽ +≈ +≒ +≡ +≤ +≥ +≦ +≧ +≪ +≫ +⊙ +⋅ +⋈ +⋯ +⌒ +① +② +③ +④ +⑤ +⑥ +⑦ +⑧ +⑨ +⑩ +⑴ +⑵ +⑶ +⑷ +⑸ +⒈ +⒉ +⒊ +⒋ +ⓒ +ⓔ +ⓘ +─ +━ +│ +┃ +┅ +┆ +┊ +┌ +└ +├ +┣ +═ +║ +╚ +╞ +╠ +╭ +╮ +╯ +╰ +╱ +╳ +▂ +▃ +▅ +▇ +█ +▉ +▋ +▌ +▍ +▎ +■ +□ +▪ +▫ +▬ +▲ +△ +▶ +► +▼ +▽ +◆ +◇ +○ +◎ +● +◕ +◠ +◢ +◤ +☀ +★ +☆ +☕ +☞ +☺ +☼ +♀ +♂ +♠ +♡ +♣ +♥ +♦ +♪ +♫ +♬ +✈ +✔ +✕ +✖ +✦ +✨ +✪ +✰ +✿ +❀ +❤ +➜ +➤ +⦿ +、 +。 +〃 +々 +〇 +〈 +〉 +《 +》 +「 +」 +『 +』 +【 +】 +〓 +〔 +〕 +〖 +〗 +〜 +〝 +〞 +ぁ +あ +ぃ +い +う +ぇ +え +お +か +き +く +け +こ +さ +し +す +せ +そ +た +ち +っ +つ +て +と +な +に +ぬ +ね +の +は +ひ +ふ +へ +ほ +ま +み +む +め +も +ゃ +や +ゅ +ゆ +ょ +よ +ら +り +る +れ +ろ +わ +を +ん +゜ +ゝ +ァ +ア +ィ +イ +ゥ +ウ +ェ +エ +ォ +オ +カ +キ +ク +ケ +コ +サ +シ +ス +セ +ソ +タ +チ +ッ +ツ +テ +ト +ナ +ニ +ヌ +ネ +ノ +ハ +ヒ +フ +ヘ +ホ +マ +ミ +ム +メ +モ +ャ +ヤ +ュ +ユ +ョ +ヨ +ラ +リ +ル +レ +ロ +ワ +ヲ +ン +ヶ +・ +ー +ヽ +ㄅ +ㄆ +ㄇ +ㄉ +ㄋ +ㄌ +ㄍ +ㄎ +ㄏ +ㄒ +ㄚ +ㄛ +ㄞ +ㄟ +ㄢ +ㄤ +ㄥ +ㄧ +ㄨ +ㆍ +㈦ +㊣ +㎡ +㗎 +一 +丁 +七 +万 +丈 +三 +上 +下 +不 +与 +丐 +丑 +专 +且 +丕 +世 +丘 +丙 +业 +丛 +东 +丝 +丞 +丟 +両 +丢 +两 +严 +並 +丧 +丨 +个 +丫 +中 +丰 +串 +临 +丶 +丸 +丹 +为 +主 +丼 +丽 +举 +丿 +乂 +乃 +久 +么 +义 +之 +乌 +乍 +乎 +乏 +乐 +乒 +乓 +乔 +乖 +乗 +乘 +乙 +乜 +九 +乞 +也 +习 +乡 +书 +乩 +买 +乱 +乳 +乾 +亀 +亂 +了 +予 +争 +事 +二 +于 +亏 +云 +互 +五 +井 +亘 +亙 +亚 +些 +亜 +亞 +亟 +亡 +亢 +交 +亥 +亦 +产 +亨 +亩 +享 +京 +亭 +亮 +亲 +亳 +亵 +人 +亿 +什 +仁 +仃 +仄 +仅 +仆 +仇 +今 +介 +仍 +从 +仏 +仑 +仓 +仔 +仕 +他 +仗 +付 +仙 +仝 +仞 +仟 +代 +令 +以 +仨 +仪 +们 +仮 +仰 +仲 +件 +价 +任 +份 +仿 +企 +伉 +伊 +伍 +伎 +伏 +伐 +休 +伕 +众 +优 +伙 +会 +伝 +伞 +伟 +传 +伢 +伤 +伦 +伪 +伫 +伯 +估 +伴 +伶 +伸 +伺 +似 +伽 +佃 +但 +佇 +佈 +位 +低 +住 +佐 +佑 +体 +佔 +何 +佗 +佘 +余 +佚 +佛 +作 +佝 +佞 +佟 +你 +佢 +佣 +佤 +佥 +佩 +佬 +佯 +佰 +佳 +併 +佶 +佻 +佼 +使 +侃 +侄 +來 +侈 +例 +侍 +侏 +侑 +侖 +侗 +供 +依 +侠 +価 +侣 +侥 +侦 +侧 +侨 +侬 +侮 +侯 +侵 +侶 +侷 +便 +係 +促 +俄 +俊 +俎 +俏 +俐 +俑 +俗 +俘 +俚 +保 +俞 +俟 +俠 +信 +俨 +俩 +俪 +俬 +俭 +修 +俯 +俱 +俳 +俸 +俺 +俾 +倆 +倉 +個 +倌 +倍 +倏 +們 +倒 +倔 +倖 +倘 +候 +倚 +倜 +借 +倡 +値 +倦 +倩 +倪 +倫 +倬 +倭 +倶 +债 +值 +倾 +偃 +假 +偈 +偉 +偌 +偎 +偏 +偕 +做 +停 +健 +側 +偵 +偶 +偷 +偻 +偽 +偿 +傀 +傅 +傍 +傑 +傘 +備 +傚 +傢 +傣 +傥 +储 +傩 +催 +傭 +傲 +傳 +債 +傷 +傻 +傾 +僅 +働 +像 +僑 +僕 +僖 +僚 +僥 +僧 +僭 +僮 +僱 +僵 +價 +僻 +儀 +儂 +億 +儆 +儉 +儋 +儒 +儕 +儘 +償 +儡 +優 +儲 +儷 +儼 +儿 +兀 +允 +元 +兄 +充 +兆 +兇 +先 +光 +克 +兌 +免 +児 +兑 +兒 +兔 +兖 +党 +兜 +兢 +入 +內 +全 +兩 +八 +公 +六 +兮 +兰 +共 +兲 +关 +兴 +兵 +其 +具 +典 +兹 +养 +兼 +兽 +冀 +内 +円 +冇 +冈 +冉 +冊 +册 +再 +冏 +冒 +冕 +冗 +写 +军 +农 +冠 +冢 +冤 +冥 +冨 +冪 +冬 +冯 +冰 +冲 +决 +况 +冶 +冷 +冻 +冼 +冽 +冾 +净 +凄 +准 +凇 +凈 +凉 +凋 +凌 +凍 +减 +凑 +凛 +凜 +凝 +几 +凡 +凤 +処 +凪 +凭 +凯 +凰 +凱 +凳 +凶 +凸 +凹 +出 +击 +函 +凿 +刀 +刁 +刃 +分 +切 +刈 +刊 +刍 +刎 +刑 +划 +列 +刘 +则 +刚 +创 +初 +删 +判 +別 +刨 +利 +刪 +别 +刮 +到 +制 +刷 +券 +刹 +刺 +刻 +刽 +剁 +剂 +剃 +則 +剉 +削 +剋 +剌 +前 +剎 +剐 +剑 +剔 +剖 +剛 +剜 +剝 +剣 +剤 +剥 +剧 +剩 +剪 +副 +割 +創 +剷 +剽 +剿 +劃 +劇 +劈 +劉 +劊 +劍 +劏 +劑 +力 +劝 +办 +功 +加 +务 +劣 +动 +助 +努 +劫 +劭 +励 +劲 +劳 +労 +劵 +効 +劾 +势 +勁 +勃 +勇 +勉 +勋 +勐 +勒 +動 +勖 +勘 +務 +勛 +勝 +勞 +募 +勢 +勤 +勧 +勳 +勵 +勸 +勺 +勻 +勾 +勿 +匀 +包 +匆 +匈 +匍 +匐 +匕 +化 +北 +匙 +匝 +匠 +匡 +匣 +匪 +匮 +匯 +匱 +匹 +区 +医 +匾 +匿 +區 +十 +千 +卅 +升 +午 +卉 +半 +卍 +华 +协 +卑 +卒 +卓 +協 +单 +卖 +南 +単 +博 +卜 +卞 +卟 +占 +卡 +卢 +卤 +卦 +卧 +卫 +卮 +卯 +印 +危 +即 +却 +卵 +卷 +卸 +卻 +卿 +厂 +厄 +厅 +历 +厉 +压 +厌 +厕 +厘 +厚 +厝 +原 +厢 +厥 +厦 +厨 +厩 +厭 +厮 +厲 +厳 +去 +县 +叁 +参 +參 +又 +叉 +及 +友 +双 +反 +収 +发 +叔 +取 +受 +变 +叙 +叛 +叟 +叠 +叡 +叢 +口 +古 +句 +另 +叨 +叩 +只 +叫 +召 +叭 +叮 +可 +台 +叱 +史 +右 +叵 +叶 +号 +司 +叹 +叻 +叼 +叽 +吁 +吃 +各 +吆 +合 +吉 +吊 +吋 +同 +名 +后 +吏 +吐 +向 +吒 +吓 +吕 +吖 +吗 +君 +吝 +吞 +吟 +吠 +吡 +否 +吧 +吨 +吩 +含 +听 +吭 +吮 +启 +吱 +吳 +吴 +吵 +吶 +吸 +吹 +吻 +吼 +吽 +吾 +呀 +呂 +呃 +呆 +呈 +告 +呋 +呎 +呐 +呓 +呕 +呗 +员 +呛 +呜 +呢 +呤 +呦 +周 +呱 +呲 +味 +呵 +呷 +呸 +呻 +呼 +命 +咀 +咁 +咂 +咄 +咆 +咋 +和 +咎 +咏 +咐 +咒 +咔 +咕 +咖 +咗 +咘 +咙 +咚 +咛 +咣 +咤 +咦 +咧 +咨 +咩 +咪 +咫 +咬 +咭 +咯 +咱 +咲 +咳 +咸 +咻 +咽 +咿 +哀 +品 +哂 +哄 +哆 +哇 +哈 +哉 +哋 +哌 +响 +哎 +哏 +哐 +哑 +哒 +哔 +哗 +哟 +員 +哥 +哦 +哧 +哨 +哩 +哪 +哭 +哮 +哲 +哺 +哼 +哽 +唁 +唄 +唆 +唇 +唉 +唏 +唐 +唑 +唔 +唠 +唤 +唧 +唬 +售 +唯 +唰 +唱 +唳 +唷 +唸 +唾 +啃 +啄 +商 +啉 +啊 +問 +啓 +啕 +啖 +啜 +啞 +啟 +啡 +啤 +啥 +啦 +啧 +啪 +啫 +啬 +啮 +啰 +啱 +啲 +啵 +啶 +啷 +啸 +啻 +啼 +啾 +喀 +喂 +喃 +善 +喆 +喇 +喉 +喊 +喋 +喎 +喏 +喔 +喘 +喙 +喚 +喜 +喝 +喟 +喧 +喪 +喫 +喬 +單 +喰 +喱 +喲 +喳 +喵 +営 +喷 +喹 +喺 +喻 +喽 +嗅 +嗆 +嗇 +嗎 +嗑 +嗒 +嗓 +嗔 +嗖 +嗚 +嗜 +嗝 +嗟 +嗡 +嗣 +嗤 +嗦 +嗨 +嗪 +嗬 +嗯 +嗰 +嗲 +嗳 +嗶 +嗷 +嗽 +嘀 +嘅 +嘆 +嘈 +嘉 +嘌 +嘍 +嘎 +嘔 +嘖 +嘗 +嘘 +嘚 +嘛 +嘜 +嘞 +嘟 +嘢 +嘣 +嘤 +嘧 +嘩 +嘭 +嘮 +嘯 +嘰 +嘱 +嘲 +嘴 +嘶 +嘸 +嘹 +嘻 +嘿 +噁 +噌 +噎 +噓 +噔 +噗 +噙 +噜 +噠 +噢 +噤 +器 +噩 +噪 +噬 +噱 +噴 +噶 +噸 +噹 +噻 +噼 +嚀 +嚇 +嚎 +嚏 +嚐 +嚓 +嚕 +嚟 +嚣 +嚥 +嚨 +嚮 +嚴 +嚷 +嚼 +囂 +囉 +囊 +囍 +囑 +囔 +囗 +囚 +四 +囝 +回 +囟 +因 +囡 +团 +団 +囤 +囧 +囪 +囫 +园 +困 +囱 +囲 +図 +围 +囹 +固 +国 +图 +囿 +圃 +圄 +圆 +圈 +國 +圍 +圏 +園 +圓 +圖 +團 +圜 +土 +圣 +圧 +在 +圩 +圭 +地 +圳 +场 +圻 +圾 +址 +坂 +均 +坊 +坍 +坎 +坏 +坐 +坑 +块 +坚 +坛 +坝 +坞 +坟 +坠 +坡 +坤 +坦 +坨 +坪 +坯 +坳 +坵 +坷 +垂 +垃 +垄 +型 +垒 +垚 +垛 +垠 +垢 +垣 +垦 +垩 +垫 +垭 +垮 +垵 +埂 +埃 +埋 +城 +埔 +埕 +埗 +域 +埠 +埤 +埵 +執 +埸 +培 +基 +埼 +堀 +堂 +堃 +堅 +堆 +堇 +堑 +堕 +堙 +堡 +堤 +堪 +堯 +堰 +報 +場 +堵 +堺 +堿 +塊 +塌 +塑 +塔 +塗 +塘 +塚 +塞 +塢 +塩 +填 +塬 +塭 +塵 +塾 +墀 +境 +墅 +墉 +墊 +墒 +墓 +増 +墘 +墙 +墜 +增 +墟 +墨 +墩 +墮 +墳 +墻 +墾 +壁 +壅 +壆 +壇 +壊 +壑 +壓 +壕 +壘 +壞 +壟 +壢 +壤 +壩 +士 +壬 +壮 +壯 +声 +売 +壳 +壶 +壹 +壺 +壽 +处 +备 +変 +复 +夏 +夔 +夕 +外 +夙 +多 +夜 +够 +夠 +夢 +夥 +大 +天 +太 +夫 +夭 +央 +夯 +失 +头 +夷 +夸 +夹 +夺 +夾 +奂 +奄 +奇 +奈 +奉 +奋 +奎 +奏 +奐 +契 +奔 +奕 +奖 +套 +奘 +奚 +奠 +奢 +奥 +奧 +奪 +奬 +奮 +女 +奴 +奶 +奸 +她 +好 +如 +妃 +妄 +妆 +妇 +妈 +妊 +妍 +妒 +妓 +妖 +妘 +妙 +妝 +妞 +妣 +妤 +妥 +妨 +妩 +妪 +妮 +妲 +妳 +妹 +妻 +妾 +姆 +姉 +姊 +始 +姍 +姐 +姑 +姒 +姓 +委 +姗 +姚 +姜 +姝 +姣 +姥 +姦 +姨 +姪 +姫 +姬 +姹 +姻 +姿 +威 +娃 +娄 +娅 +娆 +娇 +娉 +娑 +娓 +娘 +娛 +娜 +娟 +娠 +娣 +娥 +娩 +娱 +娲 +娴 +娶 +娼 +婀 +婁 +婆 +婉 +婊 +婕 +婚 +婢 +婦 +婧 +婪 +婭 +婴 +婵 +婶 +婷 +婺 +婿 +媒 +媚 +媛 +媞 +媧 +媲 +媳 +媽 +媾 +嫁 +嫂 +嫉 +嫌 +嫑 +嫔 +嫖 +嫘 +嫚 +嫡 +嫣 +嫦 +嫩 +嫲 +嫵 +嫻 +嬅 +嬉 +嬌 +嬗 +嬛 +嬢 +嬤 +嬪 +嬰 +嬴 +嬷 +嬸 +嬿 +孀 +孃 +子 +孑 +孔 +孕 +孖 +字 +存 +孙 +孚 +孛 +孜 +孝 +孟 +孢 +季 +孤 +学 +孩 +孪 +孫 +孬 +孰 +孱 +孳 +孵 +學 +孺 +孽 +孿 +宁 +它 +宅 +宇 +守 +安 +宋 +完 +宏 +宓 +宕 +宗 +官 +宙 +定 +宛 +宜 +宝 +实 +実 +宠 +审 +客 +宣 +室 +宥 +宦 +宪 +宫 +宮 +宰 +害 +宴 +宵 +家 +宸 +容 +宽 +宾 +宿 +寂 +寄 +寅 +密 +寇 +富 +寐 +寒 +寓 +寛 +寝 +寞 +察 +寡 +寢 +寥 +實 +寧 +寨 +審 +寫 +寬 +寮 +寰 +寵 +寶 +寸 +对 +寺 +寻 +导 +対 +寿 +封 +専 +射 +将 +將 +專 +尉 +尊 +尋 +對 +導 +小 +少 +尔 +尕 +尖 +尘 +尚 +尝 +尤 +尧 +尬 +就 +尴 +尷 +尸 +尹 +尺 +尻 +尼 +尽 +尾 +尿 +局 +屁 +层 +屄 +居 +屆 +屈 +屉 +届 +屋 +屌 +屍 +屎 +屏 +屐 +屑 +展 +屜 +属 +屠 +屡 +屢 +層 +履 +屬 +屯 +山 +屹 +屿 +岀 +岁 +岂 +岌 +岐 +岑 +岔 +岖 +岗 +岘 +岙 +岚 +岛 +岡 +岩 +岫 +岬 +岭 +岱 +岳 +岷 +岸 +峇 +峋 +峒 +峙 +峡 +峤 +峥 +峦 +峨 +峪 +峭 +峯 +峰 +峴 +島 +峻 +峽 +崁 +崂 +崆 +崇 +崎 +崑 +崔 +崖 +崗 +崙 +崛 +崧 +崩 +崭 +崴 +崽 +嵇 +嵊 +嵋 +嵌 +嵐 +嵘 +嵩 +嵬 +嵯 +嶂 +嶄 +嶇 +嶋 +嶙 +嶺 +嶼 +嶽 +巅 +巍 +巒 +巔 +巖 +川 +州 +巡 +巢 +工 +左 +巧 +巨 +巩 +巫 +差 +己 +已 +巳 +巴 +巷 +巻 +巽 +巾 +巿 +币 +市 +布 +帅 +帆 +师 +希 +帐 +帑 +帕 +帖 +帘 +帚 +帛 +帜 +帝 +帥 +带 +帧 +師 +席 +帮 +帯 +帰 +帳 +帶 +帷 +常 +帼 +帽 +幀 +幂 +幄 +幅 +幌 +幔 +幕 +幟 +幡 +幢 +幣 +幫 +干 +平 +年 +并 +幸 +幹 +幺 +幻 +幼 +幽 +幾 +广 +庁 +広 +庄 +庆 +庇 +床 +序 +庐 +库 +应 +底 +庖 +店 +庙 +庚 +府 +庞 +废 +庠 +度 +座 +庫 +庭 +庵 +庶 +康 +庸 +庹 +庾 +廁 +廂 +廃 +廈 +廉 +廊 +廓 +廖 +廚 +廝 +廟 +廠 +廢 +廣 +廬 +廳 +延 +廷 +建 +廿 +开 +弁 +异 +弃 +弄 +弈 +弊 +弋 +式 +弑 +弒 +弓 +弔 +引 +弗 +弘 +弛 +弟 +张 +弥 +弦 +弧 +弩 +弭 +弯 +弱 +張 +強 +弹 +强 +弼 +弾 +彅 +彆 +彈 +彌 +彎 +归 +当 +录 +彗 +彙 +彝 +形 +彤 +彥 +彦 +彧 +彩 +彪 +彫 +彬 +彭 +彰 +影 +彷 +役 +彻 +彼 +彿 +往 +征 +径 +待 +徇 +很 +徉 +徊 +律 +後 +徐 +徑 +徒 +従 +徕 +得 +徘 +徙 +徜 +從 +徠 +御 +徨 +復 +循 +徬 +微 +徳 +徴 +徵 +德 +徹 +徼 +徽 +心 +必 +忆 +忌 +忍 +忏 +忐 +忑 +忒 +忖 +志 +忘 +忙 +応 +忠 +忡 +忤 +忧 +忪 +快 +忱 +念 +忻 +忽 +忿 +怀 +态 +怂 +怅 +怆 +怎 +怏 +怒 +怔 +怕 +怖 +怙 +怜 +思 +怠 +怡 +急 +怦 +性 +怨 +怪 +怯 +怵 +总 +怼 +恁 +恃 +恆 +恋 +恍 +恐 +恒 +恕 +恙 +恚 +恢 +恣 +恤 +恥 +恨 +恩 +恪 +恫 +恬 +恭 +息 +恰 +恳 +恵 +恶 +恸 +恺 +恻 +恼 +恿 +悄 +悅 +悉 +悌 +悍 +悔 +悖 +悚 +悟 +悠 +患 +悦 +您 +悩 +悪 +悬 +悯 +悱 +悲 +悴 +悵 +悶 +悸 +悻 +悼 +悽 +情 +惆 +惇 +惊 +惋 +惑 +惕 +惘 +惚 +惜 +惟 +惠 +惡 +惦 +惧 +惨 +惩 +惫 +惬 +惭 +惮 +惯 +惰 +惱 +想 +惴 +惶 +惹 +惺 +愁 +愆 +愈 +愉 +愍 +意 +愕 +愚 +愛 +愜 +感 +愣 +愤 +愧 +愫 +愷 +愿 +慄 +慈 +態 +慌 +慎 +慑 +慕 +慘 +慚 +慟 +慢 +慣 +慧 +慨 +慫 +慮 +慰 +慳 +慵 +慶 +慷 +慾 +憂 +憊 +憋 +憎 +憐 +憑 +憔 +憚 +憤 +憧 +憨 +憩 +憫 +憬 +憲 +憶 +憾 +懂 +懇 +懈 +應 +懊 +懋 +懑 +懒 +懦 +懲 +懵 +懶 +懷 +懸 +懺 +懼 +懾 +懿 +戀 +戈 +戊 +戌 +戍 +戎 +戏 +成 +我 +戒 +戕 +或 +战 +戚 +戛 +戟 +戡 +戦 +截 +戬 +戮 +戰 +戲 +戳 +戴 +戶 +户 +戸 +戻 +戾 +房 +所 +扁 +扇 +扈 +扉 +手 +才 +扎 +扑 +扒 +打 +扔 +払 +托 +扛 +扣 +扦 +执 +扩 +扪 +扫 +扬 +扭 +扮 +扯 +扰 +扱 +扳 +扶 +批 +扼 +找 +承 +技 +抄 +抉 +把 +抑 +抒 +抓 +投 +抖 +抗 +折 +抚 +抛 +抜 +択 +抟 +抠 +抡 +抢 +护 +报 +抨 +披 +抬 +抱 +抵 +抹 +押 +抽 +抿 +拂 +拄 +担 +拆 +拇 +拈 +拉 +拋 +拌 +拍 +拎 +拐 +拒 +拓 +拔 +拖 +拗 +拘 +拙 +拚 +招 +拜 +拟 +拡 +拢 +拣 +拥 +拦 +拧 +拨 +择 +括 +拭 +拮 +拯 +拱 +拳 +拴 +拷 +拼 +拽 +拾 +拿 +持 +挂 +指 +挈 +按 +挎 +挑 +挖 +挙 +挚 +挛 +挝 +挞 +挟 +挠 +挡 +挣 +挤 +挥 +挨 +挪 +挫 +振 +挲 +挹 +挺 +挽 +挾 +捂 +捅 +捆 +捉 +捋 +捌 +捍 +捎 +捏 +捐 +捕 +捞 +损 +捡 +换 +捣 +捧 +捨 +捩 +据 +捱 +捲 +捶 +捷 +捺 +捻 +掀 +掂 +掃 +掇 +授 +掉 +掌 +掏 +掐 +排 +掖 +掘 +掙 +掛 +掠 +採 +探 +掣 +接 +控 +推 +掩 +措 +掬 +掰 +掲 +掳 +掴 +掷 +掸 +掺 +揀 +揃 +揄 +揆 +揉 +揍 +描 +提 +插 +揖 +揚 +換 +握 +揣 +揩 +揪 +揭 +揮 +援 +揶 +揸 +揹 +揽 +搀 +搁 +搂 +搅 +損 +搏 +搐 +搓 +搔 +搖 +搗 +搜 +搞 +搡 +搪 +搬 +搭 +搵 +搶 +携 +搽 +摀 +摁 +摄 +摆 +摇 +摈 +摊 +摒 +摔 +摘 +摞 +摟 +摧 +摩 +摯 +摳 +摸 +摹 +摺 +摻 +撂 +撃 +撅 +撇 +撈 +撐 +撑 +撒 +撓 +撕 +撚 +撞 +撤 +撥 +撩 +撫 +撬 +播 +撮 +撰 +撲 +撵 +撷 +撸 +撻 +撼 +撿 +擀 +擁 +擂 +擄 +擅 +擇 +擊 +擋 +操 +擎 +擒 +擔 +擘 +據 +擞 +擠 +擡 +擢 +擦 +擬 +擰 +擱 +擲 +擴 +擷 +擺 +擼 +擾 +攀 +攏 +攒 +攔 +攘 +攙 +攜 +攝 +攞 +攢 +攣 +攤 +攥 +攪 +攫 +攬 +支 +收 +攸 +改 +攻 +放 +政 +故 +效 +敌 +敍 +敎 +敏 +救 +敕 +敖 +敗 +敘 +教 +敛 +敝 +敞 +敢 +散 +敦 +敬 +数 +敲 +整 +敵 +敷 +數 +斂 +斃 +文 +斋 +斌 +斎 +斐 +斑 +斓 +斗 +料 +斛 +斜 +斟 +斡 +斤 +斥 +斧 +斩 +斫 +斬 +断 +斯 +新 +斷 +方 +於 +施 +旁 +旃 +旅 +旋 +旌 +旎 +族 +旖 +旗 +无 +既 +日 +旦 +旧 +旨 +早 +旬 +旭 +旮 +旱 +时 +旷 +旺 +旻 +昀 +昂 +昆 +昇 +昉 +昊 +昌 +明 +昏 +易 +昔 +昕 +昙 +星 +映 +春 +昧 +昨 +昭 +是 +昱 +昴 +昵 +昶 +昼 +显 +晁 +時 +晃 +晉 +晋 +晌 +晏 +晒 +晓 +晔 +晕 +晖 +晗 +晚 +晝 +晞 +晟 +晤 +晦 +晨 +晩 +普 +景 +晰 +晴 +晶 +晷 +智 +晾 +暂 +暄 +暇 +暈 +暉 +暌 +暐 +暑 +暖 +暗 +暝 +暢 +暧 +暨 +暫 +暮 +暱 +暴 +暸 +暹 +曄 +曆 +曇 +曉 +曖 +曙 +曜 +曝 +曠 +曦 +曬 +曰 +曲 +曳 +更 +書 +曹 +曼 +曾 +替 +最 +會 +月 +有 +朋 +服 +朐 +朔 +朕 +朗 +望 +朝 +期 +朦 +朧 +木 +未 +末 +本 +札 +朮 +术 +朱 +朴 +朵 +机 +朽 +杀 +杂 +权 +杆 +杈 +杉 +李 +杏 +材 +村 +杓 +杖 +杜 +杞 +束 +杠 +条 +来 +杨 +杭 +杯 +杰 +東 +杳 +杵 +杷 +杼 +松 +板 +极 +构 +枇 +枉 +枋 +析 +枕 +林 +枚 +果 +枝 +枢 +枣 +枪 +枫 +枭 +枯 +枰 +枱 +枳 +架 +枷 +枸 +柄 +柏 +某 +柑 +柒 +染 +柔 +柘 +柚 +柜 +柞 +柠 +柢 +查 +柩 +柬 +柯 +柱 +柳 +柴 +柵 +査 +柿 +栀 +栃 +栄 +栅 +标 +栈 +栉 +栋 +栎 +栏 +树 +栓 +栖 +栗 +校 +栩 +株 +样 +核 +根 +格 +栽 +栾 +桀 +桁 +桂 +桃 +桅 +框 +案 +桉 +桌 +桎 +桐 +桑 +桓 +桔 +桜 +桠 +桡 +桢 +档 +桥 +桦 +桧 +桨 +桩 +桶 +桿 +梁 +梅 +梆 +梏 +梓 +梗 +條 +梟 +梢 +梦 +梧 +梨 +梭 +梯 +械 +梳 +梵 +梶 +检 +棂 +棄 +棉 +棋 +棍 +棒 +棕 +棗 +棘 +棚 +棟 +棠 +棣 +棧 +森 +棱 +棲 +棵 +棹 +棺 +椁 +椅 +椋 +植 +椎 +椒 +検 +椪 +椭 +椰 +椹 +椽 +椿 +楂 +楊 +楓 +楔 +楚 +楝 +楞 +楠 +楣 +楨 +楫 +業 +楮 +極 +楷 +楸 +楹 +楼 +楽 +概 +榄 +榆 +榈 +榉 +榔 +榕 +榖 +榛 +榜 +榨 +榫 +榭 +榮 +榱 +榴 +榷 +榻 +槁 +槃 +構 +槌 +槍 +槎 +槐 +槓 +様 +槛 +槟 +槤 +槭 +槲 +槳 +槻 +槽 +槿 +樁 +樂 +樊 +樑 +樓 +標 +樞 +樟 +模 +樣 +権 +横 +樫 +樯 +樱 +樵 +樸 +樹 +樺 +樽 +樾 +橄 +橇 +橋 +橐 +橘 +橙 +機 +橡 +橢 +橫 +橱 +橹 +橼 +檀 +檄 +檎 +檐 +檔 +檗 +檜 +檢 +檬 +檯 +檳 +檸 +檻 +櫃 +櫚 +櫛 +櫥 +櫸 +櫻 +欄 +權 +欒 +欖 +欠 +次 +欢 +欣 +欧 +欲 +欸 +欺 +欽 +款 +歆 +歇 +歉 +歌 +歎 +歐 +歓 +歙 +歛 +歡 +止 +正 +此 +步 +武 +歧 +歩 +歪 +歯 +歲 +歳 +歴 +歷 +歸 +歹 +死 +歼 +殁 +殃 +殆 +殇 +殉 +殊 +残 +殒 +殓 +殖 +殘 +殞 +殡 +殤 +殭 +殯 +殲 +殴 +段 +殷 +殺 +殼 +殿 +毀 +毁 +毂 +毅 +毆 +毋 +母 +毎 +每 +毒 +毓 +比 +毕 +毗 +毘 +毙 +毛 +毡 +毫 +毯 +毽 +氈 +氏 +氐 +民 +氓 +气 +氖 +気 +氙 +氛 +氟 +氡 +氢 +氣 +氤 +氦 +氧 +氨 +氪 +氫 +氮 +氯 +氰 +氲 +水 +氷 +永 +氹 +氾 +汀 +汁 +求 +汆 +汇 +汉 +汎 +汐 +汕 +汗 +汙 +汛 +汝 +汞 +江 +池 +污 +汤 +汨 +汩 +汪 +汰 +汲 +汴 +汶 +汹 +決 +汽 +汾 +沁 +沂 +沃 +沅 +沈 +沉 +沌 +沏 +沐 +沒 +沓 +沖 +沙 +沛 +沟 +没 +沢 +沣 +沥 +沦 +沧 +沪 +沫 +沭 +沮 +沱 +河 +沸 +油 +治 +沼 +沽 +沾 +沿 +況 +泄 +泉 +泊 +泌 +泓 +法 +泗 +泛 +泞 +泠 +泡 +波 +泣 +泥 +注 +泪 +泫 +泮 +泯 +泰 +泱 +泳 +泵 +泷 +泸 +泻 +泼 +泽 +泾 +洁 +洄 +洋 +洒 +洗 +洙 +洛 +洞 +津 +洩 +洪 +洮 +洱 +洲 +洵 +洶 +洸 +洹 +活 +洼 +洽 +派 +流 +浃 +浄 +浅 +浆 +浇 +浊 +测 +济 +浏 +浑 +浒 +浓 +浔 +浙 +浚 +浜 +浣 +浦 +浩 +浪 +浬 +浮 +浯 +浴 +海 +浸 +涂 +涅 +涇 +消 +涉 +涌 +涎 +涓 +涔 +涕 +涙 +涛 +涝 +涞 +涟 +涠 +涡 +涣 +涤 +润 +涧 +涨 +涩 +涪 +涮 +涯 +液 +涵 +涸 +涼 +涿 +淀 +淄 +淅 +淆 +淇 +淋 +淌 +淑 +淒 +淖 +淘 +淙 +淚 +淞 +淡 +淤 +淦 +淨 +淩 +淪 +淫 +淬 +淮 +深 +淳 +淵 +混 +淹 +淺 +添 +淼 +清 +済 +渉 +渊 +渋 +渍 +渎 +渐 +渔 +渗 +渙 +渚 +減 +渝 +渠 +渡 +渣 +渤 +渥 +渦 +温 +測 +渭 +港 +渲 +渴 +游 +渺 +渾 +湃 +湄 +湊 +湍 +湖 +湘 +湛 +湟 +湧 +湫 +湮 +湯 +湳 +湾 +湿 +満 +溃 +溅 +溉 +溏 +源 +準 +溜 +溝 +溟 +溢 +溥 +溧 +溪 +溫 +溯 +溱 +溴 +溶 +溺 +溼 +滁 +滂 +滄 +滅 +滇 +滋 +滌 +滑 +滓 +滔 +滕 +滙 +滚 +滝 +滞 +滟 +满 +滢 +滤 +滥 +滦 +滨 +滩 +滬 +滯 +滲 +滴 +滷 +滸 +滾 +滿 +漁 +漂 +漆 +漉 +漏 +漓 +演 +漕 +漠 +漢 +漣 +漩 +漪 +漫 +漬 +漯 +漱 +漲 +漳 +漸 +漾 +漿 +潆 +潇 +潋 +潍 +潑 +潔 +潘 +潛 +潜 +潞 +潟 +潢 +潤 +潦 +潧 +潭 +潮 +潰 +潴 +潸 +潺 +潼 +澀 +澄 +澆 +澈 +澍 +澎 +澗 +澜 +澡 +澤 +澧 +澱 +澳 +澹 +激 +濁 +濂 +濃 +濑 +濒 +濕 +濘 +濛 +濟 +濠 +濡 +濤 +濫 +濬 +濮 +濯 +濱 +濺 +濾 +瀅 +瀆 +瀉 +瀋 +瀏 +瀑 +瀕 +瀘 +瀚 +瀛 +瀝 +瀞 +瀟 +瀧 +瀨 +瀬 +瀰 +瀾 +灌 +灏 +灑 +灘 +灝 +灞 +灣 +火 +灬 +灭 +灯 +灰 +灵 +灶 +灸 +灼 +災 +灾 +灿 +炀 +炁 +炅 +炉 +炊 +炎 +炒 +炔 +炕 +炖 +炙 +炜 +炫 +炬 +炭 +炮 +炯 +炳 +炷 +炸 +点 +為 +炼 +炽 +烁 +烂 +烃 +烈 +烊 +烏 +烘 +烙 +烛 +烟 +烤 +烦 +烧 +烨 +烩 +烫 +烬 +热 +烯 +烷 +烹 +烽 +焉 +焊 +焕 +焖 +焗 +焘 +焙 +焚 +焜 +無 +焦 +焯 +焰 +焱 +然 +焼 +煅 +煉 +煊 +煌 +煎 +煒 +煖 +煙 +煜 +煞 +煤 +煥 +煦 +照 +煨 +煩 +煮 +煲 +煸 +煽 +熄 +熊 +熏 +熒 +熔 +熙 +熟 +熠 +熨 +熬 +熱 +熵 +熹 +熾 +燁 +燃 +燄 +燈 +燉 +燊 +燎 +燒 +燔 +燕 +燙 +燜 +營 +燥 +燦 +燧 +燭 +燮 +燴 +燻 +燼 +燿 +爆 +爍 +爐 +爛 +爪 +爬 +爭 +爰 +爱 +爲 +爵 +父 +爷 +爸 +爹 +爺 +爻 +爽 +爾 +牆 +片 +版 +牌 +牍 +牒 +牙 +牛 +牝 +牟 +牠 +牡 +牢 +牦 +牧 +物 +牯 +牲 +牴 +牵 +特 +牺 +牽 +犀 +犁 +犄 +犊 +犍 +犒 +犢 +犧 +犬 +犯 +状 +犷 +犸 +犹 +狀 +狂 +狄 +狈 +狎 +狐 +狒 +狗 +狙 +狞 +狠 +狡 +狩 +独 +狭 +狮 +狰 +狱 +狸 +狹 +狼 +狽 +猎 +猕 +猖 +猗 +猙 +猛 +猜 +猝 +猥 +猩 +猪 +猫 +猬 +献 +猴 +猶 +猷 +猾 +猿 +獄 +獅 +獎 +獐 +獒 +獗 +獠 +獣 +獨 +獭 +獰 +獲 +獵 +獷 +獸 +獺 +獻 +獼 +獾 +玄 +率 +玉 +王 +玑 +玖 +玛 +玟 +玠 +玥 +玩 +玫 +玮 +环 +现 +玲 +玳 +玷 +玺 +玻 +珀 +珂 +珅 +珈 +珉 +珊 +珍 +珏 +珐 +珑 +珙 +珞 +珠 +珣 +珥 +珩 +珪 +班 +珮 +珲 +珺 +現 +球 +琅 +理 +琇 +琉 +琊 +琍 +琏 +琐 +琛 +琢 +琥 +琦 +琨 +琪 +琬 +琮 +琰 +琲 +琳 +琴 +琵 +琶 +琺 +琼 +瑀 +瑁 +瑄 +瑋 +瑕 +瑗 +瑙 +瑚 +瑛 +瑜 +瑞 +瑟 +瑠 +瑣 +瑤 +瑩 +瑪 +瑯 +瑰 +瑶 +瑾 +璀 +璁 +璃 +璇 +璉 +璋 +璎 +璐 +璜 +璞 +璟 +璧 +璨 +環 +璽 +璿 +瓊 +瓏 +瓒 +瓜 +瓢 +瓣 +瓤 +瓦 +瓮 +瓯 +瓴 +瓶 +瓷 +甄 +甌 +甕 +甘 +甙 +甚 +甜 +生 +產 +産 +甥 +甦 +用 +甩 +甫 +甬 +甭 +甯 +田 +由 +甲 +申 +电 +男 +甸 +町 +画 +甾 +畀 +畅 +界 +畏 +畑 +畔 +留 +畜 +畝 +畢 +略 +畦 +番 +畫 +異 +畲 +畳 +畴 +當 +畸 +畹 +畿 +疆 +疇 +疊 +疏 +疑 +疔 +疖 +疗 +疙 +疚 +疝 +疟 +疡 +疣 +疤 +疥 +疫 +疮 +疯 +疱 +疲 +疳 +疵 +疸 +疹 +疼 +疽 +疾 +痂 +病 +症 +痈 +痉 +痊 +痍 +痒 +痔 +痕 +痘 +痙 +痛 +痞 +痠 +痢 +痣 +痤 +痧 +痨 +痪 +痫 +痰 +痱 +痴 +痹 +痺 +痼 +痿 +瘀 +瘁 +瘋 +瘍 +瘓 +瘘 +瘙 +瘟 +瘠 +瘡 +瘢 +瘤 +瘦 +瘧 +瘩 +瘪 +瘫 +瘴 +瘸 +瘾 +療 +癇 +癌 +癒 +癖 +癜 +癞 +癡 +癢 +癣 +癥 +癫 +癬 +癮 +癱 +癲 +癸 +発 +登 +發 +白 +百 +皂 +的 +皆 +皇 +皈 +皋 +皎 +皑 +皓 +皖 +皙 +皚 +皮 +皰 +皱 +皴 +皺 +皿 +盂 +盃 +盅 +盆 +盈 +益 +盎 +盏 +盐 +监 +盒 +盔 +盖 +盗 +盘 +盛 +盜 +盞 +盟 +盡 +監 +盤 +盥 +盧 +盪 +目 +盯 +盱 +盲 +直 +相 +盹 +盼 +盾 +省 +眈 +眉 +看 +県 +眙 +眞 +真 +眠 +眦 +眨 +眩 +眯 +眶 +眷 +眸 +眺 +眼 +眾 +着 +睁 +睇 +睏 +睐 +睑 +睛 +睜 +睞 +睡 +睢 +督 +睥 +睦 +睨 +睪 +睫 +睬 +睹 +睽 +睾 +睿 +瞄 +瞅 +瞇 +瞋 +瞌 +瞎 +瞑 +瞒 +瞓 +瞞 +瞟 +瞠 +瞥 +瞧 +瞩 +瞪 +瞬 +瞭 +瞰 +瞳 +瞻 +瞼 +瞿 +矇 +矍 +矗 +矚 +矛 +矜 +矢 +矣 +知 +矩 +矫 +短 +矮 +矯 +石 +矶 +矽 +矾 +矿 +码 +砂 +砌 +砍 +砒 +研 +砖 +砗 +砚 +砝 +砣 +砥 +砧 +砭 +砰 +砲 +破 +砷 +砸 +砺 +砼 +砾 +础 +硅 +硐 +硒 +硕 +硝 +硫 +硬 +确 +硯 +硼 +碁 +碇 +碉 +碌 +碍 +碎 +碑 +碓 +碗 +碘 +碚 +碛 +碟 +碣 +碧 +碩 +碰 +碱 +碳 +碴 +確 +碼 +碾 +磁 +磅 +磊 +磋 +磐 +磕 +磚 +磡 +磨 +磬 +磯 +磲 +磷 +磺 +礁 +礎 +礙 +礡 +礦 +礪 +礫 +礴 +示 +礼 +社 +祀 +祁 +祂 +祇 +祈 +祉 +祎 +祐 +祕 +祖 +祗 +祚 +祛 +祜 +祝 +神 +祟 +祠 +祢 +祥 +票 +祭 +祯 +祷 +祸 +祺 +祿 +禀 +禁 +禄 +禅 +禍 +禎 +福 +禛 +禦 +禧 +禪 +禮 +禱 +禹 +禺 +离 +禽 +禾 +禿 +秀 +私 +秃 +秆 +秉 +秋 +种 +科 +秒 +秘 +租 +秣 +秤 +秦 +秧 +秩 +秭 +积 +称 +秸 +移 +秽 +稀 +稅 +程 +稍 +税 +稔 +稗 +稚 +稜 +稞 +稟 +稠 +稣 +種 +稱 +稲 +稳 +稷 +稹 +稻 +稼 +稽 +稿 +穀 +穂 +穆 +穌 +積 +穎 +穗 +穢 +穩 +穫 +穴 +究 +穷 +穹 +空 +穿 +突 +窃 +窄 +窈 +窍 +窑 +窒 +窓 +窕 +窖 +窗 +窘 +窜 +窝 +窟 +窠 +窥 +窦 +窨 +窩 +窪 +窮 +窯 +窺 +窿 +竄 +竅 +竇 +竊 +立 +竖 +站 +竜 +竞 +竟 +章 +竣 +童 +竭 +端 +競 +竹 +竺 +竽 +竿 +笃 +笆 +笈 +笋 +笏 +笑 +笔 +笙 +笛 +笞 +笠 +符 +笨 +第 +笹 +笺 +笼 +筆 +等 +筊 +筋 +筍 +筏 +筐 +筑 +筒 +答 +策 +筛 +筝 +筠 +筱 +筲 +筵 +筷 +筹 +签 +简 +箇 +箋 +箍 +箏 +箐 +箔 +箕 +算 +箝 +管 +箩 +箫 +箭 +箱 +箴 +箸 +節 +篁 +範 +篆 +篇 +築 +篑 +篓 +篙 +篝 +篠 +篡 +篤 +篩 +篪 +篮 +篱 +篷 +簇 +簌 +簍 +簡 +簦 +簧 +簪 +簫 +簷 +簸 +簽 +簾 +簿 +籁 +籃 +籌 +籍 +籐 +籟 +籠 +籤 +籬 +籮 +籲 +米 +类 +籼 +籽 +粄 +粉 +粑 +粒 +粕 +粗 +粘 +粟 +粤 +粥 +粧 +粪 +粮 +粱 +粲 +粳 +粵 +粹 +粼 +粽 +精 +粿 +糅 +糊 +糍 +糕 +糖 +糗 +糙 +糜 +糞 +糟 +糠 +糧 +糬 +糯 +糰 +糸 +系 +糾 +紀 +紂 +約 +紅 +紉 +紊 +紋 +納 +紐 +紓 +純 +紗 +紘 +紙 +級 +紛 +紜 +素 +紡 +索 +紧 +紫 +紮 +累 +細 +紳 +紹 +紺 +終 +絃 +組 +絆 +経 +結 +絕 +絞 +絡 +絢 +給 +絨 +絮 +統 +絲 +絳 +絵 +絶 +絹 +綁 +綏 +綑 +經 +継 +続 +綜 +綠 +綢 +綦 +綫 +綬 +維 +綱 +網 +綴 +綵 +綸 +綺 +綻 +綽 +綾 +綿 +緊 +緋 +総 +緑 +緒 +緘 +線 +緝 +緞 +締 +緣 +編 +緩 +緬 +緯 +練 +緹 +緻 +縁 +縄 +縈 +縛 +縝 +縣 +縫 +縮 +縱 +縴 +縷 +總 +績 +繁 +繃 +繆 +繇 +繋 +織 +繕 +繚 +繞 +繡 +繩 +繪 +繫 +繭 +繳 +繹 +繼 +繽 +纂 +續 +纍 +纏 +纓 +纔 +纖 +纜 +纠 +红 +纣 +纤 +约 +级 +纨 +纪 +纫 +纬 +纭 +纯 +纰 +纱 +纲 +纳 +纵 +纶 +纷 +纸 +纹 +纺 +纽 +纾 +线 +绀 +练 +组 +绅 +细 +织 +终 +绊 +绍 +绎 +经 +绑 +绒 +结 +绔 +绕 +绘 +给 +绚 +绛 +络 +绝 +绞 +统 +绡 +绢 +绣 +绥 +绦 +继 +绩 +绪 +绫 +续 +绮 +绯 +绰 +绳 +维 +绵 +绶 +绷 +绸 +绻 +综 +绽 +绾 +绿 +缀 +缄 +缅 +缆 +缇 +缈 +缉 +缎 +缓 +缔 +缕 +编 +缘 +缙 +缚 +缜 +缝 +缠 +缢 +缤 +缥 +缨 +缩 +缪 +缭 +缮 +缰 +缱 +缴 +缸 +缺 +缽 +罂 +罄 +罌 +罐 +网 +罔 +罕 +罗 +罚 +罡 +罢 +罩 +罪 +置 +罰 +署 +罵 +罷 +罹 +羁 +羅 +羈 +羊 +羌 +美 +羔 +羚 +羞 +羟 +羡 +羣 +群 +羥 +羧 +羨 +義 +羯 +羲 +羸 +羹 +羽 +羿 +翁 +翅 +翊 +翌 +翎 +習 +翔 +翘 +翟 +翠 +翡 +翦 +翩 +翰 +翱 +翳 +翹 +翻 +翼 +耀 +老 +考 +耄 +者 +耆 +耋 +而 +耍 +耐 +耒 +耕 +耗 +耘 +耙 +耦 +耨 +耳 +耶 +耷 +耸 +耻 +耽 +耿 +聂 +聆 +聊 +聋 +职 +聒 +联 +聖 +聘 +聚 +聞 +聪 +聯 +聰 +聲 +聳 +聴 +聶 +職 +聽 +聾 +聿 +肃 +肄 +肅 +肆 +肇 +肉 +肋 +肌 +肏 +肓 +肖 +肘 +肚 +肛 +肝 +肠 +股 +肢 +肤 +肥 +肩 +肪 +肮 +肯 +肱 +育 +肴 +肺 +肽 +肾 +肿 +胀 +胁 +胃 +胄 +胆 +背 +胍 +胎 +胖 +胚 +胛 +胜 +胝 +胞 +胡 +胤 +胥 +胧 +胫 +胭 +胯 +胰 +胱 +胳 +胴 +胶 +胸 +胺 +能 +脂 +脅 +脆 +脇 +脈 +脉 +脊 +脍 +脏 +脐 +脑 +脓 +脖 +脘 +脚 +脛 +脣 +脩 +脫 +脯 +脱 +脲 +脳 +脸 +脹 +脾 +腆 +腈 +腊 +腋 +腌 +腎 +腐 +腑 +腓 +腔 +腕 +腥 +腦 +腩 +腫 +腭 +腮 +腰 +腱 +腳 +腴 +腸 +腹 +腺 +腻 +腼 +腾 +腿 +膀 +膈 +膊 +膏 +膑 +膘 +膚 +膛 +膜 +膝 +膠 +膦 +膨 +膩 +膳 +膺 +膻 +膽 +膾 +膿 +臀 +臂 +臃 +臆 +臉 +臊 +臍 +臓 +臘 +臟 +臣 +臥 +臧 +臨 +自 +臬 +臭 +至 +致 +臺 +臻 +臼 +臾 +舀 +舂 +舅 +舆 +與 +興 +舉 +舊 +舌 +舍 +舎 +舐 +舒 +舔 +舖 +舗 +舛 +舜 +舞 +舟 +航 +舫 +般 +舰 +舱 +舵 +舶 +舷 +舸 +船 +舺 +舾 +艇 +艋 +艘 +艙 +艦 +艮 +良 +艰 +艱 +色 +艳 +艷 +艹 +艺 +艾 +节 +芃 +芈 +芊 +芋 +芍 +芎 +芒 +芙 +芜 +芝 +芡 +芥 +芦 +芩 +芪 +芫 +芬 +芭 +芮 +芯 +花 +芳 +芷 +芸 +芹 +芻 +芽 +芾 +苁 +苄 +苇 +苋 +苍 +苏 +苑 +苒 +苓 +苔 +苕 +苗 +苛 +苜 +苞 +苟 +苡 +苣 +若 +苦 +苫 +苯 +英 +苷 +苹 +苻 +茁 +茂 +范 +茄 +茅 +茉 +茎 +茏 +茗 +茜 +茧 +茨 +茫 +茬 +茭 +茯 +茱 +茲 +茴 +茵 +茶 +茸 +茹 +茼 +荀 +荃 +荆 +草 +荊 +荏 +荐 +荒 +荔 +荖 +荘 +荚 +荞 +荟 +荠 +荡 +荣 +荤 +荥 +荧 +荨 +荪 +荫 +药 +荳 +荷 +荸 +荻 +荼 +荽 +莅 +莆 +莉 +莊 +莎 +莒 +莓 +莖 +莘 +莞 +莠 +莢 +莧 +莪 +莫 +莱 +莲 +莴 +获 +莹 +莺 +莽 +莿 +菀 +菁 +菅 +菇 +菈 +菊 +菌 +菏 +菓 +菖 +菘 +菜 +菟 +菠 +菡 +菩 +華 +菱 +菲 +菸 +菽 +萁 +萃 +萄 +萊 +萋 +萌 +萍 +萎 +萘 +萝 +萤 +营 +萦 +萧 +萨 +萩 +萬 +萱 +萵 +萸 +萼 +落 +葆 +葉 +著 +葚 +葛 +葡 +董 +葦 +葩 +葫 +葬 +葭 +葯 +葱 +葳 +葵 +葷 +葺 +蒂 +蒋 +蒐 +蒔 +蒙 +蒜 +蒞 +蒟 +蒡 +蒨 +蒲 +蒸 +蒹 +蒻 +蒼 +蒿 +蓁 +蓄 +蓆 +蓉 +蓋 +蓑 +蓓 +蓖 +蓝 +蓟 +蓦 +蓬 +蓮 +蓼 +蓿 +蔑 +蔓 +蔔 +蔗 +蔘 +蔚 +蔡 +蔣 +蔥 +蔫 +蔬 +蔭 +蔵 +蔷 +蔺 +蔻 +蔼 +蔽 +蕁 +蕃 +蕈 +蕉 +蕊 +蕎 +蕙 +蕤 +蕨 +蕩 +蕪 +蕭 +蕲 +蕴 +蕻 +蕾 +薄 +薅 +薇 +薈 +薊 +薏 +薑 +薔 +薙 +薛 +薦 +薨 +薩 +薪 +薬 +薯 +薰 +薹 +藉 +藍 +藏 +藐 +藓 +藕 +藜 +藝 +藤 +藥 +藩 +藹 +藻 +藿 +蘆 +蘇 +蘊 +蘋 +蘑 +蘚 +蘭 +蘸 +蘼 +蘿 +虎 +虏 +虐 +虑 +虔 +處 +虚 +虛 +虜 +虞 +號 +虢 +虧 +虫 +虬 +虱 +虹 +虻 +虽 +虾 +蚀 +蚁 +蚂 +蚊 +蚌 +蚓 +蚕 +蚜 +蚝 +蚣 +蚤 +蚩 +蚪 +蚯 +蚱 +蚵 +蛀 +蛆 +蛇 +蛊 +蛋 +蛎 +蛐 +蛔 +蛙 +蛛 +蛟 +蛤 +蛭 +蛮 +蛰 +蛳 +蛹 +蛻 +蛾 +蜀 +蜂 +蜃 +蜆 +蜇 +蜈 +蜊 +蜍 +蜒 +蜓 +蜕 +蜗 +蜘 +蜚 +蜜 +蜡 +蜢 +蜥 +蜱 +蜴 +蜷 +蜻 +蜿 +蝇 +蝈 +蝉 +蝌 +蝎 +蝕 +蝗 +蝙 +蝟 +蝠 +蝦 +蝨 +蝴 +蝶 +蝸 +蝼 +螂 +螃 +融 +螞 +螢 +螨 +螯 +螳 +螺 +蟀 +蟄 +蟆 +蟋 +蟎 +蟑 +蟒 +蟠 +蟬 +蟲 +蟹 +蟻 +蟾 +蠅 +蠍 +蠔 +蠕 +蠛 +蠟 +蠡 +蠢 +蠣 +蠱 +蠶 +蠹 +蠻 +血 +衄 +衅 +衆 +行 +衍 +術 +衔 +街 +衙 +衛 +衝 +衞 +衡 +衢 +衣 +补 +表 +衩 +衫 +衬 +衮 +衰 +衲 +衷 +衹 +衾 +衿 +袁 +袂 +袄 +袅 +袈 +袋 +袍 +袒 +袖 +袜 +袞 +袤 +袪 +被 +袭 +袱 +裁 +裂 +装 +裆 +裊 +裏 +裔 +裕 +裘 +裙 +補 +裝 +裟 +裡 +裤 +裨 +裱 +裳 +裴 +裸 +裹 +製 +裾 +褂 +複 +褐 +褒 +褓 +褔 +褚 +褥 +褪 +褫 +褲 +褶 +褻 +襁 +襄 +襟 +襠 +襪 +襬 +襯 +襲 +西 +要 +覃 +覆 +覇 +見 +規 +覓 +視 +覚 +覦 +覧 +親 +覬 +観 +覷 +覺 +覽 +觀 +见 +观 +规 +觅 +视 +览 +觉 +觊 +觎 +觐 +觑 +角 +觞 +解 +觥 +触 +觸 +言 +訂 +計 +訊 +討 +訓 +訕 +訖 +託 +記 +訛 +訝 +訟 +訣 +訥 +訪 +設 +許 +訳 +訴 +訶 +診 +註 +証 +詆 +詐 +詔 +評 +詛 +詞 +詠 +詡 +詢 +詣 +試 +詩 +詫 +詬 +詭 +詮 +詰 +話 +該 +詳 +詹 +詼 +誅 +誇 +誉 +誌 +認 +誓 +誕 +誘 +語 +誠 +誡 +誣 +誤 +誥 +誦 +誨 +說 +説 +読 +誰 +課 +誹 +誼 +調 +諄 +談 +請 +諏 +諒 +論 +諗 +諜 +諡 +諦 +諧 +諫 +諭 +諮 +諱 +諳 +諷 +諸 +諺 +諾 +謀 +謁 +謂 +謄 +謊 +謎 +謐 +謔 +謗 +謙 +講 +謝 +謠 +謨 +謬 +謹 +謾 +譁 +證 +譎 +譏 +識 +譙 +譚 +譜 +警 +譬 +譯 +議 +譲 +譴 +護 +譽 +讀 +變 +讓 +讚 +讞 +计 +订 +认 +讥 +讧 +讨 +让 +讪 +讫 +训 +议 +讯 +记 +讲 +讳 +讴 +讶 +讷 +许 +讹 +论 +讼 +讽 +设 +访 +诀 +证 +诃 +评 +诅 +识 +诈 +诉 +诊 +诋 +词 +诏 +译 +试 +诗 +诘 +诙 +诚 +诛 +话 +诞 +诟 +诠 +诡 +询 +诣 +诤 +该 +详 +诧 +诩 +诫 +诬 +语 +误 +诰 +诱 +诲 +说 +诵 +诶 +请 +诸 +诺 +读 +诽 +课 +诿 +谀 +谁 +调 +谄 +谅 +谆 +谈 +谊 +谋 +谌 +谍 +谎 +谏 +谐 +谑 +谒 +谓 +谔 +谕 +谗 +谘 +谙 +谚 +谛 +谜 +谟 +谢 +谣 +谤 +谥 +谦 +谧 +谨 +谩 +谪 +谬 +谭 +谯 +谱 +谲 +谴 +谶 +谷 +豁 +豆 +豇 +豈 +豉 +豊 +豌 +豎 +豐 +豔 +豚 +象 +豢 +豪 +豫 +豬 +豹 +豺 +貂 +貅 +貌 +貓 +貔 +貘 +貝 +貞 +負 +財 +貢 +貧 +貨 +販 +貪 +貫 +責 +貯 +貰 +貳 +貴 +貶 +買 +貸 +費 +貼 +貽 +貿 +賀 +賁 +賂 +賃 +賄 +資 +賈 +賊 +賑 +賓 +賜 +賞 +賠 +賡 +賢 +賣 +賤 +賦 +質 +賬 +賭 +賴 +賺 +購 +賽 +贅 +贈 +贊 +贍 +贏 +贓 +贖 +贛 +贝 +贞 +负 +贡 +财 +责 +贤 +败 +账 +货 +质 +贩 +贪 +贫 +贬 +购 +贮 +贯 +贰 +贱 +贲 +贴 +贵 +贷 +贸 +费 +贺 +贻 +贼 +贾 +贿 +赁 +赂 +赃 +资 +赅 +赈 +赊 +赋 +赌 +赎 +赏 +赐 +赓 +赔 +赖 +赘 +赚 +赛 +赝 +赞 +赠 +赡 +赢 +赣 +赤 +赦 +赧 +赫 +赭 +走 +赳 +赴 +赵 +赶 +起 +趁 +超 +越 +趋 +趕 +趙 +趟 +趣 +趨 +足 +趴 +趵 +趸 +趺 +趾 +跃 +跄 +跆 +跋 +跌 +跎 +跑 +跖 +跚 +跛 +距 +跟 +跡 +跤 +跨 +跩 +跪 +路 +跳 +践 +跷 +跹 +跺 +跻 +踉 +踊 +踌 +踏 +踐 +踝 +踞 +踟 +踢 +踩 +踪 +踮 +踱 +踴 +踵 +踹 +蹂 +蹄 +蹇 +蹈 +蹉 +蹊 +蹋 +蹑 +蹒 +蹙 +蹟 +蹣 +蹤 +蹦 +蹩 +蹬 +蹭 +蹲 +蹴 +蹶 +蹺 +蹼 +蹿 +躁 +躇 +躉 +躊 +躋 +躍 +躏 +躪 +身 +躬 +躯 +躲 +躺 +軀 +車 +軋 +軌 +軍 +軒 +軟 +転 +軸 +軼 +軽 +軾 +較 +載 +輒 +輓 +輔 +輕 +輛 +輝 +輟 +輩 +輪 +輯 +輸 +輻 +輾 +輿 +轄 +轅 +轆 +轉 +轍 +轎 +轟 +车 +轧 +轨 +轩 +转 +轭 +轮 +软 +轰 +轲 +轴 +轶 +轻 +轼 +载 +轿 +较 +辄 +辅 +辆 +辇 +辈 +辉 +辊 +辍 +辐 +辑 +输 +辕 +辖 +辗 +辘 +辙 +辛 +辜 +辞 +辟 +辣 +辦 +辨 +辩 +辫 +辭 +辮 +辯 +辰 +辱 +農 +边 +辺 +辻 +込 +辽 +达 +迁 +迂 +迄 +迅 +过 +迈 +迎 +运 +近 +返 +还 +这 +进 +远 +违 +连 +迟 +迢 +迤 +迥 +迦 +迩 +迪 +迫 +迭 +述 +迴 +迷 +迸 +迹 +迺 +追 +退 +送 +适 +逃 +逅 +逆 +选 +逊 +逍 +透 +逐 +递 +途 +逕 +逗 +這 +通 +逛 +逝 +逞 +速 +造 +逢 +連 +逮 +週 +進 +逵 +逶 +逸 +逻 +逼 +逾 +遁 +遂 +遅 +遇 +遊 +運 +遍 +過 +遏 +遐 +遑 +遒 +道 +達 +違 +遗 +遙 +遛 +遜 +遞 +遠 +遢 +遣 +遥 +遨 +適 +遭 +遮 +遲 +遴 +遵 +遶 +遷 +選 +遺 +遼 +遽 +避 +邀 +邁 +邂 +邃 +還 +邇 +邈 +邊 +邋 +邏 +邑 +邓 +邕 +邛 +邝 +邢 +那 +邦 +邨 +邪 +邬 +邮 +邯 +邰 +邱 +邳 +邵 +邸 +邹 +邺 +邻 +郁 +郅 +郊 +郎 +郑 +郜 +郝 +郡 +郢 +郤 +郦 +郧 +部 +郫 +郭 +郴 +郵 +郷 +郸 +都 +鄂 +鄉 +鄒 +鄔 +鄙 +鄞 +鄢 +鄧 +鄭 +鄰 +鄱 +鄲 +鄺 +酉 +酊 +酋 +酌 +配 +酐 +酒 +酗 +酚 +酝 +酢 +酣 +酥 +酩 +酪 +酬 +酮 +酯 +酰 +酱 +酵 +酶 +酷 +酸 +酿 +醃 +醇 +醉 +醋 +醍 +醐 +醒 +醚 +醛 +醜 +醞 +醣 +醪 +醫 +醬 +醮 +醯 +醴 +醺 +釀 +釁 +采 +釉 +释 +釋 +里 +重 +野 +量 +釐 +金 +釗 +釘 +釜 +針 +釣 +釦 +釧 +釵 +鈀 +鈉 +鈍 +鈎 +鈔 +鈕 +鈞 +鈣 +鈦 +鈪 +鈴 +鈺 +鈾 +鉀 +鉄 +鉅 +鉉 +鉑 +鉗 +鉚 +鉛 +鉤 +鉴 +鉻 +銀 +銃 +銅 +銑 +銓 +銖 +銘 +銜 +銬 +銭 +銮 +銳 +銷 +銹 +鋁 +鋅 +鋒 +鋤 +鋪 +鋰 +鋸 +鋼 +錄 +錐 +錘 +錚 +錠 +錢 +錦 +錨 +錫 +錮 +錯 +録 +錳 +錶 +鍊 +鍋 +鍍 +鍛 +鍥 +鍰 +鍵 +鍺 +鍾 +鎂 +鎊 +鎌 +鎏 +鎔 +鎖 +鎗 +鎚 +鎧 +鎬 +鎮 +鎳 +鏈 +鏖 +鏗 +鏘 +鏞 +鏟 +鏡 +鏢 +鏤 +鏽 +鐘 +鐮 +鐲 +鐳 +鐵 +鐸 +鐺 +鑄 +鑊 +鑑 +鑒 +鑣 +鑫 +鑰 +鑲 +鑼 +鑽 +鑾 +鑿 +针 +钉 +钊 +钎 +钏 +钒 +钓 +钗 +钙 +钛 +钜 +钝 +钞 +钟 +钠 +钡 +钢 +钣 +钤 +钥 +钦 +钧 +钨 +钩 +钮 +钯 +钰 +钱 +钳 +钴 +钵 +钺 +钻 +钼 +钾 +钿 +铀 +铁 +铂 +铃 +铄 +铅 +铆 +铉 +铎 +铐 +铛 +铜 +铝 +铠 +铡 +铢 +铣 +铤 +铨 +铩 +铬 +铭 +铮 +铰 +铲 +铵 +银 +铸 +铺 +链 +铿 +销 +锁 +锂 +锄 +锅 +锆 +锈 +锉 +锋 +锌 +锏 +锐 +锑 +错 +锚 +锟 +锡 +锢 +锣 +锤 +锥 +锦 +锭 +键 +锯 +锰 +锲 +锵 +锹 +锺 +锻 +镀 +镁 +镂 +镇 +镉 +镌 +镍 +镐 +镑 +镕 +镖 +镗 +镛 +镜 +镣 +镭 +镯 +镰 +镳 +镶 +長 +长 +門 +閃 +閉 +開 +閎 +閏 +閑 +閒 +間 +閔 +閘 +閡 +関 +閣 +閥 +閨 +閩 +閱 +閲 +閹 +閻 +閾 +闆 +闇 +闊 +闌 +闍 +闔 +闕 +闖 +闘 +關 +闡 +闢 +门 +闪 +闫 +闭 +问 +闯 +闰 +闲 +间 +闵 +闷 +闸 +闹 +闺 +闻 +闽 +闾 +阀 +阁 +阂 +阅 +阆 +阇 +阈 +阉 +阎 +阐 +阑 +阔 +阕 +阖 +阙 +阚 +阜 +队 +阡 +阪 +阮 +阱 +防 +阳 +阴 +阵 +阶 +阻 +阿 +陀 +陂 +附 +际 +陆 +陇 +陈 +陋 +陌 +降 +限 +陕 +陛 +陝 +陞 +陟 +陡 +院 +陣 +除 +陨 +险 +陪 +陰 +陲 +陳 +陵 +陶 +陷 +陸 +険 +陽 +隅 +隆 +隈 +隊 +隋 +隍 +階 +随 +隐 +隔 +隕 +隘 +隙 +際 +障 +隠 +隣 +隧 +隨 +險 +隱 +隴 +隶 +隸 +隻 +隼 +隽 +难 +雀 +雁 +雄 +雅 +集 +雇 +雉 +雋 +雌 +雍 +雎 +雏 +雑 +雒 +雕 +雖 +雙 +雛 +雜 +雞 +離 +難 +雨 +雪 +雯 +雰 +雲 +雳 +零 +雷 +雹 +電 +雾 +需 +霁 +霄 +霆 +震 +霈 +霉 +霊 +霍 +霎 +霏 +霑 +霓 +霖 +霜 +霞 +霧 +霭 +霰 +露 +霸 +霹 +霽 +霾 +靂 +靄 +靈 +青 +靓 +靖 +静 +靚 +靛 +靜 +非 +靠 +靡 +面 +靥 +靦 +革 +靳 +靴 +靶 +靼 +鞅 +鞋 +鞍 +鞏 +鞑 +鞘 +鞠 +鞣 +鞦 +鞭 +韆 +韋 +韌 +韓 +韜 +韦 +韧 +韩 +韬 +韭 +音 +韵 +韶 +韻 +響 +頁 +頂 +頃 +項 +順 +須 +頌 +預 +頑 +頒 +頓 +頗 +領 +頜 +頡 +頤 +頫 +頭 +頰 +頷 +頸 +頹 +頻 +頼 +顆 +題 +額 +顎 +顏 +顔 +願 +顛 +類 +顧 +顫 +顯 +顱 +顴 +页 +顶 +顷 +项 +顺 +须 +顼 +顽 +顾 +顿 +颁 +颂 +预 +颅 +领 +颇 +颈 +颉 +颊 +颌 +颍 +颐 +频 +颓 +颔 +颖 +颗 +题 +颚 +颛 +颜 +额 +颞 +颠 +颡 +颢 +颤 +颦 +颧 +風 +颯 +颱 +颳 +颶 +颼 +飄 +飆 +风 +飒 +飓 +飕 +飘 +飙 +飚 +飛 +飞 +食 +飢 +飨 +飩 +飪 +飯 +飲 +飼 +飽 +飾 +餃 +餅 +餉 +養 +餌 +餐 +餒 +餓 +餘 +餚 +餛 +餞 +餡 +館 +餮 +餵 +餾 +饅 +饈 +饋 +饌 +饍 +饑 +饒 +饕 +饗 +饞 +饥 +饨 +饪 +饬 +饭 +饮 +饯 +饰 +饱 +饲 +饴 +饵 +饶 +饷 +饺 +饼 +饽 +饿 +馀 +馁 +馄 +馅 +馆 +馈 +馋 +馍 +馏 +馒 +馔 +首 +馗 +香 +馥 +馨 +馬 +馭 +馮 +馳 +馴 +駁 +駄 +駅 +駆 +駐 +駒 +駕 +駛 +駝 +駭 +駱 +駿 +騁 +騎 +騏 +験 +騙 +騨 +騰 +騷 +驀 +驅 +驊 +驍 +驒 +驕 +驗 +驚 +驛 +驟 +驢 +驥 +马 +驭 +驮 +驯 +驰 +驱 +驳 +驴 +驶 +驷 +驸 +驹 +驻 +驼 +驾 +驿 +骁 +骂 +骄 +骅 +骆 +骇 +骈 +骊 +骋 +验 +骏 +骐 +骑 +骗 +骚 +骛 +骜 +骞 +骠 +骡 +骤 +骥 +骧 +骨 +骯 +骰 +骶 +骷 +骸 +骼 +髂 +髅 +髋 +髏 +髒 +髓 +體 +髖 +高 +髦 +髪 +髮 +髯 +髻 +鬃 +鬆 +鬍 +鬓 +鬚 +鬟 +鬢 +鬣 +鬥 +鬧 +鬱 +鬼 +魁 +魂 +魄 +魅 +魇 +魍 +魏 +魔 +魘 +魚 +魯 +魷 +鮑 +鮨 +鮪 +鮭 +鮮 +鯉 +鯊 +鯖 +鯛 +鯨 +鯰 +鯽 +鰍 +鰓 +鰭 +鰲 +鰻 +鰾 +鱈 +鱉 +鱔 +鱗 +鱷 +鱸 +鱼 +鱿 +鲁 +鲈 +鲍 +鲑 +鲛 +鲜 +鲟 +鲢 +鲤 +鲨 +鲫 +鲱 +鲲 +鲶 +鲷 +鲸 +鳃 +鳄 +鳅 +鳌 +鳍 +鳕 +鳖 +鳗 +鳝 +鳞 +鳥 +鳩 +鳳 +鳴 +鳶 +鴉 +鴕 +鴛 +鴦 +鴨 +鴻 +鴿 +鵑 +鵜 +鵝 +鵡 +鵬 +鵰 +鵲 +鶘 +鶩 +鶯 +鶴 +鷗 +鷲 +鷹 +鷺 +鸚 +鸞 +鸟 +鸠 +鸡 +鸢 +鸣 +鸥 +鸦 +鸨 +鸪 +鸭 +鸯 +鸳 +鸵 +鸽 +鸾 +鸿 +鹂 +鹃 +鹄 +鹅 +鹈 +鹉 +鹊 +鹌 +鹏 +鹑 +鹕 +鹘 +鹜 +鹞 +鹤 +鹦 +鹧 +鹫 +鹭 +鹰 +鹳 +鹵 +鹹 +鹼 +鹽 +鹿 +麂 +麋 +麒 +麓 +麗 +麝 +麟 +麥 +麦 +麩 +麴 +麵 +麸 +麺 +麻 +麼 +麽 +麾 +黃 +黄 +黍 +黎 +黏 +黑 +黒 +黔 +默 +黛 +黜 +黝 +點 +黠 +黨 +黯 +黴 +鼋 +鼎 +鼐 +鼓 +鼠 +鼬 +鼹 +鼻 +鼾 +齁 +齊 +齋 +齐 +齒 +齡 +齢 +齣 +齦 +齿 +龄 +龅 +龈 +龊 +龋 +龌 +龍 +龐 +龔 +龕 +龙 +龚 +龛 +龜 +龟 +︰ +︱ +︶ +︿ +﹁ +﹂ +﹍ +﹏ +﹐ +﹑ +﹒ +﹔ +﹕ +﹖ +﹗ +﹙ +﹚ +﹝ +﹞ +﹡ +﹣ +！ +＂ +＃ +＄ +％ +＆ +＇ +（ +） +＊ +＋ +， +－ +． +／ +０ +１ +２ +３ +４ +５ +６ +７ +８ +９ +： +； +＜ +＝ +＞ +？ +＠ +［ +＼ +］ +＾ +＿ +｀ +ａ +ｂ +ｃ +ｄ +ｅ +ｆ +ｇ +ｈ +ｉ +ｊ +ｋ +ｌ +ｍ +ｎ +ｏ +ｐ +ｑ +ｒ +ｓ +ｔ +ｕ +ｖ +ｗ +ｘ +ｙ +ｚ +｛ +｜ +｝ +～ +｡ +｢ +｣ +､ +･ +ｯ +ｰ +ｲ +ｸ +ｼ +ｽ +ﾄ +ﾉ +ﾌ +ﾗ +ﾙ +ﾝ +ﾞ +ﾟ +￣ +￥ +👍 +🔥 +😂 +😎 +... +yam +10 +2017 +12 +11 +2016 +20 +30 +15 +06 +lofter +##s +2015 +by +16 +14 +18 +13 +24 +17 +2014 +21 +##0 +22 +19 +25 +23 +com +100 +00 +05 +2013 +##a +03 +09 +08 +28 +##2 +50 +01 +04 +##1 +27 +02 +2012 +##3 +26 +##e +07 +##8 +##5 +##6 +##4 +##9 +##7 +29 +2011 +40 +##t +2010 +##o +##d +##i +2009 +##n +app +www +the +##m +31 +##c +##l +##y +##r +##g +2008 +60 +http +200 +qq +##p +80 +##f +google +pixnet +90 +cookies +tripadvisor +500 +##er +##k +35 +##h +facebook +2007 +2000 +70 +##b +of +##x +##u +45 +300 +iphone +32 +1000 +2006 +48 +ip +36 +in +38 +3d +##w +##ing +55 +ctrip +##on +##v +33 +##の +to +34 +400 +id +2005 +it +37 +windows +llc +top +99 +42 +39 +000 +led +at +##an +41 +51 +52 +46 +49 +43 +53 +44 +##z +android +58 +and +59 +2004 +56 +vr +##か +5000 +2003 +47 +blogthis +twitter +54 +##le +150 +ok +2018 +57 +75 +cn +no +ios +##in +##mm +##00 +800 +on +te +3000 +65 +2001 +360 +95 +ig +lv +120 +##ng +##を +##us +##に +pc +てす +── +600 +##te +85 +2002 +88 +##ed +html +ncc +wifi +email +64 +blog +is +##10 +##て +mail +online +##al +dvd +##ic +studio +##は +##℃ +##ia +##と +line +vip +72 +##q +98 +##ce +##en +for +##is +##ra +##es +##j +usb +net +cp +1999 +asia +4g +##cm +diy +new +3c +##お +ta +66 +language +vs +apple +tw +86 +web +##ne +ipad +62 +you +##re +101 +68 +##tion +ps +de +bt +pony +atm +##2017 +1998 +67 +##ch +ceo +##or +go +##na +av +pro +cafe +96 +pinterest +97 +63 +pixstyleme3c +##ta +more +said +##2016 +1997 +mp3 +700 +##ll +nba +jun +##20 +92 +tv +1995 +pm +61 +76 +nbsp +250 +##ie +linux +##ma +cd +110 +hd +##17 +78 +##ion +77 +6000 +am +##th +##st +94 +##se +##et +69 +180 +gdp +my +105 +81 +abc +89 +flash +79 +one +93 +1990 +1996 +##ck +gps +##も +##ly +web885 +106 +2020 +91 +##ge +4000 +1500 +xd +boss +isbn +1994 +org +##ry +me +love +##11 +0fork +73 +##12 +3g +##ter +##ar +71 +82 +##la +hotel +130 +1970 +pk +83 +87 +140 +ie +##os +##30 +##el +74 +##50 +seo +cpu +##ml +p2p +84 +may +##る +sun +tue +internet +cc +posted +youtube +##at +##ン +##man +ii +##ル +##15 +abs +nt +pdf +yahoo +ago +1980 +##it +news +mac +104 +##てす +##me +##り +java +1992 +spa +##de +##nt +hk +all +plus +la +1993 +##mb +##16 +##ve +west +##da +160 +air +##い +##ps +から +##to +1989 +logo +htc +php +https +fi +momo +##son +sat +##ke +##80 +ebd +suv +wi +day +apk +##88 +##um +mv +galaxy +wiki +or +brake +##ス +1200 +する +this +1991 +mon +##こ +❤2017 +po +##ない +javascript +life +home +june +##ss +system +900 +##ー +##０ +pp +1988 +world +fb +4k +br +##as +ic +ai +leonardo +safari +##60 +live +free +xx +wed +win7 +kiehl +##co +lg +o2o +##go +us +235 +1949 +mm +しい +vfm +kanye +##90 +##2015 +##id +jr +##ey +123 +rss +##sa +##ro +##am +##no +thu +fri +350 +##sh +##ki +103 +comments +name +##のて +##pe +##ine +max +1987 +8000 +uber +##mi +##ton +wordpress +office +1986 +1985 +##ment +107 +bd +win10 +##ld +##li +gmail +bb +dior +##rs +##ri +##rd +##ます +up +cad +##® +dr +して +read +##21 +をお +##io +##99 +url +1984 +pvc +paypal +show +policy +##40 +##ty +##18 +with +##★ +##01 +txt +102 +##ba +dna +from +post +mini +ar +taiwan +john +##ga +privacy +agoda +##13 +##ny +word +##24 +##22 +##by +##ur +##hz +1982 +##ang +265 +cookie +netscape +108 +##ka +##～ +##ad +house +share +note +ibm +code +hello +nike +sim +survey +##016 +1979 +1950 +wikia +##32 +##017 +5g +cbc +##tor +##kg +1983 +##rt +##14 +campaign +store +2500 +os +##ct +##ts +##° +170 +api +##ns +365 +excel +##な +##ao +##ら +##し +～～ +##nd +university +163 +には +518 +##70 +##ya +##il +##25 +pierre +ipo +0020 +897 +##23 +hotels +##ian +のお +125 +years +6606 +##ers +##26 +high +##day +time +##ay +bug +##line +##く +##す +##be +xp +talk2yam +yamservice +10000 +coco +##dy +sony +##ies +1978 +microsoft +david +people +##ha +1960 +instagram +intel +その +##ot +iso +1981 +##va +115 +##mo +##land +xxx +man +co +ltxsw +##ation +baby +220 +##pa +##ol +1945 +7000 +tag +450 +##ue +msn +##31 +oppo +##ト +##ca +control +##om +st +chrome +##ure +##ん +be +##き +lol +##19 +した +##bo +240 +lady +##100 +##way +##から +4600 +##ko +##do +##un +4s +corporation +168 +##ni +herme +##28 +ｃｐ +978 +##up +##06 +ui +##ds +ppt +admin +three +します +bbc +re +128 +##48 +ca +##015 +##35 +hp +##ee +tpp +##た +##ive +×× +root +##cc +##ました +##ble +##ity +adobe +park +114 +et +oled +city +##ex +##ler +##ap +china +##book +20000 +view +##ice +global +##km +your +hong +##mg +out +##ms +ng +ebay +##29 +menu +ubuntu +##cy +rom +##view +open +ktv +do +server +##lo +if +english +##ね +##５ +##oo +1600 +##02 +step1 +kong +club +135 +july +inc +1976 +mr +hi +##net +touch +##ls +##ii +michael +lcd +##05 +##33 +phone +james +step2 +1300 +ios9 +##box +dc +##２ +##ley +samsung +111 +280 +pokemon +css +##ent +##les +いいえ +##１ +s8 +atom +play +bmw +##said +sa +etf +ctrl +♥yoyo♥ +##55 +2025 +##2014 +##66 +adidas +amazon +1958 +##ber +##ner +visa +##77 +##der +1800 +connectivity +##hi +firefox +109 +118 +hr +so +style +mark +pop +ol +skip +1975 +as +##27 +##ir +##61 +190 +mba +##う +##ai +le +##ver +1900 +cafe2017 +lte +super +113 +129 +##ron +amd +like +##☆ +are +##ster +we +##sk +paul +data +international +##ft +longchamp +ssd +good +##ート +##ti +reply +##my +↓↓↓ +apr +star +##ker +source +136 +js +112 +get +force +photo +##one +126 +##2013 +##ow +link +bbs +1972 +goods +##lin +python +119 +##ip +game +##ics +##ません +blue +##● +520 +##45 +page +itunes +##03 +1955 +260 +1968 +gt +gif +618 +##ff +##47 +group +くたさい +about +bar +ganji +##nce +music +lee +not +1977 +1971 +1973 +##per +an +faq +comment +##って +days +##ock +116 +##bs +1974 +1969 +v1 +player +1956 +xbox +sql +fm +f1 +139 +##ah +210 +##lv +##mp +##000 +melody +1957 +##３ +550 +17life +199 +1966 +xml +market +##au +##71 +999 +##04 +what +gl +##95 +##age +tips +##68 +book +##ting +mysql +can +1959 +230 +##ung +wonderland +watch +10℃ +##ction +9000 +mar +mobile +1946 +1962 +article +##db +part +▲top +party +って +1967 +1964 +1948 +##07 +##ore +##op +この +dj +##78 +##38 +010 +main +225 +1965 +##ong +art +320 +ad +134 +020 +##73 +117 +pm2 +japan +228 +##08 +ts +1963 +##ica +der +sm +##36 +2019 +##wa +ct +##７ +##や +##64 +1937 +homemesh +search +##85 +##れは +##tv +##di +macbook +##９ +##くたさい +service +##♥ +type +った +750 +##ier +##si +##75 +##います +##ok +best +##ット +goris +lock +##った +cf +3m +big +##ut +ftp +carol +##vi +１０ +1961 +happy +sd +##ac +122 +anti +pe +cnn +iii +1920 +138 +##ラ +1940 +esp +jan +tags +##98 +##51 +august +vol +##86 +154 +##™ +##fs +##れ +##sion +design +ac +##ム +press +jordan +ppp +that +key +check +##６ +##tt +##㎡ +1080p +##lt +power +##42 +1952 +##bc +vivi +##ック +he +133 +121 +jpg +##rry +201 +175 +3500 +1947 +nb +##ted +##rn +しています +1954 +usd +##t00 +master +##ンク +001 +model +##58 +al +##09 +1953 +##34 +ram +goo +ても +##ui +127 +1930 +red +##ary +rpg +item +##pm +##41 +270 +##za +project +##2012 +hot +td +blogabstract +##ger +##62 +650 +##44 +gr2 +##します +##ｍ +black +electronic +nfc +year +asus +また +html5 +cindy +##hd +m3 +132 +esc +##od +booking +##53 +fed +tvb +##81 +##ina +mit +165 +##いる +chan +192 +distribution +next +になる +peter +bios +steam +cm +1941 +にも +pk10 +##ix +##65 +##91 +dec +nasa +##ana +icecat +00z +b1 +will +##46 +li +se +##ji +##み +##ard +oct +##ain +jp +##ze +##bi +cio +##56 +smart +h5 +##39 +##port +curve +vpn +##nm +##dia +utc +##あり +12345678910 +##52 +rmvb +chanel +a4 +miss +##and +##im +media +who +##63 +she +girl +5s +124 +vera +##して +class +vivo +king +##フ +##ei +national +ab +1951 +5cm +888 +145 +ipod +ap +1100 +5mm +211 +ms +2756 +##69 +mp4 +msci +##po +##89 +131 +mg +index +380 +##bit +##out +##zz +##97 +##67 +158 +apec +##８ +photoshop +opec +￥799 +ては +##96 +##tes +##ast +2g +○○ +##ール +￥2899 +##ling +##よ +##ory +1938 +##ical +kitty +content +##43 +step3 +##cn +win8 +155 +vc +1400 +iphone7 +robert +##した +tcl +137 +beauty +##87 +en +dollars +##ys +##oc +step +pay +yy +a1 +##2011 +##lly +##ks +##♪ +1939 +188 +download +1944 +sep +exe +ph +います +school +gb +center +pr +street +##board +uv +##37 +##lan +winrar +##que +##ua +##com +1942 +1936 +480 +gpu +##４ +ettoday +fu +tom +##54 +##ren +##via +149 +##72 +b2b +144 +##79 +##tch +rose +arm +mb +##49 +##ial +##nn +nvidia +step4 +mvp +00㎡ +york +156 +##イ +how +cpi +591 +2765 +gov +kg +joe +##xx +mandy +pa +##ser +copyright +fashion +1935 +don +##け +ecu +##ist +##art +erp +wap +have +##lm +talk +##ek +##ning +##if +ch +##ite +video +1943 +cs +san +iot +look +##84 +##2010 +##ku +october +##ux +trump +##hs +##ide +box +141 +first +##ins +april +##ight +##83 +185 +angel +protected +aa +151 +162 +x1 +m2 +##fe +##× +##ho +size +143 +min +ofo +fun +gomaji +ex +hdmi +food +dns +march +chris +kevin +##のか +##lla +##pp +##ec +ag +ems +6s +720p +##rm +##ham +off +##92 +asp +team +fandom +ed +299 +▌♥ +##ell +info +されています +##82 +sina +4066 +161 +##able +##ctor +330 +399 +315 +dll +rights +ltd +idc +jul +3kg +1927 +142 +ma +surface +##76 +##ク +～～～ +304 +mall +eps +146 +green +##59 +map +space +donald +v2 +sodu +##light +1931 +148 +1700 +まて +310 +reserved +htm +##han +##57 +2d +178 +mod +##ise +##tions +152 +ti +##shi +doc +1933 +icp +055 +wang +##ram +shopping +aug +##pi +##well +now +wam +b2 +からお +##hu +236 +1928 +##gb +266 +f2 +##93 +153 +mix +##ef +##uan +bwl +##plus +##res +core +##ess +tea +5℃ +hktvmall +nhk +##ate +list +##ese +301 +feb +4m +inn +ての +nov +159 +12345 +daniel +##ci +pass +##bet +##nk +coffee +202 +ssl +airbnb +##ute +fbi +woshipm +skype +ea +cg +sp +##fc +##www +yes +edge +alt +007 +##94 +fpga +##ght +##gs +iso9001 +さい +##ile +##wood +##uo +image +lin +icon +american +##em +1932 +set +says +##king +##tive +blogger +##74 +なと +256 +147 +##ox +##zy +##red +##ium +##lf +nokia +claire +##リ +##ding +november +lohas +##500 +##tic +##マ +##cs +##ある +##che +##ire +##gy +##ult +db +january +win +##カ +166 +road +ptt +##ま +##つ +198 +##fa +##mer +anna +pchome +はい +udn +ef +420 +##time +##tte +2030 +##ア +g20 +white +かかります +1929 +308 +garden +eleven +di +##おります +chen +309b +777 +172 +young +cosplay +ちてない +4500 +bat +##123 +##tra +##ては +kindle +npc +steve +etc +##ern +##｜ +call +xperia +ces +travel +sk +s7 +##ous +1934 +##int +みいたたけます +183 +edu +file +cho +qr +##car +##our +186 +##ant +##ｄ +eric +1914 +rends +##jo +##する +mastercard +##2000 +kb +##min +290 +##ino +vista +##ris +##ud +jack +2400 +##set +169 +pos +1912 +##her +##ou +taipei +しく +205 +beta +##ませんか +232 +##fi +express +255 +body +##ill +aphojoy +user +december +meiki +##ick +tweet +richard +##av +##ᆫ +iphone6 +##dd +ちてすか +views +##mark +321 +pd +##００ +times +##▲ +level +##ash +10g +point +5l +##ome +208 +koreanmall +##ak +george +q2 +206 +wma +tcp +##200 +スタッフ +full +mlb +##lle +##watch +tm +run +179 +911 +smith +business +##und +1919 +color +##tal +222 +171 +##less +moon +4399 +##rl +update +pcb +shop +499 +157 +little +なし +end +##mhz +van +dsp +easy +660 +##house +##key +history +##ｏ +oh +##001 +##hy +##web +oem +let +was +##2009 +##gg +review +##wan +182 +##°c +203 +uc +title +##val +united +233 +2021 +##ons +doi +trivago +overdope +sbs +##ance +##ち +grand +special +573032185 +imf +216 +wx17house +##so +##ーム +audi +##he +london +william +##rp +##ake +science +beach +cfa +amp +ps4 +880 +##800 +##link +##hp +crm +ferragamo +bell +make +##eng +195 +under +zh +photos +2300 +##style +##ント +via +176 +da +##gi +company +i7 +##ray +thomas +370 +ufo +i5 +##max +plc +ben +back +research +8g +173 +mike +##pc +##ッフ +september +189 +##ace +vps +february +167 +pantos +wp +lisa +1921 +★★ +jquery +night +long +offer +##berg +##news +1911 +##いて +ray +fks +wto +せます +over +164 +340 +##all +##rus +1924 +##888 +##works +blogtitle +loftpermalink +##→ +187 +martin +test +ling +km +##め +15000 +fda +v3 +##ja +##ロ +ｗedding +かある +outlet +family +##ea +をこ +##top +story +##ness +salvatore +##lu +204 +swift +215 +room +している +oracle +##ul +1925 +sam +b2c +week +pi +rock +##のは +##ａ +##けと +##ean +##300 +##gle +cctv +after +chinese +##back +powered +x2 +##tan +1918 +##nes +##イン +canon +only +181 +##zi +##las +say +##oe +184 +##sd +221 +##bot +##world +##zo +sky +made +top100 +just +1926 +pmi +802 +234 +gap +##vr +177 +les +174 +▲topoct +ball +vogue +vi +ing +ofweek +cos +##list +##ort +▲topmay +##なら +##lon +として +last +##tc +##of +##bus +##gen +real +eva +##コ +a3 +nas +##lie +##ria +##coin +##bt +▲topapr +his +212 +cat +nata +vive +health +⋯⋯ +drive +sir +▲topmar +du +cup +##カー +##ook +##よう +##sy +alex +msg +tour +しました +3ce +##word +193 +ebooks +r8 +block +318 +##より +2200 +nice +pvp +207 +months +1905 +rewards +##ther +1917 +0800 +##xi +##チ +##sc +micro +850 +gg +blogfp +op +1922 +daily +m1 +264 +true +##bb +ml +##tar +##のお +##ky +anthony +196 +253 +##yo +state +218 +##ara +##aa +##rc +##tz +##ston +より +gear +##eo +##ade +ge +see +1923 +##win +##ura +ss +heart +##den +##ita +down +##sm +el +png +2100 +610 +rakuten +whatsapp +bay +dream +add +##use +680 +311 +pad +gucci +mpv +##ode +##fo +island +▲topjun +##▼ +223 +jason +214 +chicago +##❤ +しの +##hone +io +##れる +##ことか +sogo +be2 +##ology +990 +cloud +vcd +##con +2～3 +##ford +##joy +##kb +##こさいます +##rade +but +##ach +docker +##ful +rfid +ul +##ase +hit +ford +##star +580 +##○ +１１ +a2 +sdk +reading +edited +##are +cmos +##mc +238 +siri +light +##ella +##ため +bloomberg +##read +pizza +##ison +jimmy +##vm +college +node +journal +ba +18k +##play +245 +##cer +２０ +magic +##yu +191 +jump +288 +tt +##ings +asr +##lia +3200 +step5 +network +##cd +mc +いします +1234 +pixstyleme +273 +##600 +2800 +money +★★★★★ +1280 +１２ +430 +bl +みの +act +##tus +tokyo +##rial +##life +emba +##ae +saas +tcs +##rk +##wang +summer +##sp +ko +##ving +390 +premium +##その +netflix +##ヒ +uk +mt +##lton +right +frank +two +209 +える +##ple +##cal +021 +##んな +##sen +##ville +hold +nexus +dd +##ius +てお +##mah +##なく +tila +zero +820 +ce +##tin +resort +##ws +charles +old +p10 +5d +report +##360 +##ru +##には +bus +vans +lt +##est +pv +##レ +links +rebecca +##ツ +##dm +azure +##365 +きな +limited +bit +4gb +##mon +1910 +moto +##eam +213 +1913 +var +eos +なとの +226 +blogspot +された +699 +e3 +dos +dm +fc +##ments +##ik +##kw +boy +##bin +##ata +960 +er +##せ +219 +##vin +##tu +##ula +194 +##∥ +station +##ろ +##ature +835 +files +zara +hdr +top10 +nature +950 +magazine +s6 +marriott +##シ +avira +case +##っと +tab +##ran +tony +##home +oculus +im +##ral +jean +saint +cry +307 +rosie +##force +##ini +ice +##bert +のある +##nder +##mber +pet +2600 +##◆ +plurk +▲topdec +##sis +00kg +▲topnov +720 +##ence +tim +##ω +##nc +##ても +##name +log +ips +great +ikea +malaysia +unix +##イト +3600 +##ncy +##nie +12000 +akb48 +##ye +##oid +404 +##chi +##いた +oa +xuehai +##1000 +##orm +##rf +275 +さん +##ware +##リー +980 +ho +##pro +text +##era +560 +bob +227 +##ub +##2008 +8891 +scp +avi +##zen +2022 +mi +wu +museum +qvod +apache +lake +jcb +▲topaug +★★★ +ni +##hr +hill +302 +ne +weibo +490 +ruby +##ーシ +##ヶ +##row +4d +▲topjul +iv +##ish +github +306 +mate +312 +##スト +##lot +##ane +andrew +のハイト +##tina +t1 +rf +ed2k +##vel +##900 +way +final +りの +ns +5a +705 +197 +##メ +sweet +bytes +##ene +▲topjan +231 +##cker +##2007 +##px +100g +topapp +229 +helpapp +rs +low +14k +g4g +care +630 +ldquo +あり +##fork +leave +rm +edition +##gan +##zon +##qq +▲topsep +##google +##ism +gold +224 +explorer +##zer +toyota +category +select +visual +##labels +restaurant +##md +posts +s1 +##ico +もっと +angelababy +123456 +217 +sports +s3 +mbc +1915 +してくたさい +shell +x86 +candy +##new +kbs +face +xl +470 +##here +4a +swissinfo +v8 +▲topfeb +dram +##ual +##vice +3a +##wer +sport +q1 +ios10 +public +int +card +##ｃ +ep +au +rt +##れた +1080 +bill +##mll +kim +３０ +460 +wan +##uk +##ミ +x3 +298 +0t +scott +##ming +239 +e5 +##3d +h7n9 +worldcat +brown +##あります +##vo +##led +##580 +##ax +249 +410 +##ert +paris +##～6 +polo +925 +##lr +599 +##ナ +capital +##hing +bank +cv +1g +##chat +##ｓ +##たい +adc +##ule +2m +##ｅ +digital +hotmail +268 +##pad +870 +bbq +quot +##ring +before +wali +##まて +mcu +2k +2b +という +costco +316 +north +333 +switch +##city +##ｐ +philips +##mann +management +panasonic +##cl +##vd +##ping +##rge +alice +##lk +##ましょう +css3 +##ney +vision +alpha +##ular +##400 +##tter +lz +にお +##ありません +mode +gre +1916 +pci +##tm +237 +1～2 +##yan +##そ +について +##let +##キ +work +war +coach +ah +mary +##ᅵ +huang +##pt +a8 +pt +follow +##berry +1895 +##ew +a5 +ghost +##ション +##wn +##og +south +##code +girls +##rid +action +villa +git +r11 +table +games +##cket +error +##anonymoussaid +##ag +here +##ame +##gc +qa +##■ +##lis +gmp +##gin +vmalife +##cher +yu +wedding +##tis +demo +dragon +530 +soho +social +bye +##rant +river +orz +acer +325 +##↑ +##ース +##ats +261 +del +##ven +440 +ups +##ように +##ター +305 +value +macd +yougou +##dn +661 +##ano +ll +##urt +##rent +continue +script +##wen +##ect +paper +263 +319 +shift +##chel +##フト +##cat +258 +x5 +fox +243 +##さん +car +aaa +##blog +loading +##yn +##tp +kuso +799 +si +sns +イカせるテンマ +ヒンクテンマ3 +rmb +vdc +forest +central +prime +help +ultra +##rmb +##ような +241 +square +688 +##しい +のないフロクに +##field +##reen +##ors +##ju +c1 +start +510 +##air +##map +cdn +##wo +cba +stephen +m8 +100km +##get +opera +##base +##ood +vsa +com™ +##aw +##ail +251 +なのて +count +t2 +##ᅡ +##een +2700 +hop +##gp +vsc +tree +##eg +##ose +816 +285 +##ories +##shop +alphago +v4 +1909 +simon +##ᆼ +fluke62max +zip +スホンサー +##sta +louis +cr +bas +##～10 +bc +##yer +hadoop +##ube +##wi +1906 +0755 +hola +##low +place +centre +5v +d3 +##fer +252 +##750 +##media +281 +540 +0l +exchange +262 +series +##ハー +##san +eb +##bank +##ｋ +q3 +##nge +##mail +take +##lp +259 +1888 +client +east +cache +event +vincent +##ールを +きを +##nse +sui +855 +adchoice +##и +##stry +##なたの +246 +##zone +ga +apps +sea +##ab +248 +cisco +##タ +##rner +kymco +##care +dha +##pu +##yi +minkoff +royal +p1 +への +annie +269 +collection +kpi +playstation +257 +になります +866 +bh +##bar +queen +505 +radio +1904 +andy +armani +##xy +manager +iherb +##ery +##share +spring +raid +johnson +1908 +##ob +volvo +hall +##ball +v6 +our +taylor +##hk +bi +242 +##cp +kate +bo +water +technology +##rie +サイトは +277 +##ona +##sl +hpv +303 +gtx +hip +rdquo +jayz +stone +##lex +##rum +namespace +##やり +620 +##ale +##atic +des +##erson +##ql +##ves +##type +enter +##この +##てきます +d2 +##168 +##mix +##bian +との +a9 +jj +ky +##lc +access +movie +##hc +リストに +tower +##ration +##mit +ます +##nch +ua +tel +prefix +##o2 +1907 +##point +1901 +ott +～10 +##http +##ury +baidu +##ink +member +##logy +bigbang +nownews +##js +##shot +##tb +##こと +247 +eba +##tics +##lus +ける +v5 +spark +##ama +there +##ions +god +##lls +##down +hiv +##ress +burberry +day2 +##kv +◆◆ +jeff +related +film +edit +joseph +283 +##ark +cx +32gb +order +g9 +30000 +##ans +##tty +s5 +##bee +かあります +thread +xr +buy +sh +005 +land +spotify +mx +##ari +276 +##verse +×email +sf +why +##ことて +244 +7headlines +nego +sunny +dom +exo +401 +666 +positioning +fit +rgb +##tton +278 +kiss +alexa +adam +lp +みリストを +##ｇ +mp +##ties +##llow +amy +##du +np +002 +institute +271 +##rth +##lar +2345 +590 +##des +sidebar +１５ +imax +site +##cky +##kit +##ime +##009 +season +323 +##fun +##ンター +##ひ +gogoro +a7 +pu +lily +fire +twd600 +##ッセーシを +いて +##vis +30ml +##cture +##をお +information +##オ +close +friday +##くれる +yi +nick +てすか +##tta +##tel +6500 +##lock +cbd +economy +254 +かお +267 +tinker +double +375 +8gb +voice +##app +oops +channel +today +985 +##right +raw +xyz +##＋ +jim +edm +##cent +7500 +supreme +814 +ds +##its +##asia +dropbox +##てすか +##tti +books +272 +100ml +##tle +##ller +##ken +##more +##boy +sex +309 +##dom +t3 +##ider +##なります +##unch +1903 +810 +feel +5500 +##かった +##put +により +s2 +mo +##gh +men +ka +amoled +div +##tr +##n1 +port +howard +##tags +ken +dnf +##nus +adsense +##а +ide +##へ +buff +thunder +##town +##ique +has +##body +auto +pin +##erry +tee +てした +295 +number +##the +##013 +object +psp +cool +udnbkk +16gb +##mic +miui +##tro +most +r2 +##alk +##nity +1880 +±0 +##いました +428 +s4 +law +version +##oa +n1 +sgs +docomo +##tf +##ack +henry +fc2 +##ded +##sco +##014 +##rite +286 +0mm +linkedin +##ada +##now +wii +##ndy +ucbug +##◎ +sputniknews +legalminer +##ika +##xp +2gb +##bu +q10 +oo +b6 +come +##rman +cheese +ming +maker +##gm +nikon +##fig +ppi +kelly +##ります +jchere +てきます +ted +md +003 +fgo +tech +##tto +dan +soc +##gl +##len +hair +earth +640 +521 +img +##pper +##a1 +##てきる +##ロク +acca +##ition +##ference +suite +##ig +outlook +##mond +##cation +398 +##pr +279 +101vip +358 +##999 +282 +64gb +3800 +345 +airport +##over +284 +##おり +jones +##ith +lab +##su +##いるのて +co2 +town +piece +##llo +no1 +vmware +24h +##qi +focus +reader +##admin +##ora +tb +false +##log +1898 +know +lan +838 +##ces +f4 +##ume +motel +stop +##oper +na +flickr +netcomponents +##af +##─ +pose +williams +local +##ound +##cg +##site +##iko +いお +274 +5m +gsm +con +##ath +1902 +friends +##hip +cell +317 +##rey +780 +cream +##cks +012 +##dp +facebooktwitterpinterestgoogle +sso +324 +shtml +song +swiss +##mw +##キンク +lumia +xdd +string +tiffany +522 +marc +られた +insee +russell +sc +dell +##ations +ｏｋ +camera +289 +##vs +##flow +##late +classic +287 +##nter +stay +g1 +mtv +512 +##ever +##lab +##nger +qe +sata +ryan +d1 +50ml +cms +##cing +su +292 +3300 +editor +296 +##nap +security +sunday +association +##ens +##700 +##bra +acg +##かり +sofascore +とは +mkv +##ign +jonathan +gary +build +labels +##oto +tesla +moba +qi +gohappy +general +ajax +1024 +##かる +サイト +society +##test +##urs +wps +fedora +##ich +mozilla +328 +##480 +##dr +usa +urn +##lina +##ｒ +grace +##die +##try +##ader +1250 +##なり +elle +570 +##chen +##ᆯ +price +##ten +uhz +##ough +eq +##hen +states +push +session +balance +wow +506 +##cus +##py +when +##ward +##ep +34e +wong +library +prada +##サイト +##cle +running +##ree +313 +ck +date +q4 +##ctive +##ool +##＞ +mk +##ira +##163 +388 +die +secret +rq +dota +buffet +は１ヶ +e6 +##ez +pan +368 +ha +##card +##cha +2a +##さ +alan +day3 +eye +f3 +##end +france +keep +adi +rna +tvbs +##ala +solo +nova +##え +##tail +##ょう +support +##ries +##なる +##ved +base +copy +iis +fps +##ways +hero +hgih +profile +fish +mu +ssh +entertainment +chang +##wd +click +cake +##ond +pre +##tom +kic +pixel +##ov +##fl +product +6a +##pd +dear +##gate +es +yumi +audio +##² +##sky +echo +bin +where +##ture +329 +##ape +find +sap +isis +##なと +nand +##101 +##load +##ream +band +a6 +525 +never +##post +festival +50cm +##we +555 +guide +314 +zenfone +##ike +335 +gd +forum +jessica +strong +alexander +##ould +software +allen +##ious +program +360° +else +lohasthree +##gar +することかてきます +please +##れます +rc +##ggle +##ric +bim +50000 +##own +eclipse +355 +brian +3ds +##side +061 +361 +##other +##ける +##tech +##ator +485 +engine +##ged +##ｔ +plaza +##fit +cia +ngo +westbrook +shi +tbs +50mm +##みませんか +sci +291 +reuters +##ily +contextlink +##hn +af +##cil +bridge +very +##cel +1890 +cambridge +##ize +15g +##aid +##data +790 +frm +##head +award +butler +##sun +meta +##mar +america +ps3 +puma +pmid +##すか +lc +670 +kitchen +##lic +オーフン5 +きなしソフトサーヒス +そして +day1 +future +★★★★ +##text +##page +##rris +pm1 +##ket +fans +##っています +1001 +christian +bot +kids +trackback +##hai +c3 +display +##hl +n2 +1896 +idea +さんも +##sent +airmail +##ug +##men +pwm +けます +028 +##lution +369 +852 +awards +schemas +354 +asics +wikipedia +font +##tional +##vy +c2 +293 +##れている +##dget +##ein +っている +contact +pepper +スキル +339 +##～5 +294 +##uel +##ument +730 +##hang +みてす +q5 +##sue +rain +##ndi +wei +swatch +##cept +わせ +331 +popular +##ste +##tag +p2 +501 +trc +1899 +##west +##live +justin +honda +ping +messenger +##rap +v9 +543 +##とは +unity +appqq +はすへて +025 +leo +##tone +##テ +##ass +uniqlo +##010 +502 +her +jane +memory +moneydj +##tical +human +12306 +していると +##m2 +coc +miacare +##mn +tmt +##core +vim +kk +##may +fan +target +use +too +338 +435 +2050 +867 +737 +fast +##2c +services +##ope +omega +energy +##わ +pinkoi +1a +##なから +##rain +jackson +##ement +##シャンルの +374 +366 +そんな +p9 +rd +##ᆨ +1111 +##tier +##vic +zone +##│ +385 +690 +dl +isofix +cpa +m4 +322 +kimi +めて +davis +##lay +lulu +##uck +050 +weeks +qs +##hop +920 +##ｎ +ae +##ear +～5 +eia +405 +##fly +korea +jpeg +boost +##ship +small +##リア +1860 +eur +297 +425 +valley +##iel +simple +##ude +rn +k2 +##ena +されます +non +patrick +しているから +##ナー +feed +5757 +30g +process +well +qqmei +##thing +they +aws +lu +pink +##ters +##kin +または +board +##vertisement +wine +##ien +unicode +##dge +r1 +359 +##tant +いを +##twitter +##3c +cool1 +される +##れて +##ｌ +isp +##012 +standard +45㎡2 +402 +##150 +matt +##fu +326 +##iner +googlemsn +pixnetfacebookyahoo +##ラン +x7 +886 +##uce +メーカー +sao +##ev +##きました +##file +9678 +403 +xddd +shirt +6l +##rio +##hat +3mm +givenchy +ya +bang +##lio +monday +crystal +ロクイン +##abc +336 +head +890 +ubuntuforumwikilinuxpastechat +##vc +##～20 +##rity +cnc +7866 +ipv6 +null +1897 +##ost +yang +imsean +tiger +##fet +##ンス +352 +##＝ +dji +327 +ji +maria +##come +##んて +foundation +3100 +##beth +##なった +1m +601 +active +##aft +##don +3p +sr +349 +emma +##khz +living +415 +353 +1889 +341 +709 +457 +sas +x6 +##face +pptv +x4 +##mate +han +sophie +##jing +337 +fifa +##mand +other +sale +inwedding +##gn +てきちゃいます +##mmy +##pmlast +bad +nana +nbc +してみてくたさいね +なとはお +##wu +##かあります +##あ +note7 +single +##340 +せからこ +してくたさい♪この +しにはとんとんワークケートを +するとあなたにもっとマッチした +ならワークケートへ +もみつかっちゃうかも +ワークケートの +##bel +window +##dio +##ht +union +age +382 +１４ +##ivity +##ｙ +コメント +domain +neo +##isa +##lter +5k +f5 +steven +##cts +powerpoint +tft +self +g2 +ft +##テル +zol +##act +mwc +381 +343 +もう +nbapop +408 +てある +eds +ace +##room +previous +author +tomtom +il +##ets +hu +financial +☆☆☆ +っています +bp +5t +chi +1gb +##hg +fairmont +cross +008 +gay +h2 +function +##けて +356 +also +1b +625 +##ータ +##raph +1894 +3～5 +##ils +i3 +334 +avenue +##host +による +##bon +##tsu +message +navigation +50g +fintech +h6 +##ことを +8cm +##ject +##vas +##firm +credit +##wf +xxxx +form +##nor +##space +huawei +plan +json +sbl +##dc +machine +921 +392 +wish +##120 +##sol +windows7 +edward +##ために +development +washington +##nsis +lo +818 +##sio +##ym +##bor +planet +##～8 +##wt +ieee +gpa +##めて +camp +ann +gm +##tw +##oka +connect +##rss +##work +##atus +wall +chicken +soul +2mm +##times +fa +##ather +##cord +009 +##eep +hitachi +gui +harry +##pan +e1 +disney +##press +##ーション +wind +386 +frigidaire +##tl +liu +hsu +332 +basic +von +ev +いた +てきる +スホンサーサイト +learning +##ull +expedia +archives +change +##wei +santa +cut +ins +6gb +turbo +brand +cf1 +508 +004 +return +747 +##rip +h1 +##nis +##をこ +128gb +##にお +3t +application +しており +emc +rx +##oon +384 +quick +412 +15058 +wilson +wing +chapter +##bug +beyond +##cms +##dar +##oh +zoom +e2 +trip +sb +##nba +rcep +342 +aspx +ci +080 +gc +gnu +める +##count +advanced +dance +dv +##url +##ging +367 +8591 +am09 +shadow +battle +346 +##ｉ +##cia +##という +emily +##のてす +##tation +host +ff +techorz +sars +##mini +##mporary +##ering +nc +4200 +798 +##next +cma +##mbps +##gas +##ift +##dot +##ィ +455 +##～17 +amana +##りの +426 +##ros +ir +00㎡1 +##eet +##ible +##↓ +710 +ˋ▽ˊ +##aka +dcs +iq +##ｖ +l1 +##lor +maggie +##011 +##iu +588 +##～1 +830 +##gt +1tb +articles +create +##burg +##iki +database +fantasy +##rex +##cam +dlc +dean +##you +hard +path +gaming +victoria +maps +cb +##lee +##itor +overchicstoretvhome +systems +##xt +416 +p3 +sarah +760 +##nan +407 +486 +x9 +install +second +626 +##ann +##ph +##rcle +##nic +860 +##nar +ec +##とう +768 +metro +chocolate +##rian +～4 +##table +##しています +skin +##sn +395 +mountain +##0mm +inparadise +6m +7x24 +ib +4800 +##jia +eeworld +creative +g5 +g3 +357 +parker +ecfa +village +からの +18000 +sylvia +サーヒス +hbl +##ques +##onsored +##x2 +##きます +##v4 +##tein +ie6 +383 +##stack +389 +ver +##ads +##baby +sound +bbe +##110 +##lone +##uid +ads +022 +gundam +351 +thinkpad +006 +scrum +match +##ave +mems +##470 +##oy +##なりました +##talk +glass +lamigo +span +##eme +job +##a5 +jay +wade +kde +498 +##lace +ocean +tvg +##covery +##r3 +##ners +##rea +junior +think +##aine +cover +##ision +##sia +↓↓ +##bow +msi +413 +458 +406 +##love +711 +801 +soft +z2 +##pl +456 +1840 +mobil +mind +##uy +427 +nginx +##oi +めた +##rr +6221 +##mple +##sson +##ーシてす +371 +##nts +91tv +comhd +crv3000 +##uard +1868 +397 +deep +lost +field +gallery +##bia +rate +spf +redis +traction +930 +icloud +011 +なら +fe +jose +372 +##tory +into +sohu +fx +899 +379 +kicstart2 +##hia +すく +##～3 +##sit +ra +２４ +##walk +##xure +500g +##pact +pacific +xa +natural +carlo +##250 +##walker +1850 +##can +cto +gigi +516 +##サー +pen +##hoo +ob +matlab +##ｂ +##yy +13913459 +##iti +mango +##bbs +sense +c5 +oxford +##ニア +walker +jennifer +##ola +course +##bre +701 +##pus +##rder +lucky +075 +##ぁ +ivy +なお +##nia +sotheby +side +##ugh +joy +##orage +##ush +##bat +##dt +364 +r9 +##2d +##gio +511 +country +wear +##lax +##～7 +##moon +393 +seven +study +411 +348 +lonzo +8k +##ェ +evolution +##イフ +##kk +gs +kd +##レス +arduino +344 +b12 +##lux +arpg +##rdon +cook +##x5 +dark +five +##als +##ida +とても +sign +362 +##ちの +something +20mm +##nda +387 +##posted +fresh +tf +1870 +422 +cam +##mine +##skip +##form +##ssion +education +394 +##tee +dyson +stage +##jie +want +##night +epson +pack +あります +##ppy +テリヘル +##█ +wd +##eh +##rence +left +##lvin +golden +mhz +discovery +##trix +##n2 +loft +##uch +##dra +##sse +speed +～1 +1mdb +sorry +welcome +##urn +wave +gaga +##lmer +teddy +##160 +トラックハック +せよ +611 +##f2016 +378 +rp +##sha +rar +##あなたに +##きた +840 +holiday +##ュー +373 +074 +##vg +##nos +##rail +gartner +gi +6p +##dium +kit +488 +b3 +eco +##ろう +20g +sean +##stone +autocad +nu +##np +f16 +write +029 +m5 +##ias +images +atp +##dk +fsm +504 +1350 +ve +52kb +##xxx +##のに +##cake +414 +unit +lim +ru +1v +##ification +published +angela +16g +analytics +ak +##ｑ +##nel +gmt +##icon +again +##₂ +##bby +ios11 +445 +かこさいます +waze +いてす +##ハ +9985 +##ust +##ティー +framework +##007 +iptv +delete +52sykb +cl +wwdc +027 +30cm +##fw +##ての +1389 +##xon +brandt +##ses +##dragon +tc +vetements +anne +monte +modern +official +##へて +##ere +##nne +##oud +もちろん +５０ +etnews +##a2 +##graphy +421 +863 +##ちゃん +444 +##rtex +##てお +l2 +##gma +mount +ccd +たと +archive +morning +tan +ddos +e7 +##ホ +day4 +##ウ +gis +453 +its +495 +factory +bruce +pg +##ito +ってくたさい +guest +cdma +##lling +536 +n3 +しかし +3～4 +mega +eyes +ro +１３ +women +dac +church +##jun +singapore +##facebook +6991 +starbucks +##tos +##stin +##shine +zen +##mu +tina +20℃ +1893 +##たけて +503 +465 +request +##gence +qt +##っ +1886 +347 +363 +q7 +##zzi +diary +##tore +409 +##ead +468 +cst +##osa +canada +agent +va +##jiang +##ちは +##ーク +##lam +sg +##nix +##sday +##よって +g6 +##master +bing +##zl +charlie +１６ +8mm +nb40 +##ーン +thai +##ルフ +ln284ct +##itz +##2f +bonnie +##food +##lent +originals +##stro +##lts +418 +∟∣ +##bscribe +children +ntd +yesstyle +##かも +hmv +##tment +d5 +2cm +arts +sms +##pn +##я +##いい +topios9 +539 +lifestyle +virtual +##ague +xz +##deo +muji +024 +unt +##nnis +##ᅩ +faq1 +1884 +396 +##ette +fly +64㎡ +はしめまして +441 +curry +##pop +のこ +release +##← +##◆◆ +##cast +073 +ありな +500ml +##ews +5c +##stle +ios7 +##ima +787 +dog +lenovo +##r4 +roger +013 +cbs +vornado +100m +417 +##desk +##クok +##ald +1867 +9595 +2900 +##van +oil +##ｘ +some +break +common +##jy +##lines +g7 +twice +419 +ella +nano +belle +にこ +##mes +##self +##note +jb +##ことかてきます +benz +##との +##ova +451 +save +##wing +##ますのて +kai +りは +##hua +##rect +rainer +##unge +448 +##0m +adsl +##かな +guestname +##uma +##kins +##zu +tokichoi +##price +county +##med +##mus +rmk +391 +address +vm +えて +openload +##group +##hin +##iginal +amg +urban +##oz +jobs +emi +##public +beautiful +##sch +album +##dden +##bell +jerry +works +hostel +miller +##drive +##rmin +##１０ +376 +boot +828 +##370 +##fx +##cm～ +1885 +##nome +##ctionary +##oman +##lish +##cr +##hm +433 +##how +432 +francis +xi +c919 +b5 +evernote +##uc +vga +##3000 +coupe +##urg +##cca +##uality +019 +6g +れる +multi +##また +##ett +em +hey +##ani +##tax +##rma +inside +than +740 +leonnhurt +##jin +ict +れた +bird +notes +200mm +くの +##dical +##lli +result +442 +iu +ee +438 +smap +gopro +##last +yin +pure +998 +32g +けた +5kg +##dan +##rame +mama +##oot +bean +marketing +##hur +2l +bella +sync +xuite +##ground +515 +discuz +##getrelax +##ince +##bay +##5s +cj +##イス +gmat +apt +##pass +jing +##rix +c4 +rich +##とても +niusnews +##ello +bag +770 +##eting +##mobile +１８ +culture +015 +##のてすか +377 +1020 +area +##ience +616 +details +gp +universal +silver +dit +はお +private +ddd +u11 +kanshu +##ified +fung +##nny +dx +##520 +tai +475 +023 +##fr +##lean +3s +##pin +429 +##rin +25000 +ly +rick +##bility +usb3 +banner +##baru +##gion +metal +dt +vdf +1871 +karl +qualcomm +bear +1010 +oldid +ian +jo +##tors +population +##ernel +1882 +mmorpg +##mv +##bike +603 +##© +ww +friend +##ager +exhibition +##del +##pods +fpx +structure +##free +##tings +kl +##rley +##copyright +##mma +california +3400 +orange +yoga +4l +canmake +honey +##anda +##コメント +595 +nikkie +##ルハイト +dhl +publishing +##mall +##gnet +20cm +513 +##クセス +##┅ +e88 +970 +##dog +fishbase +##! +##" +### +##$ +##% +##& +##' +##( +##) +##* +##+ +##, +##- +##. +##/ +##: +##; +##< +##= +##> +##? +##@ +##[ +##\ +##] +##^ +##_ +##{ +##| +##} +##~ +##£ +##¤ +##¥ +##§ +##« +##± +##³ +##µ +##· +##¹ +##º +##» +##¼ +##ß +##æ +##÷ +##ø +##đ +##ŋ +##ɔ +##ə +##ɡ +##ʰ +##ˇ +##ˈ +##ˊ +##ˋ +##ˍ +##ː +##˙ +##˚ +##ˢ +##α +##β +##γ +##δ +##ε +##η +##θ +##ι +##κ +##λ +##μ +##ν +##ο +##π +##ρ +##ς +##σ +##τ +##υ +##φ +##χ +##ψ +##б +##в +##г +##д +##е +##ж +##з +##к +##л +##м +##н +##о +##п +##р +##с +##т +##у +##ф +##х +##ц +##ч +##ш +##ы +##ь +##і +##ا +##ب +##ة +##ت +##د +##ر +##س +##ع +##ل +##م +##ن +##ه +##و +##ي +##۩ +##ก +##ง +##น +##ม +##ย +##ร +##อ +##า +##เ +##๑ +##་ +##ღ +##ᄀ +##ᄁ +##ᄂ +##ᄃ +##ᄅ +##ᄆ +##ᄇ +##ᄈ +##ᄉ +##ᄋ +##ᄌ +##ᄎ +##ᄏ +##ᄐ +##ᄑ +##ᄒ +##ᅢ +##ᅣ +##ᅥ +##ᅦ +##ᅧ +##ᅨ +##ᅪ +##ᅬ +##ᅭ +##ᅮ +##ᅯ +##ᅲ +##ᅳ +##ᅴ +##ᆷ +##ᆸ +##ᆺ +##ᆻ +##ᗜ +##ᵃ +##ᵉ +##ᵍ +##ᵏ +##ᵐ +##ᵒ +##ᵘ +##‖ +##„ +##† +##• +##‥ +##‧ +##
 +##‰ +##′ +##″ +##‹ +##› +##※ +##‿ +##⁄ +##ⁱ +##⁺ +##ⁿ +##₁ +##₃ +##₄ +##€ +##№ +##ⅰ +##ⅱ +##ⅲ +##ⅳ +##ⅴ +##↔ +##↗ +##↘ +##⇒ +##∀ +##− +##∕ +##∙ +##√ +##∞ +##∟ +##∠ +##∣ +##∩ +##∮ +##∶ +##∼ +##∽ +##≈ +##≒ +##≡ +##≤ +##≥ +##≦ +##≧ +##≪ +##≫ +##⊙ +##⋅ +##⋈ +##⋯ +##⌒ +##① +##② +##③ +##④ +##⑤ +##⑥ +##⑦ +##⑧ +##⑨ +##⑩ +##⑴ +##⑵ +##⑶ +##⑷ +##⑸ +##⒈ +##⒉ +##⒊ +##⒋ +##ⓒ +##ⓔ +##ⓘ +##━ +##┃ +##┆ +##┊ +##┌ +##└ +##├ +##┣ +##═ +##║ +##╚ +##╞ +##╠ +##╭ +##╮ +##╯ +##╰ +##╱ +##╳ +##▂ +##▃ +##▅ +##▇ +##▉ +##▋ +##▌ +##▍ +##▎ +##□ +##▪ +##▫ +##▬ +##△ +##▶ +##► +##▽ +##◇ +##◕ +##◠ +##◢ +##◤ +##☀ +##☕ +##☞ +##☺ +##☼ +##♀ +##♂ +##♠ +##♡ +##♣ +##♦ +##♫ +##♬ +##✈ +##✔ +##✕ +##✖ +##✦ +##✨ +##✪ +##✰ +##✿ +##❀ +##➜ +##➤ +##⦿ +##、 +##。 +##〃 +##々 +##〇 +##〈 +##〉 +##《 +##》 +##「 +##」 +##『 +##』 +##【 +##】 +##〓 +##〔 +##〕 +##〖 +##〗 +##〜 +##〝 +##〞 +##ぃ +##ぇ +##ぬ +##ふ +##ほ +##む +##ゃ +##ゅ +##ゆ +##ょ +##゜ +##ゝ +##ァ +##ゥ +##エ +##ォ +##ケ +##サ +##セ +##ソ +##ッ +##ニ +##ヌ +##ネ +##ノ +##ヘ +##モ +##ャ +##ヤ +##ュ +##ユ +##ョ +##ヨ +##ワ +##ヲ +##・ +##ヽ +##ㄅ +##ㄆ +##ㄇ +##ㄉ +##ㄋ +##ㄌ +##ㄍ +##ㄎ +##ㄏ +##ㄒ +##ㄚ +##ㄛ +##ㄞ +##ㄟ +##ㄢ +##ㄤ +##ㄥ +##ㄧ +##ㄨ +##ㆍ +##㈦ +##㊣ +##㗎 +##一 +##丁 +##七 +##万 +##丈 +##三 +##上 +##下 +##不 +##与 +##丐 +##丑 +##专 +##且 +##丕 +##世 +##丘 +##丙 +##业 +##丛 +##东 +##丝 +##丞 +##丟 +##両 +##丢 +##两 +##严 +##並 +##丧 +##丨 +##个 +##丫 +##中 +##丰 +##串 +##临 +##丶 +##丸 +##丹 +##为 +##主 +##丼 +##丽 +##举 +##丿 +##乂 +##乃 +##久 +##么 +##义 +##之 +##乌 +##乍 +##乎 +##乏 +##乐 +##乒 +##乓 +##乔 +##乖 +##乗 +##乘 +##乙 +##乜 +##九 +##乞 +##也 +##习 +##乡 +##书 +##乩 +##买 +##乱 +##乳 +##乾 +##亀 +##亂 +##了 +##予 +##争 +##事 +##二 +##于 +##亏 +##云 +##互 +##五 +##井 +##亘 +##亙 +##亚 +##些 +##亜 +##亞 +##亟 +##亡 +##亢 +##交 +##亥 +##亦 +##产 +##亨 +##亩 +##享 +##京 +##亭 +##亮 +##亲 +##亳 +##亵 +##人 +##亿 +##什 +##仁 +##仃 +##仄 +##仅 +##仆 +##仇 +##今 +##介 +##仍 +##从 +##仏 +##仑 +##仓 +##仔 +##仕 +##他 +##仗 +##付 +##仙 +##仝 +##仞 +##仟 +##代 +##令 +##以 +##仨 +##仪 +##们 +##仮 +##仰 +##仲 +##件 +##价 +##任 +##份 +##仿 +##企 +##伉 +##伊 +##伍 +##伎 +##伏 +##伐 +##休 +##伕 +##众 +##优 +##伙 +##会 +##伝 +##伞 +##伟 +##传 +##伢 +##伤 +##伦 +##伪 +##伫 +##伯 +##估 +##伴 +##伶 +##伸 +##伺 +##似 +##伽 +##佃 +##但 +##佇 +##佈 +##位 +##低 +##住 +##佐 +##佑 +##体 +##佔 +##何 +##佗 +##佘 +##余 +##佚 +##佛 +##作 +##佝 +##佞 +##佟 +##你 +##佢 +##佣 +##佤 +##佥 +##佩 +##佬 +##佯 +##佰 +##佳 +##併 +##佶 +##佻 +##佼 +##使 +##侃 +##侄 +##來 +##侈 +##例 +##侍 +##侏 +##侑 +##侖 +##侗 +##供 +##依 +##侠 +##価 +##侣 +##侥 +##侦 +##侧 +##侨 +##侬 +##侮 +##侯 +##侵 +##侶 +##侷 +##便 +##係 +##促 +##俄 +##俊 +##俎 +##俏 +##俐 +##俑 +##俗 +##俘 +##俚 +##保 +##俞 +##俟 +##俠 +##信 +##俨 +##俩 +##俪 +##俬 +##俭 +##修 +##俯 +##俱 +##俳 +##俸 +##俺 +##俾 +##倆 +##倉 +##個 +##倌 +##倍 +##倏 +##們 +##倒 +##倔 +##倖 +##倘 +##候 +##倚 +##倜 +##借 +##倡 +##値 +##倦 +##倩 +##倪 +##倫 +##倬 +##倭 +##倶 +##债 +##值 +##倾 +##偃 +##假 +##偈 +##偉 +##偌 +##偎 +##偏 +##偕 +##做 +##停 +##健 +##側 +##偵 +##偶 +##偷 +##偻 +##偽 +##偿 +##傀 +##傅 +##傍 +##傑 +##傘 +##備 +##傚 +##傢 +##傣 +##傥 +##储 +##傩 +##催 +##傭 +##傲 +##傳 +##債 +##傷 +##傻 +##傾 +##僅 +##働 +##像 +##僑 +##僕 +##僖 +##僚 +##僥 +##僧 +##僭 +##僮 +##僱 +##僵 +##價 +##僻 +##儀 +##儂 +##億 +##儆 +##儉 +##儋 +##儒 +##儕 +##儘 +##償 +##儡 +##優 +##儲 +##儷 +##儼 +##儿 +##兀 +##允 +##元 +##兄 +##充 +##兆 +##兇 +##先 +##光 +##克 +##兌 +##免 +##児 +##兑 +##兒 +##兔 +##兖 +##党 +##兜 +##兢 +##入 +##內 +##全 +##兩 +##八 +##公 +##六 +##兮 +##兰 +##共 +##兲 +##关 +##兴 +##兵 +##其 +##具 +##典 +##兹 +##养 +##兼 +##兽 +##冀 +##内 +##円 +##冇 +##冈 +##冉 +##冊 +##册 +##再 +##冏 +##冒 +##冕 +##冗 +##写 +##军 +##农 +##冠 +##冢 +##冤 +##冥 +##冨 +##冪 +##冬 +##冯 +##冰 +##冲 +##决 +##况 +##冶 +##冷 +##冻 +##冼 +##冽 +##冾 +##净 +##凄 +##准 +##凇 +##凈 +##凉 +##凋 +##凌 +##凍 +##减 +##凑 +##凛 +##凜 +##凝 +##几 +##凡 +##凤 +##処 +##凪 +##凭 +##凯 +##凰 +##凱 +##凳 +##凶 +##凸 +##凹 +##出 +##击 +##函 +##凿 +##刀 +##刁 +##刃 +##分 +##切 +##刈 +##刊 +##刍 +##刎 +##刑 +##划 +##列 +##刘 +##则 +##刚 +##创 +##初 +##删 +##判 +##別 +##刨 +##利 +##刪 +##别 +##刮 +##到 +##制 +##刷 +##券 +##刹 +##刺 +##刻 +##刽 +##剁 +##剂 +##剃 +##則 +##剉 +##削 +##剋 +##剌 +##前 +##剎 +##剐 +##剑 +##剔 +##剖 +##剛 +##剜 +##剝 +##剣 +##剤 +##剥 +##剧 +##剩 +##剪 +##副 +##割 +##創 +##剷 +##剽 +##剿 +##劃 +##劇 +##劈 +##劉 +##劊 +##劍 +##劏 +##劑 +##力 +##劝 +##办 +##功 +##加 +##务 +##劣 +##动 +##助 +##努 +##劫 +##劭 +##励 +##劲 +##劳 +##労 +##劵 +##効 +##劾 +##势 +##勁 +##勃 +##勇 +##勉 +##勋 +##勐 +##勒 +##動 +##勖 +##勘 +##務 +##勛 +##勝 +##勞 +##募 +##勢 +##勤 +##勧 +##勳 +##勵 +##勸 +##勺 +##勻 +##勾 +##勿 +##匀 +##包 +##匆 +##匈 +##匍 +##匐 +##匕 +##化 +##北 +##匙 +##匝 +##匠 +##匡 +##匣 +##匪 +##匮 +##匯 +##匱 +##匹 +##区 +##医 +##匾 +##匿 +##區 +##十 +##千 +##卅 +##升 +##午 +##卉 +##半 +##卍 +##华 +##协 +##卑 +##卒 +##卓 +##協 +##单 +##卖 +##南 +##単 +##博 +##卜 +##卞 +##卟 +##占 +##卡 +##卢 +##卤 +##卦 +##卧 +##卫 +##卮 +##卯 +##印 +##危 +##即 +##却 +##卵 +##卷 +##卸 +##卻 +##卿 +##厂 +##厄 +##厅 +##历 +##厉 +##压 +##厌 +##厕 +##厘 +##厚 +##厝 +##原 +##厢 +##厥 +##厦 +##厨 +##厩 +##厭 +##厮 +##厲 +##厳 +##去 +##县 +##叁 +##参 +##參 +##又 +##叉 +##及 +##友 +##双 +##反 +##収 +##发 +##叔 +##取 +##受 +##变 +##叙 +##叛 +##叟 +##叠 +##叡 +##叢 +##口 +##古 +##句 +##另 +##叨 +##叩 +##只 +##叫 +##召 +##叭 +##叮 +##可 +##台 +##叱 +##史 +##右 +##叵 +##叶 +##号 +##司 +##叹 +##叻 +##叼 +##叽 +##吁 +##吃 +##各 +##吆 +##合 +##吉 +##吊 +##吋 +##同 +##名 +##后 +##吏 +##吐 +##向 +##吒 +##吓 +##吕 +##吖 +##吗 +##君 +##吝 +##吞 +##吟 +##吠 +##吡 +##否 +##吧 +##吨 +##吩 +##含 +##听 +##吭 +##吮 +##启 +##吱 +##吳 +##吴 +##吵 +##吶 +##吸 +##吹 +##吻 +##吼 +##吽 +##吾 +##呀 +##呂 +##呃 +##呆 +##呈 +##告 +##呋 +##呎 +##呐 +##呓 +##呕 +##呗 +##员 +##呛 +##呜 +##呢 +##呤 +##呦 +##周 +##呱 +##呲 +##味 +##呵 +##呷 +##呸 +##呻 +##呼 +##命 +##咀 +##咁 +##咂 +##咄 +##咆 +##咋 +##和 +##咎 +##咏 +##咐 +##咒 +##咔 +##咕 +##咖 +##咗 +##咘 +##咙 +##咚 +##咛 +##咣 +##咤 +##咦 +##咧 +##咨 +##咩 +##咪 +##咫 +##咬 +##咭 +##咯 +##咱 +##咲 +##咳 +##咸 +##咻 +##咽 +##咿 +##哀 +##品 +##哂 +##哄 +##哆 +##哇 +##哈 +##哉 +##哋 +##哌 +##响 +##哎 +##哏 +##哐 +##哑 +##哒 +##哔 +##哗 +##哟 +##員 +##哥 +##哦 +##哧 +##哨 +##哩 +##哪 +##哭 +##哮 +##哲 +##哺 +##哼 +##哽 +##唁 +##唄 +##唆 +##唇 +##唉 +##唏 +##唐 +##唑 +##唔 +##唠 +##唤 +##唧 +##唬 +##售 +##唯 +##唰 +##唱 +##唳 +##唷 +##唸 +##唾 +##啃 +##啄 +##商 +##啉 +##啊 +##問 +##啓 +##啕 +##啖 +##啜 +##啞 +##啟 +##啡 +##啤 +##啥 +##啦 +##啧 +##啪 +##啫 +##啬 +##啮 +##啰 +##啱 +##啲 +##啵 +##啶 +##啷 +##啸 +##啻 +##啼 +##啾 +##喀 +##喂 +##喃 +##善 +##喆 +##喇 +##喉 +##喊 +##喋 +##喎 +##喏 +##喔 +##喘 +##喙 +##喚 +##喜 +##喝 +##喟 +##喧 +##喪 +##喫 +##喬 +##單 +##喰 +##喱 +##喲 +##喳 +##喵 +##営 +##喷 +##喹 +##喺 +##喻 +##喽 +##嗅 +##嗆 +##嗇 +##嗎 +##嗑 +##嗒 +##嗓 +##嗔 +##嗖 +##嗚 +##嗜 +##嗝 +##嗟 +##嗡 +##嗣 +##嗤 +##嗦 +##嗨 +##嗪 +##嗬 +##嗯 +##嗰 +##嗲 +##嗳 +##嗶 +##嗷 +##嗽 +##嘀 +##嘅 +##嘆 +##嘈 +##嘉 +##嘌 +##嘍 +##嘎 +##嘔 +##嘖 +##嘗 +##嘘 +##嘚 +##嘛 +##嘜 +##嘞 +##嘟 +##嘢 +##嘣 +##嘤 +##嘧 +##嘩 +##嘭 +##嘮 +##嘯 +##嘰 +##嘱 +##嘲 +##嘴 +##嘶 +##嘸 +##嘹 +##嘻 +##嘿 +##噁 +##噌 +##噎 +##噓 +##噔 +##噗 +##噙 +##噜 +##噠 +##噢 +##噤 +##器 +##噩 +##噪 +##噬 +##噱 +##噴 +##噶 +##噸 +##噹 +##噻 +##噼 +##嚀 +##嚇 +##嚎 +##嚏 +##嚐 +##嚓 +##嚕 +##嚟 +##嚣 +##嚥 +##嚨 +##嚮 +##嚴 +##嚷 +##嚼 +##囂 +##囉 +##囊 +##囍 +##囑 +##囔 +##囗 +##囚 +##四 +##囝 +##回 +##囟 +##因 +##囡 +##团 +##団 +##囤 +##囧 +##囪 +##囫 +##园 +##困 +##囱 +##囲 +##図 +##围 +##囹 +##固 +##国 +##图 +##囿 +##圃 +##圄 +##圆 +##圈 +##國 +##圍 +##圏 +##園 +##圓 +##圖 +##團 +##圜 +##土 +##圣 +##圧 +##在 +##圩 +##圭 +##地 +##圳 +##场 +##圻 +##圾 +##址 +##坂 +##均 +##坊 +##坍 +##坎 +##坏 +##坐 +##坑 +##块 +##坚 +##坛 +##坝 +##坞 +##坟 +##坠 +##坡 +##坤 +##坦 +##坨 +##坪 +##坯 +##坳 +##坵 +##坷 +##垂 +##垃 +##垄 +##型 +##垒 +##垚 +##垛 +##垠 +##垢 +##垣 +##垦 +##垩 +##垫 +##垭 +##垮 +##垵 +##埂 +##埃 +##埋 +##城 +##埔 +##埕 +##埗 +##域 +##埠 +##埤 +##埵 +##執 +##埸 +##培 +##基 +##埼 +##堀 +##堂 +##堃 +##堅 +##堆 +##堇 +##堑 +##堕 +##堙 +##堡 +##堤 +##堪 +##堯 +##堰 +##報 +##場 +##堵 +##堺 +##堿 +##塊 +##塌 +##塑 +##塔 +##塗 +##塘 +##塚 +##塞 +##塢 +##塩 +##填 +##塬 +##塭 +##塵 +##塾 +##墀 +##境 +##墅 +##墉 +##墊 +##墒 +##墓 +##増 +##墘 +##墙 +##墜 +##增 +##墟 +##墨 +##墩 +##墮 +##墳 +##墻 +##墾 +##壁 +##壅 +##壆 +##壇 +##壊 +##壑 +##壓 +##壕 +##壘 +##壞 +##壟 +##壢 +##壤 +##壩 +##士 +##壬 +##壮 +##壯 +##声 +##売 +##壳 +##壶 +##壹 +##壺 +##壽 +##处 +##备 +##変 +##复 +##夏 +##夔 +##夕 +##外 +##夙 +##多 +##夜 +##够 +##夠 +##夢 +##夥 +##大 +##天 +##太 +##夫 +##夭 +##央 +##夯 +##失 +##头 +##夷 +##夸 +##夹 +##夺 +##夾 +##奂 +##奄 +##奇 +##奈 +##奉 +##奋 +##奎 +##奏 +##奐 +##契 +##奔 +##奕 +##奖 +##套 +##奘 +##奚 +##奠 +##奢 +##奥 +##奧 +##奪 +##奬 +##奮 +##女 +##奴 +##奶 +##奸 +##她 +##好 +##如 +##妃 +##妄 +##妆 +##妇 +##妈 +##妊 +##妍 +##妒 +##妓 +##妖 +##妘 +##妙 +##妝 +##妞 +##妣 +##妤 +##妥 +##妨 +##妩 +##妪 +##妮 +##妲 +##妳 +##妹 +##妻 +##妾 +##姆 +##姉 +##姊 +##始 +##姍 +##姐 +##姑 +##姒 +##姓 +##委 +##姗 +##姚 +##姜 +##姝 +##姣 +##姥 +##姦 +##姨 +##姪 +##姫 +##姬 +##姹 +##姻 +##姿 +##威 +##娃 +##娄 +##娅 +##娆 +##娇 +##娉 +##娑 +##娓 +##娘 +##娛 +##娜 +##娟 +##娠 +##娣 +##娥 +##娩 +##娱 +##娲 +##娴 +##娶 +##娼 +##婀 +##婁 +##婆 +##婉 +##婊 +##婕 +##婚 +##婢 +##婦 +##婧 +##婪 +##婭 +##婴 +##婵 +##婶 +##婷 +##婺 +##婿 +##媒 +##媚 +##媛 +##媞 +##媧 +##媲 +##媳 +##媽 +##媾 +##嫁 +##嫂 +##嫉 +##嫌 +##嫑 +##嫔 +##嫖 +##嫘 +##嫚 +##嫡 +##嫣 +##嫦 +##嫩 +##嫲 +##嫵 +##嫻 +##嬅 +##嬉 +##嬌 +##嬗 +##嬛 +##嬢 +##嬤 +##嬪 +##嬰 +##嬴 +##嬷 +##嬸 +##嬿 +##孀 +##孃 +##子 +##孑 +##孔 +##孕 +##孖 +##字 +##存 +##孙 +##孚 +##孛 +##孜 +##孝 +##孟 +##孢 +##季 +##孤 +##学 +##孩 +##孪 +##孫 +##孬 +##孰 +##孱 +##孳 +##孵 +##學 +##孺 +##孽 +##孿 +##宁 +##它 +##宅 +##宇 +##守 +##安 +##宋 +##完 +##宏 +##宓 +##宕 +##宗 +##官 +##宙 +##定 +##宛 +##宜 +##宝 +##实 +##実 +##宠 +##审 +##客 +##宣 +##室 +##宥 +##宦 +##宪 +##宫 +##宮 +##宰 +##害 +##宴 +##宵 +##家 +##宸 +##容 +##宽 +##宾 +##宿 +##寂 +##寄 +##寅 +##密 +##寇 +##富 +##寐 +##寒 +##寓 +##寛 +##寝 +##寞 +##察 +##寡 +##寢 +##寥 +##實 +##寧 +##寨 +##審 +##寫 +##寬 +##寮 +##寰 +##寵 +##寶 +##寸 +##对 +##寺 +##寻 +##导 +##対 +##寿 +##封 +##専 +##射 +##将 +##將 +##專 +##尉 +##尊 +##尋 +##對 +##導 +##小 +##少 +##尔 +##尕 +##尖 +##尘 +##尚 +##尝 +##尤 +##尧 +##尬 +##就 +##尴 +##尷 +##尸 +##尹 +##尺 +##尻 +##尼 +##尽 +##尾 +##尿 +##局 +##屁 +##层 +##屄 +##居 +##屆 +##屈 +##屉 +##届 +##屋 +##屌 +##屍 +##屎 +##屏 +##屐 +##屑 +##展 +##屜 +##属 +##屠 +##屡 +##屢 +##層 +##履 +##屬 +##屯 +##山 +##屹 +##屿 +##岀 +##岁 +##岂 +##岌 +##岐 +##岑 +##岔 +##岖 +##岗 +##岘 +##岙 +##岚 +##岛 +##岡 +##岩 +##岫 +##岬 +##岭 +##岱 +##岳 +##岷 +##岸 +##峇 +##峋 +##峒 +##峙 +##峡 +##峤 +##峥 +##峦 +##峨 +##峪 +##峭 +##峯 +##峰 +##峴 +##島 +##峻 +##峽 +##崁 +##崂 +##崆 +##崇 +##崎 +##崑 +##崔 +##崖 +##崗 +##崙 +##崛 +##崧 +##崩 +##崭 +##崴 +##崽 +##嵇 +##嵊 +##嵋 +##嵌 +##嵐 +##嵘 +##嵩 +##嵬 +##嵯 +##嶂 +##嶄 +##嶇 +##嶋 +##嶙 +##嶺 +##嶼 +##嶽 +##巅 +##巍 +##巒 +##巔 +##巖 +##川 +##州 +##巡 +##巢 +##工 +##左 +##巧 +##巨 +##巩 +##巫 +##差 +##己 +##已 +##巳 +##巴 +##巷 +##巻 +##巽 +##巾 +##巿 +##币 +##市 +##布 +##帅 +##帆 +##师 +##希 +##帐 +##帑 +##帕 +##帖 +##帘 +##帚 +##帛 +##帜 +##帝 +##帥 +##带 +##帧 +##師 +##席 +##帮 +##帯 +##帰 +##帳 +##帶 +##帷 +##常 +##帼 +##帽 +##幀 +##幂 +##幄 +##幅 +##幌 +##幔 +##幕 +##幟 +##幡 +##幢 +##幣 +##幫 +##干 +##平 +##年 +##并 +##幸 +##幹 +##幺 +##幻 +##幼 +##幽 +##幾 +##广 +##庁 +##広 +##庄 +##庆 +##庇 +##床 +##序 +##庐 +##库 +##应 +##底 +##庖 +##店 +##庙 +##庚 +##府 +##庞 +##废 +##庠 +##度 +##座 +##庫 +##庭 +##庵 +##庶 +##康 +##庸 +##庹 +##庾 +##廁 +##廂 +##廃 +##廈 +##廉 +##廊 +##廓 +##廖 +##廚 +##廝 +##廟 +##廠 +##廢 +##廣 +##廬 +##廳 +##延 +##廷 +##建 +##廿 +##开 +##弁 +##异 +##弃 +##弄 +##弈 +##弊 +##弋 +##式 +##弑 +##弒 +##弓 +##弔 +##引 +##弗 +##弘 +##弛 +##弟 +##张 +##弥 +##弦 +##弧 +##弩 +##弭 +##弯 +##弱 +##張 +##強 +##弹 +##强 +##弼 +##弾 +##彅 +##彆 +##彈 +##彌 +##彎 +##归 +##当 +##录 +##彗 +##彙 +##彝 +##形 +##彤 +##彥 +##彦 +##彧 +##彩 +##彪 +##彫 +##彬 +##彭 +##彰 +##影 +##彷 +##役 +##彻 +##彼 +##彿 +##往 +##征 +##径 +##待 +##徇 +##很 +##徉 +##徊 +##律 +##後 +##徐 +##徑 +##徒 +##従 +##徕 +##得 +##徘 +##徙 +##徜 +##從 +##徠 +##御 +##徨 +##復 +##循 +##徬 +##微 +##徳 +##徴 +##徵 +##德 +##徹 +##徼 +##徽 +##心 +##必 +##忆 +##忌 +##忍 +##忏 +##忐 +##忑 +##忒 +##忖 +##志 +##忘 +##忙 +##応 +##忠 +##忡 +##忤 +##忧 +##忪 +##快 +##忱 +##念 +##忻 +##忽 +##忿 +##怀 +##态 +##怂 +##怅 +##怆 +##怎 +##怏 +##怒 +##怔 +##怕 +##怖 +##怙 +##怜 +##思 +##怠 +##怡 +##急 +##怦 +##性 +##怨 +##怪 +##怯 +##怵 +##总 +##怼 +##恁 +##恃 +##恆 +##恋 +##恍 +##恐 +##恒 +##恕 +##恙 +##恚 +##恢 +##恣 +##恤 +##恥 +##恨 +##恩 +##恪 +##恫 +##恬 +##恭 +##息 +##恰 +##恳 +##恵 +##恶 +##恸 +##恺 +##恻 +##恼 +##恿 +##悄 +##悅 +##悉 +##悌 +##悍 +##悔 +##悖 +##悚 +##悟 +##悠 +##患 +##悦 +##您 +##悩 +##悪 +##悬 +##悯 +##悱 +##悲 +##悴 +##悵 +##悶 +##悸 +##悻 +##悼 +##悽 +##情 +##惆 +##惇 +##惊 +##惋 +##惑 +##惕 +##惘 +##惚 +##惜 +##惟 +##惠 +##惡 +##惦 +##惧 +##惨 +##惩 +##惫 +##惬 +##惭 +##惮 +##惯 +##惰 +##惱 +##想 +##惴 +##惶 +##惹 +##惺 +##愁 +##愆 +##愈 +##愉 +##愍 +##意 +##愕 +##愚 +##愛 +##愜 +##感 +##愣 +##愤 +##愧 +##愫 +##愷 +##愿 +##慄 +##慈 +##態 +##慌 +##慎 +##慑 +##慕 +##慘 +##慚 +##慟 +##慢 +##慣 +##慧 +##慨 +##慫 +##慮 +##慰 +##慳 +##慵 +##慶 +##慷 +##慾 +##憂 +##憊 +##憋 +##憎 +##憐 +##憑 +##憔 +##憚 +##憤 +##憧 +##憨 +##憩 +##憫 +##憬 +##憲 +##憶 +##憾 +##懂 +##懇 +##懈 +##應 +##懊 +##懋 +##懑 +##懒 +##懦 +##懲 +##懵 +##懶 +##懷 +##懸 +##懺 +##懼 +##懾 +##懿 +##戀 +##戈 +##戊 +##戌 +##戍 +##戎 +##戏 +##成 +##我 +##戒 +##戕 +##或 +##战 +##戚 +##戛 +##戟 +##戡 +##戦 +##截 +##戬 +##戮 +##戰 +##戲 +##戳 +##戴 +##戶 +##户 +##戸 +##戻 +##戾 +##房 +##所 +##扁 +##扇 +##扈 +##扉 +##手 +##才 +##扎 +##扑 +##扒 +##打 +##扔 +##払 +##托 +##扛 +##扣 +##扦 +##执 +##扩 +##扪 +##扫 +##扬 +##扭 +##扮 +##扯 +##扰 +##扱 +##扳 +##扶 +##批 +##扼 +##找 +##承 +##技 +##抄 +##抉 +##把 +##抑 +##抒 +##抓 +##投 +##抖 +##抗 +##折 +##抚 +##抛 +##抜 +##択 +##抟 +##抠 +##抡 +##抢 +##护 +##报 +##抨 +##披 +##抬 +##抱 +##抵 +##抹 +##押 +##抽 +##抿 +##拂 +##拄 +##担 +##拆 +##拇 +##拈 +##拉 +##拋 +##拌 +##拍 +##拎 +##拐 +##拒 +##拓 +##拔 +##拖 +##拗 +##拘 +##拙 +##拚 +##招 +##拜 +##拟 +##拡 +##拢 +##拣 +##拥 +##拦 +##拧 +##拨 +##择 +##括 +##拭 +##拮 +##拯 +##拱 +##拳 +##拴 +##拷 +##拼 +##拽 +##拾 +##拿 +##持 +##挂 +##指 +##挈 +##按 +##挎 +##挑 +##挖 +##挙 +##挚 +##挛 +##挝 +##挞 +##挟 +##挠 +##挡 +##挣 +##挤 +##挥 +##挨 +##挪 +##挫 +##振 +##挲 +##挹 +##挺 +##挽 +##挾 +##捂 +##捅 +##捆 +##捉 +##捋 +##捌 +##捍 +##捎 +##捏 +##捐 +##捕 +##捞 +##损 +##捡 +##换 +##捣 +##捧 +##捨 +##捩 +##据 +##捱 +##捲 +##捶 +##捷 +##捺 +##捻 +##掀 +##掂 +##掃 +##掇 +##授 +##掉 +##掌 +##掏 +##掐 +##排 +##掖 +##掘 +##掙 +##掛 +##掠 +##採 +##探 +##掣 +##接 +##控 +##推 +##掩 +##措 +##掬 +##掰 +##掲 +##掳 +##掴 +##掷 +##掸 +##掺 +##揀 +##揃 +##揄 +##揆 +##揉 +##揍 +##描 +##提 +##插 +##揖 +##揚 +##換 +##握 +##揣 +##揩 +##揪 +##揭 +##揮 +##援 +##揶 +##揸 +##揹 +##揽 +##搀 +##搁 +##搂 +##搅 +##損 +##搏 +##搐 +##搓 +##搔 +##搖 +##搗 +##搜 +##搞 +##搡 +##搪 +##搬 +##搭 +##搵 +##搶 +##携 +##搽 +##摀 +##摁 +##摄 +##摆 +##摇 +##摈 +##摊 +##摒 +##摔 +##摘 +##摞 +##摟 +##摧 +##摩 +##摯 +##摳 +##摸 +##摹 +##摺 +##摻 +##撂 +##撃 +##撅 +##撇 +##撈 +##撐 +##撑 +##撒 +##撓 +##撕 +##撚 +##撞 +##撤 +##撥 +##撩 +##撫 +##撬 +##播 +##撮 +##撰 +##撲 +##撵 +##撷 +##撸 +##撻 +##撼 +##撿 +##擀 +##擁 +##擂 +##擄 +##擅 +##擇 +##擊 +##擋 +##操 +##擎 +##擒 +##擔 +##擘 +##據 +##擞 +##擠 +##擡 +##擢 +##擦 +##擬 +##擰 +##擱 +##擲 +##擴 +##擷 +##擺 +##擼 +##擾 +##攀 +##攏 +##攒 +##攔 +##攘 +##攙 +##攜 +##攝 +##攞 +##攢 +##攣 +##攤 +##攥 +##攪 +##攫 +##攬 +##支 +##收 +##攸 +##改 +##攻 +##放 +##政 +##故 +##效 +##敌 +##敍 +##敎 +##敏 +##救 +##敕 +##敖 +##敗 +##敘 +##教 +##敛 +##敝 +##敞 +##敢 +##散 +##敦 +##敬 +##数 +##敲 +##整 +##敵 +##敷 +##數 +##斂 +##斃 +##文 +##斋 +##斌 +##斎 +##斐 +##斑 +##斓 +##斗 +##料 +##斛 +##斜 +##斟 +##斡 +##斤 +##斥 +##斧 +##斩 +##斫 +##斬 +##断 +##斯 +##新 +##斷 +##方 +##於 +##施 +##旁 +##旃 +##旅 +##旋 +##旌 +##旎 +##族 +##旖 +##旗 +##无 +##既 +##日 +##旦 +##旧 +##旨 +##早 +##旬 +##旭 +##旮 +##旱 +##时 +##旷 +##旺 +##旻 +##昀 +##昂 +##昆 +##昇 +##昉 +##昊 +##昌 +##明 +##昏 +##易 +##昔 +##昕 +##昙 +##星 +##映 +##春 +##昧 +##昨 +##昭 +##是 +##昱 +##昴 +##昵 +##昶 +##昼 +##显 +##晁 +##時 +##晃 +##晉 +##晋 +##晌 +##晏 +##晒 +##晓 +##晔 +##晕 +##晖 +##晗 +##晚 +##晝 +##晞 +##晟 +##晤 +##晦 +##晨 +##晩 +##普 +##景 +##晰 +##晴 +##晶 +##晷 +##智 +##晾 +##暂 +##暄 +##暇 +##暈 +##暉 +##暌 +##暐 +##暑 +##暖 +##暗 +##暝 +##暢 +##暧 +##暨 +##暫 +##暮 +##暱 +##暴 +##暸 +##暹 +##曄 +##曆 +##曇 +##曉 +##曖 +##曙 +##曜 +##曝 +##曠 +##曦 +##曬 +##曰 +##曲 +##曳 +##更 +##書 +##曹 +##曼 +##曾 +##替 +##最 +##會 +##月 +##有 +##朋 +##服 +##朐 +##朔 +##朕 +##朗 +##望 +##朝 +##期 +##朦 +##朧 +##木 +##未 +##末 +##本 +##札 +##朮 +##术 +##朱 +##朴 +##朵 +##机 +##朽 +##杀 +##杂 +##权 +##杆 +##杈 +##杉 +##李 +##杏 +##材 +##村 +##杓 +##杖 +##杜 +##杞 +##束 +##杠 +##条 +##来 +##杨 +##杭 +##杯 +##杰 +##東 +##杳 +##杵 +##杷 +##杼 +##松 +##板 +##极 +##构 +##枇 +##枉 +##枋 +##析 +##枕 +##林 +##枚 +##果 +##枝 +##枢 +##枣 +##枪 +##枫 +##枭 +##枯 +##枰 +##枱 +##枳 +##架 +##枷 +##枸 +##柄 +##柏 +##某 +##柑 +##柒 +##染 +##柔 +##柘 +##柚 +##柜 +##柞 +##柠 +##柢 +##查 +##柩 +##柬 +##柯 +##柱 +##柳 +##柴 +##柵 +##査 +##柿 +##栀 +##栃 +##栄 +##栅 +##标 +##栈 +##栉 +##栋 +##栎 +##栏 +##树 +##栓 +##栖 +##栗 +##校 +##栩 +##株 +##样 +##核 +##根 +##格 +##栽 +##栾 +##桀 +##桁 +##桂 +##桃 +##桅 +##框 +##案 +##桉 +##桌 +##桎 +##桐 +##桑 +##桓 +##桔 +##桜 +##桠 +##桡 +##桢 +##档 +##桥 +##桦 +##桧 +##桨 +##桩 +##桶 +##桿 +##梁 +##梅 +##梆 +##梏 +##梓 +##梗 +##條 +##梟 +##梢 +##梦 +##梧 +##梨 +##梭 +##梯 +##械 +##梳 +##梵 +##梶 +##检 +##棂 +##棄 +##棉 +##棋 +##棍 +##棒 +##棕 +##棗 +##棘 +##棚 +##棟 +##棠 +##棣 +##棧 +##森 +##棱 +##棲 +##棵 +##棹 +##棺 +##椁 +##椅 +##椋 +##植 +##椎 +##椒 +##検 +##椪 +##椭 +##椰 +##椹 +##椽 +##椿 +##楂 +##楊 +##楓 +##楔 +##楚 +##楝 +##楞 +##楠 +##楣 +##楨 +##楫 +##業 +##楮 +##極 +##楷 +##楸 +##楹 +##楼 +##楽 +##概 +##榄 +##榆 +##榈 +##榉 +##榔 +##榕 +##榖 +##榛 +##榜 +##榨 +##榫 +##榭 +##榮 +##榱 +##榴 +##榷 +##榻 +##槁 +##槃 +##構 +##槌 +##槍 +##槎 +##槐 +##槓 +##様 +##槛 +##槟 +##槤 +##槭 +##槲 +##槳 +##槻 +##槽 +##槿 +##樁 +##樂 +##樊 +##樑 +##樓 +##標 +##樞 +##樟 +##模 +##樣 +##権 +##横 +##樫 +##樯 +##樱 +##樵 +##樸 +##樹 +##樺 +##樽 +##樾 +##橄 +##橇 +##橋 +##橐 +##橘 +##橙 +##機 +##橡 +##橢 +##橫 +##橱 +##橹 +##橼 +##檀 +##檄 +##檎 +##檐 +##檔 +##檗 +##檜 +##檢 +##檬 +##檯 +##檳 +##檸 +##檻 +##櫃 +##櫚 +##櫛 +##櫥 +##櫸 +##櫻 +##欄 +##權 +##欒 +##欖 +##欠 +##次 +##欢 +##欣 +##欧 +##欲 +##欸 +##欺 +##欽 +##款 +##歆 +##歇 +##歉 +##歌 +##歎 +##歐 +##歓 +##歙 +##歛 +##歡 +##止 +##正 +##此 +##步 +##武 +##歧 +##歩 +##歪 +##歯 +##歲 +##歳 +##歴 +##歷 +##歸 +##歹 +##死 +##歼 +##殁 +##殃 +##殆 +##殇 +##殉 +##殊 +##残 +##殒 +##殓 +##殖 +##殘 +##殞 +##殡 +##殤 +##殭 +##殯 +##殲 +##殴 +##段 +##殷 +##殺 +##殼 +##殿 +##毀 +##毁 +##毂 +##毅 +##毆 +##毋 +##母 +##毎 +##每 +##毒 +##毓 +##比 +##毕 +##毗 +##毘 +##毙 +##毛 +##毡 +##毫 +##毯 +##毽 +##氈 +##氏 +##氐 +##民 +##氓 +##气 +##氖 +##気 +##氙 +##氛 +##氟 +##氡 +##氢 +##氣 +##氤 +##氦 +##氧 +##氨 +##氪 +##氫 +##氮 +##氯 +##氰 +##氲 +##水 +##氷 +##永 +##氹 +##氾 +##汀 +##汁 +##求 +##汆 +##汇 +##汉 +##汎 +##汐 +##汕 +##汗 +##汙 +##汛 +##汝 +##汞 +##江 +##池 +##污 +##汤 +##汨 +##汩 +##汪 +##汰 +##汲 +##汴 +##汶 +##汹 +##決 +##汽 +##汾 +##沁 +##沂 +##沃 +##沅 +##沈 +##沉 +##沌 +##沏 +##沐 +##沒 +##沓 +##沖 +##沙 +##沛 +##沟 +##没 +##沢 +##沣 +##沥 +##沦 +##沧 +##沪 +##沫 +##沭 +##沮 +##沱 +##河 +##沸 +##油 +##治 +##沼 +##沽 +##沾 +##沿 +##況 +##泄 +##泉 +##泊 +##泌 +##泓 +##法 +##泗 +##泛 +##泞 +##泠 +##泡 +##波 +##泣 +##泥 +##注 +##泪 +##泫 +##泮 +##泯 +##泰 +##泱 +##泳 +##泵 +##泷 +##泸 +##泻 +##泼 +##泽 +##泾 +##洁 +##洄 +##洋 +##洒 +##洗 +##洙 +##洛 +##洞 +##津 +##洩 +##洪 +##洮 +##洱 +##洲 +##洵 +##洶 +##洸 +##洹 +##活 +##洼 +##洽 +##派 +##流 +##浃 +##浄 +##浅 +##浆 +##浇 +##浊 +##测 +##济 +##浏 +##浑 +##浒 +##浓 +##浔 +##浙 +##浚 +##浜 +##浣 +##浦 +##浩 +##浪 +##浬 +##浮 +##浯 +##浴 +##海 +##浸 +##涂 +##涅 +##涇 +##消 +##涉 +##涌 +##涎 +##涓 +##涔 +##涕 +##涙 +##涛 +##涝 +##涞 +##涟 +##涠 +##涡 +##涣 +##涤 +##润 +##涧 +##涨 +##涩 +##涪 +##涮 +##涯 +##液 +##涵 +##涸 +##涼 +##涿 +##淀 +##淄 +##淅 +##淆 +##淇 +##淋 +##淌 +##淑 +##淒 +##淖 +##淘 +##淙 +##淚 +##淞 +##淡 +##淤 +##淦 +##淨 +##淩 +##淪 +##淫 +##淬 +##淮 +##深 +##淳 +##淵 +##混 +##淹 +##淺 +##添 +##淼 +##清 +##済 +##渉 +##渊 +##渋 +##渍 +##渎 +##渐 +##渔 +##渗 +##渙 +##渚 +##減 +##渝 +##渠 +##渡 +##渣 +##渤 +##渥 +##渦 +##温 +##測 +##渭 +##港 +##渲 +##渴 +##游 +##渺 +##渾 +##湃 +##湄 +##湊 +##湍 +##湖 +##湘 +##湛 +##湟 +##湧 +##湫 +##湮 +##湯 +##湳 +##湾 +##湿 +##満 +##溃 +##溅 +##溉 +##溏 +##源 +##準 +##溜 +##溝 +##溟 +##溢 +##溥 +##溧 +##溪 +##溫 +##溯 +##溱 +##溴 +##溶 +##溺 +##溼 +##滁 +##滂 +##滄 +##滅 +##滇 +##滋 +##滌 +##滑 +##滓 +##滔 +##滕 +##滙 +##滚 +##滝 +##滞 +##滟 +##满 +##滢 +##滤 +##滥 +##滦 +##滨 +##滩 +##滬 +##滯 +##滲 +##滴 +##滷 +##滸 +##滾 +##滿 +##漁 +##漂 +##漆 +##漉 +##漏 +##漓 +##演 +##漕 +##漠 +##漢 +##漣 +##漩 +##漪 +##漫 +##漬 +##漯 +##漱 +##漲 +##漳 +##漸 +##漾 +##漿 +##潆 +##潇 +##潋 +##潍 +##潑 +##潔 +##潘 +##潛 +##潜 +##潞 +##潟 +##潢 +##潤 +##潦 +##潧 +##潭 +##潮 +##潰 +##潴 +##潸 +##潺 +##潼 +##澀 +##澄 +##澆 +##澈 +##澍 +##澎 +##澗 +##澜 +##澡 +##澤 +##澧 +##澱 +##澳 +##澹 +##激 +##濁 +##濂 +##濃 +##濑 +##濒 +##濕 +##濘 +##濛 +##濟 +##濠 +##濡 +##濤 +##濫 +##濬 +##濮 +##濯 +##濱 +##濺 +##濾 +##瀅 +##瀆 +##瀉 +##瀋 +##瀏 +##瀑 +##瀕 +##瀘 +##瀚 +##瀛 +##瀝 +##瀞 +##瀟 +##瀧 +##瀨 +##瀬 +##瀰 +##瀾 +##灌 +##灏 +##灑 +##灘 +##灝 +##灞 +##灣 +##火 +##灬 +##灭 +##灯 +##灰 +##灵 +##灶 +##灸 +##灼 +##災 +##灾 +##灿 +##炀 +##炁 +##炅 +##炉 +##炊 +##炎 +##炒 +##炔 +##炕 +##炖 +##炙 +##炜 +##炫 +##炬 +##炭 +##炮 +##炯 +##炳 +##炷 +##炸 +##点 +##為 +##炼 +##炽 +##烁 +##烂 +##烃 +##烈 +##烊 +##烏 +##烘 +##烙 +##烛 +##烟 +##烤 +##烦 +##烧 +##烨 +##烩 +##烫 +##烬 +##热 +##烯 +##烷 +##烹 +##烽 +##焉 +##焊 +##焕 +##焖 +##焗 +##焘 +##焙 +##焚 +##焜 +##無 +##焦 +##焯 +##焰 +##焱 +##然 +##焼 +##煅 +##煉 +##煊 +##煌 +##煎 +##煒 +##煖 +##煙 +##煜 +##煞 +##煤 +##煥 +##煦 +##照 +##煨 +##煩 +##煮 +##煲 +##煸 +##煽 +##熄 +##熊 +##熏 +##熒 +##熔 +##熙 +##熟 +##熠 +##熨 +##熬 +##熱 +##熵 +##熹 +##熾 +##燁 +##燃 +##燄 +##燈 +##燉 +##燊 +##燎 +##燒 +##燔 +##燕 +##燙 +##燜 +##營 +##燥 +##燦 +##燧 +##燭 +##燮 +##燴 +##燻 +##燼 +##燿 +##爆 +##爍 +##爐 +##爛 +##爪 +##爬 +##爭 +##爰 +##爱 +##爲 +##爵 +##父 +##爷 +##爸 +##爹 +##爺 +##爻 +##爽 +##爾 +##牆 +##片 +##版 +##牌 +##牍 +##牒 +##牙 +##牛 +##牝 +##牟 +##牠 +##牡 +##牢 +##牦 +##牧 +##物 +##牯 +##牲 +##牴 +##牵 +##特 +##牺 +##牽 +##犀 +##犁 +##犄 +##犊 +##犍 +##犒 +##犢 +##犧 +##犬 +##犯 +##状 +##犷 +##犸 +##犹 +##狀 +##狂 +##狄 +##狈 +##狎 +##狐 +##狒 +##狗 +##狙 +##狞 +##狠 +##狡 +##狩 +##独 +##狭 +##狮 +##狰 +##狱 +##狸 +##狹 +##狼 +##狽 +##猎 +##猕 +##猖 +##猗 +##猙 +##猛 +##猜 +##猝 +##猥 +##猩 +##猪 +##猫 +##猬 +##献 +##猴 +##猶 +##猷 +##猾 +##猿 +##獄 +##獅 +##獎 +##獐 +##獒 +##獗 +##獠 +##獣 +##獨 +##獭 +##獰 +##獲 +##獵 +##獷 +##獸 +##獺 +##獻 +##獼 +##獾 +##玄 +##率 +##玉 +##王 +##玑 +##玖 +##玛 +##玟 +##玠 +##玥 +##玩 +##玫 +##玮 +##环 +##现 +##玲 +##玳 +##玷 +##玺 +##玻 +##珀 +##珂 +##珅 +##珈 +##珉 +##珊 +##珍 +##珏 +##珐 +##珑 +##珙 +##珞 +##珠 +##珣 +##珥 +##珩 +##珪 +##班 +##珮 +##珲 +##珺 +##現 +##球 +##琅 +##理 +##琇 +##琉 +##琊 +##琍 +##琏 +##琐 +##琛 +##琢 +##琥 +##琦 +##琨 +##琪 +##琬 +##琮 +##琰 +##琲 +##琳 +##琴 +##琵 +##琶 +##琺 +##琼 +##瑀 +##瑁 +##瑄 +##瑋 +##瑕 +##瑗 +##瑙 +##瑚 +##瑛 +##瑜 +##瑞 +##瑟 +##瑠 +##瑣 +##瑤 +##瑩 +##瑪 +##瑯 +##瑰 +##瑶 +##瑾 +##璀 +##璁 +##璃 +##璇 +##璉 +##璋 +##璎 +##璐 +##璜 +##璞 +##璟 +##璧 +##璨 +##環 +##璽 +##璿 +##瓊 +##瓏 +##瓒 +##瓜 +##瓢 +##瓣 +##瓤 +##瓦 +##瓮 +##瓯 +##瓴 +##瓶 +##瓷 +##甄 +##甌 +##甕 +##甘 +##甙 +##甚 +##甜 +##生 +##產 +##産 +##甥 +##甦 +##用 +##甩 +##甫 +##甬 +##甭 +##甯 +##田 +##由 +##甲 +##申 +##电 +##男 +##甸 +##町 +##画 +##甾 +##畀 +##畅 +##界 +##畏 +##畑 +##畔 +##留 +##畜 +##畝 +##畢 +##略 +##畦 +##番 +##畫 +##異 +##畲 +##畳 +##畴 +##當 +##畸 +##畹 +##畿 +##疆 +##疇 +##疊 +##疏 +##疑 +##疔 +##疖 +##疗 +##疙 +##疚 +##疝 +##疟 +##疡 +##疣 +##疤 +##疥 +##疫 +##疮 +##疯 +##疱 +##疲 +##疳 +##疵 +##疸 +##疹 +##疼 +##疽 +##疾 +##痂 +##病 +##症 +##痈 +##痉 +##痊 +##痍 +##痒 +##痔 +##痕 +##痘 +##痙 +##痛 +##痞 +##痠 +##痢 +##痣 +##痤 +##痧 +##痨 +##痪 +##痫 +##痰 +##痱 +##痴 +##痹 +##痺 +##痼 +##痿 +##瘀 +##瘁 +##瘋 +##瘍 +##瘓 +##瘘 +##瘙 +##瘟 +##瘠 +##瘡 +##瘢 +##瘤 +##瘦 +##瘧 +##瘩 +##瘪 +##瘫 +##瘴 +##瘸 +##瘾 +##療 +##癇 +##癌 +##癒 +##癖 +##癜 +##癞 +##癡 +##癢 +##癣 +##癥 +##癫 +##癬 +##癮 +##癱 +##癲 +##癸 +##発 +##登 +##發 +##白 +##百 +##皂 +##的 +##皆 +##皇 +##皈 +##皋 +##皎 +##皑 +##皓 +##皖 +##皙 +##皚 +##皮 +##皰 +##皱 +##皴 +##皺 +##皿 +##盂 +##盃 +##盅 +##盆 +##盈 +##益 +##盎 +##盏 +##盐 +##监 +##盒 +##盔 +##盖 +##盗 +##盘 +##盛 +##盜 +##盞 +##盟 +##盡 +##監 +##盤 +##盥 +##盧 +##盪 +##目 +##盯 +##盱 +##盲 +##直 +##相 +##盹 +##盼 +##盾 +##省 +##眈 +##眉 +##看 +##県 +##眙 +##眞 +##真 +##眠 +##眦 +##眨 +##眩 +##眯 +##眶 +##眷 +##眸 +##眺 +##眼 +##眾 +##着 +##睁 +##睇 +##睏 +##睐 +##睑 +##睛 +##睜 +##睞 +##睡 +##睢 +##督 +##睥 +##睦 +##睨 +##睪 +##睫 +##睬 +##睹 +##睽 +##睾 +##睿 +##瞄 +##瞅 +##瞇 +##瞋 +##瞌 +##瞎 +##瞑 +##瞒 +##瞓 +##瞞 +##瞟 +##瞠 +##瞥 +##瞧 +##瞩 +##瞪 +##瞬 +##瞭 +##瞰 +##瞳 +##瞻 +##瞼 +##瞿 +##矇 +##矍 +##矗 +##矚 +##矛 +##矜 +##矢 +##矣 +##知 +##矩 +##矫 +##短 +##矮 +##矯 +##石 +##矶 +##矽 +##矾 +##矿 +##码 +##砂 +##砌 +##砍 +##砒 +##研 +##砖 +##砗 +##砚 +##砝 +##砣 +##砥 +##砧 +##砭 +##砰 +##砲 +##破 +##砷 +##砸 +##砺 +##砼 +##砾 +##础 +##硅 +##硐 +##硒 +##硕 +##硝 +##硫 +##硬 +##确 +##硯 +##硼 +##碁 +##碇 +##碉 +##碌 +##碍 +##碎 +##碑 +##碓 +##碗 +##碘 +##碚 +##碛 +##碟 +##碣 +##碧 +##碩 +##碰 +##碱 +##碳 +##碴 +##確 +##碼 +##碾 +##磁 +##磅 +##磊 +##磋 +##磐 +##磕 +##磚 +##磡 +##磨 +##磬 +##磯 +##磲 +##磷 +##磺 +##礁 +##礎 +##礙 +##礡 +##礦 +##礪 +##礫 +##礴 +##示 +##礼 +##社 +##祀 +##祁 +##祂 +##祇 +##祈 +##祉 +##祎 +##祐 +##祕 +##祖 +##祗 +##祚 +##祛 +##祜 +##祝 +##神 +##祟 +##祠 +##祢 +##祥 +##票 +##祭 +##祯 +##祷 +##祸 +##祺 +##祿 +##禀 +##禁 +##禄 +##禅 +##禍 +##禎 +##福 +##禛 +##禦 +##禧 +##禪 +##禮 +##禱 +##禹 +##禺 +##离 +##禽 +##禾 +##禿 +##秀 +##私 +##秃 +##秆 +##秉 +##秋 +##种 +##科 +##秒 +##秘 +##租 +##秣 +##秤 +##秦 +##秧 +##秩 +##秭 +##积 +##称 +##秸 +##移 +##秽 +##稀 +##稅 +##程 +##稍 +##税 +##稔 +##稗 +##稚 +##稜 +##稞 +##稟 +##稠 +##稣 +##種 +##稱 +##稲 +##稳 +##稷 +##稹 +##稻 +##稼 +##稽 +##稿 +##穀 +##穂 +##穆 +##穌 +##積 +##穎 +##穗 +##穢 +##穩 +##穫 +##穴 +##究 +##穷 +##穹 +##空 +##穿 +##突 +##窃 +##窄 +##窈 +##窍 +##窑 +##窒 +##窓 +##窕 +##窖 +##窗 +##窘 +##窜 +##窝 +##窟 +##窠 +##窥 +##窦 +##窨 +##窩 +##窪 +##窮 +##窯 +##窺 +##窿 +##竄 +##竅 +##竇 +##竊 +##立 +##竖 +##站 +##竜 +##竞 +##竟 +##章 +##竣 +##童 +##竭 +##端 +##競 +##竹 +##竺 +##竽 +##竿 +##笃 +##笆 +##笈 +##笋 +##笏 +##笑 +##笔 +##笙 +##笛 +##笞 +##笠 +##符 +##笨 +##第 +##笹 +##笺 +##笼 +##筆 +##等 +##筊 +##筋 +##筍 +##筏 +##筐 +##筑 +##筒 +##答 +##策 +##筛 +##筝 +##筠 +##筱 +##筲 +##筵 +##筷 +##筹 +##签 +##简 +##箇 +##箋 +##箍 +##箏 +##箐 +##箔 +##箕 +##算 +##箝 +##管 +##箩 +##箫 +##箭 +##箱 +##箴 +##箸 +##節 +##篁 +##範 +##篆 +##篇 +##築 +##篑 +##篓 +##篙 +##篝 +##篠 +##篡 +##篤 +##篩 +##篪 +##篮 +##篱 +##篷 +##簇 +##簌 +##簍 +##簡 +##簦 +##簧 +##簪 +##簫 +##簷 +##簸 +##簽 +##簾 +##簿 +##籁 +##籃 +##籌 +##籍 +##籐 +##籟 +##籠 +##籤 +##籬 +##籮 +##籲 +##米 +##类 +##籼 +##籽 +##粄 +##粉 +##粑 +##粒 +##粕 +##粗 +##粘 +##粟 +##粤 +##粥 +##粧 +##粪 +##粮 +##粱 +##粲 +##粳 +##粵 +##粹 +##粼 +##粽 +##精 +##粿 +##糅 +##糊 +##糍 +##糕 +##糖 +##糗 +##糙 +##糜 +##糞 +##糟 +##糠 +##糧 +##糬 +##糯 +##糰 +##糸 +##系 +##糾 +##紀 +##紂 +##約 +##紅 +##紉 +##紊 +##紋 +##納 +##紐 +##紓 +##純 +##紗 +##紘 +##紙 +##級 +##紛 +##紜 +##素 +##紡 +##索 +##紧 +##紫 +##紮 +##累 +##細 +##紳 +##紹 +##紺 +##終 +##絃 +##組 +##絆 +##経 +##結 +##絕 +##絞 +##絡 +##絢 +##給 +##絨 +##絮 +##統 +##絲 +##絳 +##絵 +##絶 +##絹 +##綁 +##綏 +##綑 +##經 +##継 +##続 +##綜 +##綠 +##綢 +##綦 +##綫 +##綬 +##維 +##綱 +##網 +##綴 +##綵 +##綸 +##綺 +##綻 +##綽 +##綾 +##綿 +##緊 +##緋 +##総 +##緑 +##緒 +##緘 +##線 +##緝 +##緞 +##締 +##緣 +##編 +##緩 +##緬 +##緯 +##練 +##緹 +##緻 +##縁 +##縄 +##縈 +##縛 +##縝 +##縣 +##縫 +##縮 +##縱 +##縴 +##縷 +##總 +##績 +##繁 +##繃 +##繆 +##繇 +##繋 +##織 +##繕 +##繚 +##繞 +##繡 +##繩 +##繪 +##繫 +##繭 +##繳 +##繹 +##繼 +##繽 +##纂 +##續 +##纍 +##纏 +##纓 +##纔 +##纖 +##纜 +##纠 +##红 +##纣 +##纤 +##约 +##级 +##纨 +##纪 +##纫 +##纬 +##纭 +##纯 +##纰 +##纱 +##纲 +##纳 +##纵 +##纶 +##纷 +##纸 +##纹 +##纺 +##纽 +##纾 +##线 +##绀 +##练 +##组 +##绅 +##细 +##织 +##终 +##绊 +##绍 +##绎 +##经 +##绑 +##绒 +##结 +##绔 +##绕 +##绘 +##给 +##绚 +##绛 +##络 +##绝 +##绞 +##统 +##绡 +##绢 +##绣 +##绥 +##绦 +##继 +##绩 +##绪 +##绫 +##续 +##绮 +##绯 +##绰 +##绳 +##维 +##绵 +##绶 +##绷 +##绸 +##绻 +##综 +##绽 +##绾 +##绿 +##缀 +##缄 +##缅 +##缆 +##缇 +##缈 +##缉 +##缎 +##缓 +##缔 +##缕 +##编 +##缘 +##缙 +##缚 +##缜 +##缝 +##缠 +##缢 +##缤 +##缥 +##缨 +##缩 +##缪 +##缭 +##缮 +##缰 +##缱 +##缴 +##缸 +##缺 +##缽 +##罂 +##罄 +##罌 +##罐 +##网 +##罔 +##罕 +##罗 +##罚 +##罡 +##罢 +##罩 +##罪 +##置 +##罰 +##署 +##罵 +##罷 +##罹 +##羁 +##羅 +##羈 +##羊 +##羌 +##美 +##羔 +##羚 +##羞 +##羟 +##羡 +##羣 +##群 +##羥 +##羧 +##羨 +##義 +##羯 +##羲 +##羸 +##羹 +##羽 +##羿 +##翁 +##翅 +##翊 +##翌 +##翎 +##習 +##翔 +##翘 +##翟 +##翠 +##翡 +##翦 +##翩 +##翰 +##翱 +##翳 +##翹 +##翻 +##翼 +##耀 +##老 +##考 +##耄 +##者 +##耆 +##耋 +##而 +##耍 +##耐 +##耒 +##耕 +##耗 +##耘 +##耙 +##耦 +##耨 +##耳 +##耶 +##耷 +##耸 +##耻 +##耽 +##耿 +##聂 +##聆 +##聊 +##聋 +##职 +##聒 +##联 +##聖 +##聘 +##聚 +##聞 +##聪 +##聯 +##聰 +##聲 +##聳 +##聴 +##聶 +##職 +##聽 +##聾 +##聿 +##肃 +##肄 +##肅 +##肆 +##肇 +##肉 +##肋 +##肌 +##肏 +##肓 +##肖 +##肘 +##肚 +##肛 +##肝 +##肠 +##股 +##肢 +##肤 +##肥 +##肩 +##肪 +##肮 +##肯 +##肱 +##育 +##肴 +##肺 +##肽 +##肾 +##肿 +##胀 +##胁 +##胃 +##胄 +##胆 +##背 +##胍 +##胎 +##胖 +##胚 +##胛 +##胜 +##胝 +##胞 +##胡 +##胤 +##胥 +##胧 +##胫 +##胭 +##胯 +##胰 +##胱 +##胳 +##胴 +##胶 +##胸 +##胺 +##能 +##脂 +##脅 +##脆 +##脇 +##脈 +##脉 +##脊 +##脍 +##脏 +##脐 +##脑 +##脓 +##脖 +##脘 +##脚 +##脛 +##脣 +##脩 +##脫 +##脯 +##脱 +##脲 +##脳 +##脸 +##脹 +##脾 +##腆 +##腈 +##腊 +##腋 +##腌 +##腎 +##腐 +##腑 +##腓 +##腔 +##腕 +##腥 +##腦 +##腩 +##腫 +##腭 +##腮 +##腰 +##腱 +##腳 +##腴 +##腸 +##腹 +##腺 +##腻 +##腼 +##腾 +##腿 +##膀 +##膈 +##膊 +##膏 +##膑 +##膘 +##膚 +##膛 +##膜 +##膝 +##膠 +##膦 +##膨 +##膩 +##膳 +##膺 +##膻 +##膽 +##膾 +##膿 +##臀 +##臂 +##臃 +##臆 +##臉 +##臊 +##臍 +##臓 +##臘 +##臟 +##臣 +##臥 +##臧 +##臨 +##自 +##臬 +##臭 +##至 +##致 +##臺 +##臻 +##臼 +##臾 +##舀 +##舂 +##舅 +##舆 +##與 +##興 +##舉 +##舊 +##舌 +##舍 +##舎 +##舐 +##舒 +##舔 +##舖 +##舗 +##舛 +##舜 +##舞 +##舟 +##航 +##舫 +##般 +##舰 +##舱 +##舵 +##舶 +##舷 +##舸 +##船 +##舺 +##舾 +##艇 +##艋 +##艘 +##艙 +##艦 +##艮 +##良 +##艰 +##艱 +##色 +##艳 +##艷 +##艹 +##艺 +##艾 +##节 +##芃 +##芈 +##芊 +##芋 +##芍 +##芎 +##芒 +##芙 +##芜 +##芝 +##芡 +##芥 +##芦 +##芩 +##芪 +##芫 +##芬 +##芭 +##芮 +##芯 +##花 +##芳 +##芷 +##芸 +##芹 +##芻 +##芽 +##芾 +##苁 +##苄 +##苇 +##苋 +##苍 +##苏 +##苑 +##苒 +##苓 +##苔 +##苕 +##苗 +##苛 +##苜 +##苞 +##苟 +##苡 +##苣 +##若 +##苦 +##苫 +##苯 +##英 +##苷 +##苹 +##苻 +##茁 +##茂 +##范 +##茄 +##茅 +##茉 +##茎 +##茏 +##茗 +##茜 +##茧 +##茨 +##茫 +##茬 +##茭 +##茯 +##茱 +##茲 +##茴 +##茵 +##茶 +##茸 +##茹 +##茼 +##荀 +##荃 +##荆 +##草 +##荊 +##荏 +##荐 +##荒 +##荔 +##荖 +##荘 +##荚 +##荞 +##荟 +##荠 +##荡 +##荣 +##荤 +##荥 +##荧 +##荨 +##荪 +##荫 +##药 +##荳 +##荷 +##荸 +##荻 +##荼 +##荽 +##莅 +##莆 +##莉 +##莊 +##莎 +##莒 +##莓 +##莖 +##莘 +##莞 +##莠 +##莢 +##莧 +##莪 +##莫 +##莱 +##莲 +##莴 +##获 +##莹 +##莺 +##莽 +##莿 +##菀 +##菁 +##菅 +##菇 +##菈 +##菊 +##菌 +##菏 +##菓 +##菖 +##菘 +##菜 +##菟 +##菠 +##菡 +##菩 +##華 +##菱 +##菲 +##菸 +##菽 +##萁 +##萃 +##萄 +##萊 +##萋 +##萌 +##萍 +##萎 +##萘 +##萝 +##萤 +##营 +##萦 +##萧 +##萨 +##萩 +##萬 +##萱 +##萵 +##萸 +##萼 +##落 +##葆 +##葉 +##著 +##葚 +##葛 +##葡 +##董 +##葦 +##葩 +##葫 +##葬 +##葭 +##葯 +##葱 +##葳 +##葵 +##葷 +##葺 +##蒂 +##蒋 +##蒐 +##蒔 +##蒙 +##蒜 +##蒞 +##蒟 +##蒡 +##蒨 +##蒲 +##蒸 +##蒹 +##蒻 +##蒼 +##蒿 +##蓁 +##蓄 +##蓆 +##蓉 +##蓋 +##蓑 +##蓓 +##蓖 +##蓝 +##蓟 +##蓦 +##蓬 +##蓮 +##蓼 +##蓿 +##蔑 +##蔓 +##蔔 +##蔗 +##蔘 +##蔚 +##蔡 +##蔣 +##蔥 +##蔫 +##蔬 +##蔭 +##蔵 +##蔷 +##蔺 +##蔻 +##蔼 +##蔽 +##蕁 +##蕃 +##蕈 +##蕉 +##蕊 +##蕎 +##蕙 +##蕤 +##蕨 +##蕩 +##蕪 +##蕭 +##蕲 +##蕴 +##蕻 +##蕾 +##薄 +##薅 +##薇 +##薈 +##薊 +##薏 +##薑 +##薔 +##薙 +##薛 +##薦 +##薨 +##薩 +##薪 +##薬 +##薯 +##薰 +##薹 +##藉 +##藍 +##藏 +##藐 +##藓 +##藕 +##藜 +##藝 +##藤 +##藥 +##藩 +##藹 +##藻 +##藿 +##蘆 +##蘇 +##蘊 +##蘋 +##蘑 +##蘚 +##蘭 +##蘸 +##蘼 +##蘿 +##虎 +##虏 +##虐 +##虑 +##虔 +##處 +##虚 +##虛 +##虜 +##虞 +##號 +##虢 +##虧 +##虫 +##虬 +##虱 +##虹 +##虻 +##虽 +##虾 +##蚀 +##蚁 +##蚂 +##蚊 +##蚌 +##蚓 +##蚕 +##蚜 +##蚝 +##蚣 +##蚤 +##蚩 +##蚪 +##蚯 +##蚱 +##蚵 +##蛀 +##蛆 +##蛇 +##蛊 +##蛋 +##蛎 +##蛐 +##蛔 +##蛙 +##蛛 +##蛟 +##蛤 +##蛭 +##蛮 +##蛰 +##蛳 +##蛹 +##蛻 +##蛾 +##蜀 +##蜂 +##蜃 +##蜆 +##蜇 +##蜈 +##蜊 +##蜍 +##蜒 +##蜓 +##蜕 +##蜗 +##蜘 +##蜚 +##蜜 +##蜡 +##蜢 +##蜥 +##蜱 +##蜴 +##蜷 +##蜻 +##蜿 +##蝇 +##蝈 +##蝉 +##蝌 +##蝎 +##蝕 +##蝗 +##蝙 +##蝟 +##蝠 +##蝦 +##蝨 +##蝴 +##蝶 +##蝸 +##蝼 +##螂 +##螃 +##融 +##螞 +##螢 +##螨 +##螯 +##螳 +##螺 +##蟀 +##蟄 +##蟆 +##蟋 +##蟎 +##蟑 +##蟒 +##蟠 +##蟬 +##蟲 +##蟹 +##蟻 +##蟾 +##蠅 +##蠍 +##蠔 +##蠕 +##蠛 +##蠟 +##蠡 +##蠢 +##蠣 +##蠱 +##蠶 +##蠹 +##蠻 +##血 +##衄 +##衅 +##衆 +##行 +##衍 +##術 +##衔 +##街 +##衙 +##衛 +##衝 +##衞 +##衡 +##衢 +##衣 +##补 +##表 +##衩 +##衫 +##衬 +##衮 +##衰 +##衲 +##衷 +##衹 +##衾 +##衿 +##袁 +##袂 +##袄 +##袅 +##袈 +##袋 +##袍 +##袒 +##袖 +##袜 +##袞 +##袤 +##袪 +##被 +##袭 +##袱 +##裁 +##裂 +##装 +##裆 +##裊 +##裏 +##裔 +##裕 +##裘 +##裙 +##補 +##裝 +##裟 +##裡 +##裤 +##裨 +##裱 +##裳 +##裴 +##裸 +##裹 +##製 +##裾 +##褂 +##複 +##褐 +##褒 +##褓 +##褔 +##褚 +##褥 +##褪 +##褫 +##褲 +##褶 +##褻 +##襁 +##襄 +##襟 +##襠 +##襪 +##襬 +##襯 +##襲 +##西 +##要 +##覃 +##覆 +##覇 +##見 +##規 +##覓 +##視 +##覚 +##覦 +##覧 +##親 +##覬 +##観 +##覷 +##覺 +##覽 +##觀 +##见 +##观 +##规 +##觅 +##视 +##览 +##觉 +##觊 +##觎 +##觐 +##觑 +##角 +##觞 +##解 +##觥 +##触 +##觸 +##言 +##訂 +##計 +##訊 +##討 +##訓 +##訕 +##訖 +##託 +##記 +##訛 +##訝 +##訟 +##訣 +##訥 +##訪 +##設 +##許 +##訳 +##訴 +##訶 +##診 +##註 +##証 +##詆 +##詐 +##詔 +##評 +##詛 +##詞 +##詠 +##詡 +##詢 +##詣 +##試 +##詩 +##詫 +##詬 +##詭 +##詮 +##詰 +##話 +##該 +##詳 +##詹 +##詼 +##誅 +##誇 +##誉 +##誌 +##認 +##誓 +##誕 +##誘 +##語 +##誠 +##誡 +##誣 +##誤 +##誥 +##誦 +##誨 +##說 +##説 +##読 +##誰 +##課 +##誹 +##誼 +##調 +##諄 +##談 +##請 +##諏 +##諒 +##論 +##諗 +##諜 +##諡 +##諦 +##諧 +##諫 +##諭 +##諮 +##諱 +##諳 +##諷 +##諸 +##諺 +##諾 +##謀 +##謁 +##謂 +##謄 +##謊 +##謎 +##謐 +##謔 +##謗 +##謙 +##講 +##謝 +##謠 +##謨 +##謬 +##謹 +##謾 +##譁 +##證 +##譎 +##譏 +##識 +##譙 +##譚 +##譜 +##警 +##譬 +##譯 +##議 +##譲 +##譴 +##護 +##譽 +##讀 +##變 +##讓 +##讚 +##讞 +##计 +##订 +##认 +##讥 +##讧 +##讨 +##让 +##讪 +##讫 +##训 +##议 +##讯 +##记 +##讲 +##讳 +##讴 +##讶 +##讷 +##许 +##讹 +##论 +##讼 +##讽 +##设 +##访 +##诀 +##证 +##诃 +##评 +##诅 +##识 +##诈 +##诉 +##诊 +##诋 +##词 +##诏 +##译 +##试 +##诗 +##诘 +##诙 +##诚 +##诛 +##话 +##诞 +##诟 +##诠 +##诡 +##询 +##诣 +##诤 +##该 +##详 +##诧 +##诩 +##诫 +##诬 +##语 +##误 +##诰 +##诱 +##诲 +##说 +##诵 +##诶 +##请 +##诸 +##诺 +##读 +##诽 +##课 +##诿 +##谀 +##谁 +##调 +##谄 +##谅 +##谆 +##谈 +##谊 +##谋 +##谌 +##谍 +##谎 +##谏 +##谐 +##谑 +##谒 +##谓 +##谔 +##谕 +##谗 +##谘 +##谙 +##谚 +##谛 +##谜 +##谟 +##谢 +##谣 +##谤 +##谥 +##谦 +##谧 +##谨 +##谩 +##谪 +##谬 +##谭 +##谯 +##谱 +##谲 +##谴 +##谶 +##谷 +##豁 +##豆 +##豇 +##豈 +##豉 +##豊 +##豌 +##豎 +##豐 +##豔 +##豚 +##象 +##豢 +##豪 +##豫 +##豬 +##豹 +##豺 +##貂 +##貅 +##貌 +##貓 +##貔 +##貘 +##貝 +##貞 +##負 +##財 +##貢 +##貧 +##貨 +##販 +##貪 +##貫 +##責 +##貯 +##貰 +##貳 +##貴 +##貶 +##買 +##貸 +##費 +##貼 +##貽 +##貿 +##賀 +##賁 +##賂 +##賃 +##賄 +##資 +##賈 +##賊 +##賑 +##賓 +##賜 +##賞 +##賠 +##賡 +##賢 +##賣 +##賤 +##賦 +##質 +##賬 +##賭 +##賴 +##賺 +##購 +##賽 +##贅 +##贈 +##贊 +##贍 +##贏 +##贓 +##贖 +##贛 +##贝 +##贞 +##负 +##贡 +##财 +##责 +##贤 +##败 +##账 +##货 +##质 +##贩 +##贪 +##贫 +##贬 +##购 +##贮 +##贯 +##贰 +##贱 +##贲 +##贴 +##贵 +##贷 +##贸 +##费 +##贺 +##贻 +##贼 +##贾 +##贿 +##赁 +##赂 +##赃 +##资 +##赅 +##赈 +##赊 +##赋 +##赌 +##赎 +##赏 +##赐 +##赓 +##赔 +##赖 +##赘 +##赚 +##赛 +##赝 +##赞 +##赠 +##赡 +##赢 +##赣 +##赤 +##赦 +##赧 +##赫 +##赭 +##走 +##赳 +##赴 +##赵 +##赶 +##起 +##趁 +##超 +##越 +##趋 +##趕 +##趙 +##趟 +##趣 +##趨 +##足 +##趴 +##趵 +##趸 +##趺 +##趾 +##跃 +##跄 +##跆 +##跋 +##跌 +##跎 +##跑 +##跖 +##跚 +##跛 +##距 +##跟 +##跡 +##跤 +##跨 +##跩 +##跪 +##路 +##跳 +##践 +##跷 +##跹 +##跺 +##跻 +##踉 +##踊 +##踌 +##踏 +##踐 +##踝 +##踞 +##踟 +##踢 +##踩 +##踪 +##踮 +##踱 +##踴 +##踵 +##踹 +##蹂 +##蹄 +##蹇 +##蹈 +##蹉 +##蹊 +##蹋 +##蹑 +##蹒 +##蹙 +##蹟 +##蹣 +##蹤 +##蹦 +##蹩 +##蹬 +##蹭 +##蹲 +##蹴 +##蹶 +##蹺 +##蹼 +##蹿 +##躁 +##躇 +##躉 +##躊 +##躋 +##躍 +##躏 +##躪 +##身 +##躬 +##躯 +##躲 +##躺 +##軀 +##車 +##軋 +##軌 +##軍 +##軒 +##軟 +##転 +##軸 +##軼 +##軽 +##軾 +##較 +##載 +##輒 +##輓 +##輔 +##輕 +##輛 +##輝 +##輟 +##輩 +##輪 +##輯 +##輸 +##輻 +##輾 +##輿 +##轄 +##轅 +##轆 +##轉 +##轍 +##轎 +##轟 +##车 +##轧 +##轨 +##轩 +##转 +##轭 +##轮 +##软 +##轰 +##轲 +##轴 +##轶 +##轻 +##轼 +##载 +##轿 +##较 +##辄 +##辅 +##辆 +##辇 +##辈 +##辉 +##辊 +##辍 +##辐 +##辑 +##输 +##辕 +##辖 +##辗 +##辘 +##辙 +##辛 +##辜 +##辞 +##辟 +##辣 +##辦 +##辨 +##辩 +##辫 +##辭 +##辮 +##辯 +##辰 +##辱 +##農 +##边 +##辺 +##辻 +##込 +##辽 +##达 +##迁 +##迂 +##迄 +##迅 +##过 +##迈 +##迎 +##运 +##近 +##返 +##还 +##这 +##进 +##远 +##违 +##连 +##迟 +##迢 +##迤 +##迥 +##迦 +##迩 +##迪 +##迫 +##迭 +##述 +##迴 +##迷 +##迸 +##迹 +##迺 +##追 +##退 +##送 +##适 +##逃 +##逅 +##逆 +##选 +##逊 +##逍 +##透 +##逐 +##递 +##途 +##逕 +##逗 +##這 +##通 +##逛 +##逝 +##逞 +##速 +##造 +##逢 +##連 +##逮 +##週 +##進 +##逵 +##逶 +##逸 +##逻 +##逼 +##逾 +##遁 +##遂 +##遅 +##遇 +##遊 +##運 +##遍 +##過 +##遏 +##遐 +##遑 +##遒 +##道 +##達 +##違 +##遗 +##遙 +##遛 +##遜 +##遞 +##遠 +##遢 +##遣 +##遥 +##遨 +##適 +##遭 +##遮 +##遲 +##遴 +##遵 +##遶 +##遷 +##選 +##遺 +##遼 +##遽 +##避 +##邀 +##邁 +##邂 +##邃 +##還 +##邇 +##邈 +##邊 +##邋 +##邏 +##邑 +##邓 +##邕 +##邛 +##邝 +##邢 +##那 +##邦 +##邨 +##邪 +##邬 +##邮 +##邯 +##邰 +##邱 +##邳 +##邵 +##邸 +##邹 +##邺 +##邻 +##郁 +##郅 +##郊 +##郎 +##郑 +##郜 +##郝 +##郡 +##郢 +##郤 +##郦 +##郧 +##部 +##郫 +##郭 +##郴 +##郵 +##郷 +##郸 +##都 +##鄂 +##鄉 +##鄒 +##鄔 +##鄙 +##鄞 +##鄢 +##鄧 +##鄭 +##鄰 +##鄱 +##鄲 +##鄺 +##酉 +##酊 +##酋 +##酌 +##配 +##酐 +##酒 +##酗 +##酚 +##酝 +##酢 +##酣 +##酥 +##酩 +##酪 +##酬 +##酮 +##酯 +##酰 +##酱 +##酵 +##酶 +##酷 +##酸 +##酿 +##醃 +##醇 +##醉 +##醋 +##醍 +##醐 +##醒 +##醚 +##醛 +##醜 +##醞 +##醣 +##醪 +##醫 +##醬 +##醮 +##醯 +##醴 +##醺 +##釀 +##釁 +##采 +##釉 +##释 +##釋 +##里 +##重 +##野 +##量 +##釐 +##金 +##釗 +##釘 +##釜 +##針 +##釣 +##釦 +##釧 +##釵 +##鈀 +##鈉 +##鈍 +##鈎 +##鈔 +##鈕 +##鈞 +##鈣 +##鈦 +##鈪 +##鈴 +##鈺 +##鈾 +##鉀 +##鉄 +##鉅 +##鉉 +##鉑 +##鉗 +##鉚 +##鉛 +##鉤 +##鉴 +##鉻 +##銀 +##銃 +##銅 +##銑 +##銓 +##銖 +##銘 +##銜 +##銬 +##銭 +##銮 +##銳 +##銷 +##銹 +##鋁 +##鋅 +##鋒 +##鋤 +##鋪 +##鋰 +##鋸 +##鋼 +##錄 +##錐 +##錘 +##錚 +##錠 +##錢 +##錦 +##錨 +##錫 +##錮 +##錯 +##録 +##錳 +##錶 +##鍊 +##鍋 +##鍍 +##鍛 +##鍥 +##鍰 +##鍵 +##鍺 +##鍾 +##鎂 +##鎊 +##鎌 +##鎏 +##鎔 +##鎖 +##鎗 +##鎚 +##鎧 +##鎬 +##鎮 +##鎳 +##鏈 +##鏖 +##鏗 +##鏘 +##鏞 +##鏟 +##鏡 +##鏢 +##鏤 +##鏽 +##鐘 +##鐮 +##鐲 +##鐳 +##鐵 +##鐸 +##鐺 +##鑄 +##鑊 +##鑑 +##鑒 +##鑣 +##鑫 +##鑰 +##鑲 +##鑼 +##鑽 +##鑾 +##鑿 +##针 +##钉 +##钊 +##钎 +##钏 +##钒 +##钓 +##钗 +##钙 +##钛 +##钜 +##钝 +##钞 +##钟 +##钠 +##钡 +##钢 +##钣 +##钤 +##钥 +##钦 +##钧 +##钨 +##钩 +##钮 +##钯 +##钰 +##钱 +##钳 +##钴 +##钵 +##钺 +##钻 +##钼 +##钾 +##钿 +##铀 +##铁 +##铂 +##铃 +##铄 +##铅 +##铆 +##铉 +##铎 +##铐 +##铛 +##铜 +##铝 +##铠 +##铡 +##铢 +##铣 +##铤 +##铨 +##铩 +##铬 +##铭 +##铮 +##铰 +##铲 +##铵 +##银 +##铸 +##铺 +##链 +##铿 +##销 +##锁 +##锂 +##锄 +##锅 +##锆 +##锈 +##锉 +##锋 +##锌 +##锏 +##锐 +##锑 +##错 +##锚 +##锟 +##锡 +##锢 +##锣 +##锤 +##锥 +##锦 +##锭 +##键 +##锯 +##锰 +##锲 +##锵 +##锹 +##锺 +##锻 +##镀 +##镁 +##镂 +##镇 +##镉 +##镌 +##镍 +##镐 +##镑 +##镕 +##镖 +##镗 +##镛 +##镜 +##镣 +##镭 +##镯 +##镰 +##镳 +##镶 +##長 +##长 +##門 +##閃 +##閉 +##開 +##閎 +##閏 +##閑 +##閒 +##間 +##閔 +##閘 +##閡 +##関 +##閣 +##閥 +##閨 +##閩 +##閱 +##閲 +##閹 +##閻 +##閾 +##闆 +##闇 +##闊 +##闌 +##闍 +##闔 +##闕 +##闖 +##闘 +##關 +##闡 +##闢 +##门 +##闪 +##闫 +##闭 +##问 +##闯 +##闰 +##闲 +##间 +##闵 +##闷 +##闸 +##闹 +##闺 +##闻 +##闽 +##闾 +##阀 +##阁 +##阂 +##阅 +##阆 +##阇 +##阈 +##阉 +##阎 +##阐 +##阑 +##阔 +##阕 +##阖 +##阙 +##阚 +##阜 +##队 +##阡 +##阪 +##阮 +##阱 +##防 +##阳 +##阴 +##阵 +##阶 +##阻 +##阿 +##陀 +##陂 +##附 +##际 +##陆 +##陇 +##陈 +##陋 +##陌 +##降 +##限 +##陕 +##陛 +##陝 +##陞 +##陟 +##陡 +##院 +##陣 +##除 +##陨 +##险 +##陪 +##陰 +##陲 +##陳 +##陵 +##陶 +##陷 +##陸 +##険 +##陽 +##隅 +##隆 +##隈 +##隊 +##隋 +##隍 +##階 +##随 +##隐 +##隔 +##隕 +##隘 +##隙 +##際 +##障 +##隠 +##隣 +##隧 +##隨 +##險 +##隱 +##隴 +##隶 +##隸 +##隻 +##隼 +##隽 +##难 +##雀 +##雁 +##雄 +##雅 +##集 +##雇 +##雉 +##雋 +##雌 +##雍 +##雎 +##雏 +##雑 +##雒 +##雕 +##雖 +##雙 +##雛 +##雜 +##雞 +##離 +##難 +##雨 +##雪 +##雯 +##雰 +##雲 +##雳 +##零 +##雷 +##雹 +##電 +##雾 +##需 +##霁 +##霄 +##霆 +##震 +##霈 +##霉 +##霊 +##霍 +##霎 +##霏 +##霑 +##霓 +##霖 +##霜 +##霞 +##霧 +##霭 +##霰 +##露 +##霸 +##霹 +##霽 +##霾 +##靂 +##靄 +##靈 +##青 +##靓 +##靖 +##静 +##靚 +##靛 +##靜 +##非 +##靠 +##靡 +##面 +##靥 +##靦 +##革 +##靳 +##靴 +##靶 +##靼 +##鞅 +##鞋 +##鞍 +##鞏 +##鞑 +##鞘 +##鞠 +##鞣 +##鞦 +##鞭 +##韆 +##韋 +##韌 +##韓 +##韜 +##韦 +##韧 +##韩 +##韬 +##韭 +##音 +##韵 +##韶 +##韻 +##響 +##頁 +##頂 +##頃 +##項 +##順 +##須 +##頌 +##預 +##頑 +##頒 +##頓 +##頗 +##領 +##頜 +##頡 +##頤 +##頫 +##頭 +##頰 +##頷 +##頸 +##頹 +##頻 +##頼 +##顆 +##題 +##額 +##顎 +##顏 +##顔 +##願 +##顛 +##類 +##顧 +##顫 +##顯 +##顱 +##顴 +##页 +##顶 +##顷 +##项 +##顺 +##须 +##顼 +##顽 +##顾 +##顿 +##颁 +##颂 +##预 +##颅 +##领 +##颇 +##颈 +##颉 +##颊 +##颌 +##颍 +##颐 +##频 +##颓 +##颔 +##颖 +##颗 +##题 +##颚 +##颛 +##颜 +##额 +##颞 +##颠 +##颡 +##颢 +##颤 +##颦 +##颧 +##風 +##颯 +##颱 +##颳 +##颶 +##颼 +##飄 +##飆 +##风 +##飒 +##飓 +##飕 +##飘 +##飙 +##飚 +##飛 +##飞 +##食 +##飢 +##飨 +##飩 +##飪 +##飯 +##飲 +##飼 +##飽 +##飾 +##餃 +##餅 +##餉 +##養 +##餌 +##餐 +##餒 +##餓 +##餘 +##餚 +##餛 +##餞 +##餡 +##館 +##餮 +##餵 +##餾 +##饅 +##饈 +##饋 +##饌 +##饍 +##饑 +##饒 +##饕 +##饗 +##饞 +##饥 +##饨 +##饪 +##饬 +##饭 +##饮 +##饯 +##饰 +##饱 +##饲 +##饴 +##饵 +##饶 +##饷 +##饺 +##饼 +##饽 +##饿 +##馀 +##馁 +##馄 +##馅 +##馆 +##馈 +##馋 +##馍 +##馏 +##馒 +##馔 +##首 +##馗 +##香 +##馥 +##馨 +##馬 +##馭 +##馮 +##馳 +##馴 +##駁 +##駄 +##駅 +##駆 +##駐 +##駒 +##駕 +##駛 +##駝 +##駭 +##駱 +##駿 +##騁 +##騎 +##騏 +##験 +##騙 +##騨 +##騰 +##騷 +##驀 +##驅 +##驊 +##驍 +##驒 +##驕 +##驗 +##驚 +##驛 +##驟 +##驢 +##驥 +##马 +##驭 +##驮 +##驯 +##驰 +##驱 +##驳 +##驴 +##驶 +##驷 +##驸 +##驹 +##驻 +##驼 +##驾 +##驿 +##骁 +##骂 +##骄 +##骅 +##骆 +##骇 +##骈 +##骊 +##骋 +##验 +##骏 +##骐 +##骑 +##骗 +##骚 +##骛 +##骜 +##骞 +##骠 +##骡 +##骤 +##骥 +##骧 +##骨 +##骯 +##骰 +##骶 +##骷 +##骸 +##骼 +##髂 +##髅 +##髋 +##髏 +##髒 +##髓 +##體 +##髖 +##高 +##髦 +##髪 +##髮 +##髯 +##髻 +##鬃 +##鬆 +##鬍 +##鬓 +##鬚 +##鬟 +##鬢 +##鬣 +##鬥 +##鬧 +##鬱 +##鬼 +##魁 +##魂 +##魄 +##魅 +##魇 +##魍 +##魏 +##魔 +##魘 +##魚 +##魯 +##魷 +##鮑 +##鮨 +##鮪 +##鮭 +##鮮 +##鯉 +##鯊 +##鯖 +##鯛 +##鯨 +##鯰 +##鯽 +##鰍 +##鰓 +##鰭 +##鰲 +##鰻 +##鰾 +##鱈 +##鱉 +##鱔 +##鱗 +##鱷 +##鱸 +##鱼 +##鱿 +##鲁 +##鲈 +##鲍 +##鲑 +##鲛 +##鲜 +##鲟 +##鲢 +##鲤 +##鲨 +##鲫 +##鲱 +##鲲 +##鲶 +##鲷 +##鲸 +##鳃 +##鳄 +##鳅 +##鳌 +##鳍 +##鳕 +##鳖 +##鳗 +##鳝 +##鳞 +##鳥 +##鳩 +##鳳 +##鳴 +##鳶 +##鴉 +##鴕 +##鴛 +##鴦 +##鴨 +##鴻 +##鴿 +##鵑 +##鵜 +##鵝 +##鵡 +##鵬 +##鵰 +##鵲 +##鶘 +##鶩 +##鶯 +##鶴 +##鷗 +##鷲 +##鷹 +##鷺 +##鸚 +##鸞 +##鸟 +##鸠 +##鸡 +##鸢 +##鸣 +##鸥 +##鸦 +##鸨 +##鸪 +##鸭 +##鸯 +##鸳 +##鸵 +##鸽 +##鸾 +##鸿 +##鹂 +##鹃 +##鹄 +##鹅 +##鹈 +##鹉 +##鹊 +##鹌 +##鹏 +##鹑 +##鹕 +##鹘 +##鹜 +##鹞 +##鹤 +##鹦 +##鹧 +##鹫 +##鹭 +##鹰 +##鹳 +##鹵 +##鹹 +##鹼 +##鹽 +##鹿 +##麂 +##麋 +##麒 +##麓 +##麗 +##麝 +##麟 +##麥 +##麦 +##麩 +##麴 +##麵 +##麸 +##麺 +##麻 +##麼 +##麽 +##麾 +##黃 +##黄 +##黍 +##黎 +##黏 +##黑 +##黒 +##黔 +##默 +##黛 +##黜 +##黝 +##點 +##黠 +##黨 +##黯 +##黴 +##鼋 +##鼎 +##鼐 +##鼓 +##鼠 +##鼬 +##鼹 +##鼻 +##鼾 +##齁 +##齊 +##齋 +##齐 +##齒 +##齡 +##齢 +##齣 +##齦 +##齿 +##龄 +##龅 +##龈 +##龊 +##龋 +##龌 +##龍 +##龐 +##龔 +##龕 +##龙 +##龚 +##龛 +##龜 +##龟 +##︰ +##︱ +##︶ +##︿ +##﹁ +##﹂ +##﹍ +##﹏ +##﹐ +##﹑ +##﹒ +##﹔ +##﹕ +##﹖ +##﹗ +##﹙ +##﹚ +##﹝ +##﹞ +##﹡ +##﹣ +##！ +##＂ +##＃ +##＄ +##％ +##＆ +##＇ +##（ +##） +##＊ +##， +##－ +##． +##／ +##： +##； +##＜ +##？ +##＠ +##［ +##＼ +##］ +##＾ +##＿ +##｀ +##ｆ +##ｈ +##ｊ +##ｕ +##ｗ +##ｚ +##｛ +##｝ +##｡ +##｢ +##｣ +##､ +##･ +##ｯ +##ｰ +##ｲ +##ｸ +##ｼ +##ｽ +##ﾄ +##ﾉ +##ﾌ +##ﾗ +##ﾙ +##ﾝ +##ﾞ +##ﾟ +##￣ +##￥ +##👍 +##🔥 +##😂 +##😎 diff --git a/BertToSimple/BiGRU/processors/__init__.py b/BertToSimple/BiGRU/processors/__init__.py new file mode 100644 index 0000000..2e1a53e --- /dev/null +++ b/BertToSimple/BiGRU/processors/__init__.py @@ -0,0 +1,5 @@ +from .utils import InputExample, InputFeatures, DataProcessor +from .glue import (glue_output_modes, glue_processors, glue_tasks_num_labels, + glue_convert_examples_to_features,collate_fn) + + diff --git a/BertToSimple/BiGRU/processors/glue.py b/BertToSimple/BiGRU/processors/glue.py new file mode 100644 index 0000000..a3049a1 --- /dev/null +++ b/BertToSimple/BiGRU/processors/glue.py @@ -0,0 +1,534 @@ +""" GLUE processors and helpers """ + +import logging +import os +import torch +from .utils import DataProcessor, InputExample, InputFeatures + +logger = logging.getLogger(__name__) + + +def collate_fn(batch): + """ + batch should be a list of (sequence, target, length) tuples... + Returns a padded tensor of sequences sorted from longest to shortest, + """ + all_input_ids, all_attention_mask, all_token_type_ids, all_lens, all_labels = map(torch.stack, zip(*batch)) + max_len = max(all_lens).item() + all_input_ids = all_input_ids[:, :max_len] + all_attention_mask = all_attention_mask[:, :max_len] + all_token_type_ids = all_token_type_ids[:, :max_len] + return all_input_ids, all_attention_mask, all_token_type_ids, all_labels + + +def glue_convert_examples_to_features(examples, tokenizer, + max_seq_length=512, + task=None, + label_list=None, + output_mode=None): + """ + Loads a data file into a list of ``InputFeatures`` + Args: + examples: List of ``InputExamples`` or ``tf.data.Dataset`` containing the examples. + tokenizer: Instance of a tokenizer that will tokenize the examples + max_length: Maximum example length + task: GLUE task + label_list: List of labels. Can be obtained from the processor using the ``processor.get_labels()`` method + output_mode: String indicating the output mode. Either ``regression`` or ``classification`` + pad_on_left: If set to ``True``, the examples will be padded on the left rather than on the right (default) + pad_token: Padding token + pad_token_segment_id: The segment ID for the padding token (It is usually 0, but can vary such as for XLNet where it is 4) + mask_padding_with_zero: If set to ``True``, the attention mask will be filled by ``1`` for actual values + and by ``0`` for padded values. If set to ``False``, inverts it (``1`` for padded values, ``0`` for + actual values) + + Returns: + If the ``examples`` input is a ``tf.data.Dataset``, will return a ``tf.data.Dataset`` + containing the task-specific features. If the input is a list of ``InputExamples``, will return + a list of task-specific ``InputFeatures`` which can be fed to the model. + + """ + if task is not None: + processor = glue_processors[task]() + if label_list is None: + label_list = processor.get_labels() + logger.info("Using label list %s for task %s" % (label_list, task)) + if output_mode is None: + output_mode = glue_output_modes[task] + logger.info("Using output mode %s for task %s" % (output_mode, task)) + # label->index的映射 + label_map = {label: i for i, label in enumerate(label_list)} + features = [] + for (ex_index, example) in enumerate(examples): + if ex_index % 10000 == 0: + logger.info("Writing example %d" % (ex_index)) + # 把一个句子按字拆开，形成一个列表 + tokens_a = tokenizer.tokenize(example.text_a) + tokens_b =None + # 在这处理是不是有两句 + if example.text_b: + tokens_b = tokenizer.tokenize(example.text_b) + if tokens_b: + # Modifies `tokens_a` and `tokens_b` in place so that the total + # length is less than the specified length. + # Account for [CLS], [SEP], [SEP] with "- 3" + _truncate_seq_pair(tokens_a, tokens_b, max_seq_length - 3) + else: + # Account for [CLS] and [SEP] with "- 2" + if len(tokens_a) > max_seq_length - 2: + tokens_a = tokens_a[0:(max_seq_length - 2)] + + tokens = [] + token_type_ids = [] + tokens.append("[CLS]") + token_type_ids.append(0) + for token in tokens_a: + tokens.append(token) + token_type_ids.append(0) + tokens.append("[SEP]") + token_type_ids.append(0) + if tokens_b: + for token in tokens_b: + tokens.append(token) + token_type_ids.append(1) + tokens.append("[SEP]") + token_type_ids.append(1) + # 将字符转换为对应的token + input_ids = tokenizer.convert_tokens_to_ids(tokens) + # The mask has 1 for real tokens and 0 for padding tokens. Only real + # tokens are attended to. + attention_mask = [1] * len(input_ids) + input_len = len(input_ids) + + # Zero-pad up to the sequence length. + while len(input_ids) < max_seq_length: + input_ids.append(0) + attention_mask.append(0) + token_type_ids.append(0) + + assert len(input_ids) == max_seq_length + assert len(attention_mask) == max_seq_length + assert len(token_type_ids) == max_seq_length + if output_mode == "classification": + label_id = label_map[example.label[0]] + float_label = [float(i) for i in example.label[1:]] + label_id = [label_id] + float_label + + elif output_mode == "regression": + label_id = float(example.label) + else: + raise KeyError(output_mode) + if ex_index < 5: + logger.info("*** Example ***") + logger.info("guid: %s" % (example.guid)) + logger.info("input_ids: %s" % " ".join([str(x) for x in input_ids])) + logger.info("attention_mask: %s" % " ".join([str(x) for x in attention_mask])) + logger.info("token_type_ids: %s" % " ".join([str(x) for x in token_type_ids])) + #logger.info("label: %s (id = %d)" % (example.label, label_id)) + logger.info("input length: %d" % (input_len)) + # token_type_id是干嘛的？ + features.append( + InputFeatures(input_ids=input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + label=label_id, + input_len=input_len)) + return features + +def _truncate_seq_pair(tokens_a, tokens_b, max_length): + """Truncates a sequence pair in place to the maximum length.""" + + # This is a simple heuristic which will always truncate the longer sequence + # one token at a time. This makes more sense than truncating an equal percent + # of tokens from each, since if one sequence is very short then each token + # that's truncated likely contains more information than a longer sequence. + while True: + total_length = len(tokens_a) + len(tokens_b) + if total_length <= max_length: + break + if len(tokens_a) > len(tokens_b): + tokens_a.pop() + else: + tokens_b.pop() + +class MrpcProcessor(DataProcessor): + """Processor for the MRPC data set (GLUE version).""" + + def get_train_examples(self, data_dir): + """See base class.""" + logger.info("LOOKING AT {}".format(os.path.join(data_dir, "logits_train.txt"))) + return self._create_examples( + self._read_txt(os.path.join(data_dir, "logits_train.txt")), "train") + + def get_dev_examples(self, data_dir): + """See base class.""" + return self._create_examples( + self._read_txt(os.path.join(data_dir, "logits_dev.txt")), "dev") + + def get_labels(self): + """See base class.""" + labels_ = list(range(glue_tasks_num_labels["mrpc"])) + labels = [str(i) for i in labels_] + return labels + + def _create_examples(self, lines, set_type): + """Creates examples for the training and dev sets.""" + examples = [] + for (i, line) in enumerate(lines): + if i == 0: + continue + guid = "%s-%s" % (set_type, i) + # 有的数据可能出问题 + try: + text_a = line[0] + label = line[1:] + except: + print("data error:",line) + continue + # InputExample就是把数据变成json + examples.append( + InputExample(guid=guid, text_a=text_a, text_b=None, label=label)) + + return examples + + +class MnliProcessor(DataProcessor): + """Processor for the MultiNLI data set (GLUE version).""" + + def get_train_examples(self, data_dir): + """See base class.""" + return self._create_examples( + self._read_tsv(os.path.join(data_dir, "train.tsv")), "train") + + def get_dev_examples(self, data_dir): + """See base class.""" + return self._create_examples( + self._read_tsv(os.path.join(data_dir, "dev_matched.tsv")), + "dev_matched") + + def get_labels(self): + """See base class.""" + return ["contradiction", "entailment", "neutral"] + + def _create_examples(self, lines, set_type): + """Creates examples for the training and dev sets.""" + examples = [] + for (i, line) in enumerate(lines): + if i == 0: + continue + guid = "%s-%s" % (set_type, line[0]) + text_a = line[8] + text_b = line[9] + label = line[-1] + examples.append( + InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label)) + return examples + +class MnliMismatchedProcessor(MnliProcessor): + """Processor for the MultiNLI Mismatched data set (GLUE version).""" + + def get_dev_examples(self, data_dir): + """See base class.""" + return self._create_examples( + self._read_tsv(os.path.join(data_dir, "dev_mismatched.tsv")), + "dev_matched") + + +class ColaProcessor(DataProcessor): + """Processor for the cola data set (GLUE version).""" + + def get_train_examples(self, data_dir): + """See base class.""" + return self._create_examples( + self._read_tsv(os.path.join(data_dir, "train.tsv")), "train") + + def get_dev_examples(self, data_dir): + """See base class.""" + return self._create_examples( + self._read_tsv(os.path.join(data_dir, "dev.tsv")), "dev") + + def get_labels(self): + """See base class.""" + return ["0", "1"] + + def _create_examples(self, lines, set_type): + """Creates examples for the training and dev sets.""" + examples = [] + for (i, line) in enumerate(lines): + guid = "%s-%s" % (set_type, i) + text_a = line[3] + label = line[1] + examples.append( + InputExample(guid=guid, text_a=text_a, text_b=None, label=label)) + return examples + +class Sst2Processor(DataProcessor): + """Processor for the SST-2 data set (GLUE version).""" + + def get_train_examples(self, data_dir): + """See base class.""" + return self._create_examples( + self._read_tsv(os.path.join(data_dir, "train.tsv")), "train") + + def get_dev_examples(self, data_dir): + """See base class.""" + return self._create_examples( + self._read_tsv(os.path.join(data_dir, "dev.tsv")), "dev") + + def get_labels(self): + """See base class.""" + return ["0", "1"] + + def _create_examples(self, lines, set_type): + """Creates examples for the training and dev sets.""" + examples = [] + for (i, line) in enumerate(lines): + if i == 0: + continue + guid = "%s-%s" % (set_type, i) + text_a = line[0] + label = line[1] + examples.append( + InputExample(guid=guid, text_a=text_a, text_b=None, label=label)) + return examples + +class StsbProcessor(DataProcessor): + """Processor for the sts-b data set (GLUE version).""" + + def get_train_examples(self, data_dir): + """See base class.""" + return self._create_examples( + self._read_tsv(os.path.join(data_dir, "train.tsv")), "train") + + def get_dev_examples(self, data_dir): + """See base class.""" + return self._create_examples( + self._read_tsv(os.path.join(data_dir, "dev.tsv")), "dev") + + def get_labels(self): + """See base class.""" + return [None] + + def _create_examples(self, lines, set_type): + """Creates examples for the training and dev sets.""" + examples = [] + for (i, line) in enumerate(lines): + if i == 0: + continue + guid = "%s-%s" % (set_type, line[0]) + text_a = line[7] + text_b = line[8] + label = line[-1] + examples.append( + InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label)) + return examples + + +class QqpProcessor(DataProcessor): + """Processor for the QQP data set (GLUE version).""" + + def get_train_examples(self, data_dir): + """See base class.""" + return self._create_examples( + self._read_tsv(os.path.join(data_dir, "train.tsv")), "train") + + def get_dev_examples(self, data_dir): + """See base class.""" + return self._create_examples( + self._read_tsv(os.path.join(data_dir, "dev.tsv")), "dev") + + def get_labels(self): + """See base class.""" + return ["0", "1"] + + def _create_examples(self, lines, set_type): + """Creates examples for the training and dev sets.""" + examples = [] + for (i, line) in enumerate(lines): + if i == 0: + continue + guid = "%s-%s" % (set_type, line[0]) + try: + text_a = line[3] + text_b = line[4] + label = line[5] + except IndexError: + continue + examples.append( + InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label)) + return examples + + +class QnliProcessor(DataProcessor): + """Processor for the QNLI data set (GLUE version).""" + + def get_train_examples(self, data_dir): + """See base class.""" + return self._create_examples( + self._read_tsv(os.path.join(data_dir, "train.tsv")), "train") + + def get_dev_examples(self, data_dir): + """See base class.""" + return self._create_examples( + self._read_tsv(os.path.join(data_dir, "dev.tsv")), + "dev_matched") + + def get_labels(self): + """See base class.""" + return ["entailment", "not_entailment"] + + def _create_examples(self, lines, set_type): + """Creates examples for the training and dev sets.""" + examples = [] + for (i, line) in enumerate(lines): + if i == 0: + continue + guid = "%s-%s" % (set_type, line[0]) + text_a = line[1] + text_b = line[2] + label = line[-1] + examples.append( + InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label)) + return examples + + +class RteProcessor(DataProcessor): + """Processor for the RTE data set (GLUE version).""" + + def get_train_examples(self, data_dir): + """See base class.""" + return self._create_examples( + self._read_tsv(os.path.join(data_dir, "train.tsv")), "train") + + def get_dev_examples(self, data_dir): + """See base class.""" + return self._create_examples( + self._read_tsv(os.path.join(data_dir, "dev.tsv")), "dev") + + def get_labels(self): + """See base class.""" + return ["entailment", "not_entailment"] + + def _create_examples(self, lines, set_type): + """Creates examples for the training and dev sets.""" + examples = [] + for (i, line) in enumerate(lines): + if i == 0: + continue + guid = "%s-%s" % (set_type, line[0]) + text_a = line[1] + text_b = line[2] + label = line[-1] + examples.append( + InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label)) + return examples + +class LcqmcProcessor(DataProcessor): + """Processor for the LCQMC data set (GLUE version).""" + + def get_train_examples(self, data_dir): + """See base class.""" + return self._create_examples( + self._read_tsv(os.path.join(data_dir, "train.txt")), "train") + + def get_dev_examples(self, data_dir): + """See base class.""" + return self._create_examples( + self._read_tsv(os.path.join(data_dir, "dev.txt")), "dev") + + def get_test_examples(self, data_dir): + """See base class.""" + return self._create_examples( + self._read_tsv(os.path.join(data_dir, "test.txt")), "test") + + def get_labels(self): + """See base class.""" + return ["0", "1"] + + def _create_examples(self, lines, set_type): + """Creates examples for the training and dev sets.""" + examples = [] + for (i, line) in enumerate(lines): + guid = "%s-%s" % (set_type, i) + text_a = line[0] + text_b = line[1] + if set_type == 'test222': + label = '0' + else: + label = line[2] + examples.append( + InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label)) + return examples + + +class WnliProcessor(DataProcessor): + """Processor for the WNLI data set (GLUE version).""" + + def get_train_examples(self, data_dir): + """See base class.""" + return self._create_examples( + self._read_tsv(os.path.join(data_dir, "train.tsv")), "train") + + def get_dev_examples(self, data_dir): + """See base class.""" + return self._create_examples( + self._read_tsv(os.path.join(data_dir, "dev.tsv")), "dev") + + def get_labels(self): + """See base class.""" + return ["0", "1"] + + def _create_examples(self, lines, set_type): + """Creates examples for the training and dev sets.""" + examples = [] + for (i, line) in enumerate(lines): + if i == 0: + continue + guid = "%s-%s" % (set_type, line[0]) + text_a = line[1] + text_b = line[2] + label = line[-1] + examples.append( + InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label)) + return examples + + +glue_tasks_num_labels = { + "mnli": 3, + "mrpc": 21, + "sst-2": 2, + "sts-b": 1, + "qqp": 2, + "qnli": 2, + "rte": 2, + 'lcqmc': 2, + "xnli": 3, +} + +glue_processors = { + "cola": ColaProcessor, + "mnli": MnliProcessor, + "mnli-mm": MnliMismatchedProcessor, + "mrpc": MrpcProcessor, + "sst-2": Sst2Processor, + "sts-b": StsbProcessor, + "qqp": QqpProcessor, + "qnli": QnliProcessor, + "rte": RteProcessor, + 'lcqmc': LcqmcProcessor, + "wnli": WnliProcessor, +} + +glue_output_modes = { + "cola": "classification", + "mnli": "classification", + "mnli-mm": "classification", + "mrpc": "classification", + "sst-2": "classification", + "sts-b": "regression", + "qqp": "classification", + "qnli": "classification", + "rte": "classification", + "wnli": "classification", + 'lcqmc': "classification", +} diff --git a/BertToSimple/BiGRU/processors/utils.py b/BertToSimple/BiGRU/processors/utils.py new file mode 100644 index 0000000..22acaef --- /dev/null +++ b/BertToSimple/BiGRU/processors/utils.py @@ -0,0 +1,104 @@ +import csv +import sys +import copy +import json + +class InputExample(object): + """ + A single training/test example for simple sequence classification. + + Args: + guid: Unique id for the example. + text_a: string. The untokenized text of the first sequence. For single + sequence tasks, only this sequence must be specified. + text_b: (Optional) string. The untokenized text of the second sequence. + Only must be specified for sequence pair tasks. + label: (Optional) string. The label of the example. This should be + specified for train and dev examples, but not for test examples. + """ + def __init__(self, guid, text_a, text_b=None, label=None): + self.guid = guid + self.text_a = text_a + self.text_b = text_b + self.label = label + + def __repr__(self): + return str(self.to_json_string()) + + def to_dict(self): + """Serializes this instance to a Python dictionary.""" + output = copy.deepcopy(self.__dict__) + return output + + def to_json_string(self): + """Serializes this instance to a JSON string.""" + return json.dumps(self.to_dict(), indent=2, sort_keys=True) + "\n" + + +class InputFeatures(object): + """ + A single set of features of data. + + Args: + input_ids: Indices of input sequence tokens in the vocabulary. + attention_mask: Mask to avoid performing attention on padding token indices. + Mask values selected in ``[0, 1]``: + Usually ``1`` for tokens that are NOT MASKED, ``0`` for MASKED (padded) tokens. + token_type_ids: Segment token indices to indicate first and second portions of the inputs. + label: Label corresponding to the input + """ + + def __init__(self, input_ids, attention_mask, token_type_ids, label,input_len): + self.input_ids = input_ids + self.attention_mask = attention_mask + self.token_type_ids = token_type_ids + self.input_len = input_len + self.label = label + + def __repr__(self): + return str(self.to_json_string()) + + def to_dict(self): + """Serializes this instance to a Python dictionary.""" + output = copy.deepcopy(self.__dict__) + return output + + def to_json_string(self): + """Serializes this instance to a JSON string.""" + return json.dumps(self.to_dict(), indent=2, sort_keys=True) + "\n" + + +class DataProcessor(object): + """Base class for data converters for sequence classification data sets.""" + + def get_train_examples(self, data_dir): + """Gets a collection of `InputExample`s for the train set.""" + raise NotImplementedError() + + def get_dev_examples(self, data_dir): + """Gets a collection of `InputExample`s for the dev set.""" + raise NotImplementedError() + + def get_labels(self): + """Gets the list of labels for this data set.""" + raise NotImplementedError() + + @classmethod + def _read_tsv(cls, input_file, quotechar=None): + """Reads a tab separated value file.""" + with open(input_file, "r", encoding="utf-8-sig") as f: + reader = csv.reader(f, delimiter="\t", quotechar=quotechar) + lines = [] + for line in reader: + lines.append(line) + return lines + + @classmethod + def _read_txt(cls, input_file): + """Reads a tab separated value file.""" + with open(input_file, "r") as f: + reader = f.readlines() + lines = [] + for line in reader: + lines.append(line.strip().split("\t")) + return lines diff --git a/BertToSimple/BiGRU/run_classifier.py b/BertToSimple/BiGRU/run_classifier.py new file mode 100644 index 0000000..8d5b49b --- /dev/null +++ b/BertToSimple/BiGRU/run_classifier.py @@ -0,0 +1,533 @@ +""" Finetuning the library models for sequence classification .""" + +from __future__ import absolute_import, division, print_function +import pandas as pd +import argparse +import os +import glob +import numpy as np +import torch +import time +from torch.utils.data import DataLoader, RandomSampler, SequentialSampler, TensorDataset +from torch.utils.data.distributed import DistributedSampler +import my_model +from model.modeling_albert import AlbertConfig, AlbertForSequenceClassification +# from model.modeling_albert_bright import AlbertConfig, AlbertForSequenceClassification # chinese version +from model import tokenization_albert +from model.file_utils import WEIGHTS_NAME +from callback.optimization.adamw import AdamW +from callback.lr_scheduler import get_linear_schedule_with_warmup + +from metrics.glue_compute_metrics import compute_metrics +from processors import glue_output_modes as output_modes +from processors import glue_processors as processors +from processors import glue_convert_examples_to_features as convert_examples_to_features +from processors import collate_fn +from tools.common import seed_everything +from tools.common import init_logger, logger +from callback.progressbar import ProgressBar + +def train(args, train_dataset, model, tokenizer): + """ Train the model """ + # 把这个变成1 不然之后会数据并行 + # args.n_gpu = 1 + args.train_batch_size = args.per_gpu_train_batch_size * max(1, args.n_gpu) + train_sampler = RandomSampler(train_dataset) if args.local_rank == -1 else DistributedSampler(train_dataset) + train_dataloader = DataLoader(train_dataset, sampler=train_sampler, batch_size=args.train_batch_size, + collate_fn=collate_fn) + + if args.max_steps > 0: + num_training_steps = args.max_steps + args.num_train_epochs = args.max_steps // (len(train_dataloader) // args.gradient_accumulation_steps) + 1 + else: + num_training_steps = len(train_dataloader) // args.gradient_accumulation_steps * args.num_train_epochs + args.warmup_steps = int(num_training_steps * args.warmup_proportion) + # Prepare optimizer and schedule (linear warmup and decay) + no_decay = ['bias', 'LayerNorm.weight'] + optimizer_grouped_parameters = [ + {'params': [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay)], + 'weight_decay': args.weight_decay}, + {'params': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)], 'weight_decay': 0.0} + ] + # optimizer = Lamb(optimizer_grouped_parameters, lr=args.learning_rate, eps=args.adam_epsilon) + optimizer = AdamW(params=optimizer_grouped_parameters, lr=args.learning_rate, eps=args.adam_epsilon) + scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=args.warmup_steps, num_training_steps=num_training_steps) + if args.fp16: + try: + from apex import amp + except ImportError: + raise ImportError("Please install apex from https://www.github.com/nvidia/apex to use fp16 training.") + model, optimizer = amp.initialize(model, optimizer, opt_level=args.fp16_opt_level) + + # multi-gpu training (should be after apex fp16 initialization) + if args.n_gpu > 1: + model = torch.nn.DataParallel(model) + + # Distributed training (should be after apex fp16 initialization) + if args.local_rank != -1: + model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[args.local_rank], + output_device=args.local_rank, + find_unused_parameters=True) + + # Train! + logger.info("***** Running training *****") + logger.info(" Num examples = %d", len(train_dataset)) + logger.info(" Num Epochs = %d", args.num_train_epochs) + logger.info(" Instantaneous batch size per GPU = %d", args.per_gpu_train_batch_size) + logger.info(" Total train batch size (w. parallel, distributed & accumulation) = %d", + args.train_batch_size * args.gradient_accumulation_steps * ( + torch.distributed.get_world_size() if args.local_rank != -1 else 1)) + logger.info(" Gradient Accumulation steps = %d", args.gradient_accumulation_steps) + logger.info(" Total optimization steps = %d", num_training_steps) + + global_step = 0 + tr_loss, logging_loss = 0.0, 0.0 + model.zero_grad() + seed_everything(args.seed) # Added here for reproductibility (even between python 2 and 3) + for _ in range(int(args.num_train_epochs)): + # 显示进度的一个对象 + pbar = ProgressBar(n_total=len(train_dataloader), desc='Training') + for step, batch in enumerate(train_dataloader): + model.train() + batch = tuple(t.to(args.device) for t in batch) + inputs = {'input_ids': batch[0], + 'attention_mask': batch[1], + 'labels': batch[3]} + inputs['token_type_ids'] = batch[2] + # print(inputs) + loss_and_logits = model(**inputs) + loss = loss_and_logits[0] # model outputs are always tuple in transformers (see doc) + if args.n_gpu > 1: + loss = loss.mean() # mean() to average on multi-gpu parallel training + if args.gradient_accumulation_steps > 1: + loss = loss / args.gradient_accumulation_steps + + if args.fp16: + with amp.scale_loss(loss, optimizer) as scaled_loss: + scaled_loss.backward() + torch.nn.utils.clip_grad_norm_(amp.master_params(optimizer), args.max_grad_norm) + else: + loss.backward() + torch.nn.utils.clip_grad_norm_(model.parameters(), args.max_grad_norm) + + tr_loss += loss.item() + # 还能这么玩？ 多个step才更新一次梯度 + if (step + 1) % args.gradient_accumulation_steps == 0: + optimizer.step() + scheduler.step() # Update learning rate schedule + model.zero_grad() + global_step += 1 + #每隔每隔logging stepeval一次 + if args.local_rank in [-1, 0] and args.logging_steps > 0 and global_step % args.logging_steps == 0: + #Log metrics + # 1个gpu必须eval + if args.local_rank == -1: # Only evaluate when single GPU otherwise metrics may not average well + evaluate(args, model, tokenizer) + + if args.local_rank in [-1, 0] and args.save_steps > 0 and global_step % args.save_steps == 0: + # Save model checkpoint + output_dir = os.path.join(args.output_dir, 'checkpoint-{}'.format(global_step)) + if not os.path.exists(output_dir): + os.makedirs(output_dir) + #model_to_save = model.module if hasattr(model, + #'module') else model # Take care of distributed/parallel training + #model_to_save.save_pretrained(output_dir) + torch.save(model.state_dict(),os.path.join(output_dir, "pytorch_model.bin")) + torch.save(args, os.path.join(output_dir, 'training_args.bin')) + logger.info("Saving model checkpoint to %s", output_dir) + pbar(step, {'loss': loss.item()}) + print(" ") + if 'cuda' in str(args.device): + del batch,inputs,loss_and_logits + torch.cuda.empty_cache() + return global_step, tr_loss / global_step + +def evaluate(args, model, tokenizer, prefix="",log_confusion=False): + # Loop to handle MNLI double evaluation (matched, mis-matched) + eval_task_names = ("mnli", "mnli-mm") if args.task_name == "mnli" else (args.task_name,) + eval_outputs_dirs = (args.output_dir, args.output_dir + '-MM') if args.task_name == "mnli" else (args.output_dir,) + + results = {} + for eval_task, eval_output_dir in zip(eval_task_names, eval_outputs_dirs): + #　这里如果用train是需要得到训练集数据的logits，为之后蒸馏做准备。 + eval_dataset = load_and_cache_examples(args, eval_task, tokenizer, data_type='dev') + if not os.path.exists(eval_output_dir) and args.local_rank in [-1, 0]: + os.makedirs(eval_output_dir) + + args.eval_batch_size = args.per_gpu_eval_batch_size * max(1, args.n_gpu) + # Note that DistributedSampler samples randomly + eval_sampler = SequentialSampler(eval_dataset) if args.local_rank == -1 else DistributedSampler(eval_dataset) + eval_dataloader = DataLoader(eval_dataset, sampler=eval_sampler, batch_size=args.eval_batch_size, + collate_fn=collate_fn,shuffle=False) + + # Eval! + logger.info("***** Running evaluation {} *****".format(prefix)) + logger.info(" Num examples = %d", len(eval_dataset)) + logger.info(" Batch size = %d", args.eval_batch_size) + eval_loss = 0.0 + nb_eval_steps = 0 + preds = None + out_label_ids = None + pbar = ProgressBar(n_total=len(eval_dataloader), desc="Evaluating") + + + with torch.no_grad(): + for step, batch in enumerate(eval_dataloader): + model.eval() + batch = tuple(t.to(args.device) for t in batch) + with torch.no_grad(): + inputs = {'input_ids': batch[0], + 'attention_mask': batch[1], + 'labels': batch[3]} + inputs['token_type_ids'] = batch[2] + t1 = time.time() + outputs = model(**inputs) + print(time.time()-t1) + tmp_eval_loss, logits = outputs[:2] + eval_loss += tmp_eval_loss.mean().item() + nb_eval_steps += 1 + if preds is None: + preds = logits.detach().cpu().numpy() + out_label_ids = inputs['labels'].detach().cpu().numpy()[:,0] + else: + preds = np.append(preds, logits.detach().cpu().numpy(), axis=0) + out_label_ids = np.append(out_label_ids, inputs['labels'].detach().cpu().numpy()[:,0]) + pbar(step) + + print(' ') + if 'cuda' in str(args.device): + del batch,inputs,outputs + torch.cuda.empty_cache() + eval_loss = eval_loss / nb_eval_steps + # 这里重新定义了mrpc任务，是多分类的 + if args.output_mode == "classification" and eval_task == "mrpc": + if log_confusion == True: + label_pre_list = [] + confusion_list = [] + # 7个环节 每个环节是3类 + for i in range(7): + confusion_list.append(np.zeros((3,3))) + + start_idx_list = list(range(0,21,3)) + final_preds = [] + assert len(out_label_ids) == len(preds) + + + for i in range(len(preds)): + now_pred_data = preds[i] + now_label = out_label_ids[i] + for link_num,start_idx in enumerate(start_idx_list,0): + if now_label>=start_idx and now_label<(start_idx+3): + break + # 根据label找出对应区间段的预测概率 + now_pred_data = now_pred_data[start_idx:start_idx+3] + now_pred_idx = np.argmax(now_pred_data) + now_pred_idx = start_idx+now_pred_idx + final_preds.append(now_pred_idx) + # 记录混淆矩阵 + if log_confusion == True: + label_pre_list.append([now_label, now_pred_idx]) + link_pre = now_pred_idx-start_idx + link_label = now_label-start_idx + confusion_list[link_num][link_label][link_pre] += 1 + # 保存混淆矩阵 + if log_confusion == True: + save_csv = pd.DataFrame(label_pre_list) + save_csv.columns = ["label","pre"] + save_csv.index = list(range(len(save_csv))) + save_csv.to_csv("./label_and_pre.csv",header=True,index=True) + for i in range(7): + np.save("./confusion/confusion{}.npy".format(i),confusion_list[i]) + print(confusion_list[i]) + + + preds = np.array(final_preds) + elif args.output_mode == "classification": + preds = np.argmax(preds, axis=1) + elif args.output_mode == "regression": + preds = np.squeeze(preds) + #不同的任务是不同的指标，mprc任务返回的是acc和f1 score + result = compute_metrics(eval_task, preds, out_label_ids) + results.update(result) + logger.info("***** Eval results {} *****".format(prefix)) + for key in sorted(result.keys()): + logger.info(" %s = %s", key, str(result[key])) + return results + +# 直接返回dataloader +def load_and_cache_examples(args, task, tokenizer, data_type='train'): + if args.local_rank not in [-1, 0] and not evaluate: + torch.distributed.barrier() # Make sure only the first process in distributed training process the dataset, and the others will use the cache + + processor = processors[task]() + output_mode = output_modes[task] + # Load data features from cache or dataset file + cached_features_file = os.path.join(args.data_dir, 'cached_{}_{}_{}_{}'.format( + data_type, + list(filter(None, args.model_name_or_path.split('/'))).pop(), + str(args.max_seq_length), + str(task))) + # 看看有没有缓存起来的处理过格式的数据 + if os.path.exists(cached_features_file): + logger.info("Loading features from cached file %s", cached_features_file) + features = torch.load(cached_features_file) + else: + logger.info("Creating features from dataset file at %s", args.data_dir) + label_list = processor.get_labels() + # mnli：语型内匹配，判断两个句子意思相近？ mnli-mm：跨语型匹配，两个句子意思相近？ + #　roberta一种bert的改进版本 + if task in ['mnli', 'mnli-mm'] and 'roberta' in args.model_type: + # HACK(label indices are swapped in RoBERTa pretrained model) + label_list[1], label_list[2] = label_list[2], label_list[1] + + if data_type == 'train': + examples = processor.get_train_examples(args.data_dir) + # examples是一个列表，列表里边是很多个字典包括:序号，句子1，句子2，是否匹配(实际是json) + elif data_type == 'dev': + examples = processor.get_dev_examples(args.data_dir) + else: + print("no test data") + raise NameError + examples = processor.get_test_examples(args.data_dir) + # 将句子分字并用数字代替，不足max_len的用0pad token_type_ids只有0，1两个数值，标识是第一句还是第二句 + features = convert_examples_to_features(examples, + tokenizer, + label_list=label_list, + max_seq_length=args.max_seq_length, + output_mode = output_mode) + if args.local_rank in [-1, 0]: + logger.info("Saving features into cached file %s", cached_features_file) + torch.save(features, cached_features_file) + if args.local_rank == 0 and not evaluate: + torch.distributed.barrier() # Make sure only the first process in distributed training process the dataset, and the others will use the cache + # Convert to Tensors and build dataset + all_input_ids = torch.tensor([f.input_ids for f in features], dtype=torch.long) + all_attention_mask = torch.tensor([f.attention_mask for f in features], dtype=torch.long) + all_token_type_ids = torch.tensor([f.token_type_ids for f in features], dtype=torch.long) + all_lens = torch.tensor([f.input_len for f in features], dtype=torch.long) + if output_mode == "classification": + all_labels = torch.tensor([f.label for f in features], dtype=torch.float) + elif output_mode == "regression": + all_labels = torch.tensor([f.label for f in features], dtype=torch.float) + dataset = TensorDataset(all_input_ids, all_attention_mask, all_token_type_ids, all_lens, all_labels) + return dataset + +def main(): + parser = argparse.ArgumentParser() + + ## Required parameters + parser.add_argument("--data_dir", default="./dataset/cuishou/", type=str, required=False, + help="The input data dir. Should contain the .tsv files (or other data files) for the task.") + # 模型类型 貌似只有roberta才会做一些额外处理 + parser.add_argument("--model_type", default="albert", type=str, required=False, + help="Model type selected in the list: ") + parser.add_argument("--model_name_or_path", default="./prev_trained_model/albert_21_93", type=str, required=False, + help="Path to pre-trained model or shortcut name selected in the list") + # 八种glue任务之一 + parser.add_argument("--task_name", default="MRPC", type=str, required=False, + help="The name of the task to train selected in the list: " + ", ".join(processors.keys())) + # 输出路径 + parser.add_argument("--output_dir", default="./outputs", type=str, required=False, + help="The output directory where the model predictions and checkpoints will be written.") + # 词表地址 + parser.add_argument("--vocab_file",default='./prev_trained_model/albert_large_zh/vocab.txt', type=str) + # 这个也是词表？直接填NONE + parser.add_argument("--spm_model_file",default=None,type=str) + # 从头训练蒸馏模型还是加载蒸馏模型 + parser.add_argument("--continue_train_distill", default=True, type=bool, required=False, + help="是否加载蒸馏一半的模型") + # 蒸馏模型的位置 + parser.add_argument("--distill_model_dir", default="./good_result/pytorch_model-90.bin", + type=str, required=False,help="是否加载蒸馏一半的模型") + # 继续蒸馏的学习率 + parser.add_argument("--continue_learning_rate", default=5e-5, type=float, + help="The initial learning rate for Adam.") + + ## Other parameters + # 空着就行，如果没有会自动找model_name_or_path的 + parser.add_argument("--config_name", default="", type=str, + help="Pretrained config name or path if not the same as model_name") + parser.add_argument("--tokenizer_name", default="", type=str, + help="Pretrained tokenizer name or path if not the same as model_name") + parser.add_argument("--cache_dir", default="", type=str, + help="Where do you want to store the pre-trained models downloaded from s3") + parser.add_argument("--max_seq_length", default=512, type=int, + help="The maximum total input sequence length after tokenization. Sequences longer " + "than this will be truncated, sequences shorter will be padded.") + + parser.add_argument("--do_train", default=False,type=bool, + help="Whether to run training.") + parser.add_argument("--do_eval", default=True,type=bool, + help="Whether to run eval on the dev set.") + parser.add_argument("--do_predict", action='store_true', + help="Whether to run the model in inference mode on the test set.") + parser.add_argument("--do_lower_case", action='store_true', + help="Set this flag if you are using an uncased model.") + + parser.add_argument("--per_gpu_train_batch_size", default=96, type=int, + help="Batch size per GPU/CPU for training.") + parser.add_argument("--per_gpu_eval_batch_size", default=1, type=int, + help="Batch size per GPU/CPU for evaluation.") + parser.add_argument('--gradient_accumulation_steps', type=int, default=1, + help="Number of updates steps to accumulate before performing a backward/update pass.") + parser.add_argument("--learning_rate", default=1e-4, type=float, + help="The initial learning rate for Adam.") + parser.add_argument("--weight_decay", default=0.0, type=float, + help="Weight deay if we apply some.") + parser.add_argument("--adam_epsilon", default=1e-6, type=float, + help="Epsilon for Adam optimizer.") + parser.add_argument("--max_grad_norm", default=1.0, type=float, + help="Max gradient norm.") + parser.add_argument("--num_train_epochs", default=50.0, type=float, + help="Total number of training epochs to perform.") + parser.add_argument("--max_steps", default=-1, type=int, + help="If > 0: set total number of training steps to perform. Override num_train_epochs.") + parser.add_argument("--warmup_proportion", default=0.1, type=float, + help="Proportion of training to perform linear learning rate warmup for,E.g., 0.1 = 10% of training.") + # 每多少个batch评估一次eval 700 + parser.add_argument('--logging_steps', type=int, default=300, + help="Log every X updates steps.") + # 每多少batch保存一次模型 + parser.add_argument('--save_steps', type=int, default=900, + help="Save checkpoint every X updates steps.") + parser.add_argument("--eval_all_checkpoints", action='store_true', + help="Evaluate all checkpoints starting with the same prefix as model_name ending and ending with step number") + parser.add_argument("--no_cuda", action='store_true', + help="Avoid using CUDA when available") + parser.add_argument('--overwrite_output_dir', default=True,type=bool, + help="Overwrite the content of the output directory") + parser.add_argument('--overwrite_cache', action='store_true', + help="Overwrite the cached training and evaluation sets") + parser.add_argument('--seed', type=int, default=42, + help="random seed for initialization") + + parser.add_argument('--fp16', action='store_true', + help="Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit") + parser.add_argument('--fp16_opt_level', type=str, default='O1', + help="For fp16: Apex AMP optimization level selected in ['O0', 'O1', 'O2', and 'O3']." + "See details at https://nvidia.github.io/apex/amp.html") + parser.add_argument("--local_rank", type=int, default=-1, + help="For distributed training: local_rank") + parser.add_argument('--server_ip', type=str, default='', help="For distant debugging.") + parser.add_argument('--server_port', type=str, default='', help="For distant debugging.") + args = parser.parse_args() + #是否有输出文件夹 + if not os.path.exists(args.output_dir): + os.mkdir(args.output_dir) + args.output_dir = args.output_dir + '{}'.format(args.model_type) + if not os.path.exists(args.output_dir): + os.mkdir(args.output_dir) + init_logger(log_file=args.output_dir + '/{}-{}.log'.format(args.model_type, args.task_name)) + if os.path.exists(args.output_dir) and os.listdir( + args.output_dir) and args.do_train and not args.overwrite_output_dir: + raise ValueError( + "Output directory ({}) already exists and is not empty. Use --overwrite_output_dir to overcome.".format( + args.output_dir)) + + # Setup distant debugging if needed + if args.server_ip and args.server_port: + # Distant debugging - see https://code.visualstudio.com/docs/python/debugging#_attach-to-a-local-script + import ptvsd + print("Waiting for debugger attach") + ptvsd.enable_attach(address=(args.server_ip, args.server_port), redirect_output=True) + ptvsd.wait_for_attach() + + # Setup CUDA, GPU & distributed training/设置GPU或者分布式训练 + if args.local_rank == -1 or args.no_cuda: + device = torch.device("cuda:0" if torch.cuda.is_available() and not args.no_cuda else "cpu") + args.n_gpu = torch.cuda.device_count() + else: # Initializes the distributed backend which will take care of sychronizing nodes/GPUs + torch.cuda.set_device(args.local_rank) + device = torch.device("cuda", args.local_rank) + torch.distributed.init_process_group(backend='nccl') + args.n_gpu = 1 + args.n_gpu = 1 + args.device = device + # Setup logging + logger.warning("Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s", + args.local_rank, device, args.n_gpu, bool(args.local_rank != -1), args.fp16) + # Set seed + seed_everything(args.seed) + # Prepare GLUE task + args.task_name = args.task_name.lower() + # process里边存放了有什么任务的列表 + if args.task_name not in processors: + raise ValueError("Task not found: %s" % (args.task_name)) + # processors字典里边存的是函数的指针，根据任务类别的字符串索引，并构建对应的类对象 + processor = processors[args.task_name]() + print(args.task_name) + # output_mode指的是他是分类任务还是回归任务 + args.output_mode = output_modes[args.task_name] + # 得到标签列表 如二分类0，1 + label_list = processor.get_labels() + num_labels = len(label_list) + + # Load pretrained model and tokenizer + if args.local_rank not in [-1, 0]: + torch.distributed.barrier() # Make sure only the first process in distributed training will download model & vocab + + args.model_type = args.model_type.lower() + print(args.config_name) + # 返回一个字典，包含模型目录下config文件里的内容+分类数+任务名称 + config = AlbertConfig.from_pretrained(args.config_name if args.config_name else args.model_name_or_path, + num_labels=num_labels, + finetuning_task=args.task_name) + # 一个能够处理文字格式的对象，包括预处理，文字->数字的映射 + tokenizer = tokenization_albert.FullTokenizer(vocab_file=args.vocab_file, do_lower_case=args.do_lower_case, + spm_model_file=args.spm_model_file) + # 保存的预训练模型没有全链接decoder，from_pretrained加载了encoder自动在后边加一个decoder + model = my_model.DistillModel(args, config) + if args.continue_train_distill == True: + model.load_state_dict(torch.load(args.distill_model_dir)) + print("*"*50) + print("从{}加载训练一半的模型继续训练，学习率为{}".format(args.distill_model_dir,args.continue_learning_rate)) + print("*" * 50) + args.learning_rate = args.continue_learning_rate + + else: + print("*" * 50) + print("从头训练，学习率为{}".format(args.learning_rate)) + print("*" * 50) + if args.local_rank == 0: + torch.distributed.barrier() # Make sure only the first process in distributed training will download model & vocab + print(args.device) + model.to(args.device) + logger.info("Training/evaluation parameters %s", args) + + # Training + if args.do_train: + train_dataset = load_and_cache_examples(args, args.task_name, tokenizer, data_type='train') + global_step, tr_loss = train(args, train_dataset, model, tokenizer) + logger.info(" global_step = %s, average loss = %s", global_step, tr_loss) + + # Saving best-practices: if you use defaults names for the model, you can reload it using from_pretrained() + if args.do_train and (args.local_rank == -1 or torch.distributed.get_rank() == 0): + # Create output directory if needed + if not os.path.exists(args.output_dir) and args.local_rank in [-1, 0]: + os.makedirs(args.output_dir) + + logger.info("Saving model checkpoint to %s", args.output_dir) + # Save a trained model, configuration and tokenizer using `save_pretrained()`. + # They can then be reloaded using `from_pretrained()` + model_to_save = model.module if hasattr(model, + 'module') else model # Take care of distributed/parallel training + #最优模型保存在output_dir + model_to_save.save_pretrained(args.output_dir) + # Good practice: save your training arguments together with the trained model + torch.save(args, os.path.join(args.output_dir, 'training_args.bin')) + + + + + + # Evaluation + results = [] + if args.do_eval and args.local_rank in [-1, 0]: + tokenizer = tokenization_albert.FullTokenizer(vocab_file=args.vocab_file, + do_lower_case=args.do_lower_case, + spm_model_file=args.spm_model_file) + evaluate(args, model, tokenizer) + + +if __name__ == "__main__": + main() diff --git a/BertToSimple/BiGRU/run_pretraining.py b/BertToSimple/BiGRU/run_pretraining.py new file mode 100644 index 0000000..ac5959e --- /dev/null +++ b/BertToSimple/BiGRU/run_pretraining.py @@ -0,0 +1,357 @@ +import torch +import json +import time +import numpy as np +from pathlib import Path +from argparse import ArgumentParser +from collections import namedtuple +from tempfile import TemporaryDirectory +from tools.common import logger, init_logger +from torch.utils.data import DataLoader, Dataset, RandomSampler +from torch.utils.data.distributed import DistributedSampler +from tools.common import AverageMeter +from metrics.custom_metrics import LMAccuracy +from torch.nn import CrossEntropyLoss +from model.modeling_albert import AlbertForPreTraining, AlbertConfig +from model.file_utils import CONFIG_NAME +from model.tokenization_bert import BertTokenizer +from callback.optimization.adamw import AdamW +from callback.lr_scheduler import get_linear_schedule_with_warmup +from tools.common import seed_everything + +InputFeatures = namedtuple("InputFeatures", "input_ids input_mask segment_ids lm_label_ids is_next") + +def convert_example_to_features(example, tokenizer, max_seq_length): + tokens = example["tokens"] + segment_ids = example["segment_ids"] + is_random_next = example["is_random_next"] + masked_lm_positions = example["masked_lm_positions"] + masked_lm_labels = example["masked_lm_labels"] + + assert len(tokens) == len(segment_ids) <= max_seq_length # The preprocessed data should be already truncated + input_ids = tokenizer.convert_tokens_to_ids(tokens) + masked_label_ids = tokenizer.convert_tokens_to_ids(masked_lm_labels) + input_array = np.zeros(max_seq_length, dtype=np.int) + input_array[:len(input_ids)] = input_ids + mask_array = np.zeros(max_seq_length, dtype=np.bool) + mask_array[:len(input_ids)] = 1 + segment_array = np.zeros(max_seq_length, dtype=np.bool) + segment_array[:len(segment_ids)] = segment_ids + lm_label_array = np.full(max_seq_length, dtype=np.int, fill_value=-1) + lm_label_array[masked_lm_positions] = masked_label_ids + + features = InputFeatures(input_ids=input_array, + input_mask=mask_array, + segment_ids=segment_array, + lm_label_ids=lm_label_array, + is_next=is_random_next) + return features + +class PregeneratedDataset(Dataset): + def __init__(self, training_path, file_id, tokenizer, data_name, reduce_memory=False): + self.tokenizer = tokenizer + self.file_id = file_id + data_file = training_path / f"{data_name}_file_{self.file_id}.json" + metrics_file = training_path / f"{data_name}_file_{self.file_id}_metrics.json" + assert data_file.is_file() and metrics_file.is_file() + metrics = json.loads(metrics_file.read_text()) + num_samples = metrics['num_training_examples'] + seq_len = metrics['max_seq_len'] + self.temp_dir = None + self.working_dir = None + if reduce_memory: + self.temp_dir = TemporaryDirectory() + self.working_dir = Path(self.temp_dir.name) + input_ids = np.memmap(filename=self.working_dir / 'input_ids.memmap', + mode='w+', dtype=np.int32, shape=(num_samples, seq_len)) + input_masks = np.memmap(filename=self.working_dir / 'input_masks.memmap', + shape=(num_samples, seq_len), mode='w+', dtype=np.bool) + segment_ids = np.memmap(filename=self.working_dir / 'segment_ids.memmap', + shape=(num_samples, seq_len), mode='w+', dtype=np.bool) + lm_label_ids = np.memmap(filename=self.working_dir / 'lm_label_ids.memmap', + shape=(num_samples, seq_len), mode='w+', dtype=np.int32) + lm_label_ids[:] = -1 + is_nexts = np.memmap(filename=self.working_dir / 'is_nexts.memmap', + shape=(num_samples,), mode='w+', dtype=np.bool) + else: + input_ids = np.zeros(shape=(num_samples, seq_len), dtype=np.int32) + input_masks = np.zeros(shape=(num_samples, seq_len), dtype=np.bool) + segment_ids = np.zeros(shape=(num_samples, seq_len), dtype=np.bool) + lm_label_ids = np.full(shape=(num_samples, seq_len), dtype=np.int32, fill_value=-1) + is_nexts = np.zeros(shape=(num_samples,), dtype=np.bool) + logger.info(f"Loading training examples for {str(data_file)}") + with data_file.open() as f: + for i, line in enumerate(f): + line = line.strip() + example = json.loads(line) + features = convert_example_to_features(example, tokenizer, seq_len) + input_ids[i] = features.input_ids + segment_ids[i] = features.segment_ids + input_masks[i] = features.input_mask + lm_label_ids[i] = features.lm_label_ids + is_nexts[i] = features.is_next + assert i == num_samples - 1 # Assert that the sample count metric was true + logger.info("Loading complete!") + self.num_samples = num_samples + self.seq_len = seq_len + self.input_ids = input_ids + self.input_masks = input_masks + self.segment_ids = segment_ids + self.lm_label_ids = lm_label_ids + self.is_nexts = is_nexts + + def __len__(self): + return self.num_samples + + def __getitem__(self, item): + return (torch.tensor(self.input_ids[item].astype(np.int64)), + torch.tensor(self.input_masks[item].astype(np.int64)), + torch.tensor(self.segment_ids[item].astype(np.int64)), + torch.tensor(self.lm_label_ids[item].astype(np.int64)), + torch.tensor(self.is_nexts[item].astype(np.int64))) +def main(): + parser = ArgumentParser() + ## Required parameters + parser.add_argument("--data_dir", default=None, type=str, required=True, + help="The input data dir. Should contain the .tsv files (or other data files) for the task.") + parser.add_argument("--config_path", default=None, type=str, required=True) + parser.add_argument("--vocab_path",default=None,type=str,required=True) + parser.add_argument("--output_dir", default=None, type=str, required=True, + help="The output directory where the model predictions and checkpoints will be written.") + parser.add_argument("--model_path", default='', type=str) + parser.add_argument('--data_name', default='albert', type=str) + parser.add_argument("--file_num", type=int, default=10, + help="Number of dynamic masking to pregenerate (with different masks)") + parser.add_argument("--reduce_memory", action="store_true", + help="Store training data as on-disc memmaps to massively reduce memory usage") + parser.add_argument("--epochs", type=int, default=4, + help="Number of epochs to train for") + parser.add_argument("--do_lower_case", action='store_true', + help="Set this flag if you are using an uncased model.") + + parser.add_argument('--num_eval_steps', default=1000) + parser.add_argument('--num_save_steps', default=2000) + parser.add_argument("--local_rank", type=int, default=-1, + help="local_rank for distributed training on gpus") + parser.add_argument("--weight_decay", default=0.01, type=float, + help="Weight deay if we apply some.") + parser.add_argument("--no_cuda", action='store_true', + help="Whether not to use CUDA when available") + parser.add_argument('--gradient_accumulation_steps', type=int, default=1, + help="Number of updates steps to accumulate before performing a backward/update pass.") + parser.add_argument("--train_batch_size", default=16, type=int, + help="Total batch size for training.") + parser.add_argument('--loss_scale', type=float, default=0, + help="Loss scaling to improve fp16 numeric stability. Only used when fp16 set to True.\n" + "0 (default value): dynamic loss scaling.\n" + "Positive power of 2: static loss scaling value.\n") + parser.add_argument("--warmup_proportion", default=0.1, type=float, + help="Linear warmup over warmup_steps.") + parser.add_argument("--adam_epsilon", default=1e-8, type=float, + help="Epsilon for Adam optimizer.") + parser.add_argument('--max_grad_norm', default=1.0, type=float) + parser.add_argument("--learning_rate", default=0.000176, type=float, + help="The initial learning rate for Adam.") + parser.add_argument('--seed', type=int, default=42, + help="random seed for initialization") + parser.add_argument('--fp16_opt_level', type=str, default='O2', + help="For fp16: Apex AMP optimization level selected in ['O0', 'O1', 'O2', and 'O3']." + "See details at https://nvidia.github.io/apex/amp.html") + parser.add_argument('--fp16', action='store_true', + help="Whether to use 16-bit float precision instead of 32-bit") + args = parser.parse_args() + + args.data_dir = Path(args.data_dir) + args.output_dir = Path(args.output_dir) + + pregenerated_data = args.data_dir / "corpus/train" + init_logger(log_file=str(args.output_dir/ "train_albert_model.log")) + assert pregenerated_data.is_dir(), \ + "--pregenerated_data should point to the folder of files made by prepare_lm_data_mask.py!" + + samples_per_epoch = 0 + for i in range(args.file_num): + data_file = pregenerated_data / f"{args.data_name}_file_{i}.json" + metrics_file = pregenerated_data / f"{args.data_name}_file_{i}_metrics.json" + if data_file.is_file() and metrics_file.is_file(): + metrics = json.loads(metrics_file.read_text()) + samples_per_epoch += metrics['num_training_examples'] + else: + if i == 0: + exit("No training data was found!") + print(f"Warning! There are fewer epochs of pregenerated data ({i}) than training epochs ({args.epochs}).") + print("This script will loop over the available data, but training diversity may be negatively impacted.") + break + logger.info(f"samples_per_epoch: {samples_per_epoch}") + if args.local_rank == -1 or args.no_cuda: + device = torch.device(f"cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu") + args.n_gpu = torch.cuda.device_count() + else: + torch.cuda.set_device(args.local_rank) + device = torch.device("cuda", args.local_rank) + args.n_gpu = 1 + # Initializes the distributed backend which will take care of sychronizing nodes/GPUs + torch.distributed.init_process_group(backend='nccl') + logger.info( + f"device: {device} , distributed training: {bool(args.local_rank != -1)}, 16-bits training: {args.fp16}") + + if args.gradient_accumulation_steps < 1: + raise ValueError( + f"Invalid gradient_accumulation_steps parameter: {args.gradient_accumulation_steps}, should be >= 1") + args.train_batch_size = args.train_batch_size // args.gradient_accumulation_steps + + seed_everything(args.seed) + tokenizer = BertTokenizer.from_pretrained(args.vocab_path, do_lower_case=args.do_lower_case) + total_train_examples = samples_per_epoch * args.epochs + + num_train_optimization_steps = int( + total_train_examples / args.train_batch_size / args.gradient_accumulation_steps) + if args.local_rank != -1: + num_train_optimization_steps = num_train_optimization_steps // torch.distributed.get_world_size() + args.warmup_steps = int(num_train_optimization_steps * args.warmup_proportion) + + bert_config = AlbertConfig.from_pretrained(args.config_path) + model = AlbertForPreTraining(config=bert_config) + if args.model_path: + model = AlbertForPreTraining.from_pretrained(args.model_path) + model.to(device) + # Prepare optimizer + param_optimizer = list(model.named_parameters()) + no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight'] + optimizer_grouped_parameters = [ + {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': args.weight_decay}, + {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0} + ] + optimizer = AdamW(params=optimizer_grouped_parameters, lr=args.learning_rate, eps=args.adam_epsilon) + scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=args.warmup_steps, num_training_steps=num_train_optimization_steps) + # optimizer = Lamb(optimizer_grouped_parameters, lr=args.learning_rate, eps=args.adam_epsilon) + if args.model_path: + optimizer.load_state_dict(torch.load(args.model_path + "/optimizer.bin")) + if args.fp16: + try: + from apex import amp + except ImportError: + raise ImportError("Please install apex from https://www.github.com/nvidia/apex to use fp16 training.") + model, optimizer = amp.initialize(model, optimizer, opt_level=args.fp16_opt_level) + + if args.n_gpu > 1: + model = torch.nn.DataParallel(model) + + if args.local_rank != -1: + model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[args.local_rank], + output_device=args.local_rank) + global_step = 0 + mask_metric = LMAccuracy() + sop_metric = LMAccuracy() + tr_mask_acc = AverageMeter() + tr_sop_acc = AverageMeter() + tr_loss = AverageMeter() + tr_mask_loss = AverageMeter() + tr_sop_loss = AverageMeter() + loss_fct = CrossEntropyLoss(ignore_index=-1) + + train_logs = {} + logger.info("***** Running training *****") + logger.info(f" Num examples = {total_train_examples}") + logger.info(f" Batch size = {args.train_batch_size}") + logger.info(f" Num steps = {num_train_optimization_steps}") + logger.info(f" warmup_steps = {args.warmup_steps}") + start_time = time.time() + seed_everything(args.seed) # Added here for reproducibility + for epoch in range(args.epochs): + for idx in range(args.file_num): + epoch_dataset = PregeneratedDataset(file_id=idx, training_path=pregenerated_data, tokenizer=tokenizer, + reduce_memory=args.reduce_memory, data_name=args.data_name) + if args.local_rank == -1: + train_sampler = RandomSampler(epoch_dataset) + else: + train_sampler = DistributedSampler(epoch_dataset) + train_dataloader = DataLoader(epoch_dataset, sampler=train_sampler, batch_size=args.train_batch_size) + model.train() + nb_tr_examples, nb_tr_steps = 0, 0 + for step, batch in enumerate(train_dataloader): + batch = tuple(t.to(device) for t in batch) + input_ids, input_mask, segment_ids, lm_label_ids, is_next = batch + outputs = model(input_ids=input_ids, token_type_ids=segment_ids, attention_mask=input_mask) + prediction_scores = outputs[0] + seq_relationship_score = outputs[1] + + masked_lm_loss = loss_fct(prediction_scores.view(-1, bert_config.vocab_size), lm_label_ids.view(-1)) + next_sentence_loss = loss_fct(seq_relationship_score.view(-1, 2), is_next.view(-1)) + loss = masked_lm_loss + next_sentence_loss + + mask_metric(logits=prediction_scores.view(-1, bert_config.vocab_size), target=lm_label_ids.view(-1)) + sop_metric(logits=seq_relationship_score.view(-1, 2), target=is_next.view(-1)) + + if args.n_gpu > 1: + loss = loss.mean() # mean() to average on multi-gpu. + if args.gradient_accumulation_steps > 1: + loss = loss / args.gradient_accumulation_steps + if args.fp16: + with amp.scale_loss(loss, optimizer) as scaled_loss: + scaled_loss.backward() + else: + loss.backward() + + nb_tr_steps += 1 + tr_mask_acc.update(mask_metric.value(), n=input_ids.size(0)) + tr_sop_acc.update(sop_metric.value(), n=input_ids.size(0)) + tr_loss.update(loss.item(), n=1) + tr_mask_loss.update(masked_lm_loss.item(), n=1) + tr_sop_loss.update(next_sentence_loss.item(), n=1) + + if (step + 1) % args.gradient_accumulation_steps == 0: + if args.fp16: + torch.nn.utils.clip_grad_norm_(amp.master_params(optimizer), args.max_grad_norm) + else: + torch.nn.utils.clip_grad_norm_(model.parameters(), args.max_grad_norm) + scheduler.step() + optimizer.step() + optimizer.zero_grad() + global_step += 1 + + if global_step % args.num_eval_steps == 0: + now = time.time() + eta = now - start_time + if eta > 3600: + eta_format = ('%d:%02d:%02d' % (eta // 3600, (eta % 3600) // 60, eta % 60)) + elif eta > 60: + eta_format = '%d:%02d' % (eta // 60, eta % 60) + else: + eta_format = '%ds' % eta + train_logs['loss'] = tr_loss.avg + train_logs['mask_acc'] = tr_mask_acc.avg + train_logs['sop_acc'] = tr_sop_acc.avg + train_logs['mask_loss'] = tr_mask_loss.avg + train_logs['sop_loss'] = tr_sop_loss.avg + show_info = f'[Training]:[{epoch}/{args.epochs}]{global_step}/{num_train_optimization_steps} ' \ + f'- ETA: {eta_format}' + "-".join( + [f' {key}: {value:.4f} ' for key, value in train_logs.items()]) + logger.info(show_info) + tr_mask_acc.reset() + tr_sop_acc.reset() + tr_loss.reset() + tr_mask_loss.reset() + tr_sop_loss.reset() + start_time = now + + if global_step % args.num_save_steps == 0: + if args.local_rank in [-1, 0] and args.num_save_steps > 0: + # Save model checkpoint + output_dir = args.output_dir / f'lm-checkpoint-{global_step}' + if not output_dir.exists(): + output_dir.mkdir() + # save model + model_to_save = model.module if hasattr(model, + 'module') else model # Take care of distributed/parallel training + model_to_save.save_pretrained(str(output_dir)) + torch.save(args, str(output_dir / 'training_args.bin')) + logger.info("Saving model checkpoint to %s", output_dir) + + torch.save(optimizer.state_dict(), str(output_dir / "optimizer.bin")) + # save config + output_config_file = output_dir / CONFIG_NAME + with open(str(output_config_file), 'w') as f: + f.write(model_to_save.config.to_json_string()) + # save vocab + tokenizer.save_vocabulary(output_dir) diff --git a/BertToSimple/BiGRU/scripts/run_classifier_cola.sh b/BertToSimple/BiGRU/scripts/run_classifier_cola.sh new file mode 100644 index 0000000..e297c4f --- /dev/null +++ b/BertToSimple/BiGRU/scripts/run_classifier_cola.sh @@ -0,0 +1,25 @@ +CURRENT_DIR=`pwd` +export BERT_BASE_DIR=$CURRENT_DIR/prev_trained_model/albert_base_en +export DATA_DIR=$CURRENT_DIR/dataset +export OUTPUR_DIR=$CURRENT_DIR/outputs +TASK_NAME="cola" + +python run_classifier.py \ + --model_type=albert \ + --model_name_or_path=$BERT_BASE_DIR \ + --task_name=$TASK_NAME \ + --do_train \ + --do_eval \ + --do_lower_case \ + --data_dir=$DATA_DIR/${TASK_NAME}/ \ + --max_seq_length=128 \ + --per_gpu_train_batch_size=32 \ + --per_gpu_eval_batch_size=32 \ + --spm_model_file=${BERT_BASE_DIR}/30k-clean.model \ + --learning_rate=1e-5 \ + --num_train_epochs=3.0 \ + --logging_steps=134 \ + --save_steps=134 \ + --output_dir=$OUTPUR_DIR/${TASK_NAME}_output/ \ + --overwrite_output_dir \ + --seed=42 \ No newline at end of file diff --git a/BertToSimple/BiGRU/scripts/run_classifier_lcqmc.sh b/BertToSimple/BiGRU/scripts/run_classifier_lcqmc.sh new file mode 100644 index 0000000..1ff9ccd --- /dev/null +++ b/BertToSimple/BiGRU/scripts/run_classifier_lcqmc.sh @@ -0,0 +1,24 @@ +CURRENT_DIR=`pwd` +export BERT_BASE_DIR=$CURRENT_DIR/prev_trained_model/albert_large_zh +export DATA_DIR=$CURRENT_DIR/dataset +export OUTPUR_DIR=$CURRENT_DIR/outputs +TASK_NAME="lcqmc" + +python run_classifier.py \ + --model_type=albert \ + --model_name_or_path=$BERT_BASE_DIR \ + --task_name=$TASK_NAME \ + --do_train \ + --do_eval \ + --do_lower_case \ + --data_dir=$DATA_DIR/${TASK_NAME}/ \ + --vocab_file=$BERT_BASE_DIR/vocab.txt \ + --max_seq_length=128 \ + --per_gpu_train_batch_size=16 \ + --per_gpu_eval_batch_size=16 \ + --learning_rate=1e-5 \ + --num_train_epochs=3.0 \ + --logging_steps=14923 \ + --save_steps=14923 \ + --output_dir=$OUTPUR_DIR/${TASK_NAME}_output/ \ + --overwrite_output_dir diff --git a/BertToSimple/BiGRU/scripts/run_classifier_mnli.sh b/BertToSimple/BiGRU/scripts/run_classifier_mnli.sh new file mode 100644 index 0000000..70108dc --- /dev/null +++ b/BertToSimple/BiGRU/scripts/run_classifier_mnli.sh @@ -0,0 +1,24 @@ +CURRENT_DIR=`pwd` +export BERT_BASE_DIR=$CURRENT_DIR/prev_trained_model/albert_base_v2 +export DATA_DIR=$CURRENT_DIR/dataset +export OUTPUR_DIR=$CURRENT_DIR/outputs +TASK_NAME="mnli" +python run_classifier.py \ + --model_type=albert \ + --model_name_or_path=$BERT_BASE_DIR \ + --task_name=$TASK_NAME \ + --do_train \ + --do_eval \ + --do_lower_case \ + --data_dir=$DATA_DIR/${TASK_NAME}/ \ + --max_seq_length=128 \ + --per_gpu_train_batch_size=16 \ + --per_gpu_eval_batch_size=16 \ + --spm_model_file=${BERT_BASE_DIR}/30k-clean.model \ + --learning_rate=1e-5 \ + --num_train_epochs=3.0 \ + --logging_steps=24544 \ + --save_steps=24544 \ + --output_dir=$OUTPUR_DIR/${TASK_NAME}_output/ \ + --overwrite_output_dir \ + --seed=42 diff --git a/BertToSimple/BiGRU/scripts/run_classifier_qqp.sh b/BertToSimple/BiGRU/scripts/run_classifier_qqp.sh new file mode 100644 index 0000000..44c918f --- /dev/null +++ b/BertToSimple/BiGRU/scripts/run_classifier_qqp.sh @@ -0,0 +1,25 @@ +CURRENT_DIR=`pwd` +export BERT_BASE_DIR=$CURRENT_DIR/prev_trained_model/albert_base_v2 +export DATA_DIR=$CURRENT_DIR/dataset +export OUTPUR_DIR=$CURRENT_DIR/outputs +TASK_NAME="qqp" + +python run_classifier.py \ + --model_type=albert \ + --model_name_or_path=$BERT_BASE_DIR \ + --task_name=$TASK_NAME \ + --do_train \ + --do_eval \ + --do_lower_case \ + --data_dir=$DATA_DIR/${TASK_NAME}/ \ + --max_seq_length=128 \ + --per_gpu_train_batch_size=16 \ + --per_gpu_eval_batch_size=8 \ + --spm_model_file=${BERT_BASE_DIR}/30k-clean.model \ + --learning_rate=1e-5 \ + --num_train_epochs=3.0 \ + --logging_steps=22741 \ + --save_steps=22741 \ + --output_dir=$OUTPUR_DIR/${TASK_NAME}_output/ \ + --overwrite_output_dir \ + --seed=42 diff --git a/BertToSimple/BiGRU/scripts/run_classifier_sst2.sh b/BertToSimple/BiGRU/scripts/run_classifier_sst2.sh new file mode 100644 index 0000000..72ca461 --- /dev/null +++ b/BertToSimple/BiGRU/scripts/run_classifier_sst2.sh @@ -0,0 +1,25 @@ +CURRENT_DIR=`pwd` +export BERT_BASE_DIR=$CURRENT_DIR/prev_trained_model/albert_large_v2 +export DATA_DIR=$CURRENT_DIR/dataset +export OUTPUR_DIR=$CURRENT_DIR/outputs +TASK_NAME="sst-2" +python run_classifier.py \ + --model_type=albert \ + --model_name_or_path=$BERT_BASE_DIR \ + --task_name=$TASK_NAME \ + --do_train \ + --do_eval \ + --do_lower_case \ + --data_dir=$DATA_DIR/${TASK_NAME}/ \ + --max_seq_length=128 \ + --per_gpu_train_batch_size=16 \ + --per_gpu_eval_batch_size=8 \ + --spm_model_file=${BERT_BASE_DIR}/30k-clean.model \ + --learning_rate=1e-5 \ + --num_train_epochs=3.0 \ + --logging_steps=4210 \ + --save_steps=4210 \ + --output_dir=$OUTPUR_DIR/${TASK_NAME}_output/ \ + --overwrite_output_dir \ + --seed=42 + diff --git a/BertToSimple/BiGRU/scripts/run_classifier_stsb.sh b/BertToSimple/BiGRU/scripts/run_classifier_stsb.sh new file mode 100644 index 0000000..d511216 --- /dev/null +++ b/BertToSimple/BiGRU/scripts/run_classifier_stsb.sh @@ -0,0 +1,24 @@ +CURRENT_DIR=`pwd` +export BERT_BASE_DIR=$CURRENT_DIR/prev_trained_model/albert_base_v2 +export DATA_DIR=$CURRENT_DIR/dataset +export OUTPUR_DIR=$CURRENT_DIR/outputs +TASK_NAME="sts-b" +python run_classifier.py \ + --model_type=albert \ + --model_name_or_path=$BERT_BASE_DIR \ + --task_name=$TASK_NAME \ + --do_train \ + --do_eval \ + --do_lower_case \ + --data_dir=$DATA_DIR/${TASK_NAME}/ \ + --max_seq_length=128 \ + --per_gpu_train_batch_size=16 \ + --per_gpu_eval_batch_size=16 \ + --spm_model_file=${BERT_BASE_DIR}/30k-clean.model \ + --learning_rate=1e-5 \ + --num_train_epochs=3.0 \ + --logging_steps=360 \ + --save_steps=360 \ + --output_dir=$OUTPUR_DIR/${TASK_NAME}_output/ \ + --overwrite_output_dir \ + --seed=42 diff --git a/BertToSimple/BiGRU/tools/common.py b/BertToSimple/BiGRU/tools/common.py new file mode 100644 index 0000000..c833c8f --- /dev/null +++ b/BertToSimple/BiGRU/tools/common.py @@ -0,0 +1,347 @@ +import os +import random +import torch +import numpy as np +import json +import pickle +import torch.nn as nn +from collections import OrderedDict +from pathlib import Path +import logging + +logger = logging.getLogger() + +def init_logger(log_file=None, log_file_level=logging.NOTSET): + ''' + Example: + >>> init_logger(log_file) + >>> logger.info("abc'") + ''' + if isinstance(log_file,Path): + log_file = str(log_file) + + log_format = logging.Formatter(fmt='%(asctime)s - %(levelname)s - %(name)s - %(message)s', + datefmt='%m/%d/%Y %H:%M:%S') + logger = logging.getLogger() + logger.setLevel(logging.INFO) + console_handler = logging.StreamHandler() + console_handler.setFormatter(log_format) + logger.handlers = [console_handler] + if log_file and log_file != '': + file_handler = logging.FileHandler(log_file) + file_handler.setLevel(log_file_level) + file_handler.setFormatter(log_format) + logger.addHandler(file_handler) + return logger + +def seed_everything(seed=1029): + ''' + 设置整个开发环境的seed + :param seed: + :param device: + :return: + ''' + random.seed(seed) + os.environ['PYTHONHASHSEED'] = str(seed) + np.random.seed(seed) + torch.manual_seed(seed) + torch.cuda.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + # some cudnn methods can be random even after fixing the seed + # unless you tell it to be deterministic + torch.backends.cudnn.deterministic = True + + +def prepare_device(n_gpu_use): + """ + setup GPU device if available, move model into configured device + # 如果n_gpu_use为数字，则使用range生成list + # 如果输入的是一个list，则默认使用list[0]作为controller + """ + if not n_gpu_use: + device_type = 'cpu' + else: + n_gpu_use = n_gpu_use.split(",") + device_type = f"cuda:{n_gpu_use[0]}" + n_gpu = torch.cuda.device_count() + if len(n_gpu_use) > 0 and n_gpu == 0: + logger.warning("Warning: There\'s no GPU available on this machine, training will be performed on CPU.") + device_type = 'cpu' + if len(n_gpu_use) > n_gpu: + msg = f"Warning: The number of GPU\'s configured to use is {n_gpu_use}, but only {n_gpu} are available on this machine." + logger.warning(msg) + n_gpu_use = range(n_gpu) + device = torch.device(device_type) + list_ids = n_gpu_use + return device, list_ids + + +def model_device(n_gpu, model): + ''' + 判断环境 cpu还是gpu + 支持单机多卡 + :param n_gpu: + :param model: + :return: + ''' + device, device_ids = prepare_device(n_gpu) + if len(device_ids) > 1: + logger.info(f"current {len(device_ids)} GPUs") + model = torch.nn.DataParallel(model, device_ids=device_ids) + if len(device_ids) == 1: + os.environ['CUDA_VISIBLE_DEVICES'] = str(device_ids[0]) + model = model.to(device) + return model, device + + +def restore_checkpoint(resume_path, model=None): + ''' + 加载模型 + :param resume_path: + :param model: + :param optimizer: + :return: + 注意： 如果是加载Bert模型的话，需要调整，不能使用该模式 + 可以使用模块自带的Bert_model.from_pretrained(state_dict = your save state_dict) + ''' + if isinstance(resume_path, Path): + resume_path = str(resume_path) + checkpoint = torch.load(resume_path) + best = checkpoint['best'] + start_epoch = checkpoint['epoch'] + 1 + states = checkpoint['state_dict'] + if isinstance(model, nn.DataParallel): + model.module.load_state_dict(states) + else: + model.load_state_dict(states) + return [model,best,start_epoch] + + +def save_pickle(data, file_path): + ''' + 保存成pickle文件 + :param data: + :param file_name: + :param pickle_path: + :return: + ''' + if isinstance(file_path, Path): + file_path = str(file_path) + with open(file_path, 'wb') as f: + pickle.dump(data, f) + + +def load_pickle(input_file): + ''' + 读取pickle文件 + :param pickle_path: + :param file_name: + :return: + ''' + with open(str(input_file), 'rb') as f: + data = pickle.load(f) + return data + + +def save_json(data, file_path): + ''' + 保存成json文件 + :param data: + :param json_path: + :param file_name: + :return: + ''' + if not isinstance(file_path, Path): + file_path = Path(file_path) + # if isinstance(data,dict): + # data = json.dumps(data) + with open(str(file_path), 'w') as f: + json.dump(data, f) + + +def load_json(file_path): + ''' + 加载json文件 + :param json_path: + :param file_name: + :return: + ''' + if not isinstance(file_path, Path): + file_path = Path(file_path) + with open(str(file_path), 'r') as f: + data = json.load(f) + return data + +def save_model(model, model_path): + """ 存储不含有显卡信息的state_dict或model + :param model: + :param model_name: + :param only_param: + :return: + """ + if isinstance(model_path, Path): + model_path = str(model_path) + if isinstance(model, nn.DataParallel): + model = model.module + state_dict = model.state_dict() + for key in state_dict: + state_dict[key] = state_dict[key].cpu() + torch.save(state_dict, model_path) + +def load_model(model, model_path): + ''' + 加载模型 + :param model: + :param model_name: + :param model_path: + :param only_param: + :return: + ''' + if isinstance(model_path, Path): + model_path = str(model_path) + logging.info(f"loading model from {str(model_path)} .") + states = torch.load(model_path) + state = states['state_dict'] + if isinstance(model, nn.DataParallel): + model.module.load_state_dict(state) + else: + model.load_state_dict(state) + return model + + +class AverageMeter(object): + ''' + computes and stores the average and current value + Example: + >>> loss = AverageMeter() + >>> for step,batch in enumerate(train_data): + >>> pred = self.model(batch) + >>> raw_loss = self.metrics(pred,target) + >>> loss.update(raw_loss.item(),n = 1) + >>> cur_loss = loss.avg + ''' + + def __init__(self): + self.reset() + + def reset(self): + self.val = 0 + self.avg = 0 + self.sum = 0 + self.count = 0 + + def update(self, val, n=1): + self.val = val + self.sum += val * n + self.count += n + self.avg = self.sum / self.count + + +def summary(model, *inputs, batch_size=-1, show_input=True): + ''' + 打印模型结构信息 + :param model: + :param inputs: + :param batch_size: + :param show_input: + :return: + Example: + >>> print("model summary info: ") + >>> for step,batch in enumerate(train_data): + >>> summary(self.model,*batch,show_input=True) + >>> break + ''' + + def register_hook(module): + def hook(module, input, output=None): + class_name = str(module.__class__).split(".")[-1].split("'")[0] + module_idx = len(summary) + + m_key = f"{class_name}-{module_idx + 1}" + summary[m_key] = OrderedDict() + summary[m_key]["input_shape"] = list(input[0].size()) + summary[m_key]["input_shape"][0] = batch_size + + if show_input is False and output is not None: + if isinstance(output, (list, tuple)): + for out in output: + if isinstance(out, torch.Tensor): + summary[m_key]["output_shape"] = [ + [-1] + list(out.size())[1:] + ][0] + else: + summary[m_key]["output_shape"] = [ + [-1] + list(out[0].size())[1:] + ][0] + else: + summary[m_key]["output_shape"] = list(output.size()) + summary[m_key]["output_shape"][0] = batch_size + + params = 0 + if hasattr(module, "weight") and hasattr(module.weight, "size"): + params += torch.prod(torch.LongTensor(list(module.weight.size()))) + summary[m_key]["trainable"] = module.weight.requires_grad + if hasattr(module, "bias") and hasattr(module.bias, "size"): + params += torch.prod(torch.LongTensor(list(module.bias.size()))) + summary[m_key]["nb_params"] = params + + if (not isinstance(module, nn.Sequential) and not isinstance(module, nn.ModuleList) and not (module == model)): + if show_input is True: + hooks.append(module.register_forward_pre_hook(hook)) + else: + hooks.append(module.register_forward_hook(hook)) + + # create properties + summary = OrderedDict() + hooks = [] + + # register hook + model.apply(register_hook) + model(*inputs) + + # remove these hooks + for h in hooks: + h.remove() + + print("-----------------------------------------------------------------------") + if show_input is True: + line_new = f"{'Layer (type)':>25} {'Input Shape':>25} {'Param #':>15}" + else: + line_new = f"{'Layer (type)':>25} {'Output Shape':>25} {'Param #':>15}" + print(line_new) + print("=======================================================================") + + total_params = 0 + total_output = 0 + trainable_params = 0 + for layer in summary: + # input_shape, output_shape, trainable, nb_params + if show_input is True: + line_new = "{:>25} {:>25} {:>15}".format( + layer, + str(summary[layer]["input_shape"]), + "{0:,}".format(summary[layer]["nb_params"]), + ) + else: + line_new = "{:>25} {:>25} {:>15}".format( + layer, + str(summary[layer]["output_shape"]), + "{0:,}".format(summary[layer]["nb_params"]), + ) + + total_params += summary[layer]["nb_params"] + if show_input is True: + total_output += np.prod(summary[layer]["input_shape"]) + else: + total_output += np.prod(summary[layer]["output_shape"]) + if "trainable" in summary[layer]: + if summary[layer]["trainable"] == True: + trainable_params += summary[layer]["nb_params"] + + print(line_new) + + print("=======================================================================") + print(f"Total params: {total_params:0,}") + print(f"Trainable params: {trainable_params:0,}") + print(f"Non-trainable params: {(total_params - trainable_params):0,}") + print("-----------------------------------------------------------------------") diff --git a/BertToSimple/textcnn/predict.py b/BertToSimple/textcnn/predict.py deleted file mode 100644 index 80c6d98..0000000 --- a/BertToSimple/textcnn/predict.py +++ /dev/null @@ -1,83 +0,0 @@ -# -*-coding:utf-8-*- - -import pickle, numpy as np -import time -from keras.layers import * -from keras.models import Model -from keras.initializers import Constant -from keras.preprocessing import sequence -from keras.models import load_model -from keras.utils.np_utils import to_categorical -from sklearn.metrics import accuracy_score -from utils import load_data - -def get_textcnn(x_len, v_size, embs): - x = Input(shape=(x_len,),dtype='int32') - # embed = Embedding(v_size,300)(x) - embed = Embedding(v_size,300,embeddings_initializer=Constant(embs),trainable=False)(x) - cnn1 = Convolution1D(256,3,padding='same',strides=1,activation='relu')(embed) - cnn1 = MaxPool1D(pool_size=4)(cnn1) - cnn2 = Convolution1D(256,4,padding='same',strides=1,activation='relu')(embed) - cnn2 = MaxPool1D(pool_size=4)(cnn2) - cnn3 = Convolution1D(256,5,padding='same',strides=1,activation='relu')(embed) - cnn3 = MaxPool1D(pool_size=4)(cnn3) - cnn = concatenate([cnn1,cnn2,cnn3],axis=-1) - flat = Flatten()(cnn) - drop = Dropout(0.2,name='drop')(flat) - y = Dense(3,activation='softmax')(drop) - model = Model(inputs=x,outputs=y) - return model - -def get_birnn(x_len, v_size, embs): - x = Input(shape=(x_len,),dtype='int32') - # embed = Embedding(v_size,300)(x) - embed = Embedding(v_size,300,embeddings_initializer=Constant(embs),trainable=False)(x) - # bi = Bidirectional(GRU(256,activation='tanh',recurrent_dropout=0.2,dropout=0.2,return_sequences=True))(embed) - bi = Bidirectional(GRU(256,activation='tanh',recurrent_dropout=0.2,dropout=0.2))(embed) - bi_1 = Bidirectional(GRU(256, activation='tanh', recurrent_dropout=0.2, dropout=0.2))(embed) - y = Dense(3,activation='softmax')(bi_1) - model = Model(inputs=x,outputs=y) - return model - - -def predict(): - x_len = 50 - - # ----- ----- ----- ----- ----- - # from keras.datasets import imdb - # (x_tr,y_tr),(x_te,y_te) = imdb.load_data(num_words=10000) - # ----- ----- ----- ----- ----- - - name = 'hotel' # clothing, fruit, hotel, pda, shampoo - (x_tr,y_tr,_),(x_de,y_de,_),(x_te,y_te,_),v_size,embs = load_data(name) - x_tr = sequence.pad_sequences(x_tr,maxlen=x_len) - x_de = sequence.pad_sequences(x_de,maxlen=x_len) - x_te = sequence.pad_sequences(x_te,maxlen=x_len) - y_tr = to_categorical(y_tr,3) - y_de = to_categorical(y_de,3) - y_te = to_categorical(y_te,3) - #with open('data/cache/t_tr','rb') as fin: y_tr = pickle.load(fin) - #with open('data/cache/t_de','rb') as fin: y_de = pickle.load(fin) - # y_tr = to_categorical(y_tr.argmax(axis=1),3) - # y_de = to_categorical(y_de.argmax(axis=1),3) - - # ----- ----- predict ----- ----- - # 模型的加载及使用 - - print("Using loaded model to predict...") - - load_model1 = load_model("model_weight(textcnn).h5") - start= time.time() - predicted = load_model1.predict(x_te) - predict_1 = np.argmax(predicted,axis=1) - print(predict_1.shape) - end = time.time() - print('time:',end-start,' acc:',accuracy_score(y_te, predict_1)) - - load_model1.summary() - - # ----- ----- ----- ----- ----- - -if __name__ == '__main__': - # run_small() - predict() diff --git a/BertToSimple/textcnn/readme.md b/BertToSimple/textcnn/readme.md deleted file mode 100644 index c3775e3..0000000 --- a/BertToSimple/textcnn/readme.md +++ /dev/null @@ -1,10 +0,0 @@ -数据格式： -0 询问是否认识借款人,我是那个 -invalid -1 yes-2 no-3 - -时间，17000条数据预测时间为12s,每条8ms 准确率：87.44% - -训练：test.py -预测： predict.py - -测试集，验证集，训练集请放在与代码同级的data文件夹 data下设置一个hotel文件夹放置，分别test.txt dev.txt train.txt diff --git a/BertToSimple/textcnn/test.py b/BertToSimple/textcnn/test.py deleted file mode 100644 index b0084b9..0000000 --- a/BertToSimple/textcnn/test.py +++ /dev/null @@ -1,89 +0,0 @@ -# -*-coding:utf-8-*- - -import pickle, numpy as np -from keras.layers import * -from keras.models import Model -from keras.initializers import Constant -from keras.preprocessing import sequence -from keras.models import load_model -from keras.utils.np_utils import to_categorical -from utils import load_data - -def get_textcnn(x_len, v_size, embs): - x = Input(shape=(x_len,),dtype='int32') - # embed = Embedding(v_size,300)(x) - embed = Embedding(v_size,300,embeddings_initializer=Constant(embs),trainable=False)(x) - cnn1 = Convolution1D(256,3,padding='same',strides=1,activation='relu')(embed) - cnn1 = MaxPool1D(pool_size=4)(cnn1) - cnn2 = Convolution1D(256,4,padding='same',strides=1,activation='relu')(embed) - cnn2 = MaxPool1D(pool_size=4)(cnn2) - cnn3 = Convolution1D(256,5,padding='same',strides=1,activation='relu')(embed) - cnn3 = MaxPool1D(pool_size=4)(cnn3) - cnn = concatenate([cnn1,cnn2,cnn3],axis=-1) - flat = Flatten()(cnn) - drop = Dropout(0.2,name='drop')(flat) - y = Dense(3,activation='softmax')(drop) - model = Model(inputs=x,outputs=y) - return model - -def get_birnn(x_len, v_size, embs): - x = Input(shape=(x_len,),dtype='int32') - # embed = Embedding(v_size,300)(x) - embed = Embedding(v_size,300,embeddings_initializer=Constant(embs),trainable=False)(x) - # bi = Bidirectional(GRU(256,activation='tanh',recurrent_dropout=0.2,dropout=0.2,return_sequences=True))(embed) - bi = Bidirectional(GRU(256,activation='tanh',recurrent_dropout=0.2,dropout=0.2))(embed) - bi_1 = Bidirectional(GRU(256, activation='tanh', recurrent_dropout=0.2, dropout=0.2))(embed) - y = Dense(3,activation='softmax')(bi_1) - model = Model(inputs=x,outputs=y) - return model - -def run_small(): - x_len = 50 - name = 'hotel' # clothing, fruit, hotel, pda, shampoo - (x_tr,y_tr,_),_,(x_te,y_te,_),v_size,embs = load_data(name) - x_tr = sequence.pad_sequences(x_tr,maxlen=x_len) - x_te = sequence.pad_sequences(x_te,maxlen=x_len) - y_tr = to_categorical(y_tr,3) - y_te = to_categorical(y_te,3) - # model = get_textcnn(x_len,v_size,embs) - model = get_birnn(x_len,v_size,embs) - model.compile(loss='softmax_crossentropy',optimizer='adam',metrics=['accuracy']) - model.fit(x_tr,y_tr,batch_size=32,epochs=5,validation_data=(x_te,y_te)) - -def run_distill(): - x_len = 50 - - # ----- ----- ----- ----- ----- - # from keras.datasets import imdb - # (x_tr,y_tr),(x_te,y_te) = imdb.load_data(num_words=10000) - # ----- ----- ----- ----- ----- - - name = 'hotel' # clothing, fruit, hotel, pda, shampoo - (x_tr,y_tr,_),(x_de,y_de,_),(x_te,y_te,_),v_size,embs = load_data(name) - x_tr = sequence.pad_sequences(x_tr,maxlen=x_len) - x_de = sequence.pad_sequences(x_de,maxlen=x_len) - x_te = sequence.pad_sequences(x_te,maxlen=x_len) - y_tr = to_categorical(y_tr,3) - y_de = to_categorical(y_de,3) - y_te = to_categorical(y_te,3) - #with open('data/cache/t_tr','rb') as fin: y_tr = pickle.load(fin) - #with open('data/cache/t_de','rb') as fin: y_de = pickle.load(fin) - # y_tr = to_categorical(y_tr.argmax(axis=1),2) - # y_de = to_categorical(y_de.argmax(axis=1),2) - - # ----- ----- distill ----- ----- - model = get_textcnn(x_len,v_size,embs) - #model = get_birnn(x_len,v_size,embs) - x_tr = np.vstack([x_tr,x_de]) - y_tr = np.vstack([y_tr,y_de]) - model.compile(loss='mse',optimizer='adam',metrics=['accuracy']) - print(x_tr.shape,y_tr.shape,x_te.shape,y_te.shape,x_de.shape,y_de.shape) - # model.compile(loss='binary_crossentropy',optimizer='adam',metrics=['accuracy']) - model.summary() - model.fit(x_tr,y_tr,batch_size=32,epochs=5,validation_data=(x_te,y_te)) - model.save('model_weight.h5') # creates a HDF5 file 'my_model.h5' - # ----- ----- ----- ----- ----- - -if __name__ == '__main__': - # run_small() - run_distill() diff --git a/BertToSimple/textcnn/utils.py b/BertToSimple/textcnn/utils.py deleted file mode 100644 index b7a4f47..0000000 --- a/BertToSimple/textcnn/utils.py +++ /dev/null @@ -1,55 +0,0 @@ -# -*- coding: utf-8 -*- - -import jieba, random, fileinput, numpy as np -from keras.preprocessing.text import Tokenizer -from sklearn.model_selection import train_test_split - -def load_data(name): - def get_w2v(): - for line in open('data/cache/word2vec',encoding='utf-8').read().strip().split('\n'): - line = line.strip().split() - if not line: continue - yield line[0],np.array(list(map(float,line[1:]))) - tokenizer = Tokenizer(filters='',lower=True,split=' ',oov_token=1) - texts = [' '.join(jieba.cut(line.split('\t',1)[1].strip()))\ - for line in open('data/{}/train.txt'.format(name,name),encoding='utf-8' - ).read().strip().split('\n')] - tokenizer.fit_on_texts(texts) - # with open('word2vec','w') as out: - # for line in fileinput.input('sgns.sogou.word'): - # word = line.strip().split()[0] - # if word in tokenizer.word_index: - # out.write(line+'\n') - # fileinput.close() - x_train,y_train = [],[]; text_train = [] - for line in open('data/{}/train.txt'.format(name),encoding='utf-8').read().strip().split('\n'): - label,text = line.split('\t',1) - text_train.append(text.strip()) - x_train.append(' '.join(jieba.cut(text.strip()))) - y_train.append(int(label)) - x_train = tokenizer.texts_to_sequences(x_train) - x_dev,y_dev = [],[]; text_dev = [] - for line in open('data/{}/dev.txt'.format(name),encoding='utf-8').read().strip().split('\n'): - label,text = line.split('\t',1) - text_dev.append(text.strip()) - x_dev.append(' '.join(jieba.cut(text.strip()))) - y_dev.append(int(label)) - x_dev = tokenizer.texts_to_sequences(x_dev) - x_test,y_test = [],[]; text_test = [] - for line in open('data/{}/test.txt'.format(name),encoding='utf-8').read().strip().split('\n'): - label,text = line.split('\t',1) - text_test.append(text.strip()) - x_test.append(' '.join(jieba.cut(text.strip()))) - y_test.append(int(label)) - x_test = tokenizer.texts_to_sequences(x_test) - v_size = len(tokenizer.word_index)+1 - embs,w2v = np.zeros((v_size,300)),dict(get_w2v()) - for word,index in tokenizer.word_index.items(): - if word in w2v: embs[index] = w2v[word] - return (x_train,y_train,text_train),\ - (x_dev,y_dev,text_dev),\ - (x_test,y_test,text_test),\ - v_size,embs - -if __name__ == '__main__': - load_data(name='hotel') diff --git a/README.md b/README.md index d6b83c4..3805f23 100644 --- a/README.md +++ b/README.md @@ -29,15 +29,13 @@ Codes for the project - [x] Roberta_wwm - [x] Albert_zn -- [ ] TinyBert +- [x] TinyBert - [x] FastBert -- [ ] BertToSimple +- [x] BiGRU ## 效果对比 -| 模型 | 描述 | 参数量| 准确率 | 预测响应时间 | -| ---- | ---- | ---- | ---- | ---- | -|RoBerta| [科大讯飞全遮罩模型]() |100M | | | + ## 参考文献