Implement multitask training

CHANGELOG.md

@@ -1,5 +1,10 @@
 # Changelog 
 
+## 2020.3.3 - Pre-release
+
+* Adds support for a Multitask models as in the original Rodrigues paper
+* Combines artefacts into a single `indices.pickle` rather than the several previous pickles. Now the model just requires the embedding, `indices.pickle`, and `weights.h5`.
+
 ## 2020.3.2 - Pre-release
 
 * Adds parse command that can be called with `python -m deep_reference_parser parse` 

deep_reference_parser/__main__.py

@@ -12,11 +12,13 @@
     from .train import train
     from .split import split
     from .parse import parse
+    from .split_parse import split_parse
 
     commands = {
         "split": split,
         "parse": parse,
         "train": train,
+        "split_parse": split_parse,
     }
 
     if len(sys.argv) == 1:

deep_reference_parser/__version__.py

@@ -1,9 +1,10 @@
 __name__ = "deep_reference_parser"
-__version__ = "2020.3.2"
+__version__ = "2020.3.3"
 __description__ = "Deep learning model for finding and parsing references"
 __url__ = "https://github.com/wellcometrust/deep_reference_parser"
 __author__ = "Wellcome Trust DataLabs Team"
 __author_email__ = "[email protected]"
 __license__ = "MIT"
-__splitter_model_version__ = "2019.12.0_splitting"
-__parser_model_version__ = "2020.3.2_parsing"
+__splitter_model_version__ = "2020.3.6_splitting"
+__parser_model_version__ = "2020.3.8_parsing"
+__multitask_model_version__ = "2020.3.18_multitask"

deep_reference_parser/common.py

@@ -6,7 +6,7 @@
 from urllib import parse, request
 
 from .logger import logger
-from .__version__ import __splitter_model_version__, __parser_model_version__
+from .__version__ import __splitter_model_version__, __parser_model_version__, __multitask_model_version__
 
 
 def get_path(path):
@@ -15,6 +15,7 @@ def get_path(path):
 
 SPLITTER_CFG = get_path(f"configs/{__splitter_model_version__}.ini")
 PARSER_CFG = get_path(f"configs/{__parser_model_version__}.ini")
+MULTITASK_CFG = get_path(f"configs/{__multitask_model_version__}.ini")
 
 
 def download_model_artefact(artefact, s3_slug):
@@ -47,13 +48,9 @@ def download_model_artefacts(model_dir, s3_slug, artefacts=None):
     if not artefacts:
 
         artefacts = [
-            "char2ind.pickle",
-            "ind2label.pickle",
-            "ind2word.pickle",
-            "label2ind.pickle",
+            "indices.pickle"
             "maxes.pickle",
             "weights.h5",
-            "word2ind.pickle",
         ]
 
     for artefact in artefacts:

deep_reference_parser/configs/2020.3.18_multitask.ini

@@ -0,0 +1,41 @@
+[DEFAULT]
+version = 2020.3.18_multitask
+description = Multitask model trained on a combination of Reach and Rodrigues 
+    data. The Rodrigues data have been concatenated into a single continuous
+    document and then cut into sequences of length=line_length, so that the
+    Rodrigues data and Reach data have the same lengths without need for much
+    padding or truncating.
+deep_reference_parser_version = b61de984f95be36445287c40af4e65a403637692
+
+[data]
+# Note that test and valid proportion are only used for data creation steps, 
+# not when running the train command.
+test_proportion = 0.25
+valid_proportion = 0.25
+data_path = data/
+respect_line_endings = 0
+respect_doc_endings = 1
+line_limit = 150
+policy_train = data/multitask/2020.3.18_multitask_train.tsv
+policy_test = data/multitask/2020.3.18_multitask_test.tsv
+policy_valid = data/multitask/2020.3.18_multitask_valid.tsv
+s3_slug = https://datalabs-public.s3.eu-west-2.amazonaws.com/deep_reference_parser/
+
+[build]
+output_path = models/multitask/2020.3.18_multitask/
+output = crf
+word_embeddings = embeddings/2020.1.1-wellcome-embeddings-300.txt
+pretrained_embedding = 0
+dropout = 0.5
+lstm_hidden = 400
+word_embedding_size = 300
+char_embedding_size = 100
+char_embedding_type = BILSTM
+optimizer = adam
+
+[train]
+epochs = 60
+batch_size = 100
+early_stopping_patience = 5
+metric = val_f1
+

deep_reference_parser/configs/2020.3.6_splitting.ini

@@ -0,0 +1,39 @@
+[DEFAULT]
+version = 2020.3.6_splitting
+description = Splitting model trained on a combination of Reach and Rodrigues 
+    data. The Rodrigues data have been concatenated into a single continuous
+    document and then cut into sequences of length=line_length, so that the
+    Rodrigues data and Reach data have the same lengths without need for much
+    padding or truncating.
+deep_reference_parser_version = e489f7efa31072b95175be8f728f1fcf03a4cabb
+
+[data]
+test_proportion = 0.25
+valid_proportion = 0.25
+data_path = data/
+respect_line_endings = 0
+respect_doc_endings = 1
+line_limit = 250
+policy_train = data/splitting/2020.3.6_splitting_train.tsv
+policy_test = data/splitting/2020.3.6_splitting_test.tsv
+policy_valid = data/splitting/2020.3.6_splitting_valid.tsv
+s3_slug = https://datalabs-public.s3.eu-west-2.amazonaws.com/deep_reference_parser/
+
+[build]
+output_path = models/splitting/2020.3.6_splitting/
+output = crf
+word_embeddings = embeddings/2020.1.1-wellcome-embeddings-300.txt
+pretrained_embedding = 0
+dropout = 0.5
+lstm_hidden = 400
+word_embedding_size = 300
+char_embedding_size = 100
+char_embedding_type = BILSTM
+optimizer = rmsprop
+
+[train]
+epochs = 30
+batch_size = 100
+early_stopping_patience = 5
+metric = val_f1
+

deep_reference_parser/configs/2020.3.8_parsing.ini

@@ -0,0 +1,38 @@
+[DEFAULT]
+version = 2020.3.8_parsing
+description = Parsing model trained on a combination of Reach and Rodrigues 
+    data. The Rodrigues data have been concatenated into a single continuous
+    document and then cut into sequences of length=line_length, so that the
+    Rodrigues data and Reach data have the same lengths without need for much
+    padding or truncating.
+deep_reference_parser_version = e489f7efa31072b95175be8f728f1fcf03a4cabb
+
+[data]
+test_proportion = 0.25
+valid_proportion = 0.25
+data_path = data/
+respect_line_endings = 0
+respect_doc_endings = 1
+line_limit = 100
+policy_train = data/parsing/2020.3.8_parsing_train.tsv
+policy_test = data/parsing/2020.3.8_parsing_test.tsv
+policy_valid = data/parsing/2020.3.8_parsing_valid.tsv
+s3_slug = https://datalabs-public.s3.eu-west-2.amazonaws.com/deep_reference_parser/
+
+[build]
+output_path = models/parsing/2020.3.8_parsing/
+output = crf
+word_embeddings = embeddings/2020.1.1-wellcome-embeddings-300.txt
+pretrained_embedding = 0
+dropout = 0.5
+lstm_hidden = 400
+word_embedding_size = 300
+char_embedding_size = 100
+char_embedding_type = BILSTM
+optimizer = rmsprop
+
+[train]
+epochs = 30
+batch_size = 100
+early_stopping_patience = 5
+metric = val_f1