chg: Use the max_len sent at init

* Remove confusing max_words parameter
* Set max_len in init with default=250

Author: ivyleavedtoadflax

deep_reference_parser/deep_reference_parser.py

@@ -72,6 +72,7 @@ def __init__(
         y_train=None,
         y_test=None,
         y_valid=None,
+        max_len=250,
         digits_word="$NUM$",
         ukn_words="out-of-vocabulary",
         padding_style="pre",
@@ -126,9 +127,8 @@ def __init__(
         self.X_validation = list()
         self.X_testing = list()
 
-        self.max_len = int()
+        self.max_len = max_len
         self.max_char = int()
-        self.max_words = int()
 
         # Defined in prepare_data
 
@@ -156,7 +156,7 @@ def prepare_data(self, save=False):
             Save(bool): If True, then data objects will be saved to
                 `self.output_path`.
         """
-        self.max_len = max([len(xx) for xx in self.X_train])
+        #self.max_len = max([len(xx) for xx in self.X_train])
 
         self.X_train_merged, self.X_test_merged, self.X_valid_merged = merge_digits(
             [self.X_train, self.X_test, self.X_valid], self.digits_word
@@ -246,14 +246,14 @@ def prepare_data(self, save=False):
         # Create character level data
 
         # Create the character level data
-        self.char2ind, self.max_words, self.max_char = character_index(
+        self.char2ind, self.max_char = character_index(
             self.X_train, self.digits_word
         )
 
         self.X_train_char = character_data(
             self.X_train,
             self.char2ind,
-            self.max_words,
+            self.max_len,
             self.max_char,
             self.digits_word,
             self.padding_style,
@@ -262,7 +262,7 @@ def prepare_data(self, save=False):
         self.X_test_char = character_data(
             self.X_test,
             self.char2ind,
-            self.max_words,
+            self.max_len,
             self.max_char,
             self.digits_word,
             self.padding_style,
@@ -271,7 +271,7 @@ def prepare_data(self, save=False):
         self.X_valid_char = character_data(
             self.X_valid,
             self.char2ind,
-            self.max_words,
+            self.max_len,
             self.max_char,
             self.digits_word,
             self.padding_style,
@@ -292,7 +292,6 @@ def prepare_data(self, save=False):
             write_pickle(self.char2ind, "char2ind.pickle", path=self.output_path)
 
             maxes = {
-                "max_words": self.max_words,
                 "max_char": self.max_char,
                 "max_len": self.max_len,
             }
@@ -317,11 +316,9 @@ def load_data(self, out_path):
 
         self.max_len = maxes["max_len"]
         self.max_char = maxes["max_char"]
-        self.max_words = maxes["max_words"]
 
         logger.debug("Setting max_len to %s", self.max_len)
         logger.debug("Setting max_char to %s", self.max_char)
-        logger.debug("Setting max_words to %s", self.max_words)
 
     def build_model(
         self,
@@ -370,7 +367,7 @@ def build_model(
 
         if word_embeddings:
 
-            word_input = Input((self.max_words,))
+            word_input = Input((self.max_len,))
             inputs.append(word_input)
 
             # TODO: More sensible handling of options for pretrained embedding.
@@ -406,7 +403,7 @@ def build_model(
 
         if self.max_char != 0:
 
-            character_input = Input((self.max_words, self.max_char,))
+            character_input = Input((self.max_len, self.max_char,))
 
             char_embedding = self.character_embedding_layer(
                 char_embedding_type=char_embedding_type,
@@ -474,7 +471,7 @@ def build_model(
 
         self.model = model
 
-        logger.debug(self.model.summary(line_length=150))
+        #logger.debug(self.model.summary(line_length=150))
 
     def train_model(
         self, epochs=25, batch_size=100, early_stopping_patience=5, metric="val_f1"
@@ -613,10 +610,6 @@ def evaluate(
 
             # Compute classification report
 
-            # Initialise list for storing predictions which will be written
-            # to tsv file.
-
-
             for i, y_target in enumerate(self.y_valid_encoded):
 
                 # Compute predictions, flatten
@@ -970,7 +963,7 @@ def prepare_X_data(self, X):
         X_char = character_data(
             X,
             self.char2ind,
-            self.max_words,
+            self.max_len,
             self.max_char,
             self.digits_word,
             self.padding_style,

deep_reference_parser/model_utils.py

@@ -205,10 +205,9 @@ def character_index(X, digits_word):
 
     # For padding
 
-    max_words = max([len(s) for s in X])
     max_char = max([len(w) for s in X for w in s])
 
-    return char2ind, max_words, max_char
+    return char2ind, max_char
 
 
 def character_data(X, char2ind, max_words, max_char, digits_word, padding_style):

deep_reference_parser/train.py

@@ -56,6 +56,7 @@ def train(config_file):
     LSTM_HIDDEN = int(cfg["build"]["lstm_hidden"])
     WORD_EMBEDDING_SIZE = int(cfg["build"]["word_embedding_size"])
     CHAR_EMBEDDING_SIZE = int(cfg["build"]["char_embedding_size"])
+    MAX_LEN = int(cfg["data"]["line_limit"])
 
     # Train config
 
@@ -74,6 +75,20 @@ def train(config_file):
     X_test, y_test = test_data[0], test_data[1:]
     X_valid, y_valid = valid_data[0], valid_data[1:]
 
+    import statistics
+
+    logger.info("Max token length %s", max([len(i) for i in X_train]))
+    logger.info("Min token length %s", min([len(i) for i in X_train]))
+    logger.info("Mean token length %s", statistics.median([len(i) for i in X_train]))
+
+    logger.info("Max token length %s", max([len(i) for i in X_test]))
+    logger.info("Min token length %s", min([len(i) for i in X_test]))
+    logger.info("Mean token length %s", statistics.median([len(i) for i in X_test]))
+
+    logger.info("Max token length %s", max([len(i) for i in X_valid]))
+    logger.info("Min token length %s", min([len(i) for i in X_valid]))
+    logger.info("Mean token length %s", statistics.median([len(i) for i in X_valid]))
+
     logger.info("X_train, y_train examples: %s, %s", len(X_train), list(map(len, y_train)))
     logger.info("X_test, y_test examples: %s, %s", len(X_test), list(map(len, y_test)))
     logger.info("X_valid, y_valid examples: %s, %s", len(X_valid), list(map(len, y_valid)))
@@ -85,6 +100,7 @@ def train(config_file):
         y_train=y_train,
         y_test=y_test,
         y_valid=y_valid,
+        max_len=MAX_LEN,
         output_path=OUTPUT_PATH,
     )
 

tests/test_deep_reference_parser.py

@@ -77,7 +77,9 @@ def test_DeepReferenceParser_train(tmpdir, cfg):
         y_train=y_test,
         y_test=y_test,
         y_valid=y_test,
+        max_len=250,
         output_path=tmpdir,
+
     )
 
     # Prepare the data