chg: Use the max_len sent at init

Don't set it based on maximum sequence length

Author: ivyleavedtoadflax

deep_reference_parser/deep_reference_parser.py

@@ -72,6 +72,7 @@ def __init__(
         y_train=None,
         y_test=None,
         y_valid=None,
+        max_len=None,
         digits_word="$NUM$",
         ukn_words="out-of-vocabulary",
         padding_style="pre",
@@ -126,7 +127,7 @@ def __init__(
         self.X_validation = list()
         self.X_testing = list()
 
-        self.max_len = int()
+        self.max_len = max_len
         self.max_char = int()
         self.max_words = int()
 
@@ -156,7 +157,7 @@ def prepare_data(self, save=False):
             Save(bool): If True, then data objects will be saved to
                 `self.output_path`.
         """
-        self.max_len = max([len(xx) for xx in self.X_train])
+        #self.max_len = max([len(xx) for xx in self.X_train])
 
         self.X_train_merged, self.X_test_merged, self.X_valid_merged = merge_digits(
             [self.X_train, self.X_test, self.X_valid], self.digits_word
@@ -253,7 +254,7 @@ def prepare_data(self, save=False):
         self.X_train_char = character_data(
             self.X_train,
             self.char2ind,
-            self.max_words,
+            self.max_len,
             self.max_char,
             self.digits_word,
             self.padding_style,
@@ -262,7 +263,7 @@ def prepare_data(self, save=False):
         self.X_test_char = character_data(
             self.X_test,
             self.char2ind,
-            self.max_words,
+            self.max_len,
             self.max_char,
             self.digits_word,
             self.padding_style,
@@ -271,7 +272,7 @@ def prepare_data(self, save=False):
         self.X_valid_char = character_data(
             self.X_valid,
             self.char2ind,
-            self.max_words,
+            self.max_len,
             self.max_char,
             self.digits_word,
             self.padding_style,
@@ -370,7 +371,7 @@ def build_model(
 
         if word_embeddings:
 
-            word_input = Input((self.max_words,))
+            word_input = Input((self.max_len,))
             inputs.append(word_input)
 
             # TODO: More sensible handling of options for pretrained embedding.
@@ -406,7 +407,7 @@ def build_model(
 
         if self.max_char != 0:
 
-            character_input = Input((self.max_words, self.max_char,))
+            character_input = Input((self.max_len, self.max_char,))
 
             char_embedding = self.character_embedding_layer(
                 char_embedding_type=char_embedding_type,
@@ -474,7 +475,7 @@ def build_model(
 
         self.model = model
 
-        logger.debug(self.model.summary(line_length=150))
+        #logger.debug(self.model.summary(line_length=150))
 
     def train_model(
         self, epochs=25, batch_size=100, early_stopping_patience=5, metric="val_f1"
@@ -970,7 +971,7 @@ def prepare_X_data(self, X):
         X_char = character_data(
             X,
             self.char2ind,
-            self.max_words,
+            self.max_len,
             self.max_char,
             self.digits_word,
             self.padding_style,

deep_reference_parser/train.py

@@ -56,6 +56,7 @@ def train(config_file):
     LSTM_HIDDEN = int(cfg["build"]["lstm_hidden"])
     WORD_EMBEDDING_SIZE = int(cfg["build"]["word_embedding_size"])
     CHAR_EMBEDDING_SIZE = int(cfg["build"]["char_embedding_size"])
+    MAX_LEN = int(cfg["data"]["line_limit"])
 
     # Train config
 
@@ -74,6 +75,20 @@ def train(config_file):
     X_test, y_test = test_data[0], test_data[1:]
     X_valid, y_valid = valid_data[0], valid_data[1:]
 
+    import statistics
+
+    logger.info("Max token length %s", max([len(i) for i in X_train]))
+    logger.info("Min token length %s", min([len(i) for i in X_train]))
+    logger.info("Mean token length %s", statistics.median([len(i) for i in X_train]))
+
+    logger.info("Max token length %s", max([len(i) for i in X_test]))
+    logger.info("Min token length %s", min([len(i) for i in X_test]))
+    logger.info("Mean token length %s", statistics.median([len(i) for i in X_test]))
+
+    logger.info("Max token length %s", max([len(i) for i in X_valid]))
+    logger.info("Min token length %s", min([len(i) for i in X_valid]))
+    logger.info("Mean token length %s", statistics.median([len(i) for i in X_valid]))
+
     logger.info("X_train, y_train examples: %s, %s", len(X_train), list(map(len, y_train)))
     logger.info("X_test, y_test examples: %s, %s", len(X_test), list(map(len, y_test)))
     logger.info("X_valid, y_valid examples: %s, %s", len(X_valid), list(map(len, y_valid)))
@@ -85,6 +100,7 @@ def train(config_file):
         y_train=y_train,
         y_test=y_test,
         y_valid=y_valid,
+        max_len=MAX_LEN,
         output_path=OUTPUT_PATH,
     )