added condition for test to also work with lower transformers version

Author: philschmid

tests/data/huggingface/run_tf.py

@@ -7,6 +7,7 @@
 from datasets import load_dataset
 
 from transformers import AutoTokenizer, TFAutoModelForSequenceClassification
+import transformers
 
 
 if __name__ == "__main__":
@@ -57,8 +58,15 @@
     )
     train_dataset.set_format(type="tensorflow", columns=["input_ids", "attention_mask", "label"])
 
-    train_features = {x: train_dataset[x] for x in ["input_ids", "attention_mask"]}
-
+    if transformers.__version__ > "4.12.0":
+        train_features = {x: train_dataset[x] for x in ["input_ids", "attention_mask"]}
+    else:
+        train_features = {
+            x: train_dataset[x].to_tensor(default_value=0, shape=[None, tokenizer.model_max_length])
+            for x in ["input_ids", "attention_mask"]
+        }
+        
+        
     tf_train_dataset = tf.data.Dataset.from_tensor_slices(
         (train_features, train_dataset["label"])
     ).batch(args.per_device_train_batch_size)
@@ -69,7 +77,14 @@
     )
     test_dataset.set_format(type="tensorflow", columns=["input_ids", "attention_mask", "label"])
 
-    test_features = {x: test_dataset[x] for x in ["input_ids", "attention_mask"]}
+    if transformers.__version__ > "4.12.0":
+        test_features = {x: test_dataset[x] for x in ["input_ids", "attention_mask"]}
+    else:
+        test_features = {
+            x: test_dataset[x].to_tensor(default_value=0, shape=[None, tokenizer.model_max_length])
+            for x in ["input_ids", "attention_mask"]
+        }
+
 
     tf_test_dataset = tf.data.Dataset.from_tensor_slices(
         (test_features, test_dataset["label"])