add log_training_trajectory option

metaseq/dataclass/configs.py

@@ -385,6 +385,12 @@ class DatasetConfig(MetaseqDataclass):
             "help": "if set, validate language model at the beginning of training or fine-tuning process"
         },
     )
+    log_training_trajectory: bool = field(
+        default=False,
+        metadata={
+            "help": "(InstructOPT specific) if set, evaluate and save checkpoints more frequently in early stage of training"
+        },
+    )
     fixed_validation_seed: Optional[int] = field(
         default=None, metadata={"help": "specified random seed for validation"}
     )

metaseq_cli/train.py

@@ -418,6 +418,10 @@ def validate_and_save(
             and num_updates >= cfg.dataset.validate_after_updates
             and was_successful_step
         )
+        or (
+            cfg.dataset.log_training_trajectory
+            and num_updates in [10, 20, 50, 100, 200, 500]
+        )
     )
     do_validate = (
         (
@@ -432,6 +436,10 @@ def validate_and_save(
             and was_successful_step
         )
         or (cfg.dataset.validate_at_beginning and num_updates == 0)
+        or (
+            cfg.dataset.log_training_trajectory
+            and num_updates in [0, 10, 20, 50, 100, 200, 500]
+        )
     ) and not cfg.dataset.disable_validation
     valid_losses = [None]
     if do_validate: