increase lr for 8 grad acc steps

YerevaNN · Dec 13, 2023 · be375a3 · be375a3
1 parent c1c0ce1
commit be375a3
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/src/config/create_train_config.py b/src/config/create_train_config.py
@@ -12,7 +12,7 @@
 for key in model_train_configs.keys():
     model_train_configs[key]["max_learning_rate"] *= 0.08
 
-model_train_configs["125m"]["max_learning_rate"] = 1.0e-3
+model_train_configs["125m"]["max_learning_rate"] = 0.0014
 
 model_train_configs["1.3b"]["warmup_steps"] = 2000
 model_train_configs["1.3b"]["max_learning_rate"] = 1.0e-5