Upload eleutherai_cluster_zphang.yml

eleutherai_cluster_zphang.yml

@@ -0,0 +1,20 @@
+# Data paths and options when using EleutherAI cluster
+{
+  "train-data-paths": ["/fsx/pile/pile_20B_tokenizer_text_document"],
+  "valid-data-paths": ["/fsx/pile/pile_20B_tokenizer_text_document"],
+  "test-data-paths": ["/fsx/pile/pile_20B_tokenizer_text_document"],
+
+  "tokenizer_type": "HFTokenizer",
+  "vocab-file": "/fsx/pile/20B_tokenizer.json",
+
+  "save": "/fsx/shiv/gpt-neox/runs/checkpoints/pythia/13b",
+  "load": "/fsx/shiv/gpt-neox/runs/checkpoints/pythia/13b",
+  "tensorboard-dir": "/fsx/shiv/gpt-neox/runs/tensorboard",
+  "log-dir": "/fsx/shiv/gpt-neox/runs/logs",
+
+  "wandb_team": "eleutherai",
+  "wandb_project": "pythia",
+  "wandb_group": "13B Decay",
+  "launcher": "openmpi",
+  "deepspeed_mpi": true
+}