use python script to clone repo and trigger train

run_sm.py

@@ -0,0 +1,16 @@
+""" Python script that triggers sagemaker flow"""
+
+import sys
+import subprocess
+
+def main():
+    # Let's skip arg names
+    repo = sys.argv[2]
+    repo_name = repo.split('/')[-1]
+    script_name = sys.argv[4]
+    cmd = f'git clone {repo} && cd {repo_name} && sh {script_name}'
+    # subprocess.call(cmd, shell=True)
+    raise ValueError(cmd)
+
+if __name__ == '__main__':
+    main()

run_speech_recognition_seq2seq_streaming.py

@@ -19,8 +19,7 @@ with 🤗 Datasets' streaming mode.
 """
 # You can also adapt this script for your own sequence to sequence speech
 # recognition task. Pointers for this are left as comments.
-import os
-raise RuntimeError(f"{os.getcwd()}")
+
 import json
 import logging
 import os

sm.py

@@ -13,7 +13,7 @@ TEST = True
 
 
 test_sm_instances = {
-    "ml.g4dn.2xlarge":
+    "ml.g4dn.xlarge":
         {
             "num_instances": 1,
             "num_gpus": 1
@@ -30,7 +30,7 @@ full_sm_instances = {
 
 sm_instances = test_sm_instances if TEST else full_sm_instances
 
-ENTRY_POINT = "run_speech_recognition_seq2seq_streaming.py"
+ENTRY_POINT = "run_sm.py"
 RUN_SCRIPT = "test_run.sh" if TEST else "run.sh"
 IMAGE_URI = "116817510867.dkr.ecr.eu-west-1.amazonaws.com/huggingface-pytorch-training:whisper-finetuning-0223e276db78adf4ea4dc5f874793cb2"
 if IMAGE_URI is None:
@@ -66,7 +66,6 @@ def parse_run_script():
                 .replace("\n", "") \
                 .replace('"', "")
             line = line.split("=")
-            # remove '\t--'
             key = str(line[0])
             try:
                 value = line[1]
@@ -78,8 +77,8 @@ def parse_run_script():
 
 
 set_creds()
-hyperparameters = parse_run_script()
-pprint(hyperparameters)
+# hyperparameters = parse_run_script()
+# pprint(hyperparameters)
 
 hf_token = os.environ.get("HF_TOKEN")
 if hf_token is None:
@@ -93,6 +92,10 @@ env_vars = {
 }
 pprint(env_vars)
 repo = f"https://huggingface.co/marinone94/{os.getcwd().split('/')[-1]}"
+hyperparameters = {
+    "repo": repo,
+    "entrypoint": RUN_SCRIPT
+}
 for sm_instance_name, sm_instance_values in sm_instances.items():
         num_instances: int = \
             int(sm_instance_values["num_instances"])