Replaced Junk Layer with a more useful Layer, retraining still needed.

config.json

@@ -14,7 +14,7 @@
   "max_position_embeddings": 4096,
   "model_type": "mixtral",
   "num_attention_heads": 40,
-  "num_experts_per_tok": 1,
+  "num_experts_per_tok": 2,
   "num_hidden_layers": 28,
   "num_key_value_heads": 40,
   "num_local_experts": 2,

model-00001-of-00007.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e692265a0a955505a78c73caf681fbd493f2dbcb462c1a3d2c8e07d6b71fcd5b
+oid sha256:66de1061f71bf2d3b663e1f02505e1c2bf45098ffc197437b852896631cbdfff
 size 4915719208

model-00002-of-00007.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:26259565c984423b973027c3c2f665353ca8f14c0250c1b4b7c4be90c0127f74
+oid sha256:46485d98179ad546c6c3922caf9e6b60d7c57734726c733106eb5ec7324ccb8a
 size 4996656336

model-00003-of-00007.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:651ba6a6669e03d64bff77b98ac680b6ad5bb2bed75533eac3160f5e710142fc
+oid sha256:5d3b8afa6b3622dbc48e28b1f57847318c461001dac5d9e3ce7996fa6e98f824
 size 4959956208

model-00004-of-00007.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b098f773a9ec829c293faa122afbcff2831dedac133dc620a1424b30ae8ceee0
+oid sha256:61a97b9a576c16d83ad4b6bb17f71126d0c8dc809c9799460d9d8c6866efa17a
 size 4870847960

model-00005-of-00007.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6ca2fdd70fdd8d7a90acdbf6437333201c3933b71edf8174a9362f79522c8219
+oid sha256:6ff1777360db41ab9919c503782d56dd111ea4481360351a4371c76760eab03e
 size 4996656392

model-00006-of-00007.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bce3188bdc395c22b1f0ec06c6704b58eccb0c0a0e31d8e51c2ea338dd115835
+oid sha256:8078e030ed4ff4b75da52087d7949f11d2478c38230a6b819f9fe623535f238f
 size 4959956224