Native transformers configs (huggingface/transformers#50393)

chat_template.jinja

@@ -215,4 +215,12 @@
 {%- endfor -%}
 {%- if add_generation_prompt -%}
     {{- "<|im_start|>assistant\n" -}}
+    {#- `generation_mode` opens the answer with its marker; omitted or "text", the model picks the mode itself -#}
+    {%- if generation_mode == "audio" -%}
+        {{- "<|audio_start|>" -}}
+    {%- elif generation_mode == "interleaved" -%}
+        {{- "<|mixed_start|>" -}}
+    {%- elif generation_mode is defined and generation_mode not in ["text", none] -%}
+        {{- raise_exception("generation_mode must be \"text\", \"audio\" or \"interleaved\", got " ~ generation_mode) -}}
+    {%- endif -%}
 {%- endif -%}

config.json

@@ -2,221 +2,119 @@
   "architectures": [
     "Lfm3AudioForConditionalGeneration"
   ],
-  "model_type": "lfm3_audio",
-  "format_version": 1,
-  "codebooks": 12,
-  "audio_vocab_size": 2017,
-  "maskgit_steps": 3,
-  "interleaved_n_text": 6,
-  "interleaved_n_audio": 12,
-  "preprocessor": {
-    "feature_type": "log_mel",
-    "sample_rate": 16000,
-    "waveform_dtype": "float32",
-    "channel_reduction": "mean",
-    "n_mels": 128,
-    "n_fft": 400,
-    "win_length": 400,
-    "hop_length": 160,
-    "window": "hann",
-    "window_periodic": true,
-    "center": false,
-    "normalized": false,
-    "onesided": true,
-    "power": 2.0,
-    "f_min": 0.0,
-    "f_max": 8000.0,
-    "mel_scale": "slaney",
-    "mel_norm": "slaney",
-    "log_base": 10,
-    "logmel_floor": -12.0,
-    "normalization": {
-      "type": "affine",
-      "offset": 4.0,
-      "divisor": 4.0
-    },
-    "padding": {
-      "side": "right",
-      "mode": "constant",
-      "value": 0.0,
-      "minimum_samples": 400,
-      "length_alignment_offset": 400,
-      "length_alignment_multiple": 160
-    }
-  },
-  "tokenizer": {
-    "repo_id": "LiquidAI/liquid7.1_tokenizer",
-    "revision": "39badc470af6cdf6f6a6fbb4d6d1c4d1929b5b96"
-  },
-  "encoder": {
-    "dim_in": 128,
-    "dim_conv": 256,
-    "dim_out": 768,
-    "sliding_window": 32,
-    "lfm": {
-      "transformers_version": "5.19.0.dev0",
-      "architectures": null,
-      "output_hidden_states": false,
-      "return_dict": true,
-      "dtype": "float32",
-      "chunk_size_feed_forward": 0,
-      "is_encoder_decoder": false,
-      "id2label": {
-        "0": "LABEL_0",
-        "1": "LABEL_1"
-      },
-      "label2id": {
-        "LABEL_0": 0,
-        "LABEL_1": 1
-      },
-      "problem_type": null,
-      "vocab_size": 2,
-      "hidden_size": 768,
-      "intermediate_size": 2560,
-      "num_hidden_layers": 17,
-      "num_attention_heads": 8,
-      "num_key_value_heads": 4,
-      "max_position_embeddings": 128000,
-      "initializer_range": 0.02,
-      "norm_eps": 1e-05,
-      "use_cache": true,
-      "pad_token_id": 0,
-      "bos_token_id": 1,
-      "eos_token_id": null,
-      "tie_word_embeddings": true,
-      "rope_parameters": {
-        "rope_theta": 1000000.0,
-        "rope_type": "default"
-      },
-      "conv_bias": false,
-      "conv_L_cache": 3,
-      "block_multiple_of": 256,
-      "block_ffn_dim_multiplier": 1.0,
-      "block_auto_adjust_ff_dim": true,
-      "full_attn_idxs": null,
-      "layer_types": [
-        "conv",
-        "conv",
-        "full_attention",
-        "conv",
-        "conv",
-        "full_attention",
-        "conv",
-        "conv",
-        "full_attention",
-        "conv",
-        "conv",
-        "full_attention",
-        "conv",
-        "conv",
-        "full_attention",
-        "full_attention",
-        "full_attention"
-      ],
-      "_name_or_path": "",
-      "model_type": "lfm2",
-      "output_attentions": false
-    }
-  },
-  "depthformer": {
-    "transformers_version": "5.19.0.dev0",
-    "architectures": null,
-    "output_hidden_states": false,
-    "return_dict": true,
-    "dtype": null,
-    "chunk_size_feed_forward": 0,
-    "is_encoder_decoder": false,
-    "id2label": {
-      "0": "LABEL_0",
-      "1": "LABEL_1"
-    },
-    "label2id": {
-      "LABEL_0": 0,
-      "LABEL_1": 1
-    },
-    "problem_type": null,
-    "vocab_size": 1,
-    "hidden_size": 1024,
-    "intermediate_size": 2816,
-    "num_hidden_layers": 3,
-    "num_attention_heads": 32,
-    "num_key_value_heads": 8,
-    "max_position_embeddings": 12,
+  "audio_config": {
+    "attention_bias": false,
+    "attention_dropout": 0.0,
+    "conv_L_cache": 3,
+    "conv_bias": false,
+    "head_dim": 96,
+    "hidden_size": 768,
     "initializer_range": 0.02,
+    "intermediate_size": 1792,
+    "layer_types": [
+      "conv",
+      "conv",
+      "sliding_attention",
+      "conv",
+      "conv",
+      "sliding_attention",
+      "conv",
+      "conv",
+      "sliding_attention",
+      "conv",
+      "conv",
+      "sliding_attention",
+      "conv",
+      "conv",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention"
+    ],
+    "max_position_embeddings": 128000,
+    "model_type": "lfm3_audio_audio",
     "norm_eps": 1e-05,
-    "use_cache": false,
-    "pad_token_id": null,
-    "bos_token_id": null,
-    "eos_token_id": null,
-    "tie_word_embeddings": true,
+    "num_attention_heads": 8,
+    "num_hidden_layers": 17,
+    "num_key_value_heads": 4,
+    "num_mel_bins": 128,
     "rope_parameters": {
-      "full_attention": {
-        "rope_type": "default",
+      "sliding_attention": {
         "rope_theta": 1000000.0,
-        "partial_rotary_factor": 1.0
+        "rope_type": "default"
       }
     },
-    "conv_bias": false,
+    "sliding_window": 32,
+    "subsampling_conv_channels": 256,
+    "subsampling_conv_kernel_size": 3,
+    "subsampling_conv_stride": 2,
+    "subsampling_factor": 8,
+    "use_attention_gate": false,
+    "use_cache": true,
+    "use_sandwich_norm": false
+  },
+  "audio_eos_code": 2016,
+  "audio_loss_weight": 1.0,
+  "audio_out_token_id": 125032,
+  "audio_start_token_id": 125012,
+  "audio_token_id": 125031,
+  "codebook_size": 2017,
+  "depth_decoder_config": {
+    "attention_bias": false,
+    "attention_dropout": 0.0,
     "conv_L_cache": 7,
+    "conv_bias": false,
+    "head_dim": 32,
+    "hidden_size": 1024,
+    "initializer_range": 0.02,
+    "intermediate_size": 2816,
     "layer_types": [
       "full_attention",
       "full_attention",
       "full_attention"
     ],
-    "head_dim": 32,
-    "sliding_window": 256,
-    "use_sandwich_norm": false,
-    "_name_or_path": "",
-    "is_causal": false,
-    "model_type": "lfm3",
-    "output_attentions": false
-  },
-  "lfm": {
-    "transformers_version": "5.19.0.dev0",
-    "architectures": [
-      "Lfm3ForCausalLM"
-    ],
-    "output_hidden_states": false,
-    "return_dict": true,
-    "dtype": "float32",
-    "chunk_size_feed_forward": 0,
-    "is_encoder_decoder": false,
-    "id2label": {
-      "0": "LABEL_0",
-      "1": "LABEL_1"
-    },
-    "label2id": {
-      "LABEL_0": 0,
-      "LABEL_1": 1
-    },
-    "problem_type": null,
-    "vocab_size": 128000,
-    "hidden_size": 2048,
-    "intermediate_size": 7680,
-    "num_hidden_layers": 24,
-    "num_attention_heads": 16,
-    "num_key_value_heads": 4,
-    "max_position_embeddings": 128000,
-    "initializer_range": 0.02,
+    "max_position_embeddings": 12,
+    "model_type": "lfm3_audio_depth_decoder",
     "norm_eps": 1e-05,
-    "use_cache": true,
-    "pad_token_id": 124893,
-    "bos_token_id": 124894,
-    "eos_token_id": 124900,
-    "tie_word_embeddings": true,
+    "num_attention_heads": 32,
+    "num_hidden_layers": 3,
+    "num_key_value_heads": 8,
     "rope_parameters": {
       "full_attention": {
-        "partial_rotary_factor": 0.5,
+        "partial_rotary_factor": 1.0,
         "rope_theta": 1000000.0,
         "rope_type": "default"
-      },
-      "sliding_attention": {
-        "rope_theta": 10000,
-        "rope_type": "default"
       }
     },
-    "conv_bias": false,
+    "sliding_window": null,
+    "use_attention_gate": true,
+    "use_cache": false,
+    "use_sandwich_norm": false
+  },
+  "dtype": "bfloat16",
+  "initializer_range": 0.02,
+  "interleaved_n_audio": 12,
+  "interleaved_n_text": 6,
+  "maskgit_schedule": [
+    4,
+    4,
+    4
+  ],
+  "mixed_end_token_id": 125016,
+  "mixed_start_token_id": 125015,
+  "model_type": "lfm3_audio",
+  "num_codebooks": 12,
+  "projector_hidden_act": "gelu",
+  "text_config": {
+    "attention_bias": false,
+    "attention_dropout": 0.0,
+    "bos_token_id": 124894,
     "conv_L_cache": 7,
+    "conv_bias": false,
+    "eos_token_id": 124900,
+    "head_dim": 128,
+    "hidden_size": 2048,
+    "initializer_range": 0.02,
+    "intermediate_size": 7680,
     "layer_types": [
       "conv",
       "conv",
@@ -243,16 +141,33 @@
       "full_attention",
       "conv"
     ],
-    "head_dim": 128,
+    "max_position_embeddings": 128000,
+    "model_type": "lfm3",
+    "norm_eps": 1e-05,
+    "num_attention_heads": 16,
+    "num_hidden_layers": 24,
+    "num_key_value_heads": 4,
+    "pad_token_id": 124893,
+    "rope_parameters": {
+      "full_attention": {
+        "partial_rotary_factor": 0.5,
+        "rope_theta": 1000000.0,
+        "rope_type": "proportional"
+      },
+      "sliding_attention": {
+        "rope_theta": 10000.0,
+        "rope_type": "default"
+      }
+    },
     "sliding_window": 256,
+    "tie_word_embeddings": true,
+    "use_attention_gate": true,
+    "use_cache": true,
     "use_sandwich_norm": true,
-    "model_type": "lfm3",
-    "output_attentions": false
+    "vocab_size": 128000
   },
-  "auto_map": {
-    "AutoConfig": "configuration_lfm3_audio.Lfm3AudioConfig",
-    "AutoModel": "modeling_lfm3_audio.Lfm3AudioModel",
-    "AutoModelForMultimodalLM": "modeling_lfm3_audio.Lfm3AudioForConditionalGeneration",
-    "AutoModelForSeq2SeqLM": "modeling_lfm3_audio.Lfm3AudioForConditionalGeneration"
-  }
+  "text_end_token_id": 125014,
+  "text_loss_weight": 1.0,
+  "tie_word_embeddings": true,
+  "transformers_version": "5.23.0.dev0"
 }

generation_config.json

@@ -1,16 +1,19 @@
 {
   "bos_token_id": 124894,
-  "eos_token_id": [124900, 125014],
-  "pad_token_id": 124893,
-  "use_cache": true,
-  "output_attentions": false,
-  "output_hidden_states": false,
+  "depth_decoder_do_sample": true,
+  "depth_decoder_repetition_penalty": 1.1,
+  "depth_decoder_repetition_window": 4096,
+  "depth_decoder_temperature": 1.0,
+  "depth_decoder_top_k": 128,
   "do_sample": false,
-  "repetition_penalty": 1.0,
-  "temperature": 0.0,
-  "top_k": 1,
-  "audio_temperature": 1.0,
-  "audio_top_k": 128,
-  "audio_repetition_penalty": 1.1,
-  "audio_repetition_window": 4096
+  "eos_token_id": [
+    124900,
+    124895,
+    124899
+  ],
+  "max_new_tokens_last_audio_chunk": 96,
+  "max_new_tokens_per_audio_chunk": 48,
+  "pad_token_id": 124893,
+  "return_dict_in_generate": true,
+  "transformers_version": "5.23.0.dev0"
 }

processor_config.json

@@ -1,6 +1,10 @@
 {
   "audio_out_token": "<|audio_out|>",
   "audio_token": "<|audio|>",
+  "audio_tokenizer": {
+    "audio_tokenizer_class": "LiquidCodecModel",
+    "audio_tokenizer_name_or_path": "eustlb/LFM3-Audio-2B-codec"
+  },
   "feature_extractor": {
     "encoder_stride": 8,
     "feature_extractor_type": "Lfm3AudioFeatureExtractor",
@@ -14,14 +18,9 @@
     "padding_value": 0.0,
     "return_attention_mask": true,
     "sampling_rate": 16000,
-    "win_length": 400,
-    "auto_map": {
-      "AutoFeatureExtractor": "feature_extraction_lfm3_audio.Lfm3AudioFeatureExtractor"
-    }
+    "win_length": 400
   },
   "processor_class": "Lfm3AudioProcessor",
-  "auto_map": {
-    "AutoProcessor": "processing_lfm3_audio.Lfm3AudioProcessor"
-  },
-  "audio_tokenizer_subfolder": "audio_tokenizer"
+  "num_audio_tokens_per_chunk": 12,
+  "max_audio_chunks_per_context": 31
 }