Upload folder using huggingface_hub

README.md

@@ -0,0 +1,24 @@
+---
+license: apache-2.0
+language:
+- en
+- zh
+pipeline_tag: image-text-to-text
+tags:
+- ERNIE4.5
+- mlx
+library_name: transformers
+---
+
+# mlx-community/ERNIE-4.5-VL-28B-A3B-Thinking-8bit
+This model was converted to MLX format from [`baidu/ERNIE-4.5-VL-28B-A3B-Thinking`]() using mlx-vlm version **0.3.10**.
+Refer to the [original model card](https://huggingface.co/baidu/ERNIE-4.5-VL-28B-A3B-Thinking) for more details on the model.
+## Use with mlx
+
+```bash
+pip install -U mlx-vlm
+```
+
+```bash
+python -m mlx_vlm.generate --model mlx-community/ERNIE-4.5-VL-28B-A3B-Thinking-8bit --max-tokens 100 --temperature 0.0 --prompt "Describe this image." --image <path_to_image>
+```

added_tokens.json

@@ -0,0 +1,1017 @@
+{
+  "<|ASR_END|>": 101309,
+  "<|ASR_START|>": 101308,
+  "<|AUDIO_PLACEHOLDER|>": 100296,
+  "<|CROP_COL_SEP|>": 101301,
+  "<|CROP_ROW_SEP|>": 101302,
+  "<|IMAGE_END|>": 101305,
+  "<|IMAGE_PLACEHOLDER|>": 100295,
+  "<|IMAGE_SEP|>": 101303,
+  "<|IMAGE_START|>": 101304,
+  "<|LOC_0|>": 100297,
+  "<|LOC_1000|>": 101297,
+  "<|LOC_100|>": 100397,
+  "<|LOC_101|>": 100398,
+  "<|LOC_102|>": 100399,
+  "<|LOC_103|>": 100400,
+  "<|LOC_104|>": 100401,
+  "<|LOC_105|>": 100402,
+  "<|LOC_106|>": 100403,
+  "<|LOC_107|>": 100404,
+  "<|LOC_108|>": 100405,
+  "<|LOC_109|>": 100406,
+  "<|LOC_10|>": 100307,
+  "<|LOC_110|>": 100407,
+  "<|LOC_111|>": 100408,
+  "<|LOC_112|>": 100409,
+  "<|LOC_113|>": 100410,
+  "<|LOC_114|>": 100411,
+  "<|LOC_115|>": 100412,
+  "<|LOC_116|>": 100413,
+  "<|LOC_117|>": 100414,
+  "<|LOC_118|>": 100415,
+  "<|LOC_119|>": 100416,
+  "<|LOC_11|>": 100308,
+  "<|LOC_120|>": 100417,
+  "<|LOC_121|>": 100418,
+  "<|LOC_122|>": 100419,
+  "<|LOC_123|>": 100420,
+  "<|LOC_124|>": 100421,
+  "<|LOC_125|>": 100422,
+  "<|LOC_126|>": 100423,
+  "<|LOC_127|>": 100424,
+  "<|LOC_128|>": 100425,
+  "<|LOC_129|>": 100426,
+  "<|LOC_12|>": 100309,
+  "<|LOC_130|>": 100427,
+  "<|LOC_131|>": 100428,
+  "<|LOC_132|>": 100429,
+  "<|LOC_133|>": 100430,
+  "<|LOC_134|>": 100431,
+  "<|LOC_135|>": 100432,
+  "<|LOC_136|>": 100433,
+  "<|LOC_137|>": 100434,
+  "<|LOC_138|>": 100435,
+  "<|LOC_139|>": 100436,
+  "<|LOC_13|>": 100310,
+  "<|LOC_140|>": 100437,
+  "<|LOC_141|>": 100438,
+  "<|LOC_142|>": 100439,
+  "<|LOC_143|>": 100440,
+  "<|LOC_144|>": 100441,
+  "<|LOC_145|>": 100442,
+  "<|LOC_146|>": 100443,
+  "<|LOC_147|>": 100444,
+  "<|LOC_148|>": 100445,
+  "<|LOC_149|>": 100446,
+  "<|LOC_14|>": 100311,
+  "<|LOC_150|>": 100447,
+  "<|LOC_151|>": 100448,
+  "<|LOC_152|>": 100449,
+  "<|LOC_153|>": 100450,
+  "<|LOC_154|>": 100451,
+  "<|LOC_155|>": 100452,
+  "<|LOC_156|>": 100453,
+  "<|LOC_157|>": 100454,
+  "<|LOC_158|>": 100455,
+  "<|LOC_159|>": 100456,
+  "<|LOC_15|>": 100312,
+  "<|LOC_160|>": 100457,
+  "<|LOC_161|>": 100458,
+  "<|LOC_162|>": 100459,
+  "<|LOC_163|>": 100460,
+  "<|LOC_164|>": 100461,
+  "<|LOC_165|>": 100462,
+  "<|LOC_166|>": 100463,
+  "<|LOC_167|>": 100464,
+  "<|LOC_168|>": 100465,
+  "<|LOC_169|>": 100466,
+  "<|LOC_16|>": 100313,
+  "<|LOC_170|>": 100467,
+  "<|LOC_171|>": 100468,
+  "<|LOC_172|>": 100469,
+  "<|LOC_173|>": 100470,
+  "<|LOC_174|>": 100471,
+  "<|LOC_175|>": 100472,
+  "<|LOC_176|>": 100473,
+  "<|LOC_177|>": 100474,
+  "<|LOC_178|>": 100475,
+  "<|LOC_179|>": 100476,
+  "<|LOC_17|>": 100314,
+  "<|LOC_180|>": 100477,
+  "<|LOC_181|>": 100478,
+  "<|LOC_182|>": 100479,
+  "<|LOC_183|>": 100480,
+  "<|LOC_184|>": 100481,
+  "<|LOC_185|>": 100482,
+  "<|LOC_186|>": 100483,
+  "<|LOC_187|>": 100484,
+  "<|LOC_188|>": 100485,
+  "<|LOC_189|>": 100486,
+  "<|LOC_18|>": 100315,
+  "<|LOC_190|>": 100487,
+  "<|LOC_191|>": 100488,
+  "<|LOC_192|>": 100489,
+  "<|LOC_193|>": 100490,
+  "<|LOC_194|>": 100491,
+  "<|LOC_195|>": 100492,
+  "<|LOC_196|>": 100493,
+  "<|LOC_197|>": 100494,
+  "<|LOC_198|>": 100495,
+  "<|LOC_199|>": 100496,
+  "<|LOC_19|>": 100316,
+  "<|LOC_1|>": 100298,
+  "<|LOC_200|>": 100497,
+  "<|LOC_201|>": 100498,
+  "<|LOC_202|>": 100499,
+  "<|LOC_203|>": 100500,
+  "<|LOC_204|>": 100501,
+  "<|LOC_205|>": 100502,
+  "<|LOC_206|>": 100503,
+  "<|LOC_207|>": 100504,
+  "<|LOC_208|>": 100505,
+  "<|LOC_209|>": 100506,
+  "<|LOC_20|>": 100317,
+  "<|LOC_210|>": 100507,
+  "<|LOC_211|>": 100508,
+  "<|LOC_212|>": 100509,
+  "<|LOC_213|>": 100510,
+  "<|LOC_214|>": 100511,
+  "<|LOC_215|>": 100512,
+  "<|LOC_216|>": 100513,
+  "<|LOC_217|>": 100514,
+  "<|LOC_218|>": 100515,
+  "<|LOC_219|>": 100516,
+  "<|LOC_21|>": 100318,
+  "<|LOC_220|>": 100517,
+  "<|LOC_221|>": 100518,
+  "<|LOC_222|>": 100519,
+  "<|LOC_223|>": 100520,
+  "<|LOC_224|>": 100521,
+  "<|LOC_225|>": 100522,
+  "<|LOC_226|>": 100523,
+  "<|LOC_227|>": 100524,
+  "<|LOC_228|>": 100525,
+  "<|LOC_229|>": 100526,
+  "<|LOC_22|>": 100319,
+  "<|LOC_230|>": 100527,
+  "<|LOC_231|>": 100528,
+  "<|LOC_232|>": 100529,
+  "<|LOC_233|>": 100530,
+  "<|LOC_234|>": 100531,
+  "<|LOC_235|>": 100532,
+  "<|LOC_236|>": 100533,
+  "<|LOC_237|>": 100534,
+  "<|LOC_238|>": 100535,
+  "<|LOC_239|>": 100536,
+  "<|LOC_23|>": 100320,
+  "<|LOC_240|>": 100537,
+  "<|LOC_241|>": 100538,
+  "<|LOC_242|>": 100539,
+  "<|LOC_243|>": 100540,
+  "<|LOC_244|>": 100541,
+  "<|LOC_245|>": 100542,
+  "<|LOC_246|>": 100543,
+  "<|LOC_247|>": 100544,
+  "<|LOC_248|>": 100545,
+  "<|LOC_249|>": 100546,
+  "<|LOC_24|>": 100321,
+  "<|LOC_250|>": 100547,
+  "<|LOC_251|>": 100548,
+  "<|LOC_252|>": 100549,
+  "<|LOC_253|>": 100550,
+  "<|LOC_254|>": 100551,
+  "<|LOC_255|>": 100552,
+  "<|LOC_256|>": 100553,
+  "<|LOC_257|>": 100554,
+  "<|LOC_258|>": 100555,
+  "<|LOC_259|>": 100556,
+  "<|LOC_25|>": 100322,
+  "<|LOC_260|>": 100557,
+  "<|LOC_261|>": 100558,
+  "<|LOC_262|>": 100559,
+  "<|LOC_263|>": 100560,
+  "<|LOC_264|>": 100561,
+  "<|LOC_265|>": 100562,
+  "<|LOC_266|>": 100563,
+  "<|LOC_267|>": 100564,
+  "<|LOC_268|>": 100565,
+  "<|LOC_269|>": 100566,
+  "<|LOC_26|>": 100323,
+  "<|LOC_270|>": 100567,
+  "<|LOC_271|>": 100568,
+  "<|LOC_272|>": 100569,
+  "<|LOC_273|>": 100570,
+  "<|LOC_274|>": 100571,
+  "<|LOC_275|>": 100572,
+  "<|LOC_276|>": 100573,
+  "<|LOC_277|>": 100574,
+  "<|LOC_278|>": 100575,
+  "<|LOC_279|>": 100576,
+  "<|LOC_27|>": 100324,
+  "<|LOC_280|>": 100577,
+  "<|LOC_281|>": 100578,
+  "<|LOC_282|>": 100579,
+  "<|LOC_283|>": 100580,
+  "<|LOC_284|>": 100581,
+  "<|LOC_285|>": 100582,
+  "<|LOC_286|>": 100583,
+  "<|LOC_287|>": 100584,
+  "<|LOC_288|>": 100585,
+  "<|LOC_289|>": 100586,
+  "<|LOC_28|>": 100325,
+  "<|LOC_290|>": 100587,
+  "<|LOC_291|>": 100588,
+  "<|LOC_292|>": 100589,
+  "<|LOC_293|>": 100590,
+  "<|LOC_294|>": 100591,
+  "<|LOC_295|>": 100592,
+  "<|LOC_296|>": 100593,
+  "<|LOC_297|>": 100594,
+  "<|LOC_298|>": 100595,
+  "<|LOC_299|>": 100596,
+  "<|LOC_29|>": 100326,
+  "<|LOC_2|>": 100299,
+  "<|LOC_300|>": 100597,
+  "<|LOC_301|>": 100598,
+  "<|LOC_302|>": 100599,
+  "<|LOC_303|>": 100600,
+  "<|LOC_304|>": 100601,
+  "<|LOC_305|>": 100602,
+  "<|LOC_306|>": 100603,
+  "<|LOC_307|>": 100604,
+  "<|LOC_308|>": 100605,
+  "<|LOC_309|>": 100606,
+  "<|LOC_30|>": 100327,
+  "<|LOC_310|>": 100607,
+  "<|LOC_311|>": 100608,
+  "<|LOC_312|>": 100609,
+  "<|LOC_313|>": 100610,
+  "<|LOC_314|>": 100611,
+  "<|LOC_315|>": 100612,
+  "<|LOC_316|>": 100613,
+  "<|LOC_317|>": 100614,
+  "<|LOC_318|>": 100615,
+  "<|LOC_319|>": 100616,
+  "<|LOC_31|>": 100328,
+  "<|LOC_320|>": 100617,
+  "<|LOC_321|>": 100618,
+  "<|LOC_322|>": 100619,
+  "<|LOC_323|>": 100620,
+  "<|LOC_324|>": 100621,
+  "<|LOC_325|>": 100622,
+  "<|LOC_326|>": 100623,
+  "<|LOC_327|>": 100624,
+  "<|LOC_328|>": 100625,
+  "<|LOC_329|>": 100626,
+  "<|LOC_32|>": 100329,
+  "<|LOC_330|>": 100627,
+  "<|LOC_331|>": 100628,
+  "<|LOC_332|>": 100629,
+  "<|LOC_333|>": 100630,
+  "<|LOC_334|>": 100631,
+  "<|LOC_335|>": 100632,
+  "<|LOC_336|>": 100633,
+  "<|LOC_337|>": 100634,
+  "<|LOC_338|>": 100635,
+  "<|LOC_339|>": 100636,
+  "<|LOC_33|>": 100330,
+  "<|LOC_340|>": 100637,
+  "<|LOC_341|>": 100638,
+  "<|LOC_342|>": 100639,
+  "<|LOC_343|>": 100640,
+  "<|LOC_344|>": 100641,
+  "<|LOC_345|>": 100642,
+  "<|LOC_346|>": 100643,
+  "<|LOC_347|>": 100644,
+  "<|LOC_348|>": 100645,
+  "<|LOC_349|>": 100646,
+  "<|LOC_34|>": 100331,
+  "<|LOC_350|>": 100647,
+  "<|LOC_351|>": 100648,
+  "<|LOC_352|>": 100649,
+  "<|LOC_353|>": 100650,
+  "<|LOC_354|>": 100651,
+  "<|LOC_355|>": 100652,
+  "<|LOC_356|>": 100653,
+  "<|LOC_357|>": 100654,
+  "<|LOC_358|>": 100655,
+  "<|LOC_359|>": 100656,
+  "<|LOC_35|>": 100332,
+  "<|LOC_360|>": 100657,
+  "<|LOC_361|>": 100658,
+  "<|LOC_362|>": 100659,
+  "<|LOC_363|>": 100660,
+  "<|LOC_364|>": 100661,
+  "<|LOC_365|>": 100662,
+  "<|LOC_366|>": 100663,
+  "<|LOC_367|>": 100664,
+  "<|LOC_368|>": 100665,
+  "<|LOC_369|>": 100666,
+  "<|LOC_36|>": 100333,
+  "<|LOC_370|>": 100667,
+  "<|LOC_371|>": 100668,
+  "<|LOC_372|>": 100669,
+  "<|LOC_373|>": 100670,
+  "<|LOC_374|>": 100671,
+  "<|LOC_375|>": 100672,
+  "<|LOC_376|>": 100673,
+  "<|LOC_377|>": 100674,
+  "<|LOC_378|>": 100675,
+  "<|LOC_379|>": 100676,
+  "<|LOC_37|>": 100334,
+  "<|LOC_380|>": 100677,
+  "<|LOC_381|>": 100678,
+  "<|LOC_382|>": 100679,
+  "<|LOC_383|>": 100680,
+  "<|LOC_384|>": 100681,
+  "<|LOC_385|>": 100682,
+  "<|LOC_386|>": 100683,
+  "<|LOC_387|>": 100684,
+  "<|LOC_388|>": 100685,
+  "<|LOC_389|>": 100686,
+  "<|LOC_38|>": 100335,
+  "<|LOC_390|>": 100687,
+  "<|LOC_391|>": 100688,
+  "<|LOC_392|>": 100689,
+  "<|LOC_393|>": 100690,
+  "<|LOC_394|>": 100691,
+  "<|LOC_395|>": 100692,
+  "<|LOC_396|>": 100693,
+  "<|LOC_397|>": 100694,
+  "<|LOC_398|>": 100695,
+  "<|LOC_399|>": 100696,
+  "<|LOC_39|>": 100336,
+  "<|LOC_3|>": 100300,
+  "<|LOC_400|>": 100697,
+  "<|LOC_401|>": 100698,
+  "<|LOC_402|>": 100699,
+  "<|LOC_403|>": 100700,
+  "<|LOC_404|>": 100701,
+  "<|LOC_405|>": 100702,
+  "<|LOC_406|>": 100703,
+  "<|LOC_407|>": 100704,
+  "<|LOC_408|>": 100705,
+  "<|LOC_409|>": 100706,
+  "<|LOC_40|>": 100337,
+  "<|LOC_410|>": 100707,
+  "<|LOC_411|>": 100708,
+  "<|LOC_412|>": 100709,
+  "<|LOC_413|>": 100710,
+  "<|LOC_414|>": 100711,
+  "<|LOC_415|>": 100712,
+  "<|LOC_416|>": 100713,
+  "<|LOC_417|>": 100714,
+  "<|LOC_418|>": 100715,
+  "<|LOC_419|>": 100716,
+  "<|LOC_41|>": 100338,
+  "<|LOC_420|>": 100717,
+  "<|LOC_421|>": 100718,
+  "<|LOC_422|>": 100719,
+  "<|LOC_423|>": 100720,
+  "<|LOC_424|>": 100721,
+  "<|LOC_425|>": 100722,
+  "<|LOC_426|>": 100723,
+  "<|LOC_427|>": 100724,
+  "<|LOC_428|>": 100725,
+  "<|LOC_429|>": 100726,
+  "<|LOC_42|>": 100339,
+  "<|LOC_430|>": 100727,
+  "<|LOC_431|>": 100728,
+  "<|LOC_432|>": 100729,
+  "<|LOC_433|>": 100730,
+  "<|LOC_434|>": 100731,
+  "<|LOC_435|>": 100732,
+  "<|LOC_436|>": 100733,
+  "<|LOC_437|>": 100734,
+  "<|LOC_438|>": 100735,
+  "<|LOC_439|>": 100736,
+  "<|LOC_43|>": 100340,
+  "<|LOC_440|>": 100737,
+  "<|LOC_441|>": 100738,
+  "<|LOC_442|>": 100739,
+  "<|LOC_443|>": 100740,
+  "<|LOC_444|>": 100741,
+  "<|LOC_445|>": 100742,
+  "<|LOC_446|>": 100743,
+  "<|LOC_447|>": 100744,
+  "<|LOC_448|>": 100745,
+  "<|LOC_449|>": 100746,
+  "<|LOC_44|>": 100341,
+  "<|LOC_450|>": 100747,
+  "<|LOC_451|>": 100748,
+  "<|LOC_452|>": 100749,
+  "<|LOC_453|>": 100750,
+  "<|LOC_454|>": 100751,
+  "<|LOC_455|>": 100752,
+  "<|LOC_456|>": 100753,
+  "<|LOC_457|>": 100754,
+  "<|LOC_458|>": 100755,
+  "<|LOC_459|>": 100756,
+  "<|LOC_45|>": 100342,
+  "<|LOC_460|>": 100757,
+  "<|LOC_461|>": 100758,
+  "<|LOC_462|>": 100759,
+  "<|LOC_463|>": 100760,
+  "<|LOC_464|>": 100761,
+  "<|LOC_465|>": 100762,
+  "<|LOC_466|>": 100763,
+  "<|LOC_467|>": 100764,
+  "<|LOC_468|>": 100765,
+  "<|LOC_469|>": 100766,
+  "<|LOC_46|>": 100343,
+  "<|LOC_470|>": 100767,
+  "<|LOC_471|>": 100768,
+  "<|LOC_472|>": 100769,
+  "<|LOC_473|>": 100770,
+  "<|LOC_474|>": 100771,
+  "<|LOC_475|>": 100772,
+  "<|LOC_476|>": 100773,
+  "<|LOC_477|>": 100774,
+  "<|LOC_478|>": 100775,
+  "<|LOC_479|>": 100776,
+  "<|LOC_47|>": 100344,
+  "<|LOC_480|>": 100777,
+  "<|LOC_481|>": 100778,
+  "<|LOC_482|>": 100779,
+  "<|LOC_483|>": 100780,
+  "<|LOC_484|>": 100781,
+  "<|LOC_485|>": 100782,
+  "<|LOC_486|>": 100783,
+  "<|LOC_487|>": 100784,
+  "<|LOC_488|>": 100785,
+  "<|LOC_489|>": 100786,
+  "<|LOC_48|>": 100345,
+  "<|LOC_490|>": 100787,
+  "<|LOC_491|>": 100788,
+  "<|LOC_492|>": 100789,
+  "<|LOC_493|>": 100790,
+  "<|LOC_494|>": 100791,
+  "<|LOC_495|>": 100792,
+  "<|LOC_496|>": 100793,
+  "<|LOC_497|>": 100794,
+  "<|LOC_498|>": 100795,
+  "<|LOC_499|>": 100796,
+  "<|LOC_49|>": 100346,
+  "<|LOC_4|>": 100301,
+  "<|LOC_500|>": 100797,
+  "<|LOC_501|>": 100798,
+  "<|LOC_502|>": 100799,
+  "<|LOC_503|>": 100800,
+  "<|LOC_504|>": 100801,
+  "<|LOC_505|>": 100802,
+  "<|LOC_506|>": 100803,
+  "<|LOC_507|>": 100804,
+  "<|LOC_508|>": 100805,
+  "<|LOC_509|>": 100806,
+  "<|LOC_50|>": 100347,
+  "<|LOC_510|>": 100807,
+  "<|LOC_511|>": 100808,
+  "<|LOC_512|>": 100809,
+  "<|LOC_513|>": 100810,
+  "<|LOC_514|>": 100811,
+  "<|LOC_515|>": 100812,
+  "<|LOC_516|>": 100813,
+  "<|LOC_517|>": 100814,
+  "<|LOC_518|>": 100815,
+  "<|LOC_519|>": 100816,
+  "<|LOC_51|>": 100348,
+  "<|LOC_520|>": 100817,
+  "<|LOC_521|>": 100818,
+  "<|LOC_522|>": 100819,
+  "<|LOC_523|>": 100820,
+  "<|LOC_524|>": 100821,
+  "<|LOC_525|>": 100822,
+  "<|LOC_526|>": 100823,
+  "<|LOC_527|>": 100824,
+  "<|LOC_528|>": 100825,
+  "<|LOC_529|>": 100826,
+  "<|LOC_52|>": 100349,
+  "<|LOC_530|>": 100827,
+  "<|LOC_531|>": 100828,
+  "<|LOC_532|>": 100829,
+  "<|LOC_533|>": 100830,
+  "<|LOC_534|>": 100831,
+  "<|LOC_535|>": 100832,
+  "<|LOC_536|>": 100833,
+  "<|LOC_537|>": 100834,
+  "<|LOC_538|>": 100835,
+  "<|LOC_539|>": 100836,
+  "<|LOC_53|>": 100350,
+  "<|LOC_540|>": 100837,
+  "<|LOC_541|>": 100838,
+  "<|LOC_542|>": 100839,
+  "<|LOC_543|>": 100840,
+  "<|LOC_544|>": 100841,
+  "<|LOC_545|>": 100842,
+  "<|LOC_546|>": 100843,
+  "<|LOC_547|>": 100844,
+  "<|LOC_548|>": 100845,
+  "<|LOC_549|>": 100846,
+  "<|LOC_54|>": 100351,
+  "<|LOC_550|>": 100847,
+  "<|LOC_551|>": 100848,
+  "<|LOC_552|>": 100849,
+  "<|LOC_553|>": 100850,
+  "<|LOC_554|>": 100851,
+  "<|LOC_555|>": 100852,
+  "<|LOC_556|>": 100853,
+  "<|LOC_557|>": 100854,
+  "<|LOC_558|>": 100855,
+  "<|LOC_559|>": 100856,
+  "<|LOC_55|>": 100352,
+  "<|LOC_560|>": 100857,
+  "<|LOC_561|>": 100858,
+  "<|LOC_562|>": 100859,
+  "<|LOC_563|>": 100860,
+  "<|LOC_564|>": 100861,
+  "<|LOC_565|>": 100862,
+  "<|LOC_566|>": 100863,
+  "<|LOC_567|>": 100864,
+  "<|LOC_568|>": 100865,
+  "<|LOC_569|>": 100866,
+  "<|LOC_56|>": 100353,
+  "<|LOC_570|>": 100867,
+  "<|LOC_571|>": 100868,
+  "<|LOC_572|>": 100869,
+  "<|LOC_573|>": 100870,
+  "<|LOC_574|>": 100871,
+  "<|LOC_575|>": 100872,
+  "<|LOC_576|>": 100873,
+  "<|LOC_577|>": 100874,
+  "<|LOC_578|>": 100875,
+  "<|LOC_579|>": 100876,
+  "<|LOC_57|>": 100354,
+  "<|LOC_580|>": 100877,
+  "<|LOC_581|>": 100878,
+  "<|LOC_582|>": 100879,
+  "<|LOC_583|>": 100880,
+  "<|LOC_584|>": 100881,
+  "<|LOC_585|>": 100882,
+  "<|LOC_586|>": 100883,
+  "<|LOC_587|>": 100884,
+  "<|LOC_588|>": 100885,
+  "<|LOC_589|>": 100886,
+  "<|LOC_58|>": 100355,
+  "<|LOC_590|>": 100887,
+  "<|LOC_591|>": 100888,
+  "<|LOC_592|>": 100889,
+  "<|LOC_593|>": 100890,
+  "<|LOC_594|>": 100891,
+  "<|LOC_595|>": 100892,
+  "<|LOC_596|>": 100893,
+  "<|LOC_597|>": 100894,
+  "<|LOC_598|>": 100895,
+  "<|LOC_599|>": 100896,
+  "<|LOC_59|>": 100356,
+  "<|LOC_5|>": 100302,
+  "<|LOC_600|>": 100897,
+  "<|LOC_601|>": 100898,
+  "<|LOC_602|>": 100899,
+  "<|LOC_603|>": 100900,
+  "<|LOC_604|>": 100901,
+  "<|LOC_605|>": 100902,
+  "<|LOC_606|>": 100903,
+  "<|LOC_607|>": 100904,
+  "<|LOC_608|>": 100905,
+  "<|LOC_609|>": 100906,
+  "<|LOC_60|>": 100357,
+  "<|LOC_610|>": 100907,
+  "<|LOC_611|>": 100908,
+  "<|LOC_612|>": 100909,
+  "<|LOC_613|>": 100910,
+  "<|LOC_614|>": 100911,
+  "<|LOC_615|>": 100912,
+  "<|LOC_616|>": 100913,
+  "<|LOC_617|>": 100914,
+  "<|LOC_618|>": 100915,
+  "<|LOC_619|>": 100916,
+  "<|LOC_61|>": 100358,
+  "<|LOC_620|>": 100917,
+  "<|LOC_621|>": 100918,
+  "<|LOC_622|>": 100919,
+  "<|LOC_623|>": 100920,
+  "<|LOC_624|>": 100921,
+  "<|LOC_625|>": 100922,
+  "<|LOC_626|>": 100923,
+  "<|LOC_627|>": 100924,
+  "<|LOC_628|>": 100925,
+  "<|LOC_629|>": 100926,
+  "<|LOC_62|>": 100359,
+  "<|LOC_630|>": 100927,
+  "<|LOC_631|>": 100928,
+  "<|LOC_632|>": 100929,
+  "<|LOC_633|>": 100930,
+  "<|LOC_634|>": 100931,
+  "<|LOC_635|>": 100932,
+  "<|LOC_636|>": 100933,
+  "<|LOC_637|>": 100934,
+  "<|LOC_638|>": 100935,
+  "<|LOC_639|>": 100936,
+  "<|LOC_63|>": 100360,
+  "<|LOC_640|>": 100937,
+  "<|LOC_641|>": 100938,
+  "<|LOC_642|>": 100939,
+  "<|LOC_643|>": 100940,
+  "<|LOC_644|>": 100941,
+  "<|LOC_645|>": 100942,
+  "<|LOC_646|>": 100943,
+  "<|LOC_647|>": 100944,
+  "<|LOC_648|>": 100945,
+  "<|LOC_649|>": 100946,
+  "<|LOC_64|>": 100361,
+  "<|LOC_650|>": 100947,
+  "<|LOC_651|>": 100948,
+  "<|LOC_652|>": 100949,
+  "<|LOC_653|>": 100950,
+  "<|LOC_654|>": 100951,
+  "<|LOC_655|>": 100952,
+  "<|LOC_656|>": 100953,
+  "<|LOC_657|>": 100954,
+  "<|LOC_658|>": 100955,
+  "<|LOC_659|>": 100956,
+  "<|LOC_65|>": 100362,
+  "<|LOC_660|>": 100957,
+  "<|LOC_661|>": 100958,
+  "<|LOC_662|>": 100959,
+  "<|LOC_663|>": 100960,
+  "<|LOC_664|>": 100961,
+  "<|LOC_665|>": 100962,
+  "<|LOC_666|>": 100963,
+  "<|LOC_667|>": 100964,
+  "<|LOC_668|>": 100965,
+  "<|LOC_669|>": 100966,
+  "<|LOC_66|>": 100363,
+  "<|LOC_670|>": 100967,
+  "<|LOC_671|>": 100968,
+  "<|LOC_672|>": 100969,
+  "<|LOC_673|>": 100970,
+  "<|LOC_674|>": 100971,
+  "<|LOC_675|>": 100972,
+  "<|LOC_676|>": 100973,
+  "<|LOC_677|>": 100974,
+  "<|LOC_678|>": 100975,
+  "<|LOC_679|>": 100976,
+  "<|LOC_67|>": 100364,
+  "<|LOC_680|>": 100977,
+  "<|LOC_681|>": 100978,
+  "<|LOC_682|>": 100979,
+  "<|LOC_683|>": 100980,
+  "<|LOC_684|>": 100981,
+  "<|LOC_685|>": 100982,
+  "<|LOC_686|>": 100983,
+  "<|LOC_687|>": 100984,
+  "<|LOC_688|>": 100985,
+  "<|LOC_689|>": 100986,
+  "<|LOC_68|>": 100365,
+  "<|LOC_690|>": 100987,
+  "<|LOC_691|>": 100988,
+  "<|LOC_692|>": 100989,
+  "<|LOC_693|>": 100990,
+  "<|LOC_694|>": 100991,
+  "<|LOC_695|>": 100992,
+  "<|LOC_696|>": 100993,
+  "<|LOC_697|>": 100994,
+  "<|LOC_698|>": 100995,
+  "<|LOC_699|>": 100996,
+  "<|LOC_69|>": 100366,
+  "<|LOC_6|>": 100303,
+  "<|LOC_700|>": 100997,
+  "<|LOC_701|>": 100998,
+  "<|LOC_702|>": 100999,
+  "<|LOC_703|>": 101000,
+  "<|LOC_704|>": 101001,
+  "<|LOC_705|>": 101002,
+  "<|LOC_706|>": 101003,
+  "<|LOC_707|>": 101004,
+  "<|LOC_708|>": 101005,
+  "<|LOC_709|>": 101006,
+  "<|LOC_70|>": 100367,
+  "<|LOC_710|>": 101007,
+  "<|LOC_711|>": 101008,
+  "<|LOC_712|>": 101009,
+  "<|LOC_713|>": 101010,
+  "<|LOC_714|>": 101011,
+  "<|LOC_715|>": 101012,
+  "<|LOC_716|>": 101013,
+  "<|LOC_717|>": 101014,
+  "<|LOC_718|>": 101015,
+  "<|LOC_719|>": 101016,
+  "<|LOC_71|>": 100368,
+  "<|LOC_720|>": 101017,
+  "<|LOC_721|>": 101018,
+  "<|LOC_722|>": 101019,
+  "<|LOC_723|>": 101020,
+  "<|LOC_724|>": 101021,
+  "<|LOC_725|>": 101022,
+  "<|LOC_726|>": 101023,
+  "<|LOC_727|>": 101024,
+  "<|LOC_728|>": 101025,
+  "<|LOC_729|>": 101026,
+  "<|LOC_72|>": 100369,
+  "<|LOC_730|>": 101027,
+  "<|LOC_731|>": 101028,
+  "<|LOC_732|>": 101029,
+  "<|LOC_733|>": 101030,
+  "<|LOC_734|>": 101031,
+  "<|LOC_735|>": 101032,
+  "<|LOC_736|>": 101033,
+  "<|LOC_737|>": 101034,
+  "<|LOC_738|>": 101035,
+  "<|LOC_739|>": 101036,
+  "<|LOC_73|>": 100370,
+  "<|LOC_740|>": 101037,
+  "<|LOC_741|>": 101038,
+  "<|LOC_742|>": 101039,
+  "<|LOC_743|>": 101040,
+  "<|LOC_744|>": 101041,
+  "<|LOC_745|>": 101042,
+  "<|LOC_746|>": 101043,
+  "<|LOC_747|>": 101044,
+  "<|LOC_748|>": 101045,
+  "<|LOC_749|>": 101046,
+  "<|LOC_74|>": 100371,
+  "<|LOC_750|>": 101047,
+  "<|LOC_751|>": 101048,
+  "<|LOC_752|>": 101049,
+  "<|LOC_753|>": 101050,
+  "<|LOC_754|>": 101051,
+  "<|LOC_755|>": 101052,
+  "<|LOC_756|>": 101053,
+  "<|LOC_757|>": 101054,
+  "<|LOC_758|>": 101055,
+  "<|LOC_759|>": 101056,
+  "<|LOC_75|>": 100372,
+  "<|LOC_760|>": 101057,
+  "<|LOC_761|>": 101058,
+  "<|LOC_762|>": 101059,
+  "<|LOC_763|>": 101060,
+  "<|LOC_764|>": 101061,
+  "<|LOC_765|>": 101062,
+  "<|LOC_766|>": 101063,
+  "<|LOC_767|>": 101064,
+  "<|LOC_768|>": 101065,
+  "<|LOC_769|>": 101066,
+  "<|LOC_76|>": 100373,
+  "<|LOC_770|>": 101067,
+  "<|LOC_771|>": 101068,
+  "<|LOC_772|>": 101069,
+  "<|LOC_773|>": 101070,
+  "<|LOC_774|>": 101071,
+  "<|LOC_775|>": 101072,
+  "<|LOC_776|>": 101073,
+  "<|LOC_777|>": 101074,
+  "<|LOC_778|>": 101075,
+  "<|LOC_779|>": 101076,
+  "<|LOC_77|>": 100374,
+  "<|LOC_780|>": 101077,
+  "<|LOC_781|>": 101078,
+  "<|LOC_782|>": 101079,
+  "<|LOC_783|>": 101080,
+  "<|LOC_784|>": 101081,
+  "<|LOC_785|>": 101082,
+  "<|LOC_786|>": 101083,
+  "<|LOC_787|>": 101084,
+  "<|LOC_788|>": 101085,
+  "<|LOC_789|>": 101086,
+  "<|LOC_78|>": 100375,
+  "<|LOC_790|>": 101087,
+  "<|LOC_791|>": 101088,
+  "<|LOC_792|>": 101089,
+  "<|LOC_793|>": 101090,
+  "<|LOC_794|>": 101091,
+  "<|LOC_795|>": 101092,
+  "<|LOC_796|>": 101093,
+  "<|LOC_797|>": 101094,
+  "<|LOC_798|>": 101095,
+  "<|LOC_799|>": 101096,
+  "<|LOC_79|>": 100376,
+  "<|LOC_7|>": 100304,
+  "<|LOC_800|>": 101097,
+  "<|LOC_801|>": 101098,
+  "<|LOC_802|>": 101099,
+  "<|LOC_803|>": 101100,
+  "<|LOC_804|>": 101101,
+  "<|LOC_805|>": 101102,
+  "<|LOC_806|>": 101103,
+  "<|LOC_807|>": 101104,
+  "<|LOC_808|>": 101105,
+  "<|LOC_809|>": 101106,
+  "<|LOC_80|>": 100377,
+  "<|LOC_810|>": 101107,
+  "<|LOC_811|>": 101108,
+  "<|LOC_812|>": 101109,
+  "<|LOC_813|>": 101110,
+  "<|LOC_814|>": 101111,
+  "<|LOC_815|>": 101112,
+  "<|LOC_816|>": 101113,
+  "<|LOC_817|>": 101114,
+  "<|LOC_818|>": 101115,
+  "<|LOC_819|>": 101116,
+  "<|LOC_81|>": 100378,
+  "<|LOC_820|>": 101117,
+  "<|LOC_821|>": 101118,
+  "<|LOC_822|>": 101119,
+  "<|LOC_823|>": 101120,
+  "<|LOC_824|>": 101121,
+  "<|LOC_825|>": 101122,
+  "<|LOC_826|>": 101123,
+  "<|LOC_827|>": 101124,
+  "<|LOC_828|>": 101125,
+  "<|LOC_829|>": 101126,
+  "<|LOC_82|>": 100379,
+  "<|LOC_830|>": 101127,
+  "<|LOC_831|>": 101128,
+  "<|LOC_832|>": 101129,
+  "<|LOC_833|>": 101130,
+  "<|LOC_834|>": 101131,
+  "<|LOC_835|>": 101132,
+  "<|LOC_836|>": 101133,
+  "<|LOC_837|>": 101134,
+  "<|LOC_838|>": 101135,
+  "<|LOC_839|>": 101136,
+  "<|LOC_83|>": 100380,
+  "<|LOC_840|>": 101137,
+  "<|LOC_841|>": 101138,
+  "<|LOC_842|>": 101139,
+  "<|LOC_843|>": 101140,
+  "<|LOC_844|>": 101141,
+  "<|LOC_845|>": 101142,
+  "<|LOC_846|>": 101143,
+  "<|LOC_847|>": 101144,
+  "<|LOC_848|>": 101145,
+  "<|LOC_849|>": 101146,
+  "<|LOC_84|>": 100381,
+  "<|LOC_850|>": 101147,
+  "<|LOC_851|>": 101148,
+  "<|LOC_852|>": 101149,
+  "<|LOC_853|>": 101150,
+  "<|LOC_854|>": 101151,
+  "<|LOC_855|>": 101152,
+  "<|LOC_856|>": 101153,
+  "<|LOC_857|>": 101154,
+  "<|LOC_858|>": 101155,
+  "<|LOC_859|>": 101156,
+  "<|LOC_85|>": 100382,
+  "<|LOC_860|>": 101157,
+  "<|LOC_861|>": 101158,
+  "<|LOC_862|>": 101159,
+  "<|LOC_863|>": 101160,
+  "<|LOC_864|>": 101161,
+  "<|LOC_865|>": 101162,
+  "<|LOC_866|>": 101163,
+  "<|LOC_867|>": 101164,
+  "<|LOC_868|>": 101165,
+  "<|LOC_869|>": 101166,
+  "<|LOC_86|>": 100383,
+  "<|LOC_870|>": 101167,
+  "<|LOC_871|>": 101168,
+  "<|LOC_872|>": 101169,
+  "<|LOC_873|>": 101170,
+  "<|LOC_874|>": 101171,
+  "<|LOC_875|>": 101172,
+  "<|LOC_876|>": 101173,
+  "<|LOC_877|>": 101174,
+  "<|LOC_878|>": 101175,
+  "<|LOC_879|>": 101176,
+  "<|LOC_87|>": 100384,
+  "<|LOC_880|>": 101177,
+  "<|LOC_881|>": 101178,
+  "<|LOC_882|>": 101179,
+  "<|LOC_883|>": 101180,
+  "<|LOC_884|>": 101181,
+  "<|LOC_885|>": 101182,
+  "<|LOC_886|>": 101183,
+  "<|LOC_887|>": 101184,
+  "<|LOC_888|>": 101185,
+  "<|LOC_889|>": 101186,
+  "<|LOC_88|>": 100385,
+  "<|LOC_890|>": 101187,
+  "<|LOC_891|>": 101188,
+  "<|LOC_892|>": 101189,
+  "<|LOC_893|>": 101190,
+  "<|LOC_894|>": 101191,
+  "<|LOC_895|>": 101192,
+  "<|LOC_896|>": 101193,
+  "<|LOC_897|>": 101194,
+  "<|LOC_898|>": 101195,
+  "<|LOC_899|>": 101196,
+  "<|LOC_89|>": 100386,
+  "<|LOC_8|>": 100305,
+  "<|LOC_900|>": 101197,
+  "<|LOC_901|>": 101198,
+  "<|LOC_902|>": 101199,
+  "<|LOC_903|>": 101200,
+  "<|LOC_904|>": 101201,
+  "<|LOC_905|>": 101202,
+  "<|LOC_906|>": 101203,
+  "<|LOC_907|>": 101204,
+  "<|LOC_908|>": 101205,
+  "<|LOC_909|>": 101206,
+  "<|LOC_90|>": 100387,
+  "<|LOC_910|>": 101207,
+  "<|LOC_911|>": 101208,
+  "<|LOC_912|>": 101209,
+  "<|LOC_913|>": 101210,
+  "<|LOC_914|>": 101211,
+  "<|LOC_915|>": 101212,
+  "<|LOC_916|>": 101213,
+  "<|LOC_917|>": 101214,
+  "<|LOC_918|>": 101215,
+  "<|LOC_919|>": 101216,
+  "<|LOC_91|>": 100388,
+  "<|LOC_920|>": 101217,
+  "<|LOC_921|>": 101218,
+  "<|LOC_922|>": 101219,
+  "<|LOC_923|>": 101220,
+  "<|LOC_924|>": 101221,
+  "<|LOC_925|>": 101222,
+  "<|LOC_926|>": 101223,
+  "<|LOC_927|>": 101224,
+  "<|LOC_928|>": 101225,
+  "<|LOC_929|>": 101226,
+  "<|LOC_92|>": 100389,
+  "<|LOC_930|>": 101227,
+  "<|LOC_931|>": 101228,
+  "<|LOC_932|>": 101229,
+  "<|LOC_933|>": 101230,
+  "<|LOC_934|>": 101231,
+  "<|LOC_935|>": 101232,
+  "<|LOC_936|>": 101233,
+  "<|LOC_937|>": 101234,
+  "<|LOC_938|>": 101235,
+  "<|LOC_939|>": 101236,
+  "<|LOC_93|>": 100390,
+  "<|LOC_940|>": 101237,
+  "<|LOC_941|>": 101238,
+  "<|LOC_942|>": 101239,
+  "<|LOC_943|>": 101240,
+  "<|LOC_944|>": 101241,
+  "<|LOC_945|>": 101242,
+  "<|LOC_946|>": 101243,
+  "<|LOC_947|>": 101244,
+  "<|LOC_948|>": 101245,
+  "<|LOC_949|>": 101246,
+  "<|LOC_94|>": 100391,
+  "<|LOC_950|>": 101247,
+  "<|LOC_951|>": 101248,
+  "<|LOC_952|>": 101249,
+  "<|LOC_953|>": 101250,
+  "<|LOC_954|>": 101251,
+  "<|LOC_955|>": 101252,
+  "<|LOC_956|>": 101253,
+  "<|LOC_957|>": 101254,
+  "<|LOC_958|>": 101255,
+  "<|LOC_959|>": 101256,
+  "<|LOC_95|>": 100392,
+  "<|LOC_960|>": 101257,
+  "<|LOC_961|>": 101258,
+  "<|LOC_962|>": 101259,
+  "<|LOC_963|>": 101260,
+  "<|LOC_964|>": 101261,
+  "<|LOC_965|>": 101262,
+  "<|LOC_966|>": 101263,
+  "<|LOC_967|>": 101264,
+  "<|LOC_968|>": 101265,
+  "<|LOC_969|>": 101266,
+  "<|LOC_96|>": 100393,
+  "<|LOC_970|>": 101267,
+  "<|LOC_971|>": 101268,
+  "<|LOC_972|>": 101269,
+  "<|LOC_973|>": 101270,
+  "<|LOC_974|>": 101271,
+  "<|LOC_975|>": 101272,
+  "<|LOC_976|>": 101273,
+  "<|LOC_977|>": 101274,
+  "<|LOC_978|>": 101275,
+  "<|LOC_979|>": 101276,
+  "<|LOC_97|>": 100394,
+  "<|LOC_980|>": 101277,
+  "<|LOC_981|>": 101278,
+  "<|LOC_982|>": 101279,
+  "<|LOC_983|>": 101280,
+  "<|LOC_984|>": 101281,
+  "<|LOC_985|>": 101282,
+  "<|LOC_986|>": 101283,
+  "<|LOC_987|>": 101284,
+  "<|LOC_988|>": 101285,
+  "<|LOC_989|>": 101286,
+  "<|LOC_98|>": 100395,
+  "<|LOC_990|>": 101287,
+  "<|LOC_991|>": 101288,
+  "<|LOC_992|>": 101289,
+  "<|LOC_993|>": 101290,
+  "<|LOC_994|>": 101291,
+  "<|LOC_995|>": 101292,
+  "<|LOC_996|>": 101293,
+  "<|LOC_997|>": 101294,
+  "<|LOC_998|>": 101295,
+  "<|LOC_999|>": 101296,
+  "<|LOC_99|>": 100396,
+  "<|LOC_9|>": 100306,
+  "<|LOC_BEGIN|>": 101298,
+  "<|LOC_END|>": 101299,
+  "<|LOC_SEP|>": 101300,
+  "<|VIDEO_END|>": 101307,
+  "<|VIDEO_START|>": 101306
+}

chat_template.jinja

@@ -0,0 +1,201 @@
+{%- if chat_template_kwargs is defined and chat_template_kwargs.options is defined -%}
+    {%- set options = chat_template_kwargs.options -%}
+{%- endif -%}
+{#- 定义 options.thinking_mode 的默认值 -#}
+{%- if options is not defined -%}
+    {%- set options = {'thinking_mode': true} -%}
+{%- endif -%}
+{%- set thinking_enabled = options.get('thinking_mode', true) in ['open', 'true', true] -%}
+{%- set image_count = namespace(value=0) -%}
+{%- set video_count = namespace(value=0) -%}
+{% macro render_content(content_list, accumulate=True, role="user") %}
+  {%- for content_item in content_list -%}
+    {%- if content_item.type == 'text' -%}
+      {{- content_item.text }}
+    {%- elif content_item.type == 'image_url' -%}
+      {%- if accumulate -%}
+        {%- set image_count.value = image_count.value + 1 -%}
+      {%- endif -%}
+      {{ ' ' }}Picture{{ ' ' ~ image_count.value if accumulate else '' }}:<|IMAGE_START|><|image@placeholder|><|IMAGE_END|>
+    {%- elif content_item.type == 'video_url' -%}
+      {%- if accumulate -%}
+        {%- set video_count.value = video_count.value + 1 -%}
+      {%- endif -%}
+      {{ ' ' }}Video{{ ' ' ~ video_count.value if accumulate else '' }}:<|VIDEO_START|><|video@placeholder|><|VIDEO_END|>
+      {%- if content_item.video_url.subtitles is defined and content_item.video_url.subtitles -%}
+        {{ ' ' }}<Start of Video ASR>: {%- for subtitle in content_item.video_url.subtitles -%}
+          [{{ "%.1f"|format(subtitle[1]) }},{{ "%.1f"|format(subtitle[2]) }}]{{ subtitle[0] }}
+        {%- endfor -%} <End of Video ASR>{{ ' ' }}
+      {%- endif -%}
+    {%- endif -%}
+  {%- endfor -%}
+{% endmacro %}
+{#- ---- 定义 message 渲染 ---- -#}
+{%- macro build_messages(messages) -%}
+    {#- ---- 初始化 多模态计数器 ---- -#}
+    {%- for message in messages -%}
+        {%- if message.content is string -%}
+            {%- set content = message.content -%}
+        {%- elif message.content is iterable -%}
+            {%- set content = render_content(message.content, True, message.role) -%}
+        {%- else -%}
+            {%- set content = '' -%}
+        {%- endif -%}
+        {%- if (message.role == "user") -%}
+            {{- 'User: ' + content-}}
+        {%- elif (message.role == "system" and not loop.first) -%}
+            {{- content + '
+' -}}
+        {%- elif message.role == "assistant" -%}
+            {{- '
+Assistant: ' -}}
+            {%- set reasoning_content = '' -%}
+            {%- if message.reasoning_content is defined and message.reasoning_content is string -%}
+                {%- set reasoning_content = message.reasoning_content -%}
+            {%- else -%}
+                {%- if '</think>' in content -%}
+                    {%- set reasoning_content = content.split('</think>')[0].rstrip('
+').split('<think>')[-1].lstrip('
+') -%}
+                    {%- set content = content.split('</think>')[-1].lstrip('
+') -%}
+                {%- endif -%}
+            {%- endif -%}
+            {%- if loop.last or (not loop.last and reasoning_content) -%}
+                {{- '
+' + '<think>
+' -}}
+                {{- reasoning_content.strip('
+') if options is defined and thinking_enabled else '' -}}
+                {{- '
+</think>
+
+' -}}
+            {%- endif -%}
+            {%- if content|length > 0 -%}
+                {{- content -}}
+            {%- endif -%}
+            {%- if message.tool_calls -%}
+                {%- for tool_call in message.tool_calls -%}
+                    {%- if (not loop.first) -%}
+                        {{- '
+' -}}
+                    {%- endif -%}
+                    {%- if tool_call.function -%}
+                        {%- set tool_call = tool_call.function -%}
+                    {%- endif -%}
+                    {{- '<tool_call>
+{"name": "' -}}
+                    {{- tool_call.name -}}
+                    {{- '", "arguments": ' -}}
+                    {%- if tool_call.arguments is string -%}
+                        {{- tool_call.arguments -}}
+                    {%- else -%}
+                        {{- tool_call.arguments | tojson -}}
+                    {%- endif -%}
+                    {{- '}
+</tool_call>
+' -}}
+                {%- endfor -%}
+            {%- endif -%}
+            {{- '<|end_of_sentence|>' }}
+        {%- elif message.role == "tool" -%}
+            {%- if loop.first or (messages[loop.index0 - 1].role != "tool") -%}
+                {{- 'User: ' -}}
+            {%- endif -%}
+            {{- '
+<tool_output>
+' -}}
+            {{- content -}}
+            {{- '
+</tool_output>
+' -}}
+        {%- endif -%}
+    {%- endfor -%}
+{%- endmacro -%}
+{%- if not add_generation_prompt is defined -%}
+    {%- set add_generation_prompt = true -%}
+{%- endif -%}
+{{- '<|begin_of_sentence|>' -}}
+{%- if messages[0].role == 'system' -%}
+    {%- if messages and messages[0].role == 'system' -%}
+        {%- if messages[0].content is string -%}
+            {{- messages[0].content -}}
+        {%- elif messages[0].content is iterable -%}
+            {%- set sys_content = render_content(messages[0].content) %}
+            {{- sys_content -}}
+        {%- endif -%}
+    {%- endif -%}
+    {{- '
+' -}}
+{%- else -%}
+    {{- 'You are a multimodal AI assistant called ERNIE developed by Baidu based on the PaddlePaddle framework.
+' -}}
+{%- endif -%}
+{%- if options is defined and options.parallel_tool_calls is defined and (options.parallel_tool_calls == "true" or options.parallel_tool_calls == True) -%}
+    {{- '
+parallel_tool_calls=True
+' -}}
+{%- endif -%}
+{%- if tools -%}
+    {{- "
+<tool_list>" -}}
+    {{- '
+' -}}
+    {{- '[' -}}
+    {%- for tool in tools -%}
+        {{- '{"type": "function", "function": ' -}}
+        {{- (tool.function | tojson) -}}
+        {{- '}' -}}
+        {%- if not loop.last -%}
+            {{- ', ' -}}
+        {%- endif -%}
+    {%- endfor -%}
+    {{- ']' -}}
+    {{- "
+</tool_list>" -}}
+    {{- '
+' -}}
+{%- endif -%}
+{{- build_messages(messages) -}}
+{%- if add_generation_prompt -%}
+    {%- set append_think_label=False -%}
+    {%- if not thinking_enabled -%}
+         {%- set append_think_label=True -%}
+    {%- endif -%}
+    {{- "
+Assistant: 
+<think>
+" -}}
+    {%- if options is defined and options.tool_choice is defined -%}
+        {%- if options.tool_choice.mode == "required" -%}
+            {{- '系统要求我必须使用一个或多个工具，注意要认真填写参数，若必填参数存在信息缺失，需做出合理的假设，不可询问用户。' -}}
+            {%- if not thinking_enabled -%}
+                {{- '
+</think>
+
+<tool_call>
+' -}}
+                {%- set append_think_label=False -%}
+            {%- else -%}
+                {{- '现在开始分析用户需求，' -}}
+            {%- endif -%}
+        {%- endif -%}
+        {%- if options.tool_choice.mode == "force" -%}
+            {{- "系统指定必须使用" -}}
+            {{- options.tool_choice.name -}}
+            {{- '工具，因此我尝试填写合适的参数满足用户需求。
+</think>
+
+<tool_call>
+{"name": "' -}}
+            {{- options.tool_choice.name -}}
+            {{- '", "arguments":' -}}
+            {%- set append_think_label=False -%}
+        {%- endif -%}
+    {%- endif -%}
+    {{- "
+</think>
+
+" if append_think_label else '' -}}
+{%- endif -%}

config.json

@@ -0,0 +1,98 @@
+{
+    "architectures": [
+        "Ernie4_5_VLMoeForConditionalGeneration"
+    ],
+    "auto_map": {
+        "AutoConfig": "configuration_ernie4_5_vl.Ernie4_5_VLMoEConfig",
+        "AutoModel": "modeling_ernie4_5_vl.Ernie4_5_VLMoeForConditionalGeneration",
+        "AutoModelForCausalLM": "modeling_ernie4_5_vl.Ernie4_5_VLMoeForConditionalGeneration"
+    },
+    "bos_token_id": 1,
+    "eos_token_id": 2,
+    "hidden_act": "silu",
+    "hidden_size": 2560,
+    "im_patch_id": 100295,
+    "image_end_token_id": 101305,
+    "image_start_token_id": 101304,
+    "intermediate_size": 12288,
+    "max_position_embeddings": 131072,
+    "model_type": "ernie4_5_moe_vl",
+    "moe_capacity": [
+        128,
+        128,
+        128
+    ],
+    "moe_gate": "topk",
+    "moe_intermediate_size": [
+        1536,
+        512
+    ],
+    "moe_k": 6,
+    "moe_layer_end_index": [
+        29,
+        28
+    ],
+    "moe_layer_interval": 1,
+    "moe_layer_start_index": [
+        1,
+        1
+    ],
+    "moe_multimodal_dispatch_use_allgather": "v2-alltoall-unpad-text",
+    "moe_num_experts": [
+        64,
+        64
+    ],
+    "moe_num_shared_experts": 2,
+    "moe_use_aux_free": true,
+    "num_attention_heads": 20,
+    "num_hidden_layers": 28,
+    "num_key_value_heads": 4,
+    "pad_token_id": 0,
+    "pixel_hidden_size": 1280,
+    "quantization": {
+        "group_size": 64,
+        "bits": 8,
+        "mode": "affine"
+    },
+    "quantization_config": {
+        "group_size": 64,
+        "bits": 8,
+        "mode": "affine"
+    },
+    "rms_norm_eps": 1e-05,
+    "rope_3d": true,
+    "rope_scaling": {
+        "type": "default",
+        "mrope_section": [
+            22,
+            22,
+            20
+        ]
+    },
+    "rope_theta": 500000,
+    "spatial_conv_size": 2,
+    "temporal_conv_size": 2,
+    "tie_word_embeddings": true,
+    "use_bias": false,
+    "use_cache": true,
+    "use_rmsnorm": true,
+    "video_end_token_id": 101307,
+    "video_start_token_id": 101306,
+    "vision_config": {
+        "attn_implementation": "eager",
+        "depth": 32,
+        "embed_dim": 1280,
+        "hidden_act": "quick_gelu",
+        "hidden_size": 1280,
+        "in_channels": 3,
+        "in_chans": 3,
+        "mlp_ratio": 4,
+        "num_heads": 16,
+        "patch_size": 14,
+        "spatial_merge_size": 2,
+        "spatial_patch_size": 14,
+        "vit_first_fwd_bsz": 128,
+        "attn_sep": true
+    },
+    "vocab_size": 103424
+}

configuration_ernie4_5_vl.py

@@ -0,0 +1,658 @@
+# Copyright (c) 2025 Baidu, Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Ernie model configuration"""
+import copy
+
+from typing import List, Optional, Tuple, Union
+
+from transformers import PretrainedConfig
+
+
+__all__ = [
+    "ERNIE_PRETRAINED_INIT_CONFIGURATION",
+    "Ernie4_5_Config",
+    "Ernie4_5_MoEConfig",
+    "Ernie4_5_VLMoEConfig",
+]
+
+
+class DFNRopeVisionTransformerConfig(PretrainedConfig):
+    """
+    Configuration class for DFNRopeVisionTransformer model.
+    This class inherits from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+    documentation from [`PretrainedConfig`] for more information.
+    """
+
+    model_type = "DFNRope_vision_transformer"
+    base_model_tp_plan = {}
+
+    def __init__(
+        self,
+        depth=32,
+        embed_dim=1280,
+        hidden_size=3584,
+        hidden_act="quick_gelu",
+        mlp_ratio=4,
+        num_heads=16,
+        in_channels=3,
+        patch_size=14,
+        spatial_merge_size=2,
+        attn_implementation="eager",  # new added
+        pp_data_balance=False,
+        recompute=False,
+        attn_sep=False,
+        vit_first_fwd_bsz=128,
+        vit_num_recompute_layers=10000,
+        **kwargs,
+    ):
+        """
+        Initialize DFNRopeVisionTransformer model configuration with default or specified parameters.
+
+        Args:
+            depth (int): Number of transformer layers in the model.
+            embed_dim (int): Dimensionality of the embedding layer.
+            hidden_size (int): Dimensionality of the feedforward network.
+            hidden_act (str): Activation function for the feedforward network.
+            mlp_ratio (float): Ratio between the number of input features and
+                the number of output features in the feedforward network.
+            num_heads (int): Number of attention heads in each attention layer.
+            in_channels (int): Number of channels in the input image.
+            patch_size (int):
+                Size of patches in the input image. Defaults to 14.
+            spatial_merge_size (int):
+                Spatial merge size for the spatial transformer module. Defaults to 2.
+            attn_implementation (str): Attention implementation type. Defaults to "eager".
+            pp_data_balance (bool): Whether to balance data during preprocessing. Defaults to False.
+            recompute (bool): Whether to use recompute. Defaults to False.
+            attn_sep (bool): Whether to separate attention computation into two stages. Defaults to False.
+            vit_first_fwd_bsz (int): First forward batch size for ViT. Defaults to 128.
+            vit_num_recompute_layers (int): Number of recomputed layers for ViT. Defaults to
+        """
+        super().__init__(**kwargs)
+
+        self.depth = depth
+        self.embed_dim = embed_dim
+        self.hidden_size = hidden_size
+        self.hidden_act = hidden_act
+        self.mlp_ratio = mlp_ratio
+        self.num_heads = num_heads
+        self.in_channels = in_channels
+        self.patch_size = patch_size
+        self.spatial_merge_size = spatial_merge_size
+        self.attn_implementation = attn_implementation
+        self.pp_data_balance = pp_data_balance
+        self.recompute = recompute
+        self.attn_sep = attn_sep
+        self.vit_first_fwd_bsz = vit_first_fwd_bsz
+        self.vit_num_recompute_layers = vit_num_recompute_layers
+
+    def get(self, key, default=None):
+        """get config value by key"""
+        if hasattr(self, key):
+            return getattr(self, key)
+        else:
+            return default
+
+
+ERNIE_PRETRAINED_INIT_CONFIGURATION = {
+    "ernie/tiny-random-ernie": {
+        "hidden_size": 768,
+        "initializer_range": 0.02,
+        "intermediate_size": 11008,
+        "max_position_embeddings": 2048,
+        "model_type": "ernie",
+        "num_attention_heads": 2,
+        "num_hidden_layers": 2,
+        "rms_norm_eps": 1e-06,
+        "vocab_size": 32000,
+        "bos_token_id": 1,
+        "eos_token_id": 2,
+        "pad_token_id": 0,
+        "use_cache": False,
+        "recompute": False,
+        "use_flash_attn": True,
+        "use_pure_fp16": False,
+    },
+}
+
+
+class Ernie4_5_Config(PretrainedConfig):
+    """
+    Configuration class for ERNIE model.
+
+    This class stores the configuration of an ERNIE model, defining the model architecture.
+    It inherits from PretrainedConfig and can be used to control model outputs.
+    """
+
+    model_type = "ernie"
+    pretrained_init_configuration = ERNIE_PRETRAINED_INIT_CONFIGURATION
+    base_model_tp_plan = {}
+
+    def __init__(
+        self,
+        vocab_size=32000,
+        hidden_size=768,
+        intermediate_size=11008,
+        max_position_embeddings=32768,
+        num_hidden_layers=2,
+        num_attention_heads=2,
+        initializer_range=0.02,  # no use
+        rms_norm_eps=1e-6,
+        use_cache=False,
+        use_flash_attention=True,
+        use_sparse_flash_attn=True,
+        use_var_len_flash_attn=False,
+        recompute=False,
+        recompute_granularity="core_attn",
+        recompute_use_reentrant=False,
+        use_rmsnorm=True,
+        fuse_rms_norm=False,
+        fuse_ln=False,
+        pad_token_id=0,
+        bos_token_id=1,
+        eos_token_id=2,
+        fuse_swiglu=False,
+        use_bias=False,
+        rope_theta=10000,
+        fuse_rope=False,
+        fuse_softmax_mask=False,
+        use_fast_ln=False,
+        weight_share_add_bias=True,
+        fuse_linear=False,
+        max_sequence_length=None,
+        ignored_index=-100,
+        add_tail_layers=False,
+        use_recompute_lm_head=False,
+        use_recompute_loss_fn=False,
+        refined_recompute=dict(),
+        attention_probs_dropout_prob=0.0,
+        hidden_dropout_prob=0.0,
+        compression_ratio: float = 1.0,
+        num_key_value_heads=None,
+        use_sparse_head_and_loss_fn=False,
+        micro_batch_size=-1,
+        use_ep_comm_overlap=False,
+        use_fused_head_and_loss_fn=False,
+        token_balance_loss=False,
+        token_balance_seqlen=False,  # calculated based on batchsize and seqlen
+        cachekv_quant: bool = False,
+        pp_seg_method="layer:ErnieDecoderLayer|EmptyLayer",
+        **kwargs,
+    ):
+        """
+        Initialize ERNIE model configuration with default or specified parameters.
+
+        Args:
+            vocab_size (int): Size of the vocabulary (number of unique tokens)
+            hidden_size (int): Dimensionality of the encoder layers and the pooler layer
+            intermediate_size (int): Dimensionality of the "intermediate" (feed-forward) layer
+            max_position_embeddings (int): Maximum sequence length the model can handle
+            num_hidden_layers (int): Number of hidden layers in the Transformer encoder
+            num_attention_heads (int): Number of attention heads for each attention layer
+            rms_norm_eps (float): The epsilon used by the RMS normalization layers
+            use_cache (bool): Whether to use caching for faster generation (decoding)
+            use_flash_attention (bool): Whether to use FlashAttention for optimized attention computation
+            use_sparse_flash_attn (bool): Whether to use sparse FlashAttention
+            use_var_len_flash_attn (bool): Whether to use variable-length FlashAttention
+            recompute (bool): Whether to use gradient checkpointing to save memory
+            recompute_granularity (str): Granularity of recomputation ("core_attn", "full", etc.)
+            recompute_use_reentrant (bool): Whether to use reentrant checkpointing
+            use_rmsnorm (bool): Whether to use RMSNorm instead of LayerNorm
+            fuse_rms_norm (bool): Whether to fuse RMSNorm operations for optimization
+            fuse_ln (bool): Whether to fuse LayerNorm operations
+            pad_token_id (int): Token ID used for padding sequences
+            bos_token_id (int): Token ID used for beginning-of-sequence
+            eos_token_id (int): Token ID used for end-of-sequence
+            fuse_swiglu (bool): Whether to fuse SwiGLU operations
+            use_bias (bool): Whether to use bias terms in linear layers
+            rope_theta (float): The base period of the RoPE embeddings
+            fuse_rope (bool): Whether to fuse RoPE operations
+            use_fast_ln (bool): Whether to use optimized LayerNorm implementation
+            weight_share_add_bias (bool): Whether to share bias weights in certain layers
+            fuse_linear (bool): Whether to fuse linear operations
+            max_sequence_length (int): Maximum sequence length for positional embeddings
+            ignored_index (int): Target value that is ignored during loss computation
+            add_tail_layers (bool): Whether to add additional layers at the end
+            use_recompute_lm_head (bool): Whether to recompute gradients for language model head
+            use_recompute_loss_fn (bool): Whether to recompute gradients for loss function
+            refined_recompute (dict): Dictionary specifying refined recomputation settings
+            attention_probs_dropout_prob (float): Dropout probability for attention weights
+            hidden_dropout_prob (float): Dropout probability for hidden layers
+            compression_ratio (float): Ratio for KV cache compression (1.0 = no compression)
+            num_key_value_heads (int): Number of key/value heads (for Grouped Query Attention)
+            use_sparse_head_and_loss_fn (bool): Whether to use sparse attention head and loss function
+            micro_batch_size (int): Size of micro batches (-1 for automatic)
+            use_ep_comm_overlap (bool): Whether to overlap communication with computation
+            use_fused_head_loss_fn (bool): Whether to use fused head and loss function
+            token_balance_loss (bool): Whether to balance loss by token count
+            token_balance_seqlen (bool): Whether to balance sequence lengths
+            cachekv_quant (bool): Whether to quantize key-value cache
+            pp_seg_method (str): Method for pipeline parallel segmentation
+            **kwargs: Additional keyword arguments passed to parent class
+        """
+
+        # Set default for tied embeddings if not specified.
+        if "tie_word_embeddings" not in kwargs:
+            kwargs["tie_word_embeddings"] = False
+        super().__init__(
+            pad_token_id=pad_token_id,
+            bos_token_id=bos_token_id,
+            eos_token_id=eos_token_id,
+            **kwargs,
+        )
+        self.vocab_size = vocab_size
+        self.hidden_size = hidden_size
+        self.intermediate_size = intermediate_size
+        self.max_position_embeddings = max_position_embeddings
+        self.num_hidden_layers = num_hidden_layers
+        self.num_attention_heads = num_attention_heads
+        self.initializer_range = initializer_range
+        self.rms_norm_eps = rms_norm_eps
+        self.use_cache = use_cache
+        self.recompute = recompute
+        self.recompute_granularity = recompute_granularity
+        self.use_flash_attention = use_flash_attention
+        self.use_sparse_flash_attn = use_sparse_flash_attn
+        self.recompute_use_reentrant = recompute_use_reentrant
+        self.use_var_len_flash_attn = use_var_len_flash_attn
+        self.pad_token_id = pad_token_id
+        self.bos_token_id = bos_token_id
+        self.eos_token_id = eos_token_id
+        self.fuse_swiglu = fuse_swiglu
+        self.fuse_rms_norm = fuse_rms_norm
+        self.fuse_ln = fuse_ln
+        self.use_rmsnorm = use_rmsnorm
+        self.micro_batch_size = micro_batch_size
+
+        self.max_sequence_length = max_sequence_length
+        self.use_bias = use_bias
+        self.weight_share_add_bias = weight_share_add_bias
+        self.rope_theta = rope_theta
+        self.fuse_rope = fuse_rope
+        self.fuse_softmax_mask = fuse_softmax_mask
+        self.use_fast_ln = use_fast_ln
+
+        self.fuse_linear = fuse_linear
+        self.ignored_index = ignored_index
+        self.add_tail_layers = add_tail_layers
+        self.use_recompute_lm_head = use_recompute_lm_head
+        self.use_recompute_loss_fn = use_recompute_loss_fn
+
+        self.refined_recompute = refined_recompute
+        self.skip_recompute_ops = dict()
+        """
+            `refined_recompute` is a dictionary that specifies fine-grained gradient recomputation settings,
+            which currently only takes effect in Pipeline Parallel (PP) mode.
+
+            In PP mode, this dictionary populates `self.skip_recompute_ops` with the following structure:
+            - Key (`op_name`): The operation name to configure, with possible values:
+            * "mlp_row_ln" - MLP row-wise layer normalization
+            * "flash_attn" - Flash attention operation
+            * "attention_row_ln" - Attention row-wise layer normalization
+            * "attention_column_ln" - Attention column-wise layer normalization
+            * "mlp_column_ln" - MLP column-wise layer normalization
+
+            - Value (`skip_num`): Controls how many times to skip recomputation:
+            * 0: Never skip recomputation (minimum memory usage)
+            * -1: Always skip recomputation (maximum memory usage)
+            * [0,1,...,12]: Skip recomputation for specified number of times
+            * ≥12: Equivalent to -1 (always skip recomputation)
+
+            This allows precise control over memory/computation tradeoffs for different operations.
+        """
+        self.attention_probs_dropout_prob = attention_probs_dropout_prob
+        self.hidden_dropout_prob = hidden_dropout_prob
+        self.compression_ratio = compression_ratio
+        self.num_key_value_heads = num_key_value_heads
+        self.use_sparse_head_and_loss_fn = use_sparse_head_and_loss_fn
+        self.use_ep_comm_overlap = use_ep_comm_overlap
+        self.use_fused_head_and_loss_fn = use_fused_head_and_loss_fn
+        self.token_balance_loss = token_balance_loss
+        self.token_balance_seqlen = token_balance_seqlen
+        self.cachekv_quant = cachekv_quant
+        self.pp_seg_method = pp_seg_method
+
+    def get(self, key, default=None):
+        """get config value by key"""
+        if hasattr(self, key):
+            return getattr(self, key)
+        else:
+            return default
+
+
+class Ernie4_5_MoEConfig(Ernie4_5_Config):
+    r"""
+    Configuration class for ErnieMoE model architecture.
+
+    This class stores the configuration for a [`~ErnieModel`] and is used to instantiate
+    an ErnieMoE model according to the specified arguments. Inherits from [`PretrainedConfig`]
+    and can control model outputs.
+
+    Attributes:
+        Inherits all attributes from Ernie4_5_Config and adds MoE-specific configurations.
+    """
+
+    model_type = "ernie"
+    attribute_map = {
+        "n_positions": "max_position_embeddings",
+        "n_embd": "hidden_size",
+        "n_layer": "num_hidden_layers",
+        "n_head": "num_attention_heads",
+        "n_inner": "intermediate_size",
+        "activation_function": "hidden_act",
+    }
+    pretrained_init_configuration = ERNIE_PRETRAINED_INIT_CONFIGURATION
+    base_model_tp_plan = {}
+
+    def __init__(
+        self,
+        moe_num_experts: Union[int, list] = 0,
+        use_recompute_moe=False,
+        moe_capacity=(),
+        moe_layer_interval=2,
+        moe_layer_start_index=0,
+        moe_layer_end_index=-1,
+        moe_aux_loss_lambda=1e-2,
+        moe_z_loss_lambda=1e-4,
+        moe_orthogonal_loss_lambda=1e-2,
+        sinkhorn_2gate=True,
+        sinkhorn_temp=3e-2,
+        global_aux_loss=False,
+        moe_dropout_prob=0.0,
+        moe_group="world",
+        moe_gate="top2",
+        moe_intermediate_size: Union[int, list] = 0,
+        moe_num_shared_experts: int = 0,
+        moe_reverse_token_drop: bool = False,
+        moe_gate_act: str = "softmax",
+        moe_norm_gate_logits=True,
+        moe_all_to_all_dropout: float = 0.0,
+        moe_k=2,
+        moe_use_aux_free: bool = False,
+        # `moe_group_experts` must be used with `moe_use_hard_gate=True`
+        moe_group_experts: bool = False,
+        moe_group_orthogonal_loss: bool = True,
+        enable_delay_scale_loss: bool = True,
+        num_acc_steps: int = 1,
+        fuse_gate_detach_matmul: bool = False,
+        dpo_config=None,
+        moe_multimodal_dispatch_use_allgather: str = "",
+        moe_use_hard_gate=False,
+        moe_dense_experts_token_type_id=3,
+        **kwargs,
+    ):
+        """
+        Initialize ErnieMoE configuration with MoE-specific parameters.
+
+        Args:
+            moe_num_experts: Number of experts in MoE layers
+            use_recompute_moe: Whether to use recomputation for MoE layers
+            moe_capacity: Capacity configuration for MoE layers
+            moe_layer_interval: Interval between MoE layers
+            moe_layer_start_index: Starting layer index for MoE
+            moe_layer_end_index: Ending layer index for MoE (-1 means last layer)
+            moe_aux_loss_lambda: Weight for auxiliary loss
+            moe_z_loss_lambda: Weight for z-loss
+            moe_orthogonal_loss_lambda: Weight for orthogonal loss
+            sinkhorn_2gate: Whether to use sinkhorn 2-gate routing
+            sinkhorn_temp: Temperature for sinkhorn routing
+            global_aux_loss: Whether to use global auxiliary loss
+            moe_dropout_prob: Dropout probability for MoE layers
+            moe_group: Group configuration for MoE experts
+            moe_gate: Type of gating mechanism ('top2', etc.)
+            moe_intermediate_size: Intermediate size for MoE layers
+            moe_num_shared_experts: Number of shared experts
+            moe_reverse_token_drop: Whether to use reverse token dropping
+            moe_gate_act: Activation function for gating
+            moe_norm_gate_logits: Whether to normalize gate logits
+            moe_all_to_all_dropout: Dropout for all-to-all communication
+            moe_k: Number of experts to route to
+            moe_use_aux_free: Whether to use auxiliary-free routing
+            moe_group_experts: Whether to group experts (requires hard gating)
+            moe_group_orthogonal_loss: Whether to use group orthogonal loss
+            enable_delay_scale_loss: Whether to enable delayed loss scaling
+            num_acc_steps: Number of accumulation steps
+            fuse_gate_detach_matmul: Whether to fuse gate detach matmul
+            **kwargs: Additional base model configuration parameters
+
+        Note:
+            When use_recompute_moe is True, recompute_granularity will be changed to full_attn.
+        """
+
+        if use_recompute_moe:
+            logger.warning(
+                "set `use_recompute_moe`=True, disabling `recompute_granularity=full`, change to full_attn."
+            )
+            if kwargs["recompute"] and kwargs["recompute_granularity"] == "full":
+                kwargs["recompute_granularity"] = "full_attn"
+        super().__init__(**kwargs)
+
+        self.moe_num_experts = moe_num_experts
+        self.use_recompute_moe = use_recompute_moe
+        self.moe_capacity = moe_capacity
+        self.moe_aux_loss_lambda = moe_aux_loss_lambda
+        self.moe_z_loss_lambda = moe_z_loss_lambda
+        self.moe_orthogonal_loss_lambda = moe_orthogonal_loss_lambda
+        self.global_aux_loss = global_aux_loss
+        self.sinkhorn_2gate = sinkhorn_2gate
+        self.sinkhorn_temp = sinkhorn_temp
+        self.moe_layer_interval = moe_layer_interval
+        self.moe_dropout_prob = moe_dropout_prob
+        self.moe_group = moe_group
+        self.moe_gate = moe_gate
+        self.moe_intermediate_size = moe_intermediate_size
+        self.moe_num_shared_experts = moe_num_shared_experts
+        self.moe_reverse_token_drop = moe_reverse_token_drop
+        self.moe_k = moe_k
+        self.moe_all_to_all_dropout = moe_all_to_all_dropout
+        self.moe_group_experts = moe_group_experts
+        self.moe_group_orthogonal_loss = moe_group_orthogonal_loss
+        self.enable_delay_scale_loss = enable_delay_scale_loss
+        self.num_acc_steps = num_acc_steps
+        self.moe_layer_start_index = moe_layer_start_index
+        self.moe_layer_end_index = (
+            self.num_hidden_layers - 1
+            if moe_layer_end_index == -1
+            else moe_layer_end_index
+        )
+        self.moe_gate_act = moe_gate_act
+        self.moe_norm_gate_logits = moe_norm_gate_logits
+        self.moe_use_aux_free = moe_use_aux_free
+        self.fuse_gate_detach_matmul = fuse_gate_detach_matmul
+        self.dpo_config = dpo_config
+        self.moe_multimodal_dispatch_use_allgather = (
+            moe_multimodal_dispatch_use_allgather
+        )
+        self.moe_use_hard_gate = moe_use_hard_gate
+        self.moe_dense_experts_token_type_id = moe_dense_experts_token_type_id
+
+    @property
+    def multimodel_experts(self) -> bool:
+        """multimodel experts."""
+        return (
+            isinstance(self.moe_num_experts, (tuple, list))
+            and len(self.moe_num_experts) > 1
+        )
+
+    @property
+    def use_moe(self) -> bool:
+        """
+        Check if model is using MoE architecture.
+
+        Returns:
+            bool: True if moe_num_experts > 0, False otherwise
+        """
+        return self.moe_num_experts > 0
+
+
+class Ernie4_5_VLMoEConfig(Ernie4_5_MoEConfig):
+    """
+    This is the configuration class to store the configuration of a [`~ErnieModel`]. It is used to instantiate an Ernie
+    model according to the specified arguments, defining the model architecture. Instantiating a configuration with the
+    defaults will yield a similar configuration to that of the Ernie-7B.
+    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+    documentation from [`PretrainedConfig`] for more information.
+    Args:
+        vocab_size (`int`, *optional*, defaults to 32000):
+            Vocabulary size of the Ernie model. Defines the number of different tokens that can be represented by the
+            `inputs_ids` passed when calling [`~ErnieModel`] or [`~TFErnieModel`].
+        hidden_size (`int`, *optional*, defaults to 4096):
+            Dimension of the hidden representations.
+        intermediate_size (`int`, *optional*, defaults to 11008):
+            Dimension of the MLP representations.
+        num_hidden_layers (`int`, *optional*, defaults to 32):
+            Number of hidden layers in the Transformer encoder.
+        num_attention_heads (`int`, *optional*, defaults to 32):
+            Number of attention heads for each attention layer in the Transformer encoder.
+        hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):
+            The non-linear activation function (function or string) in the decoder.
+        initializer_range (`float`, *optional*, defaults to 0.02):
+            The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
+        rms_norm_eps (`float`, *optional*, defaults to 1e-12):
+            The epsilon used by the rms normalization layers.
+        use_cache (`bool`, *optional*, defaults to `True`):
+            Whether or not the model should return the last key/values attentions (not used by all models). Only
+            relevant if `config.is_decoder=True`.
+        tie_word_embeddings(`bool`, *optional*, defaults to `False`):
+            Whether to tie weight embeddings
+    """
+
+    model_type = "ernie4_5_moe_vl"
+    attribute_map = {
+        "n_positions": "max_position_embeddings",
+        "n_embd": "hidden_size",
+        "n_layer": "num_hidden_layers",
+        "n_head": "num_attention_heads",
+        "n_inner": "intermediate_size",
+        "activation_function": "hidden_act",
+    }
+    base_model_tp_plan = {
+        "model.layers.*.self_attn.q_proj": "colwise_rep",
+        "model.layers.*.self_attn.k_proj": "colwise_rep",
+        "model.layers.*.self_attn.v_proj": "colwise_rep",
+        "model.layers.*.self_attn.o_proj": "rowwise_rep",
+        "model.layers.*.mlp.experts.*.gate_proj": "colwise",
+        "model.layers.*.mlp.experts.*.up_proj": "colwise",
+        "model.layers.*.mlp.experts.*.down_proj": "rowwise",
+        "model.layers.*.mlp_text.experts.*.gate_proj": "colwise",
+        "model.layers.*.mlp_text.experts.*.up_proj": "colwise",
+        "model.layers.*.mlp_text.experts.*.down_proj": "rowwise",
+        "model.layers.*.mlp.gate_proj": "colwise",
+        "model.layers.*.mlp.up_proj": "colwise",
+        "model.layers.*.mlp.down_proj": "rowwise"
+    }
+
+    def __init__(
+        self,
+        vision_config=None,
+        im_patch_id=None,
+        image_start_token_id=None,
+        image_end_token_id=None,
+        video_start_token_id=None,
+        video_end_token_id=None,
+        pixel_hidden_size=None,
+        modality_detach=False,
+        temporal_conv_size=2,
+        spatial_conv_size=2,
+        mm_vocab_size=0,  # vocab for mm specialtokens
+        max_text_id=None,
+        use_temporal_conv=True,
+        moe_use_size_all2all=False,
+        moe_num_attn_experts=False,
+        moe_dense_experts_token_type_id: int = 3,
+        moe_use_hard_gate: bool = True,
+        moe_fuse_experts: bool = False,
+        moe_use_token_type_bias: bool = False,
+        disable_ffn_model_parallel=False,
+        fuse_attn_ffn=True,
+        rope_3d=True,
+        freq_allocation=20,
+        using_precision_check=False,
+        use_recompute_resampler=False,
+        resampler_fuse_rms_norm=False,
+        moe_layer_feed_fake_token=False,
+        tensor_parallel_degree=1,
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+        if isinstance(vision_config, dict):
+            self.vision_config = DFNRopeVisionTransformerConfig(**vision_config)
+        else:
+            self.vision_config = DFNRopeVisionTransformerConfig()
+        self.im_patch_id = im_patch_id
+        self.image_start_token_id = image_start_token_id
+        self.image_end_token_id = image_end_token_id
+        self.video_start_token_id = video_start_token_id
+        self.video_end_token_id = video_end_token_id
+        self.pixel_hidden_size = pixel_hidden_size
+        self.modality_detach = modality_detach
+        self.temporal_conv_size = temporal_conv_size
+        self.spatial_conv_size = spatial_conv_size
+        self.mm_vocab_size = mm_vocab_size
+        self.max_text_id = max_text_id
+        self.use_temporal_conv = use_temporal_conv
+
+        self.moe_use_size_all2all = moe_use_size_all2all
+        self.moe_num_attn_experts = moe_num_attn_experts
+        self.moe_dense_experts_token_type_id = moe_dense_experts_token_type_id
+        self.moe_use_hard_gate = moe_use_hard_gate
+        self.moe_fuse_experts = moe_fuse_experts
+        self.moe_use_token_type_bias = moe_use_token_type_bias
+        self.disable_ffn_model_parallel = disable_ffn_model_parallel
+
+        self.fuse_attn_ffn = fuse_attn_ffn
+        self.rope_3d = rope_3d
+        self.freq_allocation = freq_allocation
+        self.using_precision_check = using_precision_check
+        self.use_recompute_resampler = use_recompute_resampler
+        self.resampler_fuse_rms_norm = resampler_fuse_rms_norm
+        self.moe_layer_feed_fake_token = moe_layer_feed_fake_token
+
+        self.tensor_parallel_degree = tensor_parallel_degree
+
+    @property
+    def multimodel_experts(self) -> bool:
+        """Check if model is using more than 1 multimodel experts."""
+        return (
+            isinstance(self.moe_num_experts, (tuple, list))
+            and len(self.moe_num_experts) > 1
+        )
+
+    @property
+    def use_moe(self) -> bool:
+        """
+        Check if model is using MoE architecture.
+
+        Returns:
+            bool: True if moe_num_experts > 0, False otherwise
+        """
+        return (
+            sum(self.moe_num_experts) > 0
+            if self.multimodel_experts
+            else self.moe_num_experts > 0
+        )
+
+    def to_dict(self, saving_file=False):
+        """to_dict"""
+        output = copy.deepcopy(self.__dict__)
+        if self.vision_config:
+            output["vision_config"] = (
+                self.vision_config.to_dict()
+                if isinstance(self.vision_config, (DFNRopeVisionTransformerConfig))
+                else self.vision_config
+            )
+
+        output["model_type"] = self.__class__.model_type
+        return output

generation_config.json

@@ -0,0 +1,10 @@
+{
+    "top_p": 0.95,
+    "temperature": 0.6,
+    "repetition_penalty": 1.0,
+    "frequency_penalty": 0.0,
+    "presence_penalty": 0.0,
+    "pad_token_id": 0,
+    "bos_token_id": 1,
+    "eos_token_id": 2
+}

model-00001-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d69b84ef9c8adf09db46dedc7617a3c5bc5d2e01873fd6beaae7be63deca97cd
+size 5164467451

model-00002-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1249107f5d84314b259ada9f168d2220278569824a9883faba01a6421a389eb6
+size 5338233868

model-00003-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:816778dba4ac3201318ad2763b0dc655780e9b84ac8d3df4923aeb55a79d08dc
+size 5201768676

model-00004-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5d1a47ac5e7768435d69542a49dc8f1466df1f95fcf56697b6c93570206b0bf3
+size 5291072738

model-00005-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3e7184852de6f4d91694a8eb80d0f77a055e7d2d6cc11eefb86fffd7f0c91363
+size 5291072740

model-00006-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d40a03f511bf3e106819a8d259036d7d86db3c66480cc218de918713b5bce17f
+size 5338234102

model-00007-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7b5036abe8ea4e2aedbf1e60f0b234c71014d1a6db2ecf3765d192e6066a3dd2
+size 203343248

preprocessor_config.json

@@ -0,0 +1,35 @@
+{
+  "auto_map": {
+    "AutoProcessor": "processing_ernie4_5_vl.Ernie4_5_VLProcessor",
+    "AutoImageProcessor": "processing_ernie4_5_vl.Ernie4_5_VLImageProcessor"
+  },
+  "crop_size": {
+    "height": 224,
+    "width": 224
+  },
+  "do_center_crop": false,
+  "do_convert_rgb": true,
+  "do_normalize": true,
+  "do_rescale": true,
+  "do_resize": true,
+  "image_mean": [
+    0.48145466,
+    0.4578275,
+    0.40821073
+  ],
+  "image_std": [
+    0.26862954,
+    0.26130258,
+    0.27577711
+  ],
+  "resample": 3,
+  "rescale_factor": 0.00392156862745098,
+  "size": {
+    "height": 224,
+    "width": 224
+  },
+  "min_pixels": 3136,
+  "max_pixels": 4816896,
+  "processor_class": "Ernie4_5_VLProcessor",
+  "image_processor_type": "Ernie4_5_VLImageProcessor"
+}

processing_ernie4_5_vl.py

@@ -0,0 +1,1867 @@
+# Copyright (c) 2025 Baidu, Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Tokenization classes and Image processor class, Processor class for Ernie_45T_VL."""
+
+import copy
+import io
+import os
+import math
+import random
+import requests
+import base64
+import datetime
+import hashlib
+import threading
+import uuid
+import decord
+from shutil import copyfile
+from typing import Any, Dict, List, Optional, Tuple, Union
+
+import numpy as np
+import torch
+from PIL import Image, ImageDraw, ImageFont
+from PIL.ExifTags import TAGS
+from collections import defaultdict
+from pathlib import Path
+from tempfile import NamedTemporaryFile as ntf
+
+import sentencepiece as spm
+from transformers.tokenization_utils import PreTrainedTokenizer
+from transformers.tokenization_utils_base import (
+    PaddingStrategy,
+    TextInput,
+)
+from transformers.utils import TensorType, logging
+from transformers.video_utils import VideoInput
+from transformers.processing_utils import ProcessorMixin
+from transformers.feature_extraction_utils import BatchFeature
+from transformers.image_processing_utils import BaseImageProcessor, BatchFeature
+from transformers.image_transforms import (
+    convert_to_rgb,
+    normalize,
+    rescale,
+    resize,
+    to_channel_dimension_format,
+)
+from transformers.image_utils import (
+    OPENAI_CLIP_MEAN,
+    OPENAI_CLIP_STD,
+    ChannelDimension,
+    ImageInput,
+    PILImageResampling,
+    get_image_size,
+    infer_channel_dimension_format,
+    is_valid_image,
+    make_list_of_images,
+    to_numpy_array,
+    valid_images,
+)
+
+logger = logging.get_logger(__name__)
+
+
+class Ernie4_5_VLTokenizer(PreTrainedTokenizer):
+    """
+    Ernie4_5_VLTokenizer
+    """
+
+    vocab_files_names = {
+        "vocab_file": "tokenizer.model",
+    }
+    # Model input names expected by the tokenizer
+    model_input_names = ["input_ids", "position_ids", "attention_mask", "labels"]
+    # Padding side (where to add padding tokens)
+    padding_side = "right"
+
+    def __init__(
+        self,
+        vocab_file,
+        bos_token="<s>",
+        cls_token="<cls>",
+        eos_token="</s>",
+        mask_token="<mask:0>",
+        pad_token="<pad>",
+        sep_token="<sep>",
+        unk_token="<unk>",
+        additional_special_tokens=None,
+        **kwargs,
+    ):
+        """
+        Initialize the Ernie4_5_VLTokenizer
+
+        Args:
+            vocab_file (str): Path to the tokenizer vocabulary model.
+            bos_token (str, optional): The beginning of sequence token. Defaults to `"<s>"`.
+            cls_token (str, optional): The classifier token. Defaults to `"<cls>"`.
+            eos_token (str, optional): The end of sequence token. Defaults to `"</s>"`.
+            mask_token (str, optional): The masking token. Defaults to `"<mask:0>"`.
+            pad_token (str, optional): The padding token. Defaults to `"<pad>"`.
+            sep_token (str, optional): The separation token. Defaults to `"<sep>"`.
+            unk_token (str, optional): The unknown tokens symbol. Defaults to `"<unk>"`.
+            additional_special_tokens (List[str], optional): Additional special tokens to use.
+                Defaults to `["<mask:1>", "<mask:7>"]`.
+            **kwargs (dict): Additional keyword arguments passed along to the superclass.
+        """
+
+        # Store vocabulary file path
+        self.vocab_file = vocab_file
+        # Initialize SentencePiece processor
+        self.sp_model = spm.SentencePieceProcessor()
+        # Load the vocabulary model
+        self.sp_model.Load(vocab_file)
+
+        # Set default additional special tokens if none provided
+        if additional_special_tokens is None:
+            additional_special_tokens = ["<mask:1>", "<mask:7>"]
+        super().__init__(
+            bos_token=bos_token,
+            cls_token=cls_token,
+            eos_token=eos_token,
+            mask_token=mask_token,
+            pad_token=pad_token,
+            sep_token=sep_token,
+            unk_token=unk_token,
+            additional_special_tokens=additional_special_tokens,
+            **kwargs,
+        )
+
+    @property
+    def space_token(self):
+        """Return the space token"""
+        return "<mask:1>"
+
+    @property
+    def space_token_id(self):
+        """Return the ID of the space token"""
+        return self.sp_model.piece_to_id("<mask:1>")
+
+    @property
+    def gend_token(self):
+        """Return the gender token"""
+        return "<mask:7>"
+
+    @property
+    def gend_token_id(self):
+        """Return the ID of the gender token"""
+        return self.sp_model.piece_to_id("<mask:7>")
+
+    @property
+    def im_start_id(self):
+        """Return the ID of the image start token"""
+        return self.sp_model.piece_to_id("<|im_start|>")
+
+    @property
+    def im_end_id(self):
+        """Return the ID of the image end token"""
+        return self.sp_model.piece_to_id("<|im_end|>")
+
+    @property
+    def vocab_size(self):
+        """Return the size of the vocabulary"""
+        return self.sp_model.vocab_size()
+
+    def get_vocab(self):
+        """Return the vocabulary as a dictionary mapping tokens to IDs"""
+        vocab = {self.convert_ids_to_tokens(i): i for i in range(self.vocab_size)}
+        vocab.update(self.added_tokens_encoder)
+        return vocab
+
+    def _tokenize(self, text):
+        """Tokenize the input text into pieces"""
+        return self.sp_model.encode_as_pieces(text)
+
+    def _convert_token_to_id(self, token):
+        """Convert a token to its corresponding ID"""
+        return self.sp_model.piece_to_id(token)
+
+    def _convert_id_to_token(self, id):
+        """Convert an ID to its corresponding token"""
+        return self.sp_model.id_to_piece(id)
+
+    def convert_tokens_to_string(self, tokens):
+        """Convert a sequence of tokens back to a string"""
+        current_sub_tokens = []
+        out_string = ""
+
+        for token in tokens:
+            # Handle special tokens differently
+            if token in self.all_special_tokens:
+                out_string += self.sp_model.decode(current_sub_tokens) + token
+                current_sub_tokens = []
+            else:
+                current_sub_tokens.append(token)
+
+        # Add any remaining sub-tokens
+        out_string += self.sp_model.decode(current_sub_tokens)
+        return out_string
+
+    def prepare_for_model(self, *args, **kwargs):
+        """Prepare the tokenized inputs for the model"""
+        # Remove add_special_tokens if present (not supported)
+        if "add_special_tokens" in kwargs:
+            kwargs.pop("add_special_tokens")
+        return super().prepare_for_model(*args, **kwargs)
+
+    def save_vocabulary(
+        self, save_directory, filename_prefix: Optional[str] = None
+    ) -> Tuple[str]:
+        """
+        Save the vocabulary and special tokens file to a directory.
+
+        Args:
+            save_directory (`str`): The directory to save the vocabulary to
+            filename_prefix (`str`, optional): Prefix to add to the filename
+
+        Returns:
+            `Tuple(str)`: Paths to the saved files
+        """
+        if not os.path.isdir(save_directory):
+            logger.error(f"Vocabulary path ({save_directory}) should be a directory")
+            return
+
+        # Construct output vocabulary file path
+        out_vocab_file = os.path.join(
+            save_directory,
+            (filename_prefix + "-" if filename_prefix else "")
+            + self.vocab_files_names["vocab_file"],
+        )
+
+        # Copy or create vocabulary file
+        if os.path.abspath(self.vocab_file) != os.path.abspath(
+            out_vocab_file
+        ) and os.path.isfile(self.vocab_file):
+            copyfile(self.vocab_file, out_vocab_file)
+        elif not os.path.isfile(self.vocab_file):
+            with open(out_vocab_file, "wb") as fi:
+                content_spiece_model = self.sp_model.serialized_model_proto()
+                fi.write(content_spiece_model)
+
+        return (out_vocab_file,)
+
+    def _decode(self, *args, **kwargs):
+        """Decode token_id back to text"""
+        # Remove some parameters that aren't used
+        kwargs.pop("clean_up_tokenization_spaces", None)
+        kwargs.pop("spaces_between_special_tokens", None)
+
+        # Call parent decode method with specific parameters
+        return super()._decode(
+            *args,
+            **kwargs,
+            clean_up_tokenization_spaces=False,
+            spaces_between_special_tokens=False,
+        )
+
+    def _pad(
+        self,
+        encoded_inputs: Dict,
+        max_length: Optional[int] = None,
+        padding_strategy=PaddingStrategy.DO_NOT_PAD,
+        pad_to_multiple_of: Optional[int] = None,
+        return_attention_mask: Optional[bool] = None,
+        **kwargs
+    ) -> dict:
+        """Pad the encoded inputs to the specified length"""
+        if return_attention_mask is None:
+            return_attention_mask = "attention_mask" in self.model_input_names
+        if return_attention_mask:
+            required_input = encoded_inputs[self.model_input_names[0]]
+            if padding_strategy == PaddingStrategy.LONGEST:
+                max_length = len(required_input)
+
+            # Adjust max_length if needed for multiple of padding
+            if (
+                max_length is not None
+                and pad_to_multiple_of is not None
+                and (max_length % pad_to_multiple_of != 0)
+            ):
+                max_length = (
+                    (max_length // pad_to_multiple_of) + 1
+                ) * pad_to_multiple_of
+
+            # Check if padding is needed
+            needs_to_be_padded = (
+                padding_strategy != PaddingStrategy.DO_NOT_PAD
+                and len(required_input) != max_length
+            )
+
+            # Handle attention mask if present
+            if (
+                "attention_mask" in encoded_inputs
+                and encoded_inputs["attention_mask"] is not None
+            ):
+                attention_mask = encoded_inputs.pop("attention_mask")
+                if isinstance(attention_mask, torch.Tensor):
+                    attention_mask = attention_mask.numpy()
+                elif isinstance(attention_mask, list):
+                    attention_mask = np.array(attention_mask)
+                elif not isinstance(attention_mask, np.ndarray):
+                    raise ValueError(
+                        f"Unexpected type {type(attention_mask)} of attention_mask, "
+                    )
+            else:
+                # Create default attention mask if none provided
+                attention_mask = np.tril(
+                    np.ones((len(required_input), len(required_input)), dtype=np.int64)
+                )
+                attention_mask = np.expand_dims(attention_mask, axis=0)
+
+            # Perform padding if needed
+            if needs_to_be_padded:
+                difference = max_length - len(required_input)
+                if self.padding_side == "right":
+                    if attention_mask.ndim == 1:
+                        pad_width = [(0, difference)]
+                    else:
+                        pad_width = [(0, 0), (0, difference), (0, difference)]
+                elif self.padding_side == "left":
+                    if attention_mask.ndim == 1:
+                        pad_width = [(difference, 0)]
+                    else:
+                        pad_width = [(0, 0), (difference, 0), (difference, 0)]
+                else:
+                    raise ValueError(
+                        "Invalid padding strategy:" + str(self.padding_side)
+                    )
+
+                attention_mask = np.pad(
+                    attention_mask,
+                    pad_width=pad_width,
+                    mode="constant",
+                    constant_values=0,
+                )
+
+        # Call parent padding method
+        encoded_inputs = super()._pad(
+            encoded_inputs,
+            max_length,
+            padding_strategy=padding_strategy,
+            pad_to_multiple_of=pad_to_multiple_of,
+            return_attention_mask=False,
+        )
+
+        # Add attention mask back if needed
+        if return_attention_mask:
+            encoded_inputs["attention_mask"] = attention_mask.tolist()
+
+        return encoded_inputs
+
+
+def round_by_factor(number: int, factor: int) -> int:
+    """Returns the closest integer to 'number' that is divisible by 'factor'."""
+    return round(number / factor) * factor
+
+
+def ceil_by_factor(number: int, factor: int) -> int:
+    """Returns the smallest integer greater than or equal to 'number' that is divisible by 'factor'."""
+    return math.ceil(number / factor) * factor
+
+
+def floor_by_factor(number: int, factor: int) -> int:
+    """Returns the largest integer less than or equal to 'number' that is divisible by 'factor'."""
+    return math.floor(number / factor) * factor
+
+
+def smart_resize(
+    height: int,
+    width: int,
+    factor: int = 28,
+    min_pixels: int = 4 * 28 * 28,
+    max_pixels: int = 16384 * 28 * 28,
+):
+    """
+    Rescales the image so that the following conditions are met:
+
+    1. Both dimensions (height and width) are divisible by 'factor'.
+
+    2. The total number of pixels is within the range ['min_pixels', 'max_pixels'].
+
+    3. The aspect ratio of the image is maintained as closely as possible.
+    """
+    MAX_RATIO = 200
+    if max(height, width) / min(height, width) > MAX_RATIO:
+        if height > width:
+            new_width = max(factor, round_by_factor(width, factor))
+            new_height = floor_by_factor(new_width * MAX_RATIO, factor)
+        else:
+            new_height = max(factor, round_by_factor(height, factor))
+            new_width = floor_by_factor(new_height * MAX_RATIO, factor)
+
+        logger.info(
+            f"absolute aspect ratio must be smaller than {MAX_RATIO}, got {max(height, width) / min(height, width)},\
+              resize to {max(new_height, new_width) / min(new_height, new_width)}"
+        )
+
+        height = new_height
+        width = new_width
+
+    h_bar = max(factor, round_by_factor(height, factor))
+    w_bar = max(factor, round_by_factor(width, factor))
+    if h_bar * w_bar > max_pixels:
+        beta = math.sqrt((height * width) / max_pixels)
+        h_bar = floor_by_factor(height / beta, factor)
+        w_bar = floor_by_factor(width / beta, factor)
+    elif h_bar * w_bar < min_pixels:
+        beta = math.sqrt(min_pixels / (height * width))
+        h_bar = ceil_by_factor(height * beta, factor)
+        w_bar = ceil_by_factor(width * beta, factor)
+
+    if min_pixels > h_bar * w_bar or h_bar * w_bar > max_pixels:
+        raise ValueError(f"encounter invalid h_bar: {h_bar}, w_bar: {w_bar}")
+
+    return h_bar, w_bar
+
+
+def is_scaled_image(image: np.ndarray) -> bool:
+    """
+    Checks to see whether the pixel values have already been rescaled to [0, 1].
+    """
+    if image.dtype == np.uint8:
+        return False
+
+    # It's possible the image has pixel values in [0, 255] but is of floating type
+    return np.min(image) >= 0 and np.max(image) <= 1
+
+
+def make_batched_images(images) -> List[List[ImageInput]]:
+    """
+    Accepts images in list or nested list format, and makes a list of images for preprocessing.
+
+    Args:
+        images (`Union[List[List[ImageInput]], List[ImageInput], ImageInput]`):
+            The input image.
+
+    Returns:
+        list: A list of images.
+    """
+    if (
+        isinstance(images, (list, tuple))
+        and isinstance(images[0], (list, tuple))
+        and is_valid_image(images[0][0])
+    ):
+        return [img for img_list in images for img in img_list]
+
+    elif isinstance(images, (list, tuple)) and is_valid_image(images[0]):
+        return images
+
+    elif is_valid_image(images):
+        return [images]
+
+    raise ValueError(f"Could not make batched images from {images}")
+
+
+# Copied from transformers.models.llava_next_video.image_processing_llava_next_video.make_batched_videos
+def make_batched_videos(videos) -> List[VideoInput]:
+    """dummy"""
+    if (
+        isinstance(videos, (list, tuple))
+        and isinstance(videos[0], (list, tuple))
+        and is_valid_image(videos[0][0])
+    ):
+        return videos
+
+    elif isinstance(videos, (list, tuple)) and is_valid_image(videos[0]):
+        if isinstance(videos[0], Image.Image):
+            return [videos]
+        elif len(videos[0].shape) == 4:
+            return [list(video) for video in videos]
+
+    elif is_valid_image(videos) and len(videos.shape) == 4:
+        return [list(videos)]
+
+    raise ValueError(f"Could not make batched video from {videos}")
+
+
+class Ernie4_5_VLImageProcessor(BaseImageProcessor):
+    r"""
+    Constructs a adaptive image processor that dynamically resizes images based on the original images.
+
+    Args:
+        do_resize (`bool`, *optional*, defaults to `True`):
+            Whether to resize the image's (height, width) dimensions.
+        resample (`PILImageResampling`, *optional*, defaults to `Resampling.BICUBIC`):
+            Resampling filter to use when resizing the image.
+        do_rescale (`bool`, *optional*, defaults to `True`):
+            Whether to rescale the image by the specified scale `rescale_factor`.
+        rescale_factor (`int` or `float`, *optional*, defaults to `1/255`):
+            Scale factor to use if rescaling the image.
+        do_normalize (`bool`, *optional*, defaults to `True`):
+            Whether to normalize the image.
+        image_mean (`float` or `List[float]`, *optional*, defaults to `[0.48145466, 0.4578275, 0.40821073]`):
+            Mean to use if normalizing the image. This is a float or list of floats for each channel in the image.
+        image_std (`float` or `List[float]`, *optional*, defaults to `[0.26862954, 0.26130258, 0.27577711]`):
+            Standard deviation to use if normalizing the image. This is a float or list of floats for each channel
+            in the image.
+        do_convert_rgb (`bool`, *optional*, defaults to `True`):
+            Whether to convert the image to RGB.
+        min_pixels (`int`, *optional*, defaults to `56 * 56`):
+            The min pixels of the image to resize the image.
+        max_pixels (`int`, *optional*, defaults to `28 * 28 * 1280`):
+            The max pixels of the image to resize the image.
+        patch_size (`int`, *optional*, defaults to 14):
+            The spacial patch size of the vision encoder.
+        temporal_conv_size (`int`, *optional*, defaults to 2):
+            The temporal conv size in resampler.
+        merge_size (`int`, *optional*, defaults to 2):
+            The merge size of the vision encoder to llm encoder.
+    """
+
+    model_input_names = [
+        "pixel_values",
+        "image_grid_thw",
+        "pixel_values_videos",
+        "video_grid_thw",
+    ]
+
+    def __init__(
+        self,
+        do_resize: bool = True,
+        resample: PILImageResampling = PILImageResampling.BICUBIC,
+        do_rescale: bool = True,
+        rescale_factor: Union[float, List[float]] = 1 / 255,
+        do_normalize: bool = True,
+        image_mean: Optional[Union[float, List[float]]] = None,
+        image_std: Optional[Union[float, List[float]]] = None,
+        do_convert_rgb: bool = True,
+        min_pixels: int = 56 * 56,
+        max_pixels: int = 28 * 28 * 1280,
+        patch_size: int = 14,
+        temporal_conv_size: int = 2,
+        merge_size: int = 2,
+        **kwargs,
+    ) -> None:
+        """init"""
+        super().__init__(**kwargs)
+        self.do_resize = do_resize
+        self.resample = resample
+        self.do_rescale = do_rescale
+        self.rescale_factor = rescale_factor
+        self.do_normalize = do_normalize
+        self.image_mean = image_mean if image_mean is not None else OPENAI_CLIP_MEAN
+        self.image_std = image_std if image_std is not None else OPENAI_CLIP_STD
+        self.min_pixels = min_pixels
+        self.max_pixels = max_pixels
+        self.patch_size = patch_size
+        self.temporal_conv_size = temporal_conv_size
+        self.merge_size = merge_size
+        self.size = {"min_pixels": min_pixels, "max_pixels": max_pixels}
+        self.do_convert_rgb = do_convert_rgb
+
+    def set_pixels(self, min_pixels=None, max_pixels=None, msg=""):
+        """set_pixels"""
+        if min_pixels is not None:
+            assert (
+                isinstance(min_pixels, int) and min_pixels >= 0
+            ), "min_pixels must be positive int"
+            logger.info(
+                f"{msg} Ernie4_5_VLImageProcessor set min_pixels = {min_pixels}"
+            )
+            self.min_pixels = min_pixels
+            self.size["min_pixels"] = int(min_pixels)
+        if max_pixels is not None:
+            assert (
+                isinstance(max_pixels, int) and max_pixels > 0
+            ), "max_pixels must be positive int"
+            logger.info(
+                f"{msg} Ernie4_5_VLImageProcessor set max_pixels = {max_pixels}"
+            )
+            self.max_pixels = max_pixels
+            self.size["max_pixels"] = int(max_pixels)
+
+    def get_smarted_resize(self, height, width, min_pixels=None, max_pixels=None):
+        """dummy"""
+        actual_min_pixels = min_pixels if min_pixels is not None else self.min_pixels
+        actual_max_pixels = max_pixels if max_pixels is not None else self.max_pixels
+        resized_height, resized_width = smart_resize(
+            height,
+            width,
+            factor=self.patch_size * self.merge_size,
+            min_pixels=actual_min_pixels,
+            max_pixels=actual_max_pixels,
+        )
+        return (resized_height, resized_width), (
+            resized_height // self.patch_size,
+            resized_width // self.patch_size,
+        )
+
+    def _preprocess(
+        self,
+        images: Union[ImageInput, VideoInput],
+        do_resize: bool = True,
+        resample: PILImageResampling = None,
+        do_rescale: bool = True,
+        rescale_factor: float = 1 / 255,
+        do_normalize: bool = True,
+        image_mean: Optional[Union[float, List[float]]] = None,
+        image_std: Optional[Union[float, List[float]]] = None,
+        do_convert_rgb: bool = False,
+        data_format: Optional[ChannelDimension] = ChannelDimension.FIRST,
+        input_data_format: Optional[Union[str, ChannelDimension]] = None,
+        predetermined_grid_thw=None,
+    ):
+        """
+        Preprocess an image or batch of images. Copy of the `preprocess` method from `CLIPImageProcessor`.
+
+        Args:
+            images (`ImageInput` or `VideoInput`):
+                Image or batch of images to preprocess. Expects pixel values ranging from 0 to 255.
+                If pixel values range from 0 to 1, set `do_rescale=False`.
+            do_resize (`bool`, *optional*, defaults to `self.do_resize`):
+                Whether to resize the image.
+            resample (`PILImageResampling`, *optional*, defaults to `self.resample`):
+                Resampling filter to use if resizing the image. This can be one of the `PILImageResampling` enums.
+            do_rescale (`bool`, *optional*, defaults to `self.do_rescale`):
+                Whether to rescale the image.
+            rescale_factor (`float`, *optional*, defaults to `self.rescale_factor`):
+                Scale factor to use if rescaling the image.
+            do_normalize (`bool`, *optional*, defaults to `self.do_normalize`):
+                Whether to normalize the image.
+            image_mean (`float` or `List[float]`, *optional*, defaults to `self.image_mean`):
+                Mean to use if normalizing the image.
+                Can be a float or a list of floats corresponding to the number of channels in the image.
+            image_std (`float` or `List[float]`, *optional*, defaults to `self.image_std`):
+                Standard deviation to use if normalizing the image.
+                Can be a float or a list of floats corresponding to the number of channels in the image.
+            do_convert_rgb (`bool`, *optional*, defaults to `self.do_convert_rgb`):
+                Whether to convert the image to RGB.
+            data_format (`ChannelDimension`, *optional*, defaults to `ChannelDimension.FIRST`):
+                The channel dimension format for the output image. Can be one of:
+                - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format.
+                - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format.
+                - Unset: Use the channel dimension format of the input image.
+            input_data_format (`ChannelDimension` or `str`, *optional*):
+                The channel dimension format for the input image. Can be one of:
+                - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format.
+                - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format.
+                - `"none"` or `ChannelDimension.NONE`: image in (height, width) format.
+                - `"none"` or `ChannelDimension.NONE`: image in (height, width) format.
+        """
+        images = make_list_of_images(images)
+
+        if do_convert_rgb:
+            images = [convert_to_rgb(image) for image in images]
+
+        # All transformations expect numpy arrays.
+        images = [to_numpy_array(image) for image in images]
+
+        if is_scaled_image(images[0]) and do_rescale:
+            logger.warning_once(
+                "It looks like you are trying to rescale already rescaled images. If the input"
+                " images have pixel values between 0 and 1, set `do_rescale=False` to avoid rescaling them again."
+            )
+        if input_data_format is None:
+            # We assume that all images have the same channel dimension format.
+            input_data_format = infer_channel_dimension_format(images[0])
+
+        height, width = get_image_size(images[0], channel_dim=input_data_format)
+        resized_height, resized_width = height, width
+        processed_images = []
+
+        if predetermined_grid_thw is not None:
+            assert len(predetermined_grid_thw) == len(
+                images
+            ), f"len(predetermined_grid_thw) {len(predetermined_grid_thw)} == len(images) {len(images)}"
+
+        for img_idx, image in enumerate(images):
+            if do_resize:
+                if predetermined_grid_thw is not None:
+                    (resized_height, resized_width) = predetermined_grid_thw[img_idx]
+                    resized_height *= self.patch_size
+                    resized_width *= self.patch_size
+                else:
+                    resized_height, resized_width = smart_resize(
+                        height,
+                        width,
+                        factor=self.patch_size * self.merge_size,
+                        min_pixels=self.min_pixels,
+                        max_pixels=self.max_pixels,
+                    )
+
+                image = resize(
+                    image,
+                    size=(resized_height, resized_width),
+                    resample=resample,
+                    data_format=input_data_format,
+                )
+            if do_rescale:
+                image = rescale(
+                    image, scale=rescale_factor, data_format=input_data_format
+                )
+
+            if do_normalize:
+                image = normalize(
+                    image=image,
+                    mean=image_mean,
+                    std=image_std,
+                    data_format=input_data_format,
+                )
+
+            image = to_channel_dimension_format(
+                image, data_format, input_channel_dim=input_data_format
+            )  # [C, H, W]
+
+            processed_images.append(image)
+        patches = np.array(processed_images)
+        if data_format == ChannelDimension.LAST:
+            patches = patches.transpose([0, 3, 1, 2])
+
+        channel = patches.shape[1]  # [time, C, H, W]
+        grid_t = patches.shape[0]
+        grid_h, grid_w = (
+            resized_height // self.patch_size,
+            resized_width // self.patch_size,
+        )
+        patches = patches.reshape(
+            [
+                grid_t,
+                channel,
+                grid_h // self.merge_size,
+                self.merge_size,
+                self.patch_size,
+                grid_w // self.merge_size,
+                self.merge_size,
+                self.patch_size,
+            ]
+        )
+        # [grid_t, grid_h/merge_size, grid_w/merge_size, merge_size, merge_size, C, psz, psz]
+        patches = patches.transpose([0, 2, 5, 3, 6, 1, 4, 7])
+
+        flatten_patches = patches.reshape(
+            [grid_t * grid_h * grid_w, channel * self.patch_size * self.patch_size]
+        )  # [grid_t * grid_h * grid_w, C * psz * psz]
+
+        return flatten_patches, (grid_t, grid_h, grid_w)
+
+    def preprocess(
+        self,
+        images: ImageInput,
+        videos: VideoInput = None,
+        do_resize: bool = True,
+        size: Optional[Union[int, List[int]]] = None,
+        resample: PILImageResampling = None,
+        do_rescale: bool = True,
+        rescale_factor: float = 1 / 255,
+        do_normalize: bool = True,
+        image_mean: Optional[Union[float, List[float]]] = None,
+        image_std: Optional[Union[float, List[float]]] = None,
+        do_convert_rgb: bool = False,
+        return_tensors: Optional[Union[str, TensorType]] = None,
+        data_format: Optional[ChannelDimension] = ChannelDimension.FIRST,
+        input_data_format: Optional[Union[str, ChannelDimension]] = None,
+        predetermined_grid_thw=None,
+    ):
+        """
+        Args:
+            images (`ImageInput`):
+                Image to preprocess. Expects a single or batch of images with pixel values ranging from 0 to 255. If
+                passing in images with pixel values between 0 and 1, set `do_rescale=False`.
+            videos (`VideoInput`):
+                Video to preprocess. Expects a single or batch of videos with pixel values ranging from 0 to 255. If
+                passing in videos with pixel values between 0 and 1, set `do_rescale=False`.
+            do_resize (`bool`, *optional*, defaults to `self.do_resize`):
+                Whether to resize the image.
+            size (`Dict[str, int]`, *optional*, defaults to `self.size`):
+                Size of the image after resizing. Shortest edge of the image is resized to size["shortest_edge"], with
+                the longest edge resized to keep the input aspect ratio.
+            resample (`int`, *optional*, defaults to `self.resample`):
+                Resampling filter to use if resizing the image. This can be one of the enum `PILImageResampling`. Only
+                has an effect if `do_resize` is set to `True`.
+            do_rescale (`bool`, *optional*, defaults to `self.do_rescale`):
+                Whether to rescale the image.
+            rescale_factor (`float`, *optional*, defaults to `self.rescale_factor`):
+                Rescale factor to rescale the image by if `do_rescale` is set to `True`.
+            do_normalize (`bool`, *optional*, defaults to `self.do_normalize`):
+                Whether to normalize the image.
+            image_mean (`float` or `List[float]`, *optional*, defaults to `self.image_mean`):
+                Image mean to use for normalization. Only has an effect if `do_normalize` is set to `True`.
+            image_std (`float` or `List[float]`, *optional*, defaults to `self.image_std`):
+                Image standard deviation to use for normalization. Only has an effect if `do_normalize` is set to
+                `True`.
+            do_convert_rgb (`bool`, *optional*, defaults to `self.do_convert_rgb`):
+                Whether to convert the image to RGB.
+            return_tensors (`str` or `TensorType`, *optional*):
+                The type of tensors to return. Can be one of:
+                - Unset: Return a list of `np.ndarray`.
+                - `TensorType.PYTORCH` or `'pt'`: Return a batch of type `torch.Tensor`.
+                - `TensorType.NUMPY` or `'np'`: Return a batch of type `np.ndarray`.
+            data_format (`ChannelDimension` or `str`, *optional*, defaults to `ChannelDimension.FIRST`):
+                The channel dimension format for the output image. Can be one of:
+                - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format.
+                - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format.
+                - Unset: Use the channel dimension format of the input image.
+            input_data_format (`ChannelDimension` or `str`, *optional*):
+                The channel dimension format for the input image. If unset, the channel dimension format is inferred
+                from the input image. Can be one of:
+                - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format.
+                - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format.
+                - `"none"` or `ChannelDimension.NONE`: image in (height, width) format.
+
+        """
+        do_resize = do_resize if do_resize is not None else self.do_resize
+        size = size if size is not None else self.size
+        resample = resample if resample is not None else self.resample
+        do_rescale = do_rescale if do_rescale is not None else self.do_rescale
+        rescale_factor = (
+            rescale_factor if rescale_factor is not None else self.rescale_factor
+        )
+        do_normalize = do_normalize if do_normalize is not None else self.do_normalize
+        image_mean = image_mean if image_mean is not None else self.image_mean
+        image_std = image_std if image_std is not None else self.image_std
+        do_convert_rgb = (
+            do_convert_rgb if do_convert_rgb is not None else self.do_convert_rgb
+        )
+
+        if images is not None:
+            images = make_batched_images(images)
+
+        if images is not None and not valid_images(images):
+            raise ValueError(
+                "Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, "
+                "torch.Tensor."
+            )
+
+        data = {}
+        if images is not None:
+            pixel_values, vision_grid_thws = [], []
+            for img_idx, image in enumerate(images):
+                if predetermined_grid_thw is not None:
+                    predetermined_grid_thw_one = [predetermined_grid_thw[img_idx]]
+                else:
+                    predetermined_grid_thw_one = None
+                patches, image_grid_thw = self._preprocess(
+                    image,
+                    do_resize=do_resize,
+                    resample=resample,
+                    do_rescale=do_rescale,
+                    rescale_factor=rescale_factor,
+                    do_normalize=do_normalize,
+                    image_mean=image_mean,
+                    image_std=image_std,
+                    data_format=data_format,
+                    do_convert_rgb=do_convert_rgb,
+                    input_data_format=input_data_format,
+                    predetermined_grid_thw=predetermined_grid_thw_one,
+                )
+                pixel_values.extend(patches)
+                vision_grid_thws.append(image_grid_thw)
+            pixel_values = np.array(pixel_values)
+            vision_grid_thws = np.array(vision_grid_thws)
+            data.update(
+                {"pixel_values": pixel_values, "image_grid_thw": vision_grid_thws}
+            )
+
+        if videos is not None:
+            videos = make_batched_videos(videos)
+            pixel_values, vision_grid_thws = [], []
+            for images in videos:
+                patches, video_grid_thw = self._preprocess(
+                    images,
+                    do_resize=do_resize,
+                    resample=resample,
+                    do_rescale=do_rescale,
+                    rescale_factor=rescale_factor,
+                    do_normalize=do_normalize,
+                    image_mean=image_mean,
+                    image_std=image_std,
+                    data_format=data_format,
+                    do_convert_rgb=do_convert_rgb,
+                    input_data_format=input_data_format,
+                    predetermined_grid_thw=predetermined_grid_thw,
+                )
+                pixel_values.extend(patches)
+                vision_grid_thws.append(video_grid_thw)
+            pixel_values = np.array(pixel_values)
+            vision_grid_thws = np.array(vision_grid_thws)
+
+            data.update(
+                {
+                    "pixel_values_videos": pixel_values,
+                    "video_grid_thw": vision_grid_thws,
+                }
+            )
+
+        return BatchFeature(data=data, tensor_type=return_tensors)
+
+
+RAW_VIDEO_DIR = "./download_tmp/raw_video/"
+RAW_IMAGE_DIR = "./download_tmp/raw_images/"
+EXTRACTED_FRAME_DIR = "./download_tmp/extracted_frames/"
+TMP_DIR = "./download_tmp/upload_tmp/"
+
+FONT_PATH = os.path.join(Path(__file__).parent.absolute(), "Roboto-Regular.ttf")
+if not os.path.exists(FONT_PATH):
+    ttf = requests.get("https://paddlenlp.bj.bcebos.com/vision-language-models/materials/Roboto-Regular.ttf")
+    open(FONT_PATH, "wb").write(ttf.content)
+
+
+def is_gif(data: bytes) -> bool:
+    """
+    check if a bytes is a gif based on the magic head
+    """
+    return data[:6] in (b"GIF87a", b"GIF89a")
+
+
+class VideoReaderWrapper(decord.VideoReader):
+    """
+    Solving memory leak bug
+
+    https://github.com/dmlc/decord/issues/208
+    """
+
+    def __init__(self, video_path, *args, **kwargs):
+        with ntf(delete=True, suffix=".gif") as gif_file:
+            gif_input = None
+            self.original_file = None
+            if isinstance(video_path, str):
+                self.original_file = video_path
+                if video_path.lower().endswith(".gif"):
+                    gif_input = video_path
+            elif isinstance(video_path, bytes):
+                if is_gif(video_path):
+                    gif_file.write(video_path)
+                    gif_input = gif_file.name
+            elif isinstance(video_path, io.BytesIO):
+                video_path.seek(0)
+                tmp_bytes = video_path.read()
+                video_path.seek(0)
+                if is_gif(tmp_bytes):
+                    gif_file.write(tmp_bytes)
+                    gif_input = gif_file.name
+
+            if gif_input is not None:
+                try:
+                    # moviepy 1.0
+                    import moviepy.editor as mp
+                except:
+                    # moviepy 2.0
+                    import moviepy as mp
+                clip = mp.VideoFileClip(gif_input)
+                mp4_file = ntf(delete=False, suffix=".mp4")
+                clip.write_videofile(mp4_file.name, logger=None)
+                clip.close()
+                video_path = mp4_file.name
+                self.original_file = video_path
+
+            super().__init__(video_path, *args, **kwargs)
+            self.seek(0)
+
+    def __getitem__(self, key):
+        frames = super().__getitem__(key)
+        self.seek(0)
+        return frames
+
+    def __del__(self):
+        if self.original_file and os.path.exists(self.original_file):
+            os.remove(self.original_file)
+
+
+def get_filename(url=None):
+    """
+    Get Filename
+    """
+    if url is None:
+        return str(uuid.uuid4()).replace("-", "")
+    t = datetime.datetime.now()
+    if not isinstance(url, bytes):
+        url = url.encode("utf-8")
+
+    md5_hash = hashlib.md5(url).hexdigest()
+    pid = os.getpid()
+    tid = threading.get_ident()
+
+    # Remove the suffix to prevent save-jpg from reporting errors
+    image_filname = f"{t.year}-{t.month:02d}-{t.day:02d}-{pid}-{tid}-{md5_hash}"
+    return image_filname
+
+
+def file_download(url, download_dir, save_to_disk=False, retry=0, retry_interval=3):
+    """
+    Description: Download url, if url is PIL, return directly
+    Args:
+        url(str, PIL): http/local path/io.Bytes, note that io.Bytes is the image byte stream
+        download_path: when save_to_disk=True, return the saved address
+        save_to_disk: whether to save in the local path
+    """
+
+    if isinstance(url, Image.Image):
+        return url
+    elif isinstance(url, VideoReaderWrapper):
+        return url
+    elif url.startswith("http"):
+        response = requests.get(url)
+        bytes_data = response.content
+    elif os.path.isfile(url):
+        if save_to_disk:
+            return url
+        bytes_data = open(url, "rb").read()
+    else:
+        bytes_data = base64.b64decode(url)
+    if not save_to_disk:
+        return bytes_data
+
+    download_path = os.path.join(download_dir, get_filename(url))
+    Path(download_path).parent.mkdir(parents=True, exist_ok=True)
+    with open(download_path, "wb") as f:
+        f.write(bytes_data)
+    return download_path
+
+
+def get_downloadable(
+    url, download_dir=RAW_VIDEO_DIR, save_to_disk=False, retry=0, retry_interval=3
+):
+    """download video and store it in the disk
+
+    return downloaded **path** if save_to_disk is set to true
+    return downloaded **bytes** if save_to_disk is set to false
+    """
+
+    if not os.path.exists(download_dir):
+        os.makedirs(download_dir)
+    downloaded_path = file_download(
+        url,
+        download_dir,
+        save_to_disk=save_to_disk,
+        retry=retry,
+        retry_interval=retry_interval,
+    )
+    return downloaded_path
+
+
+def get_downloadable_image(
+    download_path, need_exif_info, retry_max_time=0, retry_interval=3
+):
+    """
+    Get downloadable with exif info and image processing
+    """
+
+    def get_image_exif(image):
+        exif_data = image._getexif()
+        exif_info = {}
+        if exif_data is not None:
+            for tag, value in exif_data.items():
+                tag_name = TAGS.get(tag, tag)
+                exif_info[tag_name] = value.strip()
+        return exif_info
+
+    def has_transparent_background(img):
+        """has_transparent_background"""
+        if img.mode in ("RGBA", "LA") or (
+            img.mode == "P" and "transparency" in img.info
+        ):
+            # Check for any pixel with alpha channel less than 255 (fully opaque)
+            alpha = img.convert("RGBA").split()[-1]
+            if alpha.getextrema()[0] < 255:
+                return True
+        return False
+
+    def add_white_background(img):
+        """
+        Add a white background to a transparent background image
+        """
+        if img.mode != "RGBA":
+            img = img.convert("RGBA")
+        # Create an image with a white background and the same size as the original image
+        img_white_background = Image.new("RGBA", img.size, (255, 255, 255))
+
+        # Paste the original image onto a white background
+        img_white_background.paste(img, (0, 0), img)
+
+        return img_white_background
+
+    def change_I16_to_L(img):
+        """
+        Convert image from I;16 mode to L mode
+        """
+        # Since the point function in I mode only supports addition, subtraction, and multiplication,
+        # the following * (1 / 256) cannot be changed to division.
+        return img.point(lambda i: i * (1 / 256)).convert("L")
+
+    image = get_downloadable(
+        download_path,
+        save_to_disk=False,
+        retry=retry_max_time,
+        retry_interval=retry_interval,
+    )
+    if isinstance(image, Image.Image):
+        pil_image = image
+    else:
+        pil_image = Image.open(io.BytesIO(image))
+    if need_exif_info:
+        try:
+            exif_info = get_image_exif(pil_image)
+        except Exception as why:
+            exif_info = {}
+    else:
+        exif_info = {}
+
+    try:
+        if pil_image.mode == "I;16":
+            pil_image = change_I16_to_L(pil_image)
+        if has_transparent_background(pil_image):
+            pil_image = add_white_background(pil_image)
+    except Exception as e:
+        pass
+
+    return pil_image.convert("RGB"), exif_info
+
+
+def read_video_decord(video_path, save_to_disk):
+    """get reader and meta by decord"""
+    video_path = get_downloadable(video_path, save_to_disk=save_to_disk)
+    if isinstance(video_path, VideoReaderWrapper):
+        video_reader = video_path
+    else:
+        if isinstance(video_path, bytes):
+            video_path = io.BytesIO(video_path)
+        video_reader = VideoReaderWrapper(video_path, num_threads=1)
+    vlen = len(video_reader)
+    fps = video_reader.get_avg_fps()
+    duration = vlen / float(fps)
+
+    video_meta = {"fps": fps, "duration": duration, "num_of_frame": vlen}
+
+    return video_reader, video_meta, video_path
+
+
+def get_frame_indices(
+    vlen,
+    target_frames=-1,
+    target_fps=-1,
+    frames_sample="middle",
+    fix_start=None,
+    input_fps=-1,
+):
+    """get_frame_indices"""
+    assert frames_sample in ["rand", "middle", "leading"]
+    if target_frames > 0:
+        assert target_fps <= 0, "target_fps must be negative if target_frames is given."
+        if target_frames > vlen:
+            acc_samples = vlen
+            logger.info(
+                f"target_frames={target_frames} is larger than video length {vlen}, "
+                f"will sample {acc_samples} frames."
+            )
+        else:
+            acc_samples = target_frames
+            logger.debug(
+                f"sampling at target_frames={target_frames}, frames_sample={frames_sample}"
+            )
+
+        # split the video into `acc_samples` intervals, and sample from each interval.
+        intervals = np.linspace(start=0, stop=vlen, num=acc_samples + 1).astype(int)
+        ranges = []
+        for idx, interv in enumerate(intervals[:-1]):
+            ranges.append((interv, intervals[idx + 1] - 1))
+        if frames_sample == "rand":
+            try:
+                frame_indices = [random.choice(range(x[0], x[1])) for x in ranges]
+            except Exception as e:
+                frame_indices = np.random.permutation(vlen)[:acc_samples]
+                frame_indices.sort()
+                frame_indices = list(frame_indices)
+        elif fix_start is not None:
+            frame_indices = [x[0] + fix_start for x in ranges]
+        elif frames_sample == "leading":
+            frame_indices = [x[0] for x in ranges]
+        elif frames_sample == "middle":
+            frame_indices = [(x[0] + x[1]) // 2 for x in ranges]
+        else:
+            raise NotImplementedError
+
+    elif target_fps > 0:
+        assert (
+            target_frames <= 0
+        ), "target_frames must be negative if target_fps is given."
+        assert input_fps > 0, "input_fps must be provided if target_fps is given."
+        logger.info(f"sampling at fps={target_fps}, frames_sample={frames_sample}")
+        duration = float(vlen) / input_fps
+        delta = (
+            1 / target_fps
+        )  # gap between frames, this is also the clip length each frame represents
+        if frames_sample == "middle":
+            frame_seconds = np.arange(0 + delta / 2, duration + delta / 2, delta)
+        elif frames_sample == "leading":
+            frame_seconds = np.arange(0, duration, delta)
+        if frames_sample == "rand":
+            frame_seconds = np.arange(0 + delta / 2, duration + delta / 2, delta)
+            rand_offset = np.random.rand(*(frame_seconds.shape)) - 0.5
+            frame_seconds += rand_offset * delta
+        frame_indices = np.around(frame_seconds * input_fps).astype(int)
+        frame_indices = [e for e in frame_indices if e < vlen]
+
+    else:
+        raise ValueError(
+            "Must provide either positive target_fps or positive target_frames."
+        )
+
+    return frame_indices
+
+
+def read_frames_decord(
+    video_path,
+    video_reader,
+    video_meta,
+    target_frames=-1,
+    target_fps=-1,
+    frames_sample="middle",
+    fix_start=None,
+    save_to_disk=False,
+    cache_dir=EXTRACTED_FRAME_DIR,
+    frame_indices=None,
+    tol=10,
+):
+    """get frames by decord"""
+
+    if frame_indices is None:
+        frame_indices = get_frame_indices(
+            video_meta["num_of_frame"],
+            target_frames=target_frames,
+            target_fps=target_fps,
+            frames_sample=frames_sample,
+            fix_start=fix_start,
+            input_fps=video_meta["fps"],
+        )
+
+    frames = []
+    for frame_indice_index in range(0, len(frame_indices)):
+        frame_indice = frame_indices[frame_indice_index]
+        try:
+            frames.append(video_reader[frame_indice].asnumpy())  # (T, H, W, C)
+        except Exception as e:
+            logger.debug(f"encounter error when get frame: {frame_indice}, error: {e}")
+            previous_counter = 1
+            later_counter = 1
+            previous_after_flag = True
+            if frame_indice == 0 or frame_indice == len(video_reader) - 1:
+                cur_tol = tol * 2
+            else:
+                cur_tol = tol
+            while previous_counter < cur_tol or later_counter < cur_tol:
+                if previous_after_flag:
+                    if frame_indice - previous_counter < 0:
+                        previous_counter += 1
+                        previous_after_flag = not previous_after_flag
+                        continue
+                    try:
+                        frames.append(
+                            video_reader[frame_indice - previous_counter].asnumpy()
+                        )
+                        logger.info(
+                            f"replace {frame_indice}-th frame with {frame_indice-previous_counter}-th frame"
+                        )
+                        frame_indices[frame_indice_index] = (
+                            frame_indice - previous_counter
+                        )
+                        break
+                    except Exception as e:
+                        previous_counter += 1
+                else:
+                    if frame_indice + later_counter >= len(video_reader):
+                        later_counter += 1
+                        previous_after_flag = not previous_after_flag
+                        continue
+                    try:
+                        frames.append(
+                            video_reader[frame_indice + later_counter].asnumpy()
+                        )
+                        logger.info(
+                            f"replace {frame_indice}-th frame with {frame_indice+later_counter}-th frame"
+                        )
+                        frame_indices[frame_indice_index] = frame_indice + later_counter
+                        break
+                    except Exception as e:
+                        later_counter += 1
+                previous_after_flag = not previous_after_flag
+
+    frames = np.stack(frames, axis=0)
+    assert len(frames) == len(
+        frame_indices
+    ), f"len(frames): {len(frames)} != len(frame_indices): {len(frame_indices)}"
+
+    ret = []
+
+    url_sha1 = get_filename()
+    for idx, frame in enumerate(frames):
+        tmp = Image.fromarray(frame, "RGB")
+        if save_to_disk:
+            save_path = os.path.join(cache_dir, f"{url_sha1}", f"{idx}.png")
+            if not os.path.exists(os.path.dirname(save_path)):
+                os.makedirs(os.path.dirname(save_path))
+            tmp.save(save_path)
+            tmp = save_path
+        ret.append(tmp)
+
+    time_stamps = [
+        frame_idx * video_meta["duration"] / video_meta["num_of_frame"]
+        for frame_idx in frame_indices
+    ]
+
+    return ret, frame_indices, time_stamps
+
+
+def render_single_image_with_timestamp(
+    image: Image, number: str, rate: float, font_path: str = FONT_PATH
+):
+    """
+    Function: Renders a timestamp to the image of pil.image
+    The timestamp size is the rate of min(width, height)
+    The font color is black, the outline is white, and the outline size is 10% of the font
+    Returns an Image object
+    """
+    draw = ImageDraw.Draw(image)
+    width, height = image.size
+    font_size = int(min(width, height) * rate)
+    outline_size = int(font_size * 0.1)
+    font = ImageFont.truetype(font_path, font_size)
+    x = 0
+    y = 0
+
+    # Draw a black timestamp with a white border
+    draw.text(
+        (x, y),
+        number,
+        font=font,
+        fill=(0, 0, 0),
+        stroke_width=outline_size,
+        stroke_fill=(255, 255, 255),
+    )
+
+    return image
+
+
+def timestamp_converting(time_stamp_in_seconds):
+    """
+    convert timestamp format from seconds to hr:min:sec
+    """
+    # get hours
+    hours = 0
+    while time_stamp_in_seconds >= 3600:
+        hours += 1
+        time_stamp_in_seconds -= 3600
+    # get minutes
+    mins = 0
+    while time_stamp_in_seconds >= 60:
+        mins += 1
+        time_stamp_in_seconds -= 60
+    time_hours = f"{int(hours):02d}"
+    time_mins = f"{int(mins):02d}"
+    time_secs = f"{time_stamp_in_seconds:05.02f}"
+    fi_time_stamp = time_hours + ":" + time_mins + ":" + time_secs
+
+    return fi_time_stamp
+
+
+def render_frame_timestamp(frame, timestamp, font_rate=0.1):
+    """
+    Function, given a frame, render the index in order
+    Logic: render the index to the upper left corner of the image
+    frame: frame, PIL.Image object
+    timestamp: timestamp, in seconds
+    font_rate: the ratio of font size to min(wi, hei)
+    """
+    time_stamp = "time: " + timestamp_converting(timestamp)
+    new_frame = render_single_image_with_timestamp(frame, time_stamp, font_rate)
+
+    return new_frame
+
+
+IDS_TYPE_FLAG = {"text": 0, "image": 1, "video": 2, "audio": 3}
+
+
+class Ernie4_5_VLProcessor(ProcessorMixin):
+    """
+    Processes multimodal chat messages into model-ready inputs,
+    handling text, images, and videos with 3D positional embeddings.
+    """
+
+    attributes = ["image_processor", "tokenizer"]
+    valid_kwargs = [
+        "chat_template",
+        "spatial_conv_size",
+        "temporal_conv_size",
+        "image_min_pixels",
+        "image_max_pixels",
+        "video_min_pixels",
+        "video_max_pixels",
+        "video_target_frames",
+        "video_frames_sample",
+        "video_max_frames",
+        "video_min_frames",
+        "video_fps",
+    ]
+    image_processor_class = "AutoImageProcessor"
+    tokenizer_class = "AutoTokenizer"
+
+    CLS_TOKEN = "<|begin_of_sentence|>"
+    SEP_TOKEN = "<|end_of_sentence|>"
+    IMG_START = "<|IMAGE_START|>"
+    IMG_END = "<|IMAGE_END|>"
+    VID_START = "<|VIDEO_START|>"
+    VID_END = "<|VIDEO_END|>"
+
+    # Flag for vLLM to detect if this processor supports video metadata
+    supports_video_metadata = True
+
+    def __init__(
+        self,
+        image_processor=None,
+        tokenizer=None,
+        chat_template=None,
+        spatial_conv_size: int = 2,
+        temporal_conv_size: int = 2,
+        image_min_pixels: int = 4 * 28 * 28,
+        image_max_pixels: int = 6177 * 28 * 28,
+        video_min_pixels: int = 299 * 28 * 28,
+        video_max_pixels: int = 1196 * 28 * 28,
+        video_target_frames: int = -1,
+        video_frames_sample: str = "leading",
+        video_max_frames: int = 180,
+        video_min_frames: int = 16,
+        video_fps: int = 2,
+        **kwargs,
+    ):
+        super().__init__(image_processor, tokenizer, chat_template=chat_template)
+        self.tokenizer.ignored_index = -100
+
+        # Convolution sizes for patch aggregation
+        self.spatial_conv_size = spatial_conv_size
+        self.temporal_conv_size = temporal_conv_size
+
+        # Pixel constraints
+        self.image_min_pixels = image_min_pixels
+        self.image_max_pixels = image_max_pixels
+        self.video_min_pixels = video_min_pixels
+        self.video_max_pixels = video_max_pixels
+
+        # Video sampling parameters
+        self.target_frames = video_target_frames
+        self.frames_sample = video_frames_sample
+        self.max_frames = video_max_frames
+        self.min_frames = video_min_frames
+        self.fps = video_fps
+
+        # Special tokens and IDs
+        self.cls_token = self.CLS_TOKEN
+        self.sep_token = self.SEP_TOKEN
+        self.image_start = self.IMG_START
+        self.image_end = self.IMG_END
+        self.video_start = self.VID_START
+        self.video_end = self.VID_END
+        self.image_patch_id = self.tokenizer.convert_tokens_to_ids(
+            "<|IMAGE_PLACEHOLDER|>"
+        )
+
+        self.token_type_mapping = self._build_token_type_mapping()
+        self.is_training = True
+        self.role_prefixes = {"system": "", "user": "User: ", "bot": "Assistant: "}
+
+    def _build_token_type_mapping(self) -> Dict[Any, int]:
+        mapping = defaultdict(lambda: IDS_TYPE_FLAG["text"])
+        for token in (self.IMG_START, self.IMG_END, self.VID_START, self.VID_END):
+            mapping[token] = IDS_TYPE_FLAG["image"]
+        mapping[self.image_patch_id] = IDS_TYPE_FLAG["image"]
+        return mapping
+
+    def train(self) -> None:
+        """Enable training mode (produces labels)."""
+        self.is_training = True
+
+    def eval(self) -> None:
+        """Enable evaluation mode (doesn't produce labels)."""
+        self.is_training = False
+
+    def _download_image(
+        self,
+        item: Dict,
+    ):
+        """Download image from url and resize it to the specified size."""
+        url_info = item.get("image_url", {})
+        url = url_info.get("url")
+        w = url_info.get("image_width", None)
+        h = url_info.get("image_height", None)
+        data = get_downloadable(url, download_dir=RAW_IMAGE_DIR, save_to_disk=False)
+
+        img = Image.open(io.BytesIO(data) if isinstance(data, bytes) else data)
+        if w and h:
+            img = img.resize((w, h))
+        return img
+
+    def _download_video(self, item: Dict):
+        """Download video from url and resize it to the specified size."""
+        url_info = item.get("video_url", {})
+        url = url_info.get("url")
+
+        frames = self._load_and_process_video(url, item)
+
+        pixel_stack = np.stack([np.array(f.convert("RGB")) for f in frames], axis=0)
+        return pixel_stack
+
+    def process_vision_info(self, messages: List[Dict[str, Any]]):
+        """Preprocess messages into lists of text, images, and videos."""
+        images = []
+        videos = []
+
+        for msg in messages:
+            content_items = msg.get("content")
+            if not isinstance(content_items, list):
+                content_items = [content_items]
+
+            for item in content_items:
+                if item.get("type") == "image_url":
+                    img = self._download_image(item)
+                    images.append(img)
+                elif item.get("type") == "video_url":
+                    pixel_stack = self._download_video(item)
+                    videos.append(pixel_stack)
+                    
+        return images, videos
+
+    def __call__(
+        self,
+        text: Union[str, List[str]],
+        images: List[Image.Image] = None,
+        videos: List[List[Image.Image]] = None,
+        **kwargs,
+    ) -> BatchFeature:
+        """
+        Convert chat messages into model inputs.
+        Returns a dict with input_ids, token_type_ids, position_ids, images, grid_thw, image_type_ids, labels.
+        """
+        outputs = {
+            "input_ids": [],
+            "token_type_ids": [],
+            "position_ids": [],
+            "images": [],
+            "grid_thw": [],
+            "image_type_ids": [],
+            "cur_position": 0,
+            "pic_cnt": 0,
+            "video_cnt": 0,
+        }
+        if images is None:
+            images = []
+        if videos is None:
+            videos = []
+        if not isinstance(text, list):
+            text = [text]
+
+        if len(videos) > 0 and isinstance(videos[0], tuple):
+            # vLLM passes (frames, metadata) tuple, render timestamps on frames
+            rendered_videos = []
+            for vid in videos:
+                video_frames, video_metadata = vid[0], vid[1]
+
+                if isinstance(video_frames, np.ndarray):
+                    frames = [Image.fromarray(f) for f in video_frames]
+                else:
+                    raise ValueError(f"Unsupported video frames type: {type(video_frames)}")
+                if len(frames) == 0:
+                    continue
+
+                video_fps = video_metadata.get("fps", 1.0)
+                video_frames_indices = video_metadata.get("frames_indices") or list(range(len(frames)))
+                video_frames_timestamps = [float(idx) / float(video_fps) for idx in video_frames_indices]
+
+                assert len(frames) == len(video_frames_timestamps), \
+                    f"Frames count {len(frames)} != timestamps count {len(video_frames_timestamps)}"
+
+                rendered_frames = [render_frame_timestamp(f, ts) for f, ts in zip(frames, video_frames_timestamps)]
+
+                # Ensure even number of frames for temporal conv
+                if len(rendered_frames) % 2 != 0:
+                    rendered_frames.append(rendered_frames[-1].copy())
+
+                pixel_stack = np.stack([np.array(f.convert("RGB")) for f in rendered_frames], axis=0)
+                rendered_videos.append(pixel_stack)
+
+            videos = rendered_videos
+
+        elif len(videos) > 0 and isinstance(videos[0], np.ndarray):
+            # Old vLLM without metadata support, warn user to upgrade
+            logger.warning_once(
+                "Video input without metadata. Timestamps will NOT be rendered. "
+                "If using vLLM, please upgrade to support video_needs_metadata."
+            )
+            # Ensure even number of frames for temporal conv
+            for vid_idx, vid in enumerate(videos):
+                if len(vid) % 2 != 0:
+                    vid = np.concatenate([vid, vid[-1:]])
+                videos[vid_idx] = vid
+
+        texts = text[0]
+
+        new_video_seg = True
+        for text_with_image in texts.split(self.VID_START + "<|video@placeholder|>" + self.VID_END):
+            new_text_seg = True
+            if not new_video_seg:
+                self._add_video(videos[outputs["video_cnt"]], outputs)
+            for text in text_with_image.split(self.IMG_START + "<|image@placeholder|>" + self.IMG_END):
+                if not new_text_seg:
+                    self._add_image(images[outputs["pic_cnt"]], outputs)
+                self._add_text(text, outputs)
+                new_text_seg = False
+            new_video_seg = False
+
+        for key in ["cur_position", "pic_cnt", "video_cnt"]:
+            outputs.pop(key, None)
+
+        outputs = self._pack_outputs(outputs)
+        for key in outputs.keys():
+            if isinstance(outputs[key], np.ndarray):
+                if key in ["images", "grid_thw"]:
+                    outputs[key] = torch.tensor(np.array(outputs[key]))
+                else:
+                    outputs[key] = torch.tensor(np.array([outputs[key]]))
+
+        return BatchFeature(data=outputs)
+
+    def _add_special_token(self, token: Union[str, int], outputs: Dict) -> None:
+        """add special token to outputs"""
+        token_id = (
+            token
+            if isinstance(token, int)
+            else self.tokenizer.convert_tokens_to_ids(token)
+        )
+        outputs["input_ids"].append(token_id)
+        outputs["token_type_ids"].append(self.token_type_mapping[token])
+        pos = outputs["cur_position"]
+        outputs["position_ids"].append([pos] * 3)
+        outputs["cur_position"] += 1
+    
+    def _add_text(self, text: str, outputs: Dict) -> None:
+        """add text to outputs"""
+        tokens = self.tokenizer.convert_tokens_to_ids(self.tokenizer.tokenize(text))
+        outputs["input_ids"].extend(tokens)
+        outputs["token_type_ids"].extend([IDS_TYPE_FLAG["text"]] * len(tokens))
+
+        start = outputs["cur_position"]
+        for i in range(len(tokens)):
+            outputs["position_ids"].append([start + i] * 3)
+        outputs["cur_position"] += len(tokens)
+
+    def _add_image(self, img: Image.Image, outputs: Dict) -> None:
+        """add image to outputs"""
+        outputs["pic_cnt"] += 1
+        self._add_special_token(self.IMG_START, outputs)
+
+        patches_h, patches_w = self.image_processor.get_smarted_resize(
+            img.height,
+            img.width,
+            min_pixels=self.image_min_pixels,
+            max_pixels=self.image_max_pixels,
+        )[1]
+        num_tokens = (patches_h * patches_w) // (self.spatial_conv_size**2)
+
+        outputs["input_ids"].extend([self.image_patch_id] * num_tokens)
+        outputs["token_type_ids"].extend([IDS_TYPE_FLAG["image"]] * num_tokens)
+
+        pos_ids = self._compute_3d_positions(
+            1, patches_h, patches_w, outputs["cur_position"]
+        )
+        outputs["position_ids"].extend(pos_ids)
+        outputs["cur_position"] = np.max(pos_ids) + 1
+
+        # Preprocess pixels
+        ret = self.image_processor.preprocess(
+            images=[img.convert("RGB")],
+            do_normalize=False,
+            do_rescale=False,
+            predetermined_grid_thw=np.array([[patches_h, patches_w]]),
+            do_convert_rgb=True,
+            input_data_format=ChannelDimension.LAST,
+        )
+        outputs["images"].append(ret["pixel_values"])
+        outputs["grid_thw"].append(ret["image_grid_thw"])
+        outputs["image_type_ids"].append(0)
+
+        self._add_special_token(self.IMG_END, outputs)
+
+    def _add_video(
+        self, pixel_stack: np.ndarray, outputs: Dict
+    ) -> None:
+        outputs["video_cnt"] += 1
+        self._add_special_token(self.VID_START, outputs)
+
+        patches_h, patches_w = self.image_processor.get_smarted_resize(
+            pixel_stack.shape[1],
+            pixel_stack.shape[2],
+            min_pixels=self.video_min_pixels,
+            max_pixels=self.video_max_pixels,
+        )[1]
+        num_frames = pixel_stack.shape[0]
+        num_tokens = (num_frames * patches_h * patches_w) // (
+            self.spatial_conv_size**2 * self.temporal_conv_size
+        )
+
+        ret = self.image_processor.preprocess(
+            images=None,
+            videos=pixel_stack,
+            do_normalize=False,
+            do_rescale=False,
+            predetermined_grid_thw=np.array([[patches_h, patches_w]] * num_frames),
+            do_convert_rgb=True,
+            input_data_format=ChannelDimension.LAST,
+        )
+        outputs["images"].append(ret["pixel_values_videos"])
+        outputs["grid_thw"].append(ret["video_grid_thw"])
+        outputs["image_type_ids"].extend([1] * num_frames)
+
+        outputs["input_ids"].extend([self.image_patch_id] * num_tokens)
+        outputs["token_type_ids"].extend([IDS_TYPE_FLAG["video"]] * num_tokens)
+
+        pos_ids = self._compute_3d_positions(
+            num_frames, patches_h, patches_w, outputs["cur_position"]
+        )
+        outputs["position_ids"].extend(pos_ids)
+        outputs["cur_position"] = np.max(pos_ids) + 1
+
+        self._add_special_token(self.VID_END, outputs)
+
+    def _load_and_process_video(self, url: str, item: Dict) -> List[Image.Image]:
+        reader, meta, path = read_video_decord(url, save_to_disk=False)
+
+        video_frame_args = dict()
+        video_frame_args["fps"] = item.get("fps", self.fps)
+        video_frame_args["min_frames"] = item.get("min_frames", self.min_frames)
+        video_frame_args["max_frames"] = item.get("max_frames", self.max_frames)
+        video_frame_args["target_frames"] = item.get(
+            "target_frames", self.target_frames
+        )
+        video_frame_args["frames_sample"] = item.get(
+            "frames_sample", self.frames_sample
+        )
+
+        video_frame_args = self._set_video_frame_args(video_frame_args, meta)
+
+        frames_data, _, timestamps = read_frames_decord(
+            path,
+            reader,
+            meta,
+            target_frames=video_frame_args["target_frames"],
+            target_fps=video_frame_args["fps"],
+            frames_sample=video_frame_args["frames_sample"],
+            save_to_disk=False,
+        )
+
+        frames: List[Image.Image] = []
+        for img_array, ts in zip(frames_data, timestamps):
+            frames.append(render_frame_timestamp(img_array, ts))
+        # Ensure even number of frames for temporal conv
+        if len(frames) % 2 != 0:
+            frames.append(copy.deepcopy(frames[-1]))
+        return frames
+
+    def _set_video_frame_args(self, video_frame_args, video_meta):
+        """
+        Set the final frame extraction parameters based on known parameters and priorities
+        """
+        # Priority: video_target_frames > (video_min_frames, video_max_frames) > video_fps
+        if video_frame_args["target_frames"] > 0:
+            if video_frame_args["fps"] >= 0:
+                raise ValueError("fps must be negative if target_frames is given")
+            if (
+                video_frame_args["min_frames"] > 0
+                and video_frame_args["target_frames"] < video_frame_args["min_frames"]
+            ):
+                raise ValueError("target_frames must be larger than min_frames")
+            if (
+                video_frame_args["max_frames"] > 0
+                and video_frame_args["target_frames"] > video_frame_args["max_frames"]
+            ):
+                raise ValueError("target_frames must be smaller than max_frames")
+        else:
+            if video_frame_args["fps"] < 0:
+                raise ValueError(
+                    "Must provide either positive target_fps or positive target_frames."
+                )
+            # First calculate the number of frames extracted under video_fps
+            frames_to_extract = int(video_meta["duration"] * video_frame_args["fps"])
+            # Determine whether it is within the target range. If not, take target_frames as the upper or lower bound
+            if (
+                video_frame_args["min_frames"] > 0
+                and video_frame_args["max_frames"] > 0
+                and video_frame_args["min_frames"] > video_frame_args["max_frames"]
+            ):
+                raise ValueError("min_frames must be smaller than max_frames")
+            if (
+                video_frame_args["min_frames"] > 0
+                and frames_to_extract < video_frame_args["min_frames"]
+            ):
+                video_frame_args["target_frames"] = video_frame_args["min_frames"]
+                video_frame_args["fps"] = -1
+            if (
+                video_frame_args["max_frames"] > 0
+                and frames_to_extract > video_frame_args["max_frames"]
+            ):
+                video_frame_args["target_frames"] = video_frame_args["max_frames"]
+                video_frame_args["fps"] = -1
+
+        return video_frame_args
+
+    def _compute_3d_positions(
+        self, t: int, h: int, w: int, start_idx: int
+    ) -> List[List[int]]:
+        # Downsample time if needed
+        t_eff = t // self.temporal_conv_size if t != 1 else 1
+        gh, gw = h // self.spatial_conv_size, w // self.spatial_conv_size
+        time_idx = np.repeat(np.arange(t_eff), gh * gw)
+        h_idx = np.tile(np.repeat(np.arange(gh), gw), t_eff)
+        w_idx = np.tile(np.arange(gw), t_eff * gh)
+
+        coords = list(zip(time_idx, h_idx, w_idx))
+        return [
+            [start_idx + ti, start_idx + hi, start_idx + wi] for ti, hi, wi in coords
+        ]
+
+    def _pack_outputs(self, outs: Dict) -> Dict[str, Any]:
+        # Stack or nullify image-related fields
+        if not outs["images"]:
+            outs["images"] = None
+            outs["grid_thw"] = None
+            outs["image_type_ids"] = None
+        else:
+            outs["images"] = np.vstack(outs["images"])
+            outs["grid_thw"] = np.vstack(outs["grid_thw"])
+            outs["image_type_ids"] = np.array(outs["image_type_ids"])
+
+        # Convert lists to arrays
+        outs["input_ids"] = np.array(outs["input_ids"], dtype=np.int64)
+        outs["token_type_ids"] = np.array(outs["token_type_ids"], dtype=np.int64)
+        outs["position_ids"] = np.array(outs["position_ids"], dtype=np.int64)
+        return outs
+
+    def batch_decode(self, *args, **kwargs):
+        """
+        This method forwards all its arguments to Ernie4_5_VLTokenizer's [`~PreTrainedTokenizer.batch_decode`]. Please
+        refer to the docstring of this method for more information.
+        """
+        return self.tokenizer.batch_decode(*args, **kwargs)
+
+    def decode(self, *args, **kwargs):
+        """
+        This method forwards all its arguments to Ernie4_5_VLTokenizer's [`~PreTrainedTokenizer.decode`].
+        Please refer to the docstring of this method for more information.
+        """
+        return self.tokenizer.decode(*args, **kwargs)
+
+    @property
+    def model_input_names(self):
+        """get model input names"""
+        tokenizer_input_names = self.tokenizer.model_input_names
+        image_processor_input_names = self.image_processor.model_input_names
+        return list(tokenizer_input_names) + list(image_processor_input_names)
+
+
+__all__ = ["Ernie4_5_VLTokenizer", "Ernie4_5_VLImageProcessor", "Ernie4_5_VLProcessor"]

processor_config.json

@@ -0,0 +1,31 @@
+{
+  "image_processor": {
+    "data_format": "channels_first",
+    "factor": 28,
+    "image_mean": [
+      0.48145466,
+      0.4578275,
+      0.40821073
+    ],
+    "image_processor_type": "ImageProcessor",
+    "image_std": [
+      0.26862954,
+      0.26130258,
+      0.27577711
+    ],
+    "max_pixels": 1003520,
+    "merge_size": 2,
+    "min_pixels": 3136,
+    "patch_size": 14,
+    "resample": 3,
+    "rescale_factor": 0.00392156862745098,
+    "size": [
+      224,
+      224
+    ],
+    "temporal_patch_size": 2
+  },
+  "processor_class": "Ernie4_5_VLProcessor",
+  "spatial_conv_size": 2,
+  "temporal_conv_size": 2
+}

special_tokens_map.json

@@ -0,0 +1 @@
+{"bos_token": "<s>", "eos_token": "</s>", "unk_token": "<unk>", "sep_token": "<|end_of_sentence|>", "pad_token": "<unk>", "cls_token": "<|begin_of_sentence|>", "mask_token": "<mask:1>", "sys_start_token": "<mask:4>", "sys_end_token": "<mask:5>", "header_start_token": "<mask:6>", "header_end_token": "<mask:7>", "additional_special_tokens": ["<|IMAGE_PLACEHOLDER|>", "<|AUDIO_PLACEHOLDER|>", "<|LOC_0|>", "<|LOC_1|>", "<|LOC_2|>", "<|LOC_3|>", "<|LOC_4|>", "<|LOC_5|>", "<|LOC_6|>", "<|LOC_7|>", "<|LOC_8|>", "<|LOC_9|>", "<|LOC_10|>", "<|LOC_11|>", "<|LOC_12|>", "<|LOC_13|>", "<|LOC_14|>", "<|LOC_15|>", "<|LOC_16|>", "<|LOC_17|>", "<|LOC_18|>", "<|LOC_19|>", "<|LOC_20|>", "<|LOC_21|>", "<|LOC_22|>", "<|LOC_23|>", "<|LOC_24|>", "<|LOC_25|>", "<|LOC_26|>", "<|LOC_27|>", "<|LOC_28|>", "<|LOC_29|>", "<|LOC_30|>", "<|LOC_31|>", "<|LOC_32|>", "<|LOC_33|>", "<|LOC_34|>", "<|LOC_35|>", "<|LOC_36|>", "<|LOC_37|>", "<|LOC_38|>", "<|LOC_39|>", "<|LOC_40|>", "<|LOC_41|>", "<|LOC_42|>", "<|LOC_43|>", "<|LOC_44|>", "<|LOC_45|>", "<|LOC_46|>", "<|LOC_47|>", "<|LOC_48|>", "<|LOC_49|>", "<|LOC_50|>", "<|LOC_51|>", "<|LOC_52|>", "<|LOC_53|>", "<|LOC_54|>", "<|LOC_55|>", "<|LOC_56|>", "<|LOC_57|>", "<|LOC_58|>", "<|LOC_59|>", "<|LOC_60|>", "<|LOC_61|>", "<|LOC_62|>", "<|LOC_63|>", "<|LOC_64|>", "<|LOC_65|>", "<|LOC_66|>", "<|LOC_67|>", "<|LOC_68|>", "<|LOC_69|>", "<|LOC_70|>", "<|LOC_71|>", "<|LOC_72|>", "<|LOC_73|>", "<|LOC_74|>", "<|LOC_75|>", "<|LOC_76|>", "<|LOC_77|>", "<|LOC_78|>", "<|LOC_79|>", "<|LOC_80|>", "<|LOC_81|>", "<|LOC_82|>", "<|LOC_83|>", "<|LOC_84|>", "<|LOC_85|>", "<|LOC_86|>", "<|LOC_87|>", "<|LOC_88|>", "<|LOC_89|>", "<|LOC_90|>", "<|LOC_91|>", "<|LOC_92|>", "<|LOC_93|>", "<|LOC_94|>", "<|LOC_95|>", "<|LOC_96|>", "<|LOC_97|>", "<|LOC_98|>", "<|LOC_99|>", "<|LOC_100|>", "<|LOC_101|>", "<|LOC_102|>", "<|LOC_103|>", "<|LOC_104|>", "<|LOC_105|>", "<|LOC_106|>", "<|LOC_107|>", "<|LOC_108|>", "<|LOC_109|>", "<|LOC_110|>", "<|LOC_111|>", "<|LOC_112|>", "<|LOC_113|>", "<|LOC_114|>", "<|LOC_115|>", "<|LOC_116|>", "<|LOC_117|>", "<|LOC_118|>", "<|LOC_119|>", "<|LOC_120|>", "<|LOC_121|>", "<|LOC_122|>", "<|LOC_123|>", "<|LOC_124|>", "<|LOC_125|>", "<|LOC_126|>", "<|LOC_127|>", "<|LOC_128|>", "<|LOC_129|>", "<|LOC_130|>", "<|LOC_131|>", "<|LOC_132|>", "<|LOC_133|>", "<|LOC_134|>", "<|LOC_135|>", "<|LOC_136|>", "<|LOC_137|>", "<|LOC_138|>", "<|LOC_139|>", "<|LOC_140|>", "<|LOC_141|>", "<|LOC_142|>", "<|LOC_143|>", "<|LOC_144|>", "<|LOC_145|>", "<|LOC_146|>", "<|LOC_147|>", "<|LOC_148|>", "<|LOC_149|>", "<|LOC_150|>", "<|LOC_151|>", "<|LOC_152|>", "<|LOC_153|>", "<|LOC_154|>", "<|LOC_155|>", "<|LOC_156|>", "<|LOC_157|>", "<|LOC_158|>", "<|LOC_159|>", "<|LOC_160|>", "<|LOC_161|>", "<|LOC_162|>", "<|LOC_163|>", "<|LOC_164|>", "<|LOC_165|>", "<|LOC_166|>", "<|LOC_167|>", "<|LOC_168|>", "<|LOC_169|>", "<|LOC_170|>", "<|LOC_171|>", "<|LOC_172|>", "<|LOC_173|>", "<|LOC_174|>", "<|LOC_175|>", "<|LOC_176|>", "<|LOC_177|>", "<|LOC_178|>", "<|LOC_179|>", "<|LOC_180|>", "<|LOC_181|>", "<|LOC_182|>", "<|LOC_183|>", "<|LOC_184|>", "<|LOC_185|>", "<|LOC_186|>", "<|LOC_187|>", "<|LOC_188|>", "<|LOC_189|>", "<|LOC_190|>", "<|LOC_191|>", "<|LOC_192|>", "<|LOC_193|>", "<|LOC_194|>", "<|LOC_195|>", "<|LOC_196|>", "<|LOC_197|>", "<|LOC_198|>", "<|LOC_199|>", "<|LOC_200|>", "<|LOC_201|>", "<|LOC_202|>", "<|LOC_203|>", "<|LOC_204|>", "<|LOC_205|>", "<|LOC_206|>", "<|LOC_207|>", "<|LOC_208|>", "<|LOC_209|>", "<|LOC_210|>", "<|LOC_211|>", "<|LOC_212|>", "<|LOC_213|>", "<|LOC_214|>", "<|LOC_215|>", "<|LOC_216|>", "<|LOC_217|>", "<|LOC_218|>", "<|LOC_219|>", "<|LOC_220|>", "<|LOC_221|>", "<|LOC_222|>", "<|LOC_223|>", "<|LOC_224|>", "<|LOC_225|>", "<|LOC_226|>", "<|LOC_227|>", "<|LOC_228|>", "<|LOC_229|>", "<|LOC_230|>", "<|LOC_231|>", "<|LOC_232|>", "<|LOC_233|>", "<|LOC_234|>", "<|LOC_235|>", "<|LOC_236|>", "<|LOC_237|>", "<|LOC_238|>", "<|LOC_239|>", "<|LOC_240|>", "<|LOC_241|>", "<|LOC_242|>", "<|LOC_243|>", "<|LOC_244|>", "<|LOC_245|>", "<|LOC_246|>", "<|LOC_247|>", "<|LOC_248|>", "<|LOC_249|>", "<|LOC_250|>", "<|LOC_251|>", "<|LOC_252|>", "<|LOC_253|>", "<|LOC_254|>", "<|LOC_255|>", "<|LOC_256|>", "<|LOC_257|>", "<|LOC_258|>", "<|LOC_259|>", "<|LOC_260|>", "<|LOC_261|>", "<|LOC_262|>", "<|LOC_263|>", "<|LOC_264|>", "<|LOC_265|>", "<|LOC_266|>", "<|LOC_267|>", "<|LOC_268|>", "<|LOC_269|>", "<|LOC_270|>", "<|LOC_271|>", "<|LOC_272|>", "<|LOC_273|>", "<|LOC_274|>", "<|LOC_275|>", "<|LOC_276|>", "<|LOC_277|>", "<|LOC_278|>", "<|LOC_279|>", "<|LOC_280|>", "<|LOC_281|>", "<|LOC_282|>", "<|LOC_283|>", "<|LOC_284|>", "<|LOC_285|>", "<|LOC_286|>", "<|LOC_287|>", "<|LOC_288|>", "<|LOC_289|>", "<|LOC_290|>", "<|LOC_291|>", "<|LOC_292|>", "<|LOC_293|>", "<|LOC_294|>", "<|LOC_295|>", "<|LOC_296|>", "<|LOC_297|>", "<|LOC_298|>", "<|LOC_299|>", "<|LOC_300|>", "<|LOC_301|>", "<|LOC_302|>", "<|LOC_303|>", "<|LOC_304|>", "<|LOC_305|>", "<|LOC_306|>", "<|LOC_307|>", "<|LOC_308|>", "<|LOC_309|>", "<|LOC_310|>", "<|LOC_311|>", "<|LOC_312|>", "<|LOC_313|>", "<|LOC_314|>", "<|LOC_315|>", "<|LOC_316|>", "<|LOC_317|>", "<|LOC_318|>", "<|LOC_319|>", "<|LOC_320|>", "<|LOC_321|>", "<|LOC_322|>", "<|LOC_323|>", "<|LOC_324|>", "<|LOC_325|>", "<|LOC_326|>", "<|LOC_327|>", "<|LOC_328|>", "<|LOC_329|>", "<|LOC_330|>", "<|LOC_331|>", "<|LOC_332|>", "<|LOC_333|>", "<|LOC_334|>", "<|LOC_335|>", "<|LOC_336|>", "<|LOC_337|>", "<|LOC_338|>", "<|LOC_339|>", "<|LOC_340|>", "<|LOC_341|>", "<|LOC_342|>", "<|LOC_343|>", "<|LOC_344|>", "<|LOC_345|>", "<|LOC_346|>", "<|LOC_347|>", "<|LOC_348|>", "<|LOC_349|>", "<|LOC_350|>", "<|LOC_351|>", "<|LOC_352|>", "<|LOC_353|>", "<|LOC_354|>", "<|LOC_355|>", "<|LOC_356|>", "<|LOC_357|>", "<|LOC_358|>", "<|LOC_359|>", "<|LOC_360|>", "<|LOC_361|>", "<|LOC_362|>", "<|LOC_363|>", "<|LOC_364|>", "<|LOC_365|>", "<|LOC_366|>", "<|LOC_367|>", "<|LOC_368|>", "<|LOC_369|>", "<|LOC_370|>", "<|LOC_371|>", "<|LOC_372|>", "<|LOC_373|>", "<|LOC_374|>", "<|LOC_375|>", "<|LOC_376|>", "<|LOC_377|>", "<|LOC_378|>", "<|LOC_379|>", "<|LOC_380|>", "<|LOC_381|>", "<|LOC_382|>", "<|LOC_383|>", "<|LOC_384|>", "<|LOC_385|>", "<|LOC_386|>", "<|LOC_387|>", "<|LOC_388|>", "<|LOC_389|>", "<|LOC_390|>", "<|LOC_391|>", "<|LOC_392|>", "<|LOC_393|>", "<|LOC_394|>", "<|LOC_395|>", "<|LOC_396|>", "<|LOC_397|>", "<|LOC_398|>", "<|LOC_399|>", "<|LOC_400|>", "<|LOC_401|>", "<|LOC_402|>", "<|LOC_403|>", "<|LOC_404|>", "<|LOC_405|>", "<|LOC_406|>", "<|LOC_407|>", "<|LOC_408|>", "<|LOC_409|>", "<|LOC_410|>", "<|LOC_411|>", "<|LOC_412|>", "<|LOC_413|>", "<|LOC_414|>", "<|LOC_415|>", "<|LOC_416|>", "<|LOC_417|>", "<|LOC_418|>", "<|LOC_419|>", "<|LOC_420|>", "<|LOC_421|>", "<|LOC_422|>", "<|LOC_423|>", "<|LOC_424|>", "<|LOC_425|>", "<|LOC_426|>", "<|LOC_427|>", "<|LOC_428|>", "<|LOC_429|>", "<|LOC_430|>", "<|LOC_431|>", "<|LOC_432|>", "<|LOC_433|>", "<|LOC_434|>", "<|LOC_435|>", "<|LOC_436|>", "<|LOC_437|>", "<|LOC_438|>", "<|LOC_439|>", "<|LOC_440|>", "<|LOC_441|>", "<|LOC_442|>", "<|LOC_443|>", "<|LOC_444|>", "<|LOC_445|>", "<|LOC_446|>", "<|LOC_447|>", "<|LOC_448|>", "<|LOC_449|>", "<|LOC_450|>", "<|LOC_451|>", "<|LOC_452|>", "<|LOC_453|>", "<|LOC_454|>", "<|LOC_455|>", "<|LOC_456|>", "<|LOC_457|>", "<|LOC_458|>", "<|LOC_459|>", "<|LOC_460|>", "<|LOC_461|>", "<|LOC_462|>", "<|LOC_463|>", "<|LOC_464|>", "<|LOC_465|>", "<|LOC_466|>", "<|LOC_467|>", "<|LOC_468|>", "<|LOC_469|>", "<|LOC_470|>", "<|LOC_471|>", "<|LOC_472|>", "<|LOC_473|>", "<|LOC_474|>", "<|LOC_475|>", "<|LOC_476|>", "<|LOC_477|>", "<|LOC_478|>", "<|LOC_479|>", "<|LOC_480|>", "<|LOC_481|>", "<|LOC_482|>", "<|LOC_483|>", "<|LOC_484|>", "<|LOC_485|>", "<|LOC_486|>", "<|LOC_487|>", "<|LOC_488|>", "<|LOC_489|>", "<|LOC_490|>", "<|LOC_491|>", "<|LOC_492|>", "<|LOC_493|>", "<|LOC_494|>", "<|LOC_495|>", "<|LOC_496|>", "<|LOC_497|>", "<|LOC_498|>", "<|LOC_499|>", "<|LOC_500|>", "<|LOC_501|>", "<|LOC_502|>", "<|LOC_503|>", "<|LOC_504|>", "<|LOC_505|>", "<|LOC_506|>", "<|LOC_507|>", "<|LOC_508|>", "<|LOC_509|>", "<|LOC_510|>", "<|LOC_511|>", "<|LOC_512|>", "<|LOC_513|>", "<|LOC_514|>", "<|LOC_515|>", "<|LOC_516|>", "<|LOC_517|>", "<|LOC_518|>", "<|LOC_519|>", "<|LOC_520|>", "<|LOC_521|>", "<|LOC_522|>", "<|LOC_523|>", "<|LOC_524|>", "<|LOC_525|>", "<|LOC_526|>", "<|LOC_527|>", "<|LOC_528|>", "<|LOC_529|>", "<|LOC_530|>", "<|LOC_531|>", "<|LOC_532|>", "<|LOC_533|>", "<|LOC_534|>", "<|LOC_535|>", "<|LOC_536|>", "<|LOC_537|>", "<|LOC_538|>", "<|LOC_539|>", "<|LOC_540|>", "<|LOC_541|>", "<|LOC_542|>", "<|LOC_543|>", "<|LOC_544|>", "<|LOC_545|>", "<|LOC_546|>", "<|LOC_547|>", "<|LOC_548|>", "<|LOC_549|>", "<|LOC_550|>", "<|LOC_551|>", "<|LOC_552|>", "<|LOC_553|>", "<|LOC_554|>", "<|LOC_555|>", "<|LOC_556|>", "<|LOC_557|>", "<|LOC_558|>", "<|LOC_559|>", "<|LOC_560|>", "<|LOC_561|>", "<|LOC_562|>", "<|LOC_563|>", "<|LOC_564|>", "<|LOC_565|>", "<|LOC_566|>", "<|LOC_567|>", "<|LOC_568|>", "<|LOC_569|>", "<|LOC_570|>", "<|LOC_571|>", "<|LOC_572|>", "<|LOC_573|>", "<|LOC_574|>", "<|LOC_575|>", "<|LOC_576|>", "<|LOC_577|>", "<|LOC_578|>", "<|LOC_579|>", "<|LOC_580|>", "<|LOC_581|>", "<|LOC_582|>", "<|LOC_583|>", "<|LOC_584|>", "<|LOC_585|>", "<|LOC_586|>", "<|LOC_587|>", "<|LOC_588|>", "<|LOC_589|>", "<|LOC_590|>", "<|LOC_591|>", "<|LOC_592|>", "<|LOC_593|>", "<|LOC_594|>", "<|LOC_595|>", "<|LOC_596|>", "<|LOC_597|>", "<|LOC_598|>", "<|LOC_599|>", "<|LOC_600|>", "<|LOC_601|>", "<|LOC_602|>", "<|LOC_603|>", "<|LOC_604|>", "<|LOC_605|>", "<|LOC_606|>", "<|LOC_607|>", "<|LOC_608|>", "<|LOC_609|>", "<|LOC_610|>", "<|LOC_611|>", "<|LOC_612|>", "<|LOC_613|>", "<|LOC_614|>", "<|LOC_615|>", "<|LOC_616|>", "<|LOC_617|>", "<|LOC_618|>", "<|LOC_619|>", "<|LOC_620|>", "<|LOC_621|>", "<|LOC_622|>", "<|LOC_623|>", "<|LOC_624|>", "<|LOC_625|>", "<|LOC_626|>", "<|LOC_627|>", "<|LOC_628|>", "<|LOC_629|>", "<|LOC_630|>", "<|LOC_631|>", "<|LOC_632|>", "<|LOC_633|>", "<|LOC_634|>", "<|LOC_635|>", "<|LOC_636|>", "<|LOC_637|>", "<|LOC_638|>", "<|LOC_639|>", "<|LOC_640|>", "<|LOC_641|>", "<|LOC_642|>", "<|LOC_643|>", "<|LOC_644|>", "<|LOC_645|>", "<|LOC_646|>", "<|LOC_647|>", "<|LOC_648|>", "<|LOC_649|>", "<|LOC_650|>", "<|LOC_651|>", "<|LOC_652|>", "<|LOC_653|>", "<|LOC_654|>", "<|LOC_655|>", "<|LOC_656|>", "<|LOC_657|>", "<|LOC_658|>", "<|LOC_659|>", "<|LOC_660|>", "<|LOC_661|>", "<|LOC_662|>", "<|LOC_663|>", "<|LOC_664|>", "<|LOC_665|>", "<|LOC_666|>", "<|LOC_667|>", "<|LOC_668|>", "<|LOC_669|>", "<|LOC_670|>", "<|LOC_671|>", "<|LOC_672|>", "<|LOC_673|>", "<|LOC_674|>", "<|LOC_675|>", "<|LOC_676|>", "<|LOC_677|>", "<|LOC_678|>", "<|LOC_679|>", "<|LOC_680|>", "<|LOC_681|>", "<|LOC_682|>", "<|LOC_683|>", "<|LOC_684|>", "<|LOC_685|>", "<|LOC_686|>", "<|LOC_687|>", "<|LOC_688|>", "<|LOC_689|>", "<|LOC_690|>", "<|LOC_691|>", "<|LOC_692|>", "<|LOC_693|>", "<|LOC_694|>", "<|LOC_695|>", "<|LOC_696|>", "<|LOC_697|>", "<|LOC_698|>", "<|LOC_699|>", "<|LOC_700|>", "<|LOC_701|>", "<|LOC_702|>", "<|LOC_703|>", "<|LOC_704|>", "<|LOC_705|>", "<|LOC_706|>", "<|LOC_707|>", "<|LOC_708|>", "<|LOC_709|>", "<|LOC_710|>", "<|LOC_711|>", "<|LOC_712|>", "<|LOC_713|>", "<|LOC_714|>", "<|LOC_715|>", "<|LOC_716|>", "<|LOC_717|>", "<|LOC_718|>", "<|LOC_719|>", "<|LOC_720|>", "<|LOC_721|>", "<|LOC_722|>", "<|LOC_723|>", "<|LOC_724|>", "<|LOC_725|>", "<|LOC_726|>", "<|LOC_727|>", "<|LOC_728|>", "<|LOC_729|>", "<|LOC_730|>", "<|LOC_731|>", "<|LOC_732|>", "<|LOC_733|>", "<|LOC_734|>", "<|LOC_735|>", "<|LOC_736|>", "<|LOC_737|>", "<|LOC_738|>", "<|LOC_739|>", "<|LOC_740|>", "<|LOC_741|>", "<|LOC_742|>", "<|LOC_743|>", "<|LOC_744|>", "<|LOC_745|>", "<|LOC_746|>", "<|LOC_747|>", "<|LOC_748|>", "<|LOC_749|>", "<|LOC_750|>", "<|LOC_751|>", "<|LOC_752|>", "<|LOC_753|>", "<|LOC_754|>", "<|LOC_755|>", "<|LOC_756|>", "<|LOC_757|>", "<|LOC_758|>", "<|LOC_759|>", "<|LOC_760|>", "<|LOC_761|>", "<|LOC_762|>", "<|LOC_763|>", "<|LOC_764|>", "<|LOC_765|>", "<|LOC_766|>", "<|LOC_767|>", "<|LOC_768|>", "<|LOC_769|>", "<|LOC_770|>", "<|LOC_771|>", "<|LOC_772|>", "<|LOC_773|>", "<|LOC_774|>", "<|LOC_775|>", "<|LOC_776|>", "<|LOC_777|>", "<|LOC_778|>", "<|LOC_779|>", "<|LOC_780|>", "<|LOC_781|>", "<|LOC_782|>", "<|LOC_783|>", "<|LOC_784|>", "<|LOC_785|>", "<|LOC_786|>", "<|LOC_787|>", "<|LOC_788|>", "<|LOC_789|>", "<|LOC_790|>", "<|LOC_791|>", "<|LOC_792|>", "<|LOC_793|>", "<|LOC_794|>", "<|LOC_795|>", "<|LOC_796|>", "<|LOC_797|>", "<|LOC_798|>", "<|LOC_799|>", "<|LOC_800|>", "<|LOC_801|>", "<|LOC_802|>", "<|LOC_803|>", "<|LOC_804|>", "<|LOC_805|>", "<|LOC_806|>", "<|LOC_807|>", "<|LOC_808|>", "<|LOC_809|>", "<|LOC_810|>", "<|LOC_811|>", "<|LOC_812|>", "<|LOC_813|>", "<|LOC_814|>", "<|LOC_815|>", "<|LOC_816|>", "<|LOC_817|>", "<|LOC_818|>", "<|LOC_819|>", "<|LOC_820|>", "<|LOC_821|>", "<|LOC_822|>", "<|LOC_823|>", "<|LOC_824|>", "<|LOC_825|>", "<|LOC_826|>", "<|LOC_827|>", "<|LOC_828|>", "<|LOC_829|>", "<|LOC_830|>", "<|LOC_831|>", "<|LOC_832|>", "<|LOC_833|>", "<|LOC_834|>", "<|LOC_835|>", "<|LOC_836|>", "<|LOC_837|>", "<|LOC_838|>", "<|LOC_839|>", "<|LOC_840|>", "<|LOC_841|>", "<|LOC_842|>", "<|LOC_843|>", "<|LOC_844|>", "<|LOC_845|>", "<|LOC_846|>", "<|LOC_847|>", "<|LOC_848|>", "<|LOC_849|>", "<|LOC_850|>", "<|LOC_851|>", "<|LOC_852|>", "<|LOC_853|>", "<|LOC_854|>", "<|LOC_855|>", "<|LOC_856|>", "<|LOC_857|>", "<|LOC_858|>", "<|LOC_859|>", "<|LOC_860|>", "<|LOC_861|>", "<|LOC_862|>", "<|LOC_863|>", "<|LOC_864|>", "<|LOC_865|>", "<|LOC_866|>", "<|LOC_867|>", "<|LOC_868|>", "<|LOC_869|>", "<|LOC_870|>", "<|LOC_871|>", "<|LOC_872|>", "<|LOC_873|>", "<|LOC_874|>", "<|LOC_875|>", "<|LOC_876|>", "<|LOC_877|>", "<|LOC_878|>", "<|LOC_879|>", "<|LOC_880|>", "<|LOC_881|>", "<|LOC_882|>", "<|LOC_883|>", "<|LOC_884|>", "<|LOC_885|>", "<|LOC_886|>", "<|LOC_887|>", "<|LOC_888|>", "<|LOC_889|>", "<|LOC_890|>", "<|LOC_891|>", "<|LOC_892|>", "<|LOC_893|>", "<|LOC_894|>", "<|LOC_895|>", "<|LOC_896|>", "<|LOC_897|>", "<|LOC_898|>", "<|LOC_899|>", "<|LOC_900|>", "<|LOC_901|>", "<|LOC_902|>", "<|LOC_903|>", "<|LOC_904|>", "<|LOC_905|>", "<|LOC_906|>", "<|LOC_907|>", "<|LOC_908|>", "<|LOC_909|>", "<|LOC_910|>", "<|LOC_911|>", "<|LOC_912|>", "<|LOC_913|>", "<|LOC_914|>", "<|LOC_915|>", "<|LOC_916|>", "<|LOC_917|>", "<|LOC_918|>", "<|LOC_919|>", "<|LOC_920|>", "<|LOC_921|>", "<|LOC_922|>", "<|LOC_923|>", "<|LOC_924|>", "<|LOC_925|>", "<|LOC_926|>", "<|LOC_927|>", "<|LOC_928|>", "<|LOC_929|>", "<|LOC_930|>", "<|LOC_931|>", "<|LOC_932|>", "<|LOC_933|>", "<|LOC_934|>", "<|LOC_935|>", "<|LOC_936|>", "<|LOC_937|>", "<|LOC_938|>", "<|LOC_939|>", "<|LOC_940|>", "<|LOC_941|>", "<|LOC_942|>", "<|LOC_943|>", "<|LOC_944|>", "<|LOC_945|>", "<|LOC_946|>", "<|LOC_947|>", "<|LOC_948|>", "<|LOC_949|>", "<|LOC_950|>", "<|LOC_951|>", "<|LOC_952|>", "<|LOC_953|>", "<|LOC_954|>", "<|LOC_955|>", "<|LOC_956|>", "<|LOC_957|>", "<|LOC_958|>", "<|LOC_959|>", "<|LOC_960|>", "<|LOC_961|>", "<|LOC_962|>", "<|LOC_963|>", "<|LOC_964|>", "<|LOC_965|>", "<|LOC_966|>", "<|LOC_967|>", "<|LOC_968|>", "<|LOC_969|>", "<|LOC_970|>", "<|LOC_971|>", "<|LOC_972|>", "<|LOC_973|>", "<|LOC_974|>", "<|LOC_975|>", "<|LOC_976|>", "<|LOC_977|>", "<|LOC_978|>", "<|LOC_979|>", "<|LOC_980|>", "<|LOC_981|>", "<|LOC_982|>", "<|LOC_983|>", "<|LOC_984|>", "<|LOC_985|>", "<|LOC_986|>", "<|LOC_987|>", "<|LOC_988|>", "<|LOC_989|>", "<|LOC_990|>", "<|LOC_991|>", "<|LOC_992|>", "<|LOC_993|>", "<|LOC_994|>", "<|LOC_995|>", "<|LOC_996|>", "<|LOC_997|>", "<|LOC_998|>", "<|LOC_999|>", "<|LOC_1000|>", "<|LOC_BEGIN|>", "<|LOC_END|>", "<|LOC_SEP|>", "<|CROP_COL_SEP|>", "<|CROP_ROW_SEP|>", "<|IMAGE_SEP|>", "<|IMAGE_START|>", "<|IMAGE_END|>", "<|VIDEO_START|>", "<|VIDEO_END|>", "<|ASR_START|>", "<|ASR_END|>"]}

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:964f353bd1d5dba660a795b8fda8b4fe3ce847be2fa9ec5c1f670fbee4da3faf
+size 1614375