Не работает в vLLM server
#15
by
pimenovdv
- opened
Запускаю в kebernetes в качестве подика, подкладываю папку /models куда скачал заранее модель с помощью либы transformers
команда на запуск:
vllm serve yandex/YandexGPT-5-Lite-8B-pretrain --max_model_len 8192 --dtype=half --gpu-memory-utilization 0.75 --download-dir /models --trust-remote-code --tensor-parallel-size 2
При загрузке выдаёт ошибку:
INFO 02-27 18:38:37 llm_engine.py:234] Initializing a V0 LLM engine (v0.7.2) with config: model='yandex/YandexGPT-5-Lite-8B-pretrain', speculative_config=None, tokenizer='yandex/YandexGPT-5-Lite-8B-pretrain', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, override_neuron_config=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.float16, max_seq_len=32768, download_dir='/models', load_format=LoadFormat.AUTO, tensor_parallel_size=2, pipeline_parallel_size=1, disable_custom_all_reduce=False, quantization=None, enforce_eager=False, kv_cache_dtype=auto, device_config=cuda, decoding_config=DecodingConfig(guided_decoding_backend='xgrammar'), observability_config=ObservabilityConfig(otlp_traces_endpoint=None, collect_model_forward_time=False, collect_model_execute_time=False), seed=0, served_model_name=yandex/YandexGPT-5-Lite-8B-pretrain, num_scheduler_steps=1, multi_step_stream_outputs=True, enable_prefix_caching=False, chunked_prefill_enabled=False, use_async_output_proc=True, disable_mm_preprocessor_cache=False, mm_processor_kwargs=None, pooler_config=None, compilation_config={"splitting_ops":[],"compile_sizes":[],"cudagraph_capture_sizes":[256,248,240,232,224,216,208,200,192,184,176,168,160,152,144,136,128,120,112,104,96,88,80,72,64,56,48,40,32,24,16,8,4,2,1],"max_capture_size":256}, use_cached_outputs=True,
Traceback (most recent call last):
File "/usr/local/bin/vllm", line 8, in <module>
sys.exit(main())
^^^^^^
File "/usr/local/lib/python3.12/dist-packages/vllm/scripts.py", line 204, in main
args.dispatch_function(args)
File "/usr/local/lib/python3.12/dist-packages/vllm/scripts.py", line 44, in serve
uvloop.run(run_server(args))
File "/usr/local/lib/python3.12/dist-packages/uvloop/__init__.py", line 109, in run
return __asyncio.run(
^^^^^^^^^^^^^^
File "/usr/lib/python3.12/asyncio/runners.py", line 195, in run
return runner.run(main)
^^^^^^^^^^^^^^^^
File "/usr/lib/python3.12/asyncio/runners.py", line 118, in run
return self._loop.run_until_complete(task)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "uvloop/loop.pyx", line 1518, in uvloop.loop.Loop.run_until_complete
File "/usr/local/lib/python3.12/dist-packages/uvloop/__init__.py", line 61, in wrapper
return await main
^^^^^^^^^^
File "/usr/local/lib/python3.12/dist-packages/vllm/entrypoints/openai/api_server.py", line 875, in run_server
async with build_async_engine_client(args) as engine_client:
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/lib/python3.12/contextlib.py", line 210, in __aenter__
return await anext(self.gen)
^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/dist-packages/vllm/entrypoints/openai/api_server.py", line 136, in build_async_engine_client
async with build_async_engine_client_from_engine_args(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/lib/python3.12/contextlib.py", line 210, in __aenter__
return await anext(self.gen)
^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/dist-packages/vllm/entrypoints/openai/api_server.py", line 220, in build_async_engine_client_from_engine_args
mq_engine_client = await asyncio.get_running_loop().run_in_executor(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/lib/python3.12/concurrent/futures/thread.py", line 59, in run
result = self.fn(*self.args, **self.kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/dist-packages/vllm/engine/multiprocessing/client.py", line 98, in __init__
self.tokenizer = init_tokenizer_from_configs(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/dist-packages/vllm/transformers_utils/tokenizer_group/__init__.py", line 32, in init_tokenizer_from_configs
return get_tokenizer_group(parallel_config.tokenizer_pool_config,
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/dist-packages/vllm/transformers_utils/tokenizer_group/__init__.py", line 53, in get_tokenizer_group
return tokenizer_cls.from_config(tokenizer_pool_config, **init_kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/dist-packages/vllm/transformers_utils/tokenizer_group/tokenizer_group.py", line 33, in from_config
return cls(**init_kwargs)
^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/dist-packages/vllm/transformers_utils/tokenizer_group/tokenizer_group.py", line 25, in __init__
self.tokenizer = get_tokenizer(self.tokenizer_id, **tokenizer_config)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/dist-packages/vllm/transformers_utils/tokenizer.py", line 191, in get_tokenizer
tokenizer = AutoTokenizer.from_pretrained(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/dist-packages/transformers/models/auto/tokenization_auto.py", line 934, in from_pretrained
return tokenizer_class.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/dist-packages/transformers/tokenization_utils_base.py", line 2036, in from_pretrained
return cls._from_pretrained(
^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/dist-packages/transformers/tokenization_utils_base.py", line 2074, in _from_pretrained
slow_tokenizer = (cls.slow_tokenizer_class)._from_pretrained(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/dist-packages/transformers/tokenization_utils_base.py", line 2276, in _from_pretrained
tokenizer = cls(*init_inputs, **init_kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/dist-packages/transformers/models/llama/tokenization_llama.py", line 169, in __init__
self.sp_model = self.get_spm_processor(kwargs.pop("from_slow", False))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/dist-packages/transformers/models/llama/tokenization_llama.py", line 199, in get_spm_processor
with open(self.vocab_file, "rb") as f:
^^^^^^^^^^^^^^^^^^^^^^^^^^^
TypeError: expected str, bytes or os.PathLike object, not NoneType
ERROR 02-27 18:38:38 engine.py:389] expected str, bytes or os.PathLike object, not NoneType
ERROR 02-27 18:38:38 engine.py:389] Traceback (most recent call last):
ERROR 02-27 18:38:38 engine.py:389] File "/usr/local/lib/python3.12/dist-packages/vllm/engine/multiprocessing/engine.py", line 380, in run_mp_engine
ERROR 02-27 18:38:38 engine.py:389] engine = MQLLMEngine.from_engine_args(engine_args=engine_args,
ERROR 02-27 18:38:38 engine.py:389] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
ERROR 02-27 18:38:38 engine.py:389] File "/usr/local/lib/python3.12/dist-packages/vllm/engine/multiprocessing/engine.py", line 123, in from_engine_args
ERROR 02-27 18:38:38 engine.py:389] return cls(ipc_path=ipc_path,
ERROR 02-27 18:38:38 engine.py:389] ^^^^^^^^^^^^^^^^^^^^^^
ERROR 02-27 18:38:38 engine.py:389] File "/usr/local/lib/python3.12/dist-packages/vllm/engine/multiprocessing/engine.py", line 75, in __init__
ERROR 02-27 18:38:38 engine.py:389] self.engine = LLMEngine(*args, **kwargs)
Process SpawnProcess-1:
ERROR 02-27 18:38:38 engine.py:389] ^^^^^^^^^^^^^^^^^^^^^^^^^^
ERROR 02-27 18:38:38 engine.py:389] File "/usr/local/lib/python3.12/dist-packages/vllm/engine/llm_engine.py", line 246, in __init__
ERROR 02-27 18:38:38 engine.py:389] self.tokenizer = self._init_tokenizer()
ERROR 02-27 18:38:38 engine.py:389] ^^^^^^^^^^^^^^^^^^^^^^
ERROR 02-27 18:38:38 engine.py:389] File "/usr/local/lib/python3.12/dist-packages/vllm/engine/llm_engine.py", line 528, in _init_tokenizer
ERROR 02-27 18:38:38 engine.py:389] return init_tokenizer_from_configs(
ERROR 02-27 18:38:38 engine.py:389] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
ERROR 02-27 18:38:38 engine.py:389] File "/usr/local/lib/python3.12/dist-packages/vllm/transformers_utils/tokenizer_group/__init__.py", line 32, in init_tokenizer_from_configs
ERROR 02-27 18:38:38 engine.py:389] return get_tokenizer_group(parallel_config.tokenizer_pool_config,
ERROR 02-27 18:38:38 engine.py:389] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
ERROR 02-27 18:38:38 engine.py:389] File "/usr/local/lib/python3.12/dist-packages/vllm/transformers_utils/tokenizer_group/__init__.py", line 53, in get_tokenizer_group
ERROR 02-27 18:38:38 engine.py:389] return tokenizer_cls.from_config(tokenizer_pool_config, **init_kwargs)
ERROR 02-27 18:38:38 engine.py:389] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
ERROR 02-27 18:38:38 engine.py:389] File "/usr/local/lib/python3.12/dist-packages/vllm/transformers_utils/tokenizer_group/tokenizer_group.py", line 33, in from_config
ERROR 02-27 18:38:38 engine.py:389] return cls(**init_kwargs)
ERROR 02-27 18:38:38 engine.py:389] ^^^^^^^^^^^^^^^^^^
ERROR 02-27 18:38:38 engine.py:389] File "/usr/local/lib/python3.12/dist-packages/vllm/transformers_utils/tokenizer_group/tokenizer_group.py", line 25, in __init__
ERROR 02-27 18:38:38 engine.py:389] self.tokenizer = get_tokenizer(self.tokenizer_id, **tokenizer_config)
ERROR 02-27 18:38:38 engine.py:389] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
ERROR 02-27 18:38:38 engine.py:389] File "/usr/local/lib/python3.12/dist-packages/vllm/transformers_utils/tokenizer.py", line 191, in get_tokenizer
ERROR 02-27 18:38:38 engine.py:389] tokenizer = AutoTokenizer.from_pretrained(
ERROR 02-27 18:38:38 engine.py:389] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
ERROR 02-27 18:38:38 engine.py:389] File "/usr/local/lib/python3.12/dist-packages/transformers/models/auto/tokenization_auto.py", line 934, in from_pretrained
ERROR 02-27 18:38:38 engine.py:389] return tokenizer_class.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)
ERROR 02-27 18:38:38 engine.py:389] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
ERROR 02-27 18:38:38 engine.py:389] File "/usr/local/lib/python3.12/dist-packages/transformers/tokenization_utils_base.py", line 2036, in from_pretrained
ERROR 02-27 18:38:38 engine.py:389] return cls._from_pretrained(
ERROR 02-27 18:38:38 engine.py:389] ^^^^^^^^^^^^^^^^^^^^^
ERROR 02-27 18:38:38 engine.py:389] File "/usr/local/lib/python3.12/dist-packages/transformers/tokenization_utils_base.py", line 2074, in _from_pretrained
ERROR 02-27 18:38:38 engine.py:389] slow_tokenizer = (cls.slow_tokenizer_class)._from_pretrained(
ERROR 02-27 18:38:38 engine.py:389] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
ERROR 02-27 18:38:38 engine.py:389] File "/usr/local/lib/python3.12/dist-packages/transformers/tokenization_utils_base.py", line 2276, in _from_pretrained
ERROR 02-27 18:38:38 engine.py:389] tokenizer = cls(*init_inputs, **init_kwargs)
ERROR 02-27 18:38:38 engine.py:389] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
ERROR 02-27 18:38:38 engine.py:389] File "/usr/local/lib/python3.12/dist-packages/transformers/models/llama/tokenization_llama.py", line 169, in __init__
ERROR 02-27 18:38:38 engine.py:389] self.sp_model = self.get_spm_processor(kwargs.pop("from_slow", False))
ERROR 02-27 18:38:38 engine.py:389] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
ERROR 02-27 18:38:38 engine.py:389] File "/usr/local/lib/python3.12/dist-packages/transformers/models/llama/tokenization_llama.py", line 199, in get_spm_processor
ERROR 02-27 18:38:38 engine.py:389] with open(self.vocab_file, "rb") as f:
ERROR 02-27 18:38:38 engine.py:389] ^^^^^^^^^^^^^^^^^^^^^^^^^^^
ERROR 02-27 18:38:38 engine.py:389] TypeError: expected str, bytes or os.PathLike object, not NoneType
Traceback (most recent call last):
File "/usr/lib/python3.12/multiprocessing/process.py", line 314, in _bootstrap
self.run()
File "/usr/lib/python3.12/multiprocessing/process.py", line 108, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.12/dist-packages/vllm/engine/multiprocessing/engine.py", line 391, in run_mp_engine
raise e
File "/usr/local/lib/python3.12/dist-packages/vllm/engine/multiprocessing/engine.py", line 380, in run_mp_engine
engine = MQLLMEngine.from_engine_args(engine_args=engine_args,
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/dist-packages/vllm/engine/multiprocessing/engine.py", line 123, in from_engine_args
return cls(ipc_path=ipc_path,
^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/dist-packages/vllm/engine/multiprocessing/engine.py", line 75, in __init__
self.engine = LLMEngine(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/dist-packages/vllm/engine/llm_engine.py", line 246, in __init__
self.tokenizer = self._init_tokenizer()
^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/dist-packages/vllm/engine/llm_engine.py", line 528, in _init_tokenizer
return init_tokenizer_from_configs(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/dist-packages/vllm/transformers_utils/tokenizer_group/__init__.py", line 32, in init_tokenizer_from_configs
return get_tokenizer_group(parallel_config.tokenizer_pool_config,
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/dist-packages/vllm/transformers_utils/tokenizer_group/__init__.py", line 53, in get_tokenizer_group
return tokenizer_cls.from_config(tokenizer_pool_config, **init_kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/dist-packages/vllm/transformers_utils/tokenizer_group/tokenizer_group.py", line 33, in from_config
return cls(**init_kwargs)
^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/dist-packages/vllm/transformers_utils/tokenizer_group/tokenizer_group.py", line 25, in __init__
self.tokenizer = get_tokenizer(self.tokenizer_id, **tokenizer_config)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/dist-packages/vllm/transformers_utils/tokenizer.py", line 191, in get_tokenizer
tokenizer = AutoTokenizer.from_pretrained(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/dist-packages/transformers/models/auto/tokenization_auto.py", line 934, in from_pretrained
return tokenizer_class.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/dist-packages/transformers/tokenization_utils_base.py", line 2036, in from_pretrained
return cls._from_pretrained(
^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/dist-packages/transformers/tokenization_utils_base.py", line 2074, in _from_pretrained
slow_tokenizer = (cls.slow_tokenizer_class)._from_pretrained(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/dist-packages/transformers/tokenization_utils_base.py", line 2276, in _from_pretrained
tokenizer = cls(*init_inputs, **init_kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/dist-packages/transformers/models/llama/tokenization_llama.py", line 169, in __init__
self.sp_model = self.get_spm_processor(kwargs.pop("from_slow", False))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/dist-packages/transformers/models/llama/tokenization_llama.py", line 199, in get_spm_processor
with open(self.vocab_file, "rb") as f:
^^^^^^^^^^^^^^^^^^^^^^^^^^^
TypeError: expected str, bytes or os.PathLike object, not NoneType