LLM推論の解説：モデルをサービングするとき実際に何が起きているのか

3.7G model-00002-of-00003.safetensors
3.7G model-00001-of-00003.safetensors
 95M model-00003-of-00003.safetensors
 11M tokenizer.json
2.6M vocab.json
1.6M merges.txt
 32K model.safetensors.index.json
 16K README.md
 11K LICENSE
9.5K tokenizer_config.json
726B config.json
239B generation_config.json

{
  "architectures": [
    "Qwen3ForCausalLM"
  ],
  ...
  "max_window_layers": 36,
  "model_type": "qwen3",
  "num_attention_heads": 32,
  "num_hidden_layers": 36,
  "num_key_value_heads": 8,
  "torch_dtype": "bfloat16",
  "transformers_version": "4.51.0",
  "use_cache": true,
  "use_sliding_window": false,
  "vocab_size": 151936
}

{
  "version": "1.0",
  "truncation": null,
  "padding": null,
  "added_tokens": [
    {
      "id": 151643,
      "content": "<|endoftext|>",
      "single_word": false,
      "lstrip": false,
      "rstrip": false,
      "normalized": false,
      "special": true
    },
    ...
  ],
  "normalizer": {...},
  "decoder": {...},
  "model": {
    "type": "BPE",
    "vocab": {
      "!": 0,
      "\"": 1,
      "#": 2,
      "$": 3,
      ...
    }
  },
  "merges": [...]
}

{
  "chat_template": "{%- if tools %}\n    {{- '<|im_start|>system\\n' }}\n    {%- if messages[0].role == 'system' %}\n        {{- messages[0].content + '\\n\\n' }}\n    {%- endif %}\n    {{- \"# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n    {%- for tool in tools %}\n        {{- \"\\n\" }}\n        {{- tool | tojson }}\n    {%- endfor %}\n    {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n    {%- if messages[0].role == 'system' %}\n        {{- '<|im_start|>system\\n' + messages[0].content + '<|im_end|>\\n' }}\n    {%- endif %}\n{%- endif %}\n{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}\n{%- for message in messages[::-1] %}\n    {%- set index = (messages|length - 1) - loop.index0 %}\n    {%- if ns.multi_step_tool and message.role == \"user\" and message.content is string and not(message.content.startswith('<tool_response>') and message.content.endswith('</tool_response>')) %}\n        {%- set ns.multi_step_tool = false %}\n        {%- set ns.last_query_index = index %}\n    {%- endif %}\n{%- endfor %}\n{%- for message in messages %}\n    {%- if message.content is string %}\n        {%- set content = message.content %}\n    {%- else %}\n        {%- set content = '' %}\n    {%- endif %}\n    {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) %}\n        {{- '<|im_start|>' + message.role + '\\n' + content + '<|im_end|>' + '\\n' }}\n    {%- elif message.role == \"assistant\" %}\n        {%- set reasoning_content = '' %}\n        {%- if message.reasoning_content is string %}\n            {%- set reasoning_content = message.reasoning_content %}\n        {%- else %}\n            {%- if '</think>' in content %}\n                {%- set reasoning_content = content.split('</think>')[0].rstrip('\\n').split('<think>')[-1].lstrip('\\n') %}\n                {%- set content = content.split('</think>')[-1].lstrip('\\n') %}\n            {%- endif %}\n        {%- endif %}\n        {%- if loop.index0 > ns.last_query_index %}\n            {%- if loop.last or (not loop.last and reasoning_content) %}\n                {{- '<|im_start|>' + message.role + '\\n<think>\\n' + reasoning_content.strip('\\n') + '\\n</think>\\n\\n' + content.lstrip('\\n') }}\n            {%- else %}\n                {{- '<|im_start|>' + message.role + '\\n' + content }}\n            {%- endif %}\n        {%- else %}\n            {{- '<|im_start|>' + message.role + '\\n' + content }}\n        {%- endif %}\n        {%- if message.tool_calls %}\n            {%- for tool_call in message.tool_calls %}\n                {%- if (loop.first and content) or (not loop.first) %}\n                    {{- '\\n' }}\n                {%- endif %}\n                {%- if tool_call.function %}\n                    {%- set tool_call = tool_call.function %}\n                {%- endif %}\n                {{- '<tool_call>\\n{\"name\": \"' }}\n                {{- tool_call.name }}\n                {{- '\", \"arguments\": ' }}\n                {%- if tool_call.arguments is string %}\n                    {{- tool_call.arguments }}\n                {%- else %}\n                    {{- tool_call.arguments | tojson }}\n                {%- endif %}\n                {{- '}\\n</tool_call>' }}\n            {%- endfor %}\n        {%- endif %}\n        {{- '<|im_end|>\\n' }}\n    {%- elif message.role == \"tool\" %}\n        {%- if loop.first or (messages[loop.index0 - 1].role != \"tool\") %}\n            {{- '<|im_start|>user' }}\n        {%- endif %}\n        {{- '\\n<tool_response>\\n' }}\n        {{- content }}\n        {{- '\\n</tool_response>' }}\n        {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n            {{- '<|im_end|>\\n' }}\n        {%- endif %}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|im_start|>assistant\\n' }}\n    {%- if enable_thinking is defined and enable_thinking is false %}\n        {{- '<think>\\n\\n</think>\\n\\n' }}\n    {%- endif %}\n{%- endif %}",
  ...
}

{
    "bos_token_id": 151643,
    "do_sample": true,
    "eos_token_id": [
        151645,
        151643
    ],
    "pad_token_id": 151643,
    "temperature": 0.6,
    "top_k": 20,
    "top_p": 0.95,
    "transformers_version": "4.51.0"
}

model.embed_tokens.weight
tensor([[-0.0287,  0.0117,  0.0104,  ..., -0.0055, -0.0270, -0.0096],
        [-0.0291, -0.0212,  0.0109,  ..., -0.0033, -0.0062, -0.0004],
        [ 0.0035,  0.0177,  0.0012,  ..., -0.0058,  0.0022,  0.0130],
        ...,
        [ 0.0060,  0.0131,  0.0190,  ...,  0.0068, -0.0049, -0.0040],
        [ 0.0060,  0.0131,  0.0190,  ...,  0.0068, -0.0049, -0.0040],
        [ 0.0060,  0.0131,  0.0190,  ...,  0.0068, -0.0049, -0.0040]],
       dtype=torch.bfloat16)
model.layers.0.input_layernorm.weight
tensor([0.0598, 0.0288, 0.0576,  ..., 0.0184, 0.0222, 0.0223],
       dtype=torch.bfloat16)
model.layers.0.mlp.down_proj.weight
tensor([[ 0.1396,  0.0056,  0.0054,  ..., -0.0084,  0.0781, -0.0693],
        [-0.0859, -0.0074, -0.0142,  ..., -0.0161, -0.0022, -0.0403],
        [ 0.0649, -0.0021,  0.0081,  ...,  0.0110,  0.0223,  0.0283],
        ...,
        [ 0.0106,  0.0208,  0.0176,  ...,  0.0060, -0.0349, -0.0060],
        [-0.0079, -0.0007, -0.0603,  ..., -0.0059, -0.0271,  0.0085],
        [-0.0221, -0.0306,  0.0121,  ..., -0.0089, -0.0233,  0.0089]],
       dtype=torch.bfloat16)
model.layers.0.mlp.gate_proj.weight
tensor([[ 0.0014,  0.0057, -0.0032,  ..., -0.0415, -0.0276, -0.0154],
        [-0.0178,  0.0078,  0.0043,  ..., -0.0186, -0.0028, -0.0018],
        [-0.0042, -0.0018, -0.0016,  ...,  0.0330, -0.0026,  0.0244],
        ...,
        [-0.0081, -0.0007, -0.0052,  ..., -0.0076, -0.0114,  0.0062],
        [-0.0061,  0.0118, -0.0116,  ..., -0.0164, -0.0015,  0.0237],
        [ 0.0130,  0.0046, -0.0036,  ..., -0.0105,  0.0297, -0.0347]],
       dtype=torch.bfloat16)
model.layers.0.mlp.up_proj.weight
tensor([[ 0.0137,  0.0089,  0.0046,  ..., -0.0339,  0.0061, -0.0117],
        [ 0.0077,  0.0078,  0.0036,  ...,  0.0134,  0.0216, -0.0040],
        [-0.0105,  0.0057,  0.0053,  ...,  0.0251,  0.0044,  0.0092],
        ...,
        [ 0.0067, -0.0003, -0.0082,  ..., -0.0049,  0.0247, -0.0181],
        [-0.0021,  0.0044, -0.0056,  ..., -0.0111,  0.0133,  0.0135],
        [ 0.0046, -0.0028,  0.0004,  ...,  0.0167,  0.0505,  0.0347]],
       dtype=torch.bfloat16)
model.layers.0.post_attention_layernorm.weight
tensor([-2.0862e-05,  2.7466e-04, -5.5313e-05,  ...,  2.1582e-01,
         2.3145e-01,  2.1484e-01], dtype=torch.bfloat16)
model.layers.0.self_attn.k_norm.weight
tensor([ 1.9453e+00,  1.1172e+00,  1.7031e+00,  1.7969e+00,  1.4688e+00,
         1.9141e+00,  1.9844e+00,  1.8281e+00,  1.8203e+00,  1.6406e+00,
         ...,
         6.4844e-01,  1.8203e+00,  2.3750e+00,  2.9062e+00,  4.0938e+00,
         2.0625e+00,  2.1094e+00,  3.0781e+00,  1.5234e+00,  2.3125e+00,
         1.9141e+00,  2.0469e+00,  2.0156e+00], dtype=torch.bfloat16)

from urllib.request import urlopen
from tokenizers import Tokenizer
TOK_URL = "https://huggingface.co/Qwen/Qwen3-4B/resolve/main/tokenizer.json"
tok = Tokenizer.from_buffer(urlopen(TOK_URL).read())
for text in ["Why is the sky blue?", "我喜欢吃火锅!", "김치찌개를 좋아해요!"]:
    enc = tok.encode(text)
    pieces = [tok.decode([i]) for i in enc.ids]
    print(f"{text!r}")
    print(f"  {len(enc.ids):>2} tokens -> {pieces}")
 
'Why is the sky blue?'
   6 tokens -> ['Why', ' is', ' the', ' sky', ' blue', '?']
 
'我喜欢吃火锅!'
   4 tokens -> ['我喜欢', '吃', '火锅', '!']
 
'김치찌개를 좋아해요!'
   9 tokens -> ['김', '치', '찌', '개', '를', ' 좋아', '해', '요', '!']

POST /v1/chat/completions HTTP/1.1
Host: localhost:11434
Content-Type: application/json
{
  "messages": [
    { "role": "user", "content": "What is the capital of Belgium?" },
    { "role": "assistant", "content": "The capital of Belgium is **Brussels**..." },
    { "role": "user", "content": "How many people live in that city?" }
  ],
  "model": "qwen3:8b"
}
 
HTTP/1.1 200 OK
Content-Type: application/json
{
  ...
  "choices": [
    {
      "message": {
        "role": "assistant",
        "content": "As of 2023, the **Brussels-Capital Region** has a population
                    of approximately **1.2 million people**..."
      },
      ...
    }
  ],
  "usage": { "prompt_tokens": 134, "completion_tokens": 843, "total_tokens": 977 }
}

vllm bench serve \
  --model Qwen/Qwen3-4B-AWQ \
  --base-url http://localhost:8000 \
  --dataset-name random \
  --random-input-len 512 --random-output-len 128 \
  --num-prompts 64 --max-concurrency 8

================= Serving Benchmark Result =================
Successful requests:                     64        
Failed requests:                         0         
Maximum request concurrency:             8         
Benchmark duration (s):                  38.83     
Total input tokens:                      32707     
Total generated tokens:                  8192      
Request throughput (req/s):              1.65      
Output token throughput (tok/s):         210.95    
Peak output token throughput (tok/s):    392.00    
Peak concurrent requests:                16.00     
Total token throughput (tok/s):          1053.20   
--------------------Time to First Token---------------------
Mean TTFT (ms):                          1632.54   
Median TTFT (ms):                        1355.88   
P99 TTFT (ms):                           5376.73   
P90 TTFT (ms):                           4301.80   
----------Time per Output Token (excl. 1st token)-----------
Mean TPOT (ms):                          25.34     
Median TPOT (ms):                        24.11     
P99 TPOT (ms):                           40.39     
P90 TPOT (ms):                           32.41     
--------------------Inter-token Latency---------------------
Mean ITL (ms):                           25.36     
Median ITL (ms):                         20.61     
P99 ITL (ms):                            29.10     
P90 ITL (ms):                            21.39     
---------------------End-to-end Latency---------------------
Mean E2EL (ms):                          4851.09   
Median E2EL (ms):                        4414.25   
P99 E2EL (ms):                           7989.19   
P90 E2EL (ms):                           7964.52   
============================================================