How to use logprobs hyperparameters?

[FocusLiwen]

Hi,

logprobs is one of the sampling parameters. How should we use it exactly?

Thanks.

[mspronesti]

Hi there. By adding the logprobs parameter you can see the log-probabilities of the most likely tokens, as well as the chosen token.
Consider the following example, which uses vLLM's OpenAI-compatible API

import openai

# Modify OpenAI's API key and API base to use vLLM's API server.
openai.api_key = "EMPTY"
openai.api_base = "http://localhost:8000/v1"

# List models API
models = openai.Model.list()
print("Models:", models)

model = models["data"][0]["id"]

# Completion API
stream = False
completion = openai.Completion.create(
    model=model,
    prompt="A robot may not injure a human being",
    echo=False,
    stream=stream,
    logprobs=5)

print("Completion results:")
if stream:
    for c in completion:
        print(c)
else:
    print(completion)

The response should look like something like this

{
  "id": "cmpl-bdcf2446e4f54a9ba8626af60425e1fd",
  "object": "text_completion",
  "created": 1695939768,
  "model": "mosaicml/mpt-7b",
  "choices": [
    {
      "index": 0,
      "text": " or, through inaction, allow a human being to come to harm.\n",
      "logprobs": {
        "text_offset": [
          0,
          3,
          4,
          12,
          15,
          21,
          22,
          28,
          30,
          36,
          42,
          45,
          50,
          53,
          58,
          59
        ],
        "token_logprobs": [
          -0.12413621693849564,
          -0.046535659581422806,
          -0.04955923184752464,
          -0.009703610092401505,
          -0.002873817225918174,
          -0.016049010679125786,
          -0.01629580371081829,
          -0.019824113696813583,
          -0.005483941175043583,
          -0.025897447019815445,
          -0.005838601849973202,
          -0.025539720430970192,
          -0.0021288369316607714,
          -0.01958426833152771,
          -0.13800686597824097,
          -0.496429443359375
        ],
        "tokens": [
          "\u0120or",
          ",",
          "\u0120through",
          "\u0120in",
          "action",
          ",",
          "\u0120allow",
          "\u0120a",
          "\u0120human",
          "\u0120being",
          "\u0120to",
          "\u0120come",
          "\u0120to",
          "\u0120harm",
          ".",
          "\u010a"
        ],
        "top_logprobs": [
          {
            "\u0120or": -0.12413621693849564,
            ",": -2.624136209487915,
            ".": -4.374136447906494,
            "\u0120(": -5.624136447906494,
            ";": -6.124136447906494
          },
          {
            ",": -0.046535659581422806,
            "\u0120through": -3.7965357303619385,
            "\u0120cause": -6.046535491943359,
            "\u0120as": -6.046535491943359,
            "\u0120or": -6.796535491943359
          },
          {
            "\u0120through": -0.04955923184752464,
            "\u0120as": -3.2995591163635254,
            "\u0120via": -6.299559116363525,
            "\u0120thru": -6.299559116363525,
            "through": -6.799559116363525
          },
          {
            "\u0120in": -0.009703610092401505,
            "\u0120its": -5.759703636169434,
            "\u0120the": -7.259703636169434,
            "\u0120his": -7.759703636169434,
            ",": -8.259703636169434
          },
          {
            "action": -0.002873817225918174,
            "activity": -6.12787389755249,
            "actions": -8.002873420715332,
            "\u0120action": -9.127873420715332,
            "attention": -9.627873420715332
          },
          {
            ",": -0.016049010679125786,
            "\u0120or": -4.766048908233643,
            "\u0120cause": -6.516048908233643,
            "\u0120(": -7.016048908233643,
            "\u0120allow": -7.516048908233643
          },
          {
            "\u0120allow": -0.01629580371081829,
            "\u0120permit": -5.766295909881592,
            "\u0120cause": -6.516295909881592,
            "\u0120allowing": -6.766295909881592,
            "\u0120allowed": -6.766295909881592
          },
          {
            "\u0120a": -0.019824113696813583,
            "\u0120human": -5.769824028015137,
            "\u0120such": -6.269824028015137,
            "\u0120the": -6.269824028015137,
            "\u0120one": -6.519824028015137
          },
          {
            "\u0120human": -0.005483941175043583,
            "\u0120person": -7.005484104156494,
            "\u0120dangerous": -7.505484104156494,
            "\u0120robot": -8.005483627319336,
            "\u0120humans": -8.005483627319336
          },
          {
            "\u0120being": -0.025897447019815445,
            "\u0120to": -3.77589750289917,
            "\u0120beings": -7.77589750289917,
            "\u0120be": -7.77589750289917,
            "being": -8.275897026062012
          },
          {
            "\u0120to": -0.005838601849973202,
            "\u00e2\u0122\u013b": -7.755838394165039,
            "\u0120or": -7.755838394165039,
            "\u0120(": -8.005838394165039,
            ",": -8.255838394165039
          },
          {
            "\u0120come": -0.025539720430970192,
            "\u0120be": -3.7755396366119385,
            "\u0120inj": -7.275539875030518,
            "\u0120to": -9.52553939819336,
            ".": -9.52553939819336
          },
          {
            "\u0120to": -0.0021288369316607714,
            "\u0120into": -7.502128601074219,
            "\u0120in": -8.752128601074219,
            "\u0120t": -9.252128601074219,
            ".": -9.252128601074219
          },
          {
            "\u0120harm": -0.01958426833152771,
            "\u0120injury": -5.0195841789245605,
            "\u0120physical": -6.0195841789245605,
            "\u0120any": -6.0195841789245605,
            "\u0120serious": -6.7695841789245605
          },
          {
            ".": -0.13800686597824097,
            ",": -3.6380069255828857,
            ".\"": -3.8880069255828857,
            ";": -4.138006687164307,
            "\u0120(": -4.888006687164307
          },
          {
            "\u010a": -0.496429443359375,
            "\u0120A": -2.496429443359375,
            "\u0120The": -3.496429443359375,
            "\u0120It": -3.496429443359375,
            "\u0120This": -3.746429443359375
          }
        ]
      },
      "finish_reason": "length"
    }
  ],
  "usage": {
    "prompt_tokens": 9,
    "total_tokens": 25,
    "completion_tokens": 16
  }
}

For each token, you can see the top K (5 in this example) most probable tokens and their log probabilities.
Hope this helps :)

[FocusLiwen]

Thanks for the example!