Tokenize

POST/v1/tokenizeTokenize

This endpoint splits input text into smaller units called tokens using byte-pair encoding (BPE). To learn more about tokenization and byte pair encoding, see the tokens page.

Parameters

X-Client-Namestringheader

The name of the project that is making the request.

Request body

application/json

object

modelstringrequired

The input will be tokenized by the tokenizer that is used by this model.

textstringrequired

The string to be tokenized, the minimum text length is 1 character, and the maximum text length is 65536 characters.

maxLength 65536 · minLength 1

Example request

{
  "model": "command",
  "text": "tokenize me! :D"
}

Responses

Headers

X-API-Warning

The name of the project that is making the request.

object

metaobject

Show child attributes

api_versionobject

Show child attributes

is_deprecatedboolean

is_experimentalboolean

versionstringrequired

billed_unitsobject

Show child attributes

classificationsnumber

The number of billed classifications units.

image_tokensnumber

The number of billed image tokens.

imagesnumber

The number of billed images.

input_tokensnumber

The number of billed input tokens.

output_tokensnumber

The number of billed output tokens.

pagesnumber

The number of billed pages parsed.

search_unitsnumber

The number of billed search units.

cached_tokensnumber

The number of prompt tokens that hit the inference cache.

tokensobject

Show child attributes

input_tokensnumber

The number of tokens used as input to the model.

output_tokensnumber

The number of tokens produced by the model.

warningsarray of string

Show child attributes

token_stringsarray of stringrequired

Show child attributes

tokensarray of integerrequired

An array of tokens, where each token is an integer.

Show child attributes

Example response

{
  "meta": {
    "api_version": {
      "version": "1"
    }
  },
  "token_strings": [
    "token",
    "ize'",
    " me",
    "!",
    " :",
    "D"
  ],
  "tokens": [
    34160,
    974,
    514,
    34,
    1420,
    69
  ]
}

object

idstring

messagestring

Example response

{
  "id": "string",
  "message": "string"
}

object

idstring

messagestring

Example response

{
  "id": "string",
  "message": "string"
}

object

idstring

messagestring

Example response

{
  "id": "string",
  "message": "string"
}

object

idstring

messagestring

Example response

{
  "id": "string",
  "message": "string"
}

object

idstring

messagestring

Example response

{
  "id": "string",
  "message": "string"
}

object

idstring

messagestring

Example response

{
  "id": "string",
  "message": "string"
}

object

idstring

messagestring

Example response

{
  "id": "string",
  "message": "string"
}

object

idstring

messagestring

Example response

{
  "id": "string",
  "message": "string"
}

object

idstring

messagestring

Example response

{
  "id": "string",
  "message": "string"
}

object

idstring

messagestring

Example response

{
  "id": "string",
  "message": "string"
}

object

idstring

messagestring

Example response

{
  "id": "string",
  "message": "string"
}

object

idstring

messagestring

Example response

{
  "id": "string",
  "message": "string"
}

Examples

X-Client-Nameheader parameter

curl -X POST 'https://api.cohere.com/v1/tokenize' \
  -H 'Content-Type: application/json' \
  -d '{"model":"string","text":"string"}'

const response = await fetch('https://api.cohere.com/v1/tokenize', {
  method: 'POST',
  headers: { 'Content-Type': 'application/json' },
  body: JSON.stringify({"model":"string","text":"string"}),
});
const data = await response.json();

const response = await fetch('https://api.cohere.com/v1/tokenize', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: "{\"model\":\"string\",\"text\":\"string\"}" });
console.log(await response.json());

import requests

response = requests.request(
    "POST",
    "https://api.cohere.com/v1/tokenize",
    headers={"Content-Type": "application/json"},
    data='{"model":"string","text":"string"}',
)
print(response.json())

package main

import (
	"context"
	"net/http"
	"strings"
)

func main() {
	ctx := context.Background()
	body := strings.NewReader(`{"model":"string","text":"string"}`)
	req, err := http.NewRequestWithContext(ctx, "POST", "https://api.cohere.com/v1/tokenize", body)
	if err != nil {
		panic(err)
	}
	req.Header.Set("Content-Type", "application/json")
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		panic(err)
	}
	defer resp.Body.Close()
}

<?php
$request = curl_init("https://api.cohere.com/v1/tokenize");
curl_setopt($request, CURLOPT_CUSTOMREQUEST, "POST");
curl_setopt($request, CURLOPT_POSTFIELDS, "{\"model\":\"string\",\"text\":\"string\"}");
curl_setopt($request, CURLOPT_HTTPHEADER, ['Content-Type: application/json']);
$response = curl_exec($request);
curl_close($request);

var request = HttpRequest.newBuilder(URI.create("https://api.cohere.com/v1/tokenize"))
    .method("POST", HttpRequest.BodyPublishers.noBody())
    .header("Content-Type", "application/json")
    .method("POST", HttpRequest.BodyPublishers.ofString("{\"model\":\"string\",\"text\":\"string\"}"))
    .build();
var response = HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString());

require 'net/http'

uri = URI("https://api.cohere.com/v1/tokenize")
request = Net::HTTPGenericRequest.new("POST", true, true, uri)
request['Content-Type'] = 'application/json'
request.body = "{\"model\":\"string\",\"text\":\"string\"}"
response = Net::HTTP.start(uri.hostname, uri.port, use_ssl: true) { |http| http.request(request) }

using var client = new HttpClient();
using var request = new HttpRequestMessage(HttpMethod.Post, "https://api.cohere.com/v1/tokenize");
request.Content = new StringContent("{\"model\":\"string\",\"text\":\"string\"}", Encoding.UTF8, "application/json");
using var response = await client.SendAsync(request);

{
  "meta": {
    "api_version": {
      "version": "1"
    }
  },
  "token_strings": [
    "token",
    "ize'",
    " me",
    "!",
    " :",
    "D"
  ],
  "tokens": [
    34160,
    974,
    514,
    34,
    1420,
    69
  ]
}

{
  "id": "string",
  "message": "string"
}

{
  "id": "string",
  "message": "string"
}

{
  "id": "string",
  "message": "string"
}

{
  "id": "string",
  "message": "string"
}

{
  "id": "string",
  "message": "string"
}

{
  "id": "string",
  "message": "string"
}

{
  "id": "string",
  "message": "string"
}

{
  "id": "string",
  "message": "string"
}

{
  "id": "string",
  "message": "string"
}

{
  "id": "string",
  "message": "string"
}

{
  "id": "string",
  "message": "string"
}

{
  "id": "string",
  "message": "string"
}

Search documentation

/datasets

/connectors

/embed-jobs

/finetuning

/models

/audio/transcriptions

/batches

Operations

Parameters

Request body

Responses

Headers

/datasets

/connectors

/embed-jobs

/finetuning

/models

/audio/transcriptions

/batches

Operations