Meta Llama 3 文本编码为 token
flyfish
tiktoken 是一个用于 OpenAI 模型的快速 BPE 分词器，这里用在Meta Llama 3上。主要功能包括将文本编码为token，以及将token解码回文本。这个过程通常使用BPE（Byte Pair Encoding）算法或其他类似的子词分割方法。
参考网址
https://github.com/openai/tiktoken
https://github.com/karpathy/minbpe
什么是BPE（Byte Pair Encoding）？
BPE（Byte Pair Encoding）是一种用于文本分词的子词（subword）分割算法。它通过逐步合并最常见的字符或字符序列来减少词汇表的大小，从而能够更高效地处理和表示文本数据。
BPE在tiktoken中的应用
简单的应用
py
import tiktoken
# 获取GPT-2编码器
enc = tiktoken.get_encoding("gpt2")
# 示例文本
text = "This is an example text."
# 将文本编码为tokens
tokens = enc.encode(text)
print(f"Encoded tokens: {tokens}")
# 将tokens解码为原文本
decoded_text = enc.decode(tokens)
print(f"Decoded text: {decoded_text}")Meta Llama 3的使用方式 - load_tiktoken_bpe函数
在tiktoken库中，BPE用于将文本编码成模型可以处理的tokens。load_tiktoken_bpe函数会加载BPE编码的词汇表和规则，以便将文本分解成子词单位。
代码示例
py
import os
from logging import getLogger
from pathlib import Path
from typing import (
AbstractSet,
cast,
Collection,
Dict,
Iterator,
List,
Literal,
Sequence,
TypedDict,
Union,
)
import tiktoken
from tiktoken.load import load_tiktoken_bpe
logger = getLogger(__name__)
Role = Literal["system", "user", "assistant"]
class Message(TypedDict):
role: Role
content: str
Dialog = Sequence[Message]
class Tokenizer:
"""
Tokenizing and encoding/decoding text using the Tiktoken tokenizer.
"""
special_tokens: Dict[str, int]
num_reserved_special_tokens = 256
pat_str = r"(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}{1,3}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+" # noqa: E501
def __init__(self, model_path: str):
"""
Initializes the Tokenizer with a Tiktoken model.
Args:
model_path (str): The path to the Tiktoken model file.
"""
assert os.path.isfile(model_path), model_path
mergeable_ranks = load_tiktoken_bpe(model_path)
num_base_tokens = len(mergeable_ranks)
special_tokens = [
"<|begin_of_text|>",
"<|end_of_text|>",
"<|reserved_special_token_0|>",
"<|reserved_special_token_1|>",
"<|reserved_special_token_2|>",
"<|reserved_special_token_3|>",
"<|start_header_id|>",
"<|end_header_id|>",
"<|reserved_special_token_4|>",
"<|eot_id|>", # end of turn
] + [
f"<|reserved_special_token_{i}|>"
for i in range(5, self.num_reserved_special_tokens - 5)
]
self.special_tokens = {
token: num_base_tokens + i for i, token in enumerate(special_tokens)
}
self.model = tiktoken.Encoding(
name=Path(model_path).name,
pat_str=self.pat_str,
mergeable_ranks=mergeable_ranks,
special_tokens=self.special_tokens,
)
logger.info(f"Reloaded tiktoken model from {model_path}")
self.n_words: int = self.model.n_vocab
# BOS / EOS token IDs
self.bos_id: int = self.special_tokens["<|begin_of_text|>"]
self.eos_id: int = self.special_tokens["<|end_of_text|>"]
self.pad_id: int = -1
self.stop_tokens = {
self.special_tokens["<|end_of_text|>"],
self.special_tokens["<|eot_id|>"],
}
logger.info(
f"#words: {self.n_words} - BOS ID: {self.bos_id} - EOS ID: {self.eos_id}"
)
def encode(
self,
s: str,
*,
bos: bool,
eos: bool,
allowed_special: Union[Literal["all"], AbstractSet[str]] = set(),
disallowed_special: Union[Literal["all"], Collection[str]] = (),
) -> List[int]:
assert type(s) is str
TIKTOKEN_MAX_ENCODE_CHARS = 400_000
MAX_NO_WHITESPACES_CHARS = 25_000
substrs = (
substr
for i in range(0, len(s), TIKTOKEN_MAX_ENCODE_CHARS)
for substr in self._split_whitespaces_or_nonwhitespaces(
s[i : i + TIKTOKEN_MAX_ENCODE_CHARS], MAX_NO_WHITESPACES_CHARS
)
)
t: List[int] = []
for substr in substrs:
t.extend(
self.model.encode(
substr,
allowed_special=allowed_special,
disallowed_special=disallowed_special,
)
)
if bos:
t.insert(0, self.bos_id)
if eos:
t.append(self.eos_id)
return t
def decode(self, t: Sequence[int]) -> str:
return self.model.decode(cast(List[int], t))
@staticmethod
def _split_whitespaces_or_nonwhitespaces(
s: str, max_consecutive_slice_len: int
) -> Iterator[str]:
current_slice_len = 0
current_slice_is_space = s[0].isspace() if len(s) > 0 else False
slice_start = 0
for i in range(len(s)):
is_now_space = s[i].isspace()
if current_slice_is_space ^ is_now_space:
current_slice_len = 1
current_slice_is_space = is_now_space
else:
current_slice_len += 1
if current_slice_len > max_consecutive_slice_len:
yield s[slice_start:i]
slice_start = i
current_slice_len = 1
yield s[slice_start:]
model_path = "Meta-Llama-3-8B-Instruct/tokenizer.model"
tokenizer = Tokenizer(model_path)
print(tokenizer.encode( "This is a test sentence.", bos=True,eos=True))
print(tokenizer.decode( [128000, 2028, 374, 264, 1296, 11914, 13, 128001]))
输出
[128000, 2028, 374, 264, 1296, 11914, 13, 128001]
<|begin_of_text|>This is a test sentence.<|end_of_text|>再测试一个
py
print(tokenizer.encode( "This is Ji'nan in the winter", bos=True,eos=True))
print(tokenizer.decode( [128000, 2028, 374, 55551, 6, 19285, 304, 279, 12688, 128001]))输出
py
[128000, 2028, 374, 55551, 6, 19285, 304, 279, 12688, 128001]
<|begin_of_text|>This is Ji'nan in the winter<|end_of_text|>在这个例子中，load_tiktoken_bpe函数加载了一个预训练的BPE词汇表和规则，然后使用这些规则将输入的文本分割成tokens。之后，这些tokens可以被解码回原文本。
扩展
py
import tiktoken
cl100k_base = tiktoken.get_encoding("cl100k_base")
# In production, load the arguments directly instead of accessing private attributes
# See openai_public.py for examples of arguments for specific encodings
enc = tiktoken.Encoding(
# If you're changing the set of special tokens, make sure to use a different name
# It should be clear from the name what behaviour to expect.
name="cl100k_im",
pat_str=cl100k_base._pat_str,
mergeable_ranks=cl100k_base._mergeable_ranks,
special_tokens={
**cl100k_base._special_tokens,
"<|im_start|>": 100264,
"<|im_end|>": 100265,
}
)
print(enc)#<Encoding 'cl100k_im'>