tokenizer.h File Reference

#include "nndeploy/base/any.h"
#include "nndeploy/base/common.h"
#include "nndeploy/base/glic_stl_include.h"
#include "nndeploy/base/log.h"
#include "nndeploy/base/macro.h"
#include "nndeploy/base/object.h"
#include "nndeploy/base/param.h"
#include "nndeploy/base/status.h"
#include "nndeploy/base/string.h"
#include "nndeploy/base/type.h"
#include "nndeploy/dag/edge.h"
#include "nndeploy/dag/graph.h"
#include "nndeploy/dag/node.h"
#include "nndeploy/device/buffer.h"
#include "nndeploy/device/device.h"
#include "nndeploy/device/memory_pool.h"
#include "nndeploy/device/tensor.h"

Include dependency graph for tokenizer.h:

This graph shows which files directly or indirectly include this file:

Go to the source code of this file.

Classes
class	nndeploy::tokenizer::TokenizerDecode
	TokenizerDecode. More...
class	nndeploy::tokenizer::TokenizerEncode
	TokenizerEncode. More...
class	nndeploy::tokenizer::TokenizerIds
class	nndeploy::tokenizer::TokenizerPraram
class	nndeploy::tokenizer::TokenizerText

Namespaces
	nndeploy
	nndeploy::tokenizer

Enumerations
enum	nndeploy::tokenizer::TokenizerType : int {   nndeploy::tokenizer::kTokenizerTypeHF = 0x0000 , nndeploy::tokenizer::kTokenizerTypeBPE , nndeploy::tokenizer::kTokenizerTypeSentencePiece , nndeploy::tokenizer::kTokenizerTypeRWKVWorld ,   nndeploy::tokenizer::kTokenizerTypeNotSupport }

Functions
TokenizerType	nndeploy::tokenizer::stringToTokenizerType (const std::string &src)
std::string	nndeploy::tokenizer::tokenizerTypeToString (TokenizerType type)