nndeploy C++ API  0.2.0
nndeploy C++ API
Public Member Functions | Public Attributes | List of all members
nndeploy::tokenizer::TokenizerPraram Class Reference

#include <tokenizer.h>

Inheritance diagram for nndeploy::tokenizer::TokenizerPraram:
[legend]
Collaboration diagram for nndeploy::tokenizer::TokenizerPraram:
[legend]

Public Member Functions

virtual base::Status deserialize (rapidjson::Value &json) override
 
std::string getShareKey ()
 
TokenizerPraram & operator= (const TokenizerPraram &tp)
 
virtual base::Status serialize (rapidjson::Value &json, rapidjson::Document::AllocatorType &allocator) override
 
 TokenizerPraram ()
 
virtual ~TokenizerPraram ()
 
- Public Member Functions inherited from nndeploy::base::Param
base::Status addDropdownParam (const std::string &dropdown_param, const std::vector< std::string > &dropdown_values)
 
base::Status addIoParam (const std::string &io_param)
 
base::Status addRequiredParam (const std::string &required_param)
 
base::Status addUiParam (const std::string &ui_param)
 
base::Status clearDropdownParams ()
 
base::Status clearIoParams ()
 
base::Status clearRequiredParams ()
 
base::Status clearUiParams ()
 
virtual base::Status deserialize (const std::string &json_str)
 
virtual base::Status get (const std::string &key, base::Any &any)
 
std::map< std::string, std::vector< std::string > > getDropdownParams ()
 
std::vector< std::string > getIoParams ()
 
std::vector< std::string > getRequiredParams ()
 
std::vector< std::string > getUiParams ()
 
virtual base::Status loadFile (const std::string &path)
 
 Param ()
 
base::Status removeDropdownParam (const std::string &dropdown_param)
 
base::Status removeIoParam (const std::string &io_param)
 
base::Status removeRequiredParam (const std::string &required_param)
 
base::Status removeUiParam (const std::string &ui_param)
 
virtual base::Status saveFile (const std::string &path)
 
virtual std::string serialize ()
 
virtual base::Status set (const std::string &key, base::Any &any)
 
base::Status setDropdownParams (const std::map< std::string, std::vector< std::string >> &dropdown_params)
 
base::Status setIoParams (const std::vector< std::string > &io_params)
 
base::Status setRequiredParams (const std::vector< std::string > &required_params)
 
base::Status setUiParams (const std::vector< std::string > &ui_params)
 
virtual ~Param ()
 

Public Attributes

std::string added_tokens_
 
bool is_path_ = true
 
std::string json_blob_
 Create HF tokenizer from a single in-memory json blob. More...
 
int max_length_ = 77
 
std::string merges_blob_
 
std::string model_blob_
 Create SentencePiece. More...
 
TokenizerType tokenizer_type_ = kTokenizerTypeHF
 
std::string vocab_blob_
 Create BPE tokenizer. More...
 
- Public Attributes inherited from nndeploy::base::Param
std::map< std::string, std::vector< std::string > > dropdown_params_
 
std::vector< std::string > io_params_
 
std::vector< std::string > required_params_
 
std::vector< std::string > ui_params_
 

Detailed Description

Definition at line 69 of file tokenizer.h.

Constructor & Destructor Documentation

◆ TokenizerPraram()

nndeploy::tokenizer::TokenizerPraram::TokenizerPraram ( )
inline

Definition at line 71 of file tokenizer.h.

◆ ~TokenizerPraram()

virtual nndeploy::tokenizer::TokenizerPraram::~TokenizerPraram ( )
inlinevirtual

Definition at line 72 of file tokenizer.h.

Member Function Documentation

◆ deserialize()

virtual base::Status nndeploy::tokenizer::TokenizerPraram::deserialize ( rapidjson::Value &  json)
inlineoverridevirtual

Reimplemented from nndeploy::base::Param.

Definition at line 175 of file tokenizer.h.

Here is the call graph for this function:

◆ getShareKey()

std::string nndeploy::tokenizer::TokenizerPraram::getShareKey ( )
inline

Definition at line 132 of file tokenizer.h.

Here is the call graph for this function:

◆ operator=()

TokenizerPraram& nndeploy::tokenizer::TokenizerPraram::operator= ( const TokenizerPraram &  tp)
inline

Definition at line 77 of file tokenizer.h.

◆ serialize()

virtual base::Status nndeploy::tokenizer::TokenizerPraram::serialize ( rapidjson::Value &  json,
rapidjson::Document::AllocatorType &  allocator 
)
inlineoverridevirtual

Reimplemented from nndeploy::base::Param.

Definition at line 144 of file tokenizer.h.

Here is the call graph for this function:

Member Data Documentation

◆ added_tokens_

std::string nndeploy::tokenizer::TokenizerPraram::added_tokens_

Definition at line 128 of file tokenizer.h.

◆ is_path_

bool nndeploy::tokenizer::TokenizerPraram::is_path_ = true

Definition at line 94 of file tokenizer.h.

◆ json_blob_

std::string nndeploy::tokenizer::TokenizerPraram::json_blob_

Create HF tokenizer from a single in-memory json blob.

Parameters
json_blobThe json blob.
Returns
The created tokenzier.

Definition at line 104 of file tokenizer.h.

◆ max_length_

int nndeploy::tokenizer::TokenizerPraram::max_length_ = 77

Definition at line 130 of file tokenizer.h.

◆ merges_blob_

std::string nndeploy::tokenizer::TokenizerPraram::merges_blob_

Definition at line 127 of file tokenizer.h.

◆ model_blob_

std::string nndeploy::tokenizer::TokenizerPraram::model_blob_

Create SentencePiece.

Parameters
model_blobThe blob that contains vocabs.
Returns
The created tokenizer.

Create RWKVWorldTokenizer.

Parameters
model_blobThe blob that contains vocabs.
Returns
The created tokenizer.

Definition at line 117 of file tokenizer.h.

◆ tokenizer_type_

TokenizerType nndeploy::tokenizer::TokenizerPraram::tokenizer_type_ = kTokenizerTypeHF

Definition at line 96 of file tokenizer.h.

◆ vocab_blob_

std::string nndeploy::tokenizer::TokenizerPraram::vocab_blob_

Create BPE tokenizer.

Parameters
vocab_blobThe blob that contains vocabs.
merges_blobThe blob that contains the merges.
added_tokensThe added tokens.
Returns
The created tokenizer.

Definition at line 126 of file tokenizer.h.

The documentation for this class was generated from the following file:
Generated by doxygen 1.9.1