Class Token

Analyzed token with morphological data from its dictionary.

Inheritance

object

Token

Inherited Members

object.Equals(object)

object.Equals(object, object)

object.GetHashCode()

object.GetType()

object.MemberwiseClone()

object.ReferenceEquals(object, object)

Namespace: Lucene.Net.Analysis.Ja

Assembly: Lucene.Net.Analysis.Kuromoji.dll

Syntax

public class Token

Constructors

Token(int, char[], int, int, JapaneseTokenizerType, int, IDictionary)

Analyzed token with morphological data from its dictionary.

Declaration

public Token(int wordId, char[] surfaceForm, int offset, int length, JapaneseTokenizerType type, int position, IDictionary dictionary)

Parameters

Type	Name	Description
int	wordId
char[]	surfaceForm
int	offset
int	length
JapaneseTokenizerType	type
int	position
IDictionary	dictionary

Properties

IsKnown

Returns true if this token is known word.

Declaration

public virtual bool IsKnown { get; }

Property Value

Type	Description
bool	true if this token is in standard dictionary. false if not.

IsUnknown

Returns true if this token is unknown word.

Declaration

public virtual bool IsUnknown { get; }

Property Value

Type	Description
bool	true if this token is unknown word. false if not.

IsUser

Returns true if this token is defined in user dictionary.

Declaration

public virtual bool IsUser { get; }

Property Value

Type	Description
bool	true if this token is in user dictionary. false if not.

Length

length of surfaceForm

Declaration

public virtual int Length { get; }

Property Value

Type	Description
int

Offset

offset into surfaceForm

Declaration

public virtual int Offset { get; }

Property Value

Type	Description
int

Position

Get index of this token in input text. Returns position of token.

Declaration

public virtual int Position { get; }

Property Value

Type	Description
int

PositionLength

Gets or Sets the length (in tokens) of this token. For normal tokens this is 1; for compound tokens it's > 1.

Declaration

public virtual int PositionLength { get; set; }

Property Value

Type	Description
int

SurfaceForm

surfaceForm

Declaration

public virtual char[] SurfaceForm { get; }

Property Value

Type	Description
char[]

Methods

GetBaseForm()

base form or null if token is not inflected

Declaration

public virtual string GetBaseForm()

Returns

Type	Description
string	base form or null if token is not inflected

GetInflectionForm()

inflection form or null

Declaration

public virtual string GetInflectionForm()

Returns

Type	Description
string	inflection form or null

GetInflectionType()

inflection type or null

Declaration

public virtual string GetInflectionType()

Returns

Type	Description
string	inflection type or null

GetPartOfSpeech()

part of speech.

Declaration

public virtual string GetPartOfSpeech()

Returns

Type	Description
string	part of speech.

GetPronunciation()

pronunciation. null if token doesn't have pronunciation.

Declaration

public virtual string GetPronunciation()

Returns

Type	Description
string	pronunciation. null if token doesn't have pronunciation.

GetReading()

reading. null if token doesn't have reading.

Declaration

public virtual string GetReading()

Returns

Type	Description
string	reading. null if token doesn't have reading.

GetSurfaceFormString()

surfaceForm as a String

Declaration

public virtual string GetSurfaceFormString()

Returns

Type	Description
string	surfaceForm as a String

ToString()

Returns a string that represents the current object.

Declaration

public override string ToString()

Returns

Type	Description
string	A string that represents the current object.

Overrides

object.ToString()