Token.cs

Go to the documentation of this file.

/* 
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 * 
 * http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

using System;
using Lucene.Net.Analysis.Tokenattributes;
using Lucene.Net.Support;
using Lucene.Net.Util;
using Payload = Lucene.Net.Index.Payload;
using TermPositions = Lucene.Net.Index.TermPositions;
using ArrayUtil = Lucene.Net.Util.ArrayUtil;
using Attribute = Lucene.Net.Util.Attribute;

namespace Lucene.Net.Analysis
{
    
    [Serializable]
    public class Token : Attribute, ITermAttribute, ITypeAttribute, IPositionIncrementAttribute, IFlagsAttribute, IOffsetAttribute, IPayloadAttribute
    {
        public const String DEFAULT_TYPE = "word";

        private const int MIN_BUFFER_SIZE = 10;

        private char[] termBuffer;
        private int termLength;
        private int startOffset, endOffset;
        private string type = DEFAULT_TYPE;
        private int flags;
        private Payload payload;
        private int positionIncrement = 1;
        
        public Token()
        {
        }
        
        public Token(int start, int end)
        {
            startOffset = start;
            endOffset = end;
        }
        
        public Token(int start, int end, String typ)
        {
            startOffset = start;
            endOffset = end;
            type = typ;
        }
        
        public Token(int start, int end, int flags)
        {
            startOffset = start;
            endOffset = end;
            this.flags = flags;
        }
        
        public Token(String text, int start, int end)
        {
            SetTermBuffer(text);
            startOffset = start;
            endOffset = end;
        }
        
        public Token(System.String text, int start, int end, System.String typ)
        {
            SetTermBuffer(text);
            startOffset = start;
            endOffset = end;
            type = typ;
        }
        
        public Token(System.String text, int start, int end, int flags)
        {
            SetTermBuffer(text);
            startOffset = start;
            endOffset = end;
            this.flags = flags;
        }
        
        public Token(char[] startTermBuffer, int termBufferOffset, int termBufferLength, int start, int end)
        {
            SetTermBuffer(startTermBuffer, termBufferOffset, termBufferLength);
            startOffset = start;
            endOffset = end;
        }

        public virtual int PositionIncrement
        {
            set
            {
                if (value < 0)
                    throw new System.ArgumentException("Increment must be zero or greater: " + value);
                this.positionIncrement = value;
            }
            get { return positionIncrement; }
        }

        public string Term
        {
            get
            {
                InitTermBuffer();
                return new System.String(termBuffer, 0, termLength);
            }
        }

        public void  SetTermBuffer(char[] buffer, int offset, int length)
        {
            GrowTermBuffer(length);
            Array.Copy(buffer, offset, termBuffer, 0, length);
            termLength = length;
        }
        
        public void  SetTermBuffer(System.String buffer)
        {
            int length = buffer.Length;
            GrowTermBuffer(length);
            TextSupport.GetCharsFromString(buffer, 0, length, termBuffer, 0);
            termLength = length;
        }
        
        public void  SetTermBuffer(System.String buffer, int offset, int length)
        {
            System.Diagnostics.Debug.Assert(offset <= buffer.Length);
            System.Diagnostics.Debug.Assert(offset + length <= buffer.Length);
            GrowTermBuffer(length);
            TextSupport.GetCharsFromString(buffer, offset, offset + length, termBuffer, 0);
            termLength = length;
        }
        
        public char[] TermBuffer()
        {
            InitTermBuffer();
            return termBuffer;
        }
        
        public virtual char[] ResizeTermBuffer(int newSize)
        {
            if (termBuffer == null)
            {
                termBuffer = new char[ArrayUtil.GetNextSize(newSize < MIN_BUFFER_SIZE ? MIN_BUFFER_SIZE : newSize)];
            }
            else
            {
                if (termBuffer.Length < newSize)
                {
                    // Not big enough; create a new array with slight
                    // over allocation and preserve content
                    var newCharBuffer = new char[ArrayUtil.GetNextSize(newSize)];
                    Array.Copy(termBuffer, 0, newCharBuffer, 0, termBuffer.Length);
                    termBuffer = newCharBuffer;
                }
            }
            return termBuffer;
        }
        
        private void  GrowTermBuffer(int newSize)
        {
            if (termBuffer == null)
            {
                // The buffer is always at least MIN_BUFFER_SIZE    
                termBuffer = new char[ArrayUtil.GetNextSize(newSize < MIN_BUFFER_SIZE?MIN_BUFFER_SIZE:newSize)];
            }
            else
            {
                if (termBuffer.Length < newSize)
                {
                    // Not big enough; create a new array with slight
                    // over allocation:
                    termBuffer = new char[ArrayUtil.GetNextSize(newSize)];
                }
            }
        }
        
        private void  InitTermBuffer()
        {
            if (termBuffer == null)
            {
                termBuffer = new char[ArrayUtil.GetNextSize(MIN_BUFFER_SIZE)];
                termLength = 0;
            }
        }
        
        public int TermLength()
        {
            InitTermBuffer();
            return termLength;
        }
        
        public void  SetTermLength(int length)
        {
            InitTermBuffer();
            if (length > termBuffer.Length)
                throw new System.ArgumentException("length " + length + " exceeds the size of the termBuffer (" + termBuffer.Length + ")");
            termLength = length;
        }

        public virtual int StartOffset
        {
            get { return startOffset; }
            set { this.startOffset = value; }
        }

        public virtual int EndOffset
        {
            get { return endOffset; }
            set { this.endOffset = value; }
        }

        public virtual void  SetOffset(int startOffset, int endOffset)
        {
            this.startOffset = startOffset;
            this.endOffset = endOffset;
        }

        public string Type
        {
            get { return type; }
            set { this.type = value; }
        }

        public virtual int Flags
        {
            get { return flags; }
            set { flags = value; }
        }

        public virtual Payload Payload
        {
            get { return payload; }
            set { payload = value; }
        }

        public override String ToString()
        {
            var sb = new System.Text.StringBuilder();
            sb.Append('(');
            InitTermBuffer();
            if (termBuffer == null)
                sb.Append("null");
            else
                sb.Append(termBuffer, 0, termLength);
            sb.Append(',').Append(startOffset).Append(',').Append(endOffset);
            if (!type.Equals("word"))
                sb.Append(",type=").Append(type);
            if (positionIncrement != 1)
                sb.Append(",posIncr=").Append(positionIncrement);
            sb.Append(')');
            return sb.ToString();
        }
        
        public override void  Clear()
        {
            payload = null;
            // Leave termBuffer to allow re-use
            termLength = 0;
            positionIncrement = 1;
            flags = 0;
            startOffset = endOffset = 0;
            type = DEFAULT_TYPE;
        }
        
        public override System.Object Clone()
        {
            var t = (Token) base.Clone();
            // Do a deep clone
            if (termBuffer != null)
            {
                t.termBuffer = new char[termBuffer.Length];
                termBuffer.CopyTo(t.termBuffer, 0);
            }
            if (payload != null)
            {
                t.payload = (Payload) payload.Clone();
            }
            return t;
        }
        
        public virtual Token Clone(char[] newTermBuffer, int newTermOffset, int newTermLength, int newStartOffset, int newEndOffset)
        {
            var t = new Token(newTermBuffer, newTermOffset, newTermLength, newStartOffset, newEndOffset)
                        {positionIncrement = positionIncrement, flags = flags, type = type};
            if (payload != null)
                t.payload = (Payload) payload.Clone();
            return t;
        }
        
        public  override bool Equals(Object obj)
        {
            if (obj == this)
                return true;

            var other = obj as Token;
            if (other == null)
                return false;
            
            InitTermBuffer();
            other.InitTermBuffer();

            if (termLength == other.termLength && startOffset == other.startOffset && endOffset == other.endOffset &&
                flags == other.flags && positionIncrement == other.positionIncrement && SubEqual(type, other.type) &&
                SubEqual(payload, other.payload))
            {
                for (int i = 0; i < termLength; i++)
                    if (termBuffer[i] != other.termBuffer[i])
                        return false;
                return true;
            }
            return false;
        }
        
        private bool SubEqual(System.Object o1, System.Object o2)
        {
            if (o1 == null)
                return o2 == null;
            return o1.Equals(o2);
        }

        public override int GetHashCode()
        {
            InitTermBuffer();
            int code = termLength;
            code = code * 31 + startOffset;
            code = code * 31 + endOffset;
            code = code * 31 + flags;
            code = code * 31 + positionIncrement;
            code = code * 31 + type.GetHashCode();
            code = (payload == null?code:code * 31 + payload.GetHashCode());
            code = code * 31 + ArrayUtil.HashCode(termBuffer, 0, termLength);
            return code;
        }
        
        // like clear() but doesn't clear termBuffer/text
        private void  ClearNoTermBuffer()
        {
            payload = null;
            positionIncrement = 1;
            flags = 0;
            startOffset = endOffset = 0;
            type = DEFAULT_TYPE;
        }
        
        public virtual Token Reinit(char[] newTermBuffer, int newTermOffset, int newTermLength, int newStartOffset, int newEndOffset, System.String newType)
        {
            ClearNoTermBuffer();
            payload = null;
            positionIncrement = 1;
            SetTermBuffer(newTermBuffer, newTermOffset, newTermLength);
            startOffset = newStartOffset;
            endOffset = newEndOffset;
            type = newType;
            return this;
        }
        
        public virtual Token Reinit(char[] newTermBuffer, int newTermOffset, int newTermLength, int newStartOffset, int newEndOffset)
        {
            ClearNoTermBuffer();
            SetTermBuffer(newTermBuffer, newTermOffset, newTermLength);
            startOffset = newStartOffset;
            endOffset = newEndOffset;
            type = DEFAULT_TYPE;
            return this;
        }
        
        public virtual Token Reinit(System.String newTerm, int newStartOffset, int newEndOffset, System.String newType)
        {
            ClearNoTermBuffer();
            SetTermBuffer(newTerm);
            startOffset = newStartOffset;
            endOffset = newEndOffset;
            type = newType;
            return this;
        }
        
        public virtual Token Reinit(System.String newTerm, int newTermOffset, int newTermLength, int newStartOffset, int newEndOffset, System.String newType)
        {
            ClearNoTermBuffer();
            SetTermBuffer(newTerm, newTermOffset, newTermLength);
            startOffset = newStartOffset;
            endOffset = newEndOffset;
            type = newType;
            return this;
        }
        
        public virtual Token Reinit(System.String newTerm, int newStartOffset, int newEndOffset)
        {
            ClearNoTermBuffer();
            SetTermBuffer(newTerm);
            startOffset = newStartOffset;
            endOffset = newEndOffset;
            type = DEFAULT_TYPE;
            return this;
        }
        
        public virtual Token Reinit(System.String newTerm, int newTermOffset, int newTermLength, int newStartOffset, int newEndOffset)
        {
            ClearNoTermBuffer();
            SetTermBuffer(newTerm, newTermOffset, newTermLength);
            startOffset = newStartOffset;
            endOffset = newEndOffset;
            type = DEFAULT_TYPE;
            return this;
        }
        
        public virtual void  Reinit(Token prototype)
        {
            prototype.InitTermBuffer();
            SetTermBuffer(prototype.termBuffer, 0, prototype.termLength);
            positionIncrement = prototype.positionIncrement;
            flags = prototype.flags;
            startOffset = prototype.startOffset;
            endOffset = prototype.endOffset;
            type = prototype.type;
            payload = prototype.payload;
        }
        
        public virtual void  Reinit(Token prototype, System.String newTerm)
        {
            SetTermBuffer(newTerm);
            positionIncrement = prototype.positionIncrement;
            flags = prototype.flags;
            startOffset = prototype.startOffset;
            endOffset = prototype.endOffset;
            type = prototype.type;
            payload = prototype.payload;
        }
        
        public virtual void  Reinit(Token prototype, char[] newTermBuffer, int offset, int length)
        {
            SetTermBuffer(newTermBuffer, offset, length);
            positionIncrement = prototype.positionIncrement;
            flags = prototype.flags;
            startOffset = prototype.startOffset;
            endOffset = prototype.endOffset;
            type = prototype.type;
            payload = prototype.payload;
        }
        
        public override void  CopyTo(Attribute target)
        {
            if (target is Token)
            {
                var to = (Token) target;
                to.Reinit(this);
                // reinit shares the payload, so clone it:
                if (payload != null)
                {
                    to.payload = (Payload) payload.Clone();
                }
            }
            else
            {
                InitTermBuffer();
                ((ITermAttribute) target).SetTermBuffer(termBuffer, 0, termLength);
                ((IOffsetAttribute) target).SetOffset(startOffset, endOffset);
                ((IPositionIncrementAttribute) target).PositionIncrement = positionIncrement;
                ((IPayloadAttribute) target).Payload = (payload == null)?null:(Payload) payload.Clone();
                ((IFlagsAttribute) target).Flags = flags;
                ((ITypeAttribute) target).Type = type;
            }
        }
       
        public static AttributeSource.AttributeFactory TOKEN_ATTRIBUTE_FACTORY =
            new TokenAttributeFactory(AttributeSource.AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY);
  
        public class TokenAttributeFactory : AttributeSource.AttributeFactory
        {

            private readonly AttributeSource.AttributeFactory _delegateFactory;

            public TokenAttributeFactory(AttributeSource.AttributeFactory delegateFactory)
            {
                this._delegateFactory = delegateFactory;
            }

            public override Attribute CreateAttributeInstance<T>()
            {
                return typeof(T).IsAssignableFrom(typeof(Token))
                           ? new Token()
                           : _delegateFactory.CreateAttributeInstance<T>();
            }

            public override bool Equals(Object other)
            {
                if (this == other) return true;

                var af = other as TokenAttributeFactory;
                return af != null && _delegateFactory.Equals(af._delegateFactory);
            }

            public override int GetHashCode()
            {
                return _delegateFactory.GetHashCode() ^ 0x0a45aa31;
            }
        }
    }
}