HunspellStemmer.cs

Go to the documentation of this file.

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;

namespace Lucene.Net.Analysis.Hunspell {
    public class HunspellStemmer {
        private static Int32 RECURSION_CAP = 2;
        private readonly HunspellDictionary _dictionary;

        public HunspellStemmer(HunspellDictionary dictionary) {
            if (dictionary == null) throw new ArgumentNullException("dictionary");
            _dictionary = dictionary;
        }

        public IEnumerable<HunspellStem> Stem(String word) {
            if (word == null) throw new ArgumentNullException("word");

            var stems = new List<HunspellStem>();
            if (_dictionary.LookupWord(word) != null)
                stems.Add(new HunspellStem(word));

            stems.AddRange(Stem(word, null, 0));
            return stems;
        }

        public IEnumerable<HunspellStem> UniqueStems(String word) {
            if (word == null) throw new ArgumentNullException("word");

            var stems = new List<HunspellStem>();
            var terms = new CharArraySet(8, false);
            if (_dictionary.LookupWord(word) != null) {
                stems.Add(new HunspellStem(word));
                terms.Add(word);
            }

            var otherStems = Stem(word, null, 0);
            foreach (var s in otherStems) {
                if (!terms.Contains(s.Stem)) {
                    stems.Add(s);
                    terms.Add(s.Stem);
                }
            }

            return stems;
        }

        private IEnumerable<HunspellStem> Stem(String word, Char[] flags, Int32 recursionDepth) {
            if (word == null) throw new ArgumentNullException("word");

            var stems = new List<HunspellStem>();
            var chars = word.ToCharArray();
            var length = word.Length;

            for (var i = 0; i < length; i++) {
                var suffixes = _dictionary.LookupSuffix(chars, i, length - i);
                if (suffixes != null) {
                    foreach (var suffix in suffixes) {
                        if (HasCrossCheckedFlag(suffix.Flag, flags)) {
                            var deAffixedLength = length - suffix.Append.Length;

                            // TODO: can we do this in-place?
                            var strippedWord = new StringBuilder()
                                .Append(word, 0, deAffixedLength)
                                .Append(suffix.Strip)
                                .ToString();

                            var stemList = ApplyAffix(strippedWord, suffix, recursionDepth);
                            foreach (var stem in stemList) {
                                stem.AddSuffix(suffix);
                            }

                            stems.AddRange(stemList);
                        }
                    }
                }
            }

            for (var i = length - 1; i >= 0; i--) {
                var prefixes = _dictionary.LookupPrefix(chars, 0, i);
                if (prefixes != null) {
                    foreach (var prefix in prefixes) {
                        if (HasCrossCheckedFlag(prefix.Flag, flags)) {
                            var deAffixedStart = prefix.Append.Length;
                            var deAffixedLength = length - deAffixedStart;

                            var strippedWord = new StringBuilder()
                                .Append(prefix.Strip)
                                .Append(word, deAffixedStart, deAffixedLength)
                                .ToString();

                            var stemList = ApplyAffix(strippedWord, prefix, recursionDepth);
                            foreach (var stem in stemList) {
                                stem.AddPrefix(prefix);
                            }

                            stems.AddRange(stemList);
                        }
                    }
                }
            }

            return stems;
        }

        public IEnumerable<HunspellStem> ApplyAffix(String strippedWord, HunspellAffix affix, Int32 recursionDepth) {
            if (strippedWord == null) throw new ArgumentNullException("strippedWord");
            if (affix == null) throw new ArgumentNullException("affix");

            if (!affix.CheckCondition(strippedWord)) {
                return new List<HunspellStem>();
            }

            var words = _dictionary.LookupWord(strippedWord);
            if (words == null) {
                return new List<HunspellStem>();
            }

            var stems = new List<HunspellStem>();

            foreach (var hunspellWord in words) {
                if (hunspellWord.HasFlag(affix.Flag)) {
                    if (affix.IsCrossProduct && recursionDepth < RECURSION_CAP) {
                        var recursiveStems = Stem(strippedWord, affix.AppendFlags, ++recursionDepth);
                        if (recursiveStems.Any()) {
                            stems.AddRange(recursiveStems);
                        } else {
                            stems.Add(new HunspellStem(strippedWord));
                        }
                    } else {
                        stems.Add(new HunspellStem(strippedWord));
                    }
                }
            }

            return stems;
        }

        private static Boolean HasCrossCheckedFlag(Char flag, Char[] flags) {
            return flags == null || Array.BinarySearch(flags, flag) >= 0;
        }
    }
}