MoreLikeThis.cs

Go to the documentation of this file.

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 * 
 * http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using Lucene.Net.Index;
using Lucene.Net.Store;
using Lucene.Net.Support;
using Lucene.Net.Util;
using IndexReader = Lucene.Net.Index.IndexReader;
using Term = Lucene.Net.Index.Term;
using BooleanClause = Lucene.Net.Search.BooleanClause;
using DefaultSimilarity = Lucene.Net.Search.DefaultSimilarity;
using TermQuery = Lucene.Net.Search.TermQuery;
using BooleanQuery = Lucene.Net.Search.BooleanQuery;
using IndexSearcher = Lucene.Net.Search.IndexSearcher;
using Query = Lucene.Net.Search.Query;
using Analyzer = Lucene.Net.Analysis.Analyzer;
using TokenStream = Lucene.Net.Analysis.TokenStream;
using StandardAnalyzer = Lucene.Net.Analysis.Standard.StandardAnalyzer;
using Document = Lucene.Net.Documents.Document;
using Lucene.Net.Analysis.Tokenattributes;

namespace Lucene.Net.Search.Similar
{
    public sealed class MoreLikeThis
    {

        public const int DEFAULT_MAX_NUM_TOKENS_PARSED = 5000;


        public static readonly Analyzer DEFAULT_ANALYZER = new StandardAnalyzer(Util.Version.LUCENE_CURRENT);

        public const int DEFAULT_MIN_TERM_FREQ = 2;

        public const int DEFAULT_MIN_DOC_FREQ = 5;

        public const int DEFAULT_MAX_DOC_FREQ = int.MaxValue;

        public const bool DEFAULT_BOOST = false;

        public static readonly System.String[] DEFAULT_FIELD_NAMES = new System.String[] { "contents" };

        public const int DEFAULT_MIN_WORD_LENGTH = 0;

        public const int DEFAULT_MAX_WORD_LENGTH = 0;

        public static readonly ISet<string> DEFAULT_STOP_WORDS = null;

        private ISet<string> stopWords = DEFAULT_STOP_WORDS;

        public const int DEFAULT_MAX_QUERY_TERMS = 25;

        private Analyzer analyzer = DEFAULT_ANALYZER;

        private int minTermFreq = DEFAULT_MIN_TERM_FREQ;

        private int minDocFreq = DEFAULT_MIN_DOC_FREQ;

        private int maxDocfreq = DEFAULT_MAX_DOC_FREQ;

        private bool boost = DEFAULT_BOOST;

        private System.String[] fieldNames = DEFAULT_FIELD_NAMES;

        private int maxNumTokensParsed = DEFAULT_MAX_NUM_TOKENS_PARSED;

        private int minWordLen = DEFAULT_MIN_WORD_LENGTH;

        private int maxWordLen = DEFAULT_MAX_WORD_LENGTH;

        private int maxQueryTerms = DEFAULT_MAX_QUERY_TERMS;

        private Lucene.Net.Search.Similarity similarity = null;

        private IndexReader ir;

        private float boostFactor = 1;

        public float BoostFactor
        {
            get { return boostFactor; }
            set { this.boostFactor = value; }
        }

        public MoreLikeThis(IndexReader ir) : this(ir,new DefaultSimilarity())
        {
        }

        public MoreLikeThis(IndexReader ir, Lucene.Net.Search.Similarity sim)
        {
            this.ir = ir;
            this.similarity = sim;
        }

        public Similarity Similarity
        {
            get { return similarity; }
            set { this.similarity = value; }
        }

        public Analyzer Analyzer
        {
            get { return analyzer; }
            set { this.analyzer = value; }
        }

        public int MinTermFreq
        {
            get { return minTermFreq; }
            set { this.minTermFreq = value; }
        }

        public int MinDocFreq
        {
            get { return minDocFreq; }
            set { this.minDocFreq = value; }
        }

        public int MaxDocFreq
        {
            get { return this.maxDocfreq; }
            set { this.maxDocfreq = value; }
        }

        public void SetMaxDocFreqPct(int maxPercentage)
        {
            this.maxDocfreq = maxPercentage * ir.NumDocs() / 100;
        }

        public bool Boost
        {
            get { return boost; }
            set { this.boost = value; }
        }

        public System.String[] GetFieldNames()
        {
            return fieldNames;
        }

        public void SetFieldNames(System.String[] fieldNames)
        {
            this.fieldNames = fieldNames;
        }

        public int MinWordLen
        {
            get { return minWordLen; }
            set { this.minWordLen = value; }
        }

        public int MaxWordLen
        {
            get { return maxWordLen; }
            set { this.maxWordLen = value; }
        }

        public void SetStopWords(ISet<string> stopWords)
        {
            this.stopWords = stopWords;
        }

        public ISet<string> GetStopWords()
        {
            return stopWords;
        }


        public int MaxQueryTerms
        {
            get { return maxQueryTerms; }
            set { this.maxQueryTerms = value; }
        }

        public int MaxNumTokensParsed
        {
            get { return maxNumTokensParsed; }
            set { maxNumTokensParsed = value; }
        }

        public Query Like(int docNum)
        {
            if (fieldNames == null)
            {
                // gather list of valid fields from lucene
                ICollection<string> fields = ir.GetFieldNames(IndexReader.FieldOption.INDEXED);
                fieldNames = fields.ToArray();
            }

            return CreateQuery(RetrieveTerms(docNum));
        }

        public Query Like(System.IO.FileInfo f)
        {
            if (fieldNames == null)
            {
                // gather list of valid fields from lucene
                ICollection<string> fields = ir.GetFieldNames(IndexReader.FieldOption.INDEXED);
                fieldNames = fields.ToArray();
            }

            return Like(new System.IO.StreamReader(f.FullName, System.Text.Encoding.Default));
        }

        public Query Like(System.Uri u)
        {
            return Like(new System.IO.StreamReader((System.Net.WebRequest.Create(u)).GetResponse().GetResponseStream(), System.Text.Encoding.Default));
        }

        public Query Like(System.IO.Stream is_Renamed)
        {
            return Like(new System.IO.StreamReader(is_Renamed, System.Text.Encoding.Default));
        }

        public Query Like(System.IO.TextReader r)
        {
            return CreateQuery(RetrieveTerms(r));
        }

        private Query CreateQuery(PriorityQueue<object[]> q)
        {
            BooleanQuery query = new BooleanQuery();
            System.Object cur;
            int qterms = 0;
            float bestScore = 0;

            while (((cur = q.Pop()) != null))
            {
                System.Object[] ar = (System.Object[])cur;
                TermQuery tq = new TermQuery(new Term((System.String)ar[1], (System.String)ar[0]));

                if (boost)
                {
                    if (qterms == 0)
                    {
                        bestScore = (float)ar[2];
                    }
                    float myScore = (float)ar[2];

                    tq.Boost = boostFactor * myScore / bestScore;
                }

                try
                {
                    query.Add(tq, Occur.SHOULD);
                }
                catch (BooleanQuery.TooManyClauses ignore)
                {
                    break;
                }

                qterms++;
                if (maxQueryTerms > 0 && qterms >= maxQueryTerms)
                {
                    break;
                }
            }

            return query;
        }

        private PriorityQueue<object[]> CreateQueue(IDictionary<string,Int> words)
        {
            // have collected all words in doc and their freqs
            int numDocs = ir.NumDocs();
            FreqQ res = new FreqQ(words.Count); // will order words by score

            var it = words.Keys.GetEnumerator();
            while (it.MoveNext())
            {
                // for every word
                System.String word = it.Current;

                int tf = words[word].x; // term freq in the source doc
                if (minTermFreq > 0 && tf < minTermFreq)
                {
                    continue; // filter out words that don't occur enough times in the source
                }

                // go through all the fields and find the largest document frequency
                System.String topField = fieldNames[0];
                int docFreq = 0;
                for (int i = 0; i < fieldNames.Length; i++)
                {
                    int freq = ir.DocFreq(new Term(fieldNames[i], word));
                    topField = (freq > docFreq) ? fieldNames[i] : topField;
                    docFreq = (freq > docFreq) ? freq : docFreq;
                }

                if (minDocFreq > 0 && docFreq < minDocFreq)
                {
                    continue; // filter out words that don't occur in enough docs
                }

                if (docFreq > maxDocfreq)
                {
                    continue; // filter out words that occur in too many docs
                }

                if (docFreq == 0)
                {
                    continue; // index update problem?
                }

                float idf = similarity.Idf(docFreq, numDocs);
                float score = tf * idf;

                // only really need 1st 3 entries, other ones are for troubleshooting
                res.InsertWithOverflow(new System.Object[] { word, topField, score, idf, docFreq, tf });
            }
            return res;
        }

        public System.String DescribeParams()
        {
            System.Text.StringBuilder sb = new System.Text.StringBuilder();
            sb.Append("\t" + "maxQueryTerms  : " + maxQueryTerms + "\n");
            sb.Append("\t" + "minWordLen     : " + minWordLen + "\n");
            sb.Append("\t" + "maxWordLen     : " + maxWordLen + "\n");
            sb.Append("\t" + "fieldNames     : \"");
            System.String delim = "";
            for (int i = 0; i < fieldNames.Length; i++)
            {
                System.String fieldName = fieldNames[i];
                sb.Append(delim).Append(fieldName);
                delim = ", ";
            }
            sb.Append("\n");
            sb.Append("\t" + "boost          : " + boost + "\n");
            sb.Append("\t" + "minTermFreq    : " + minTermFreq + "\n");
            sb.Append("\t" + "minDocFreq     : " + minDocFreq + "\n");
            return sb.ToString();
        }

        [STAThread]
        public static void Main(System.String[] a)
        {
            System.String indexName = "localhost_index";
            System.String fn = "c:/Program Files/Apache Group/Apache/htdocs/manual/vhosts/index.html.en";
            System.Uri url = null;
            for (int i = 0; i < a.Length; i++)
            {
                if (a[i].Equals("-i"))
                {
                    indexName = a[++i];
                }
                else if (a[i].Equals("-f"))
                {
                    fn = a[++i];
                }
                else if (a[i].Equals("-url"))
                {
                    url = new System.Uri(a[++i]);
                }
            }

            System.IO.StreamWriter temp_writer;
            temp_writer = new System.IO.StreamWriter(System.Console.OpenStandardOutput(), System.Console.Out.Encoding);
            temp_writer.AutoFlush = true;
            System.IO.StreamWriter o = temp_writer;
            FSDirectory dir = FSDirectory.Open(new DirectoryInfo(indexName));
            IndexReader r = IndexReader.Open(dir, true);
            o.WriteLine("Open index " + indexName + " which has " + r.NumDocs() + " docs");

            MoreLikeThis mlt = new MoreLikeThis(r);

            o.WriteLine("Query generation parameters:");
            o.WriteLine(mlt.DescribeParams());
            o.WriteLine();

            Query query = null;
            if (url != null)
            {
                o.WriteLine("Parsing URL: " + url);
                query = mlt.Like(url);
            }
            else if (fn != null)
            {
                o.WriteLine("Parsing file: " + fn);
                query = mlt.Like(new System.IO.FileInfo(fn));
            }

            o.WriteLine("q: " + query);
            o.WriteLine();
            IndexSearcher searcher = new IndexSearcher(dir, true);

            TopDocs hits = searcher.Search(query, null, 25);
            int len = hits.TotalHits;
            o.WriteLine("found: " + len + " documents matching");
            o.WriteLine();
            ScoreDoc[] scoreDocs = hits.ScoreDocs;
            for (int i = 0; i < System.Math.Min(25, len); i++)
            {
                Document d = searcher.Doc(scoreDocs[i].Doc);
                System.String summary = d.Get("summary");
                o.WriteLine("score  : " + scoreDocs[i].Score);
                o.WriteLine("url    : " + d.Get("url"));
                o.WriteLine("\ttitle  : " + d.Get("title"));
                if (summary != null)
                    o.WriteLine("\tsummary: " + d.Get("summary"));
                o.WriteLine();
            }
        }

        private PriorityQueue<object[]> RetrieveTerms(int docNum)
        {
            IDictionary<string,Int> termFreqMap = new HashMap<string,Int>();
            for (int i = 0; i < fieldNames.Length; i++)
            {
                System.String fieldName = fieldNames[i];
                ITermFreqVector vector = ir.GetTermFreqVector(docNum, fieldName);

                // field does not store term vector info
                if (vector == null)
                {
                    Document d = ir.Document(docNum);
                    System.String[] text = d.GetValues(fieldName);
                    if (text != null)
                    {
                        for (int j = 0; j < text.Length; j++)
                        {
                            AddTermFrequencies(new System.IO.StringReader(text[j]), termFreqMap, fieldName);
                        }
                    }
                }
                else
                {
                    AddTermFrequencies(termFreqMap, vector);
                }
            }

            return CreateQueue(termFreqMap);
        }

        private void AddTermFrequencies(IDictionary<string, Int> termFreqMap, ITermFreqVector vector)
        {
            System.String[] terms = vector.GetTerms();
            int[] freqs = vector.GetTermFrequencies();
            for (int j = 0; j < terms.Length; j++)
            {
                System.String term = terms[j];

                if (IsNoiseWord(term))
                {
                    continue;
                }
                // increment frequency
                Int cnt = termFreqMap[term];
                if (cnt == null)
                {
                    cnt = new Int();
                    termFreqMap[term] = cnt;
                    cnt.x = freqs[j];
                }
                else
                {
                    cnt.x += freqs[j];
                }
            }
        }
        private void AddTermFrequencies(System.IO.TextReader r, IDictionary<string,Int> termFreqMap, System.String fieldName)
        {
            TokenStream ts = analyzer.TokenStream(fieldName, r);
            int tokenCount=0;
            // for every token
            ITermAttribute termAtt = ts.AddAttribute<ITermAttribute>();
            
            while (ts.IncrementToken()) {
                string word = termAtt.Term;
                tokenCount++;
                if(tokenCount>maxNumTokensParsed)
                {
                    break;
                }
                if(IsNoiseWord(word)){
                    continue;
                }
                
                // increment frequency
                Int cnt = termFreqMap[word];
                if (cnt == null) {
                    termFreqMap[word] = new Int();
                }
                else {
                    cnt.x++;
                }
            }
        }


        private bool IsNoiseWord(System.String term)
        {
            int len = term.Length;
            if (minWordLen > 0 && len < minWordLen)
            {
                return true;
            }
            if (maxWordLen > 0 && len > maxWordLen)
            {
                return true;
            }
            if (stopWords != null && stopWords.Contains(term))
            {
                return true;
            }
            return false;
        }


        public PriorityQueue<object[]> RetrieveTerms(System.IO.TextReader r)
        {
            IDictionary<string, Int> words = new HashMap<string,Int>();
            for (int i = 0; i < fieldNames.Length; i++)
            {
                System.String fieldName = fieldNames[i];
                AddTermFrequencies(r, words, fieldName);
            }
            return CreateQueue(words);
        }


        public System.String[] RetrieveInterestingTerms(int docNum)
        {
            List<object> al = new List<object>(maxQueryTerms);
            PriorityQueue<object[]> pq = RetrieveTerms(docNum);
            System.Object cur;
            int lim = maxQueryTerms; // have to be careful, retrieveTerms returns all words but that's probably not useful to our caller...
            // we just want to return the top words
            while (((cur = pq.Pop()) != null) && lim-- > 0)
            {
                System.Object[] ar = (System.Object[])cur;
                al.Add(ar[0]); // the 1st entry is the interesting word
            }
            //System.String[] res = new System.String[al.Count];
            //return al.toArray(res);
            return al.Select(x => x.ToString()).ToArray();
        }

        public System.String[] RetrieveInterestingTerms(System.IO.TextReader r)
        {
            List<object> al = new List<object>(maxQueryTerms);
            PriorityQueue<object[]> pq = RetrieveTerms(r);
            System.Object cur;
            int lim = maxQueryTerms; // have to be careful, retrieveTerms returns all words but that's probably not useful to our caller...
            // we just want to return the top words
            while (((cur = pq.Pop()) != null) && lim-- > 0)
            {
                System.Object[] ar = (System.Object[])cur;
                al.Add(ar[0]); // the 1st entry is the interesting word
            }
            //System.String[] res = new System.String[al.Count];
            // return (System.String[]) SupportClass.ICollectionSupport.ToArray(al, res);
            return al.Select(x => x.ToString()).ToArray();
        }

        private class FreqQ : PriorityQueue<object[]>
        {
            internal FreqQ(int s)
            {
                Initialize(s);
            }

            override public bool LessThan(System.Object[] aa, System.Object[] bb)
            {
                float fa = (float)aa[2];
                float fb = (float)bb[2];
                return (float)fa > (float)fb;
            }
        }

        private class Int
        {
            internal int x;

            internal Int()
            {
                x = 1;
            }
        }
    }
}