FieldCacheSanityChecker.cs

Go to the documentation of this file.

/* 
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 * 
 * http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

using System;
using System.Collections.Generic;

using IndexReader = Lucene.Net.Index.IndexReader;
using FieldCache = Lucene.Net.Search.FieldCache;
using CacheEntry = Lucene.Net.Search.CacheEntry;

namespace Lucene.Net.Util
{
    
    public sealed class FieldCacheSanityChecker
    {
        
        private RamUsageEstimator ramCalc = null;
        public FieldCacheSanityChecker()
        {
            /* NOOP */
        }
        public void  SetRamUsageEstimator(RamUsageEstimator r)
        {
            ramCalc = r;
        }
        
        
        public static Insanity[] CheckSanity(FieldCache cache)
        {
            return CheckSanity(cache.GetCacheEntries());
        }
        
        public static Insanity[] CheckSanity(params CacheEntry[] cacheEntries)
        {
            FieldCacheSanityChecker sanityChecker = new FieldCacheSanityChecker();
            // doesn't check for interned
            sanityChecker.SetRamUsageEstimator(new RamUsageEstimator(false));
            return sanityChecker.Check(cacheEntries);
        }
        
        
        public Insanity[] Check(params CacheEntry[] cacheEntries)
        {
            if (null == cacheEntries || 0 == cacheEntries.Length)
                return new Insanity[0];
            
            if (null != ramCalc)
            {
                for (int i = 0; i < cacheEntries.Length; i++)
                {
                    cacheEntries[i].EstimateSize(ramCalc);
                }
            }
            
            // the indirect mapping lets MapOfSet dedup identical valIds for us
            //
            // maps the (valId) identityhashCode of cache values to 
            // sets of CacheEntry instances
            MapOfSets<int,CacheEntry> valIdToItems = new MapOfSets<int,CacheEntry>(new Dictionary<int,HashSet<CacheEntry>>(17));
            // maps ReaderField keys to Sets of ValueIds
            MapOfSets<ReaderField,int> readerFieldToValIds = new MapOfSets<ReaderField,int>(new Dictionary<ReaderField,HashSet<int>>(17));
            //
            
            // any keys that we know result in more then one valId
            HashSet<ReaderField> valMismatchKeys = new HashSet<ReaderField>();
            
            // iterate over all the cacheEntries to get the mappings we'll need
            for (int i = 0; i < cacheEntries.Length; i++)
            {
                CacheEntry item = cacheEntries[i];
                System.Object val = item.Value;
                
                if (val is Lucene.Net.Search.CreationPlaceholder)
                    continue;
                
                ReaderField rf = new ReaderField(item.ReaderKey, item.FieldName);
                
                System.Int32 valId = val.GetHashCode();
                
                // indirect mapping, so the MapOfSet will dedup identical valIds for us
                valIdToItems.Put(valId, item);
                if (1 < readerFieldToValIds.Put(rf, valId))
                {
                    valMismatchKeys.Add(rf);
                }
            }
            
            List<Insanity> insanity = new List<Insanity>(valMismatchKeys.Count * 3);
            
            insanity.AddRange(CheckValueMismatch(valIdToItems, readerFieldToValIds, valMismatchKeys));
            insanity.AddRange(CheckSubreaders(valIdToItems, readerFieldToValIds));
            
            return insanity.ToArray();
        }
        
        private List<Insanity> CheckValueMismatch(MapOfSets<int,CacheEntry> valIdToItems, 
                                                  MapOfSets<ReaderField,int> readerFieldToValIds, 
                                                  HashSet<ReaderField> valMismatchKeys)
        {
            
            List<Insanity> insanity = new List<Insanity>(valMismatchKeys.Count * 3);
            
            if (!(valMismatchKeys.Count == 0))
            {
                // we have multiple values for some ReaderFields
                
                IDictionary<ReaderField,HashSet<int>> rfMap = readerFieldToValIds.Map;
                IDictionary<int,HashSet<CacheEntry>> valMap = valIdToItems.Map;
                foreach (ReaderField rf in valMismatchKeys)
                {
                    List<CacheEntry> badEntries = new List<CacheEntry>(valMismatchKeys.Count * 2);
                    foreach (int val in rfMap[rf])
                    {
                        foreach (CacheEntry entry in valMap[val])
                        {
                            badEntries.Add(entry);
                        }
                    }

                    insanity.Add(new Insanity(InsanityType.VALUEMISMATCH, "Multiple distinct value objects for " + rf.ToString(), badEntries.ToArray()));
                }
            }
            return insanity;
        }
        
        private List<Insanity> CheckSubreaders(MapOfSets<int,CacheEntry> valIdToItems, 
                                               MapOfSets<ReaderField,int> readerFieldToValIds)
        {
            List<Insanity> insanity = new List<Insanity>(23);

            Dictionary<ReaderField, HashSet<ReaderField>> badChildren = new Dictionary<ReaderField, HashSet<ReaderField>>(17);
            MapOfSets<ReaderField, ReaderField> badKids = new MapOfSets<ReaderField, ReaderField>(badChildren); // wrapper

            IDictionary<int, HashSet<CacheEntry>> viToItemSets = valIdToItems.Map;
            IDictionary<ReaderField, HashSet<int>> rfToValIdSets = readerFieldToValIds.Map;

            HashSet<ReaderField> seen = new HashSet<ReaderField>();

            foreach (ReaderField rf in rfToValIdSets.Keys)
            {
                if (seen.Contains(rf))
                    continue;

                System.Collections.IList kids = GetAllDecendentReaderKeys(rf.readerKey);
                foreach (Object kidKey in kids)
                {
                    ReaderField kid = new ReaderField(kidKey, rf.fieldName);

                    if (badChildren.ContainsKey(kid))
                    {
                        // we've already process this kid as RF and found other problems
                        // track those problems as our own
                        badKids.Put(rf, kid);
                        badKids.PutAll(rf, badChildren[kid]);
                        badChildren.Remove(kid);
                    }
                    else if (rfToValIdSets.ContainsKey(kid))
                    {
                        // we have cache entries for the kid
                        badKids.Put(rf, kid);
                    }
                    seen.Add(kid);
                }
                seen.Add(rf);
            }
            
            // every mapping in badKids represents an Insanity
            foreach (ReaderField parent in badChildren.Keys)
            {
                HashSet<ReaderField> kids = badChildren[parent];
                
                List<CacheEntry> badEntries = new List<CacheEntry>(kids.Count * 2);
                
                // put parent entr(ies) in first
                {
                    foreach (int val in rfToValIdSets[parent])
                    {
                        badEntries.AddRange(viToItemSets[val]);
                    }
                }
                
                // now the entries for the descendants
                foreach (ReaderField kid in kids)
                {
                    foreach (int val in rfToValIdSets[kid])
                    {
                        badEntries.AddRange(viToItemSets[val]);
                    }
                }
                
                insanity.Add(new Insanity(InsanityType.SUBREADER, "Found caches for decendents of " + parent.ToString(), badEntries.ToArray()));
            }
            
            return insanity;
        }
        
        private System.Collections.IList GetAllDecendentReaderKeys(System.Object seed)
        {
            List<object> all = new List<object>(17); // will grow as we iter
            all.Add(seed);
            for (int i = 0; i < all.Count; i++)
            {
                System.Object obj = all[i];
                if (obj is IndexReader)
                {
                    IndexReader[] subs = ((IndexReader) obj).GetSequentialSubReaders();
                    for (int j = 0; (null != subs) && (j < subs.Length); j++)
                    {
                        all.Add(subs[j].FieldCacheKey);
                    }
                }
            }
            // need to skip the first, because it was the seed
            return all.GetRange(1, all.Count - 1);
        }
        
        private sealed class ReaderField
        {
            public System.Object readerKey;
            public System.String fieldName;
            public ReaderField(System.Object readerKey, System.String fieldName)
            {
                this.readerKey = readerKey;
                this.fieldName = fieldName;
            }
            public override int GetHashCode()
            {
                return readerKey.GetHashCode() * fieldName.GetHashCode();
            }
            public  override bool Equals(System.Object that)
            {
                if (!(that is ReaderField))
                    return false;
                
                ReaderField other = (ReaderField) that;
                return (this.readerKey == other.readerKey && this.fieldName.Equals(other.fieldName));
            }
            public override System.String ToString()
            {
                return readerKey.ToString() + "+" + fieldName;
            }
        }
        
        public sealed class Insanity
        {
            private InsanityType type;
            private System.String msg;
            private CacheEntry[] entries;
            public Insanity(InsanityType type, System.String msg, params CacheEntry[] entries)
            {
                if (null == type)
                {
                    throw new System.ArgumentException("Insanity requires non-null InsanityType");
                }
                if (null == entries || 0 == entries.Length)
                {
                    throw new System.ArgumentException("Insanity requires non-null/non-empty CacheEntry[]");
                }
                this.type = type;
                this.msg = msg;
                this.entries = entries;
            }

            public InsanityType Type
            {
                get { return type; }
            }

            public string Msg
            {
                get { return msg; }
            }

            public CacheEntry[] GetCacheEntries()
            {
                return entries;
            }
            public override System.String ToString()
            {
                System.Text.StringBuilder buf = new System.Text.StringBuilder();
                buf.Append(Type).Append(": ");
                
                System.String m = Msg;
                if (null != m)
                    buf.Append(m);
                
                buf.Append('\n');
                
                CacheEntry[] ce = GetCacheEntries();
                for (int i = 0; i < ce.Length; i++)
                {
                    buf.Append('\t').Append(ce[i].ToString()).Append('\n');
                }
                
                return buf.ToString();
            }
        }
        
        public sealed class InsanityType
        {
            private System.String label;
            internal InsanityType(System.String label)
            {
                this.label = label;
            }
            public override System.String ToString()
            {
                return label;
            }
            
            public static readonly InsanityType SUBREADER = new InsanityType("SUBREADER");
            
            public static readonly InsanityType VALUEMISMATCH = new InsanityType("VALUEMISMATCH");
            
            public static readonly InsanityType EXPECTED = new InsanityType("EXPECTED");
        }
    }
}