CheckIndex.cs

Go to the documentation of this file.

/* 
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 * 
 * http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

using System;
using System.Collections.Generic;
using System.IO;
using Lucene.Net.Support;
using AbstractField = Lucene.Net.Documents.AbstractField;
using Document = Lucene.Net.Documents.Document;
using Directory = Lucene.Net.Store.Directory;
using FSDirectory = Lucene.Net.Store.FSDirectory;
using IndexInput = Lucene.Net.Store.IndexInput;

namespace Lucene.Net.Index
{
    
    /// <summary> Basic tool and API to check the health of an index and
    /// write a new segments file that removes reference to
    /// problematic segments.
    /// 
    /// <p/>As this tool checks every byte in the index, on a large
    /// index it can take quite a long time to run.
    /// 
    /// <p/><b>WARNING</b>: this tool and API is new and
    /// experimental and is subject to suddenly change in the
    /// next release.  Please make a complete backup of your
    /// index before using this to fix your index!
    /// </summary>
    public class CheckIndex
    {
        private StreamWriter infoStream;
        private readonly Directory dir;
        
        /// <summary> Returned from <see cref="CheckIndex_Renamed_Method()" /> detailing the health and status of the index.
        /// 
        /// <p/><b>WARNING</b>: this API is new and experimental and is
        /// subject to suddenly change in the next release.
        /// 
        /// </summary>
        
        public class Status
        {
            
            /// <summary>True if no problems were found with the index. </summary>
            public bool clean;
            
            /// <summary>True if we were unable to locate and load the segments_N file. </summary>
            public bool missingSegments;
            
            /// <summary>True if we were unable to open the segments_N file. </summary>
            public bool cantOpenSegments;
            
            /// <summary>True if we were unable to read the version number from segments_N file. </summary>
            public bool missingSegmentVersion;
            
            /// <summary>Name of latest segments_N file in the index. </summary>
            public System.String segmentsFileName;
            
            /// <summary>Number of segments in the index. </summary>
            public int numSegments;
            
            /// <summary>String description of the version of the index. </summary>
            public System.String segmentFormat;

            /// <summary>Empty unless you passed specific segments list to check as optional 3rd argument.</summary>
            /// <seealso>
            ///   <cref>CheckIndex.CheckIndex_Renamed_Method(System.Collections.IList)</cref>
            /// </seealso>
            public List<string> segmentsChecked = new List<string>();
            
            /// <summary>True if the index was created with a newer version of Lucene than the CheckIndex tool. </summary>
            public bool toolOutOfDate;
            
            /// <summary>List of <see cref="SegmentInfoStatus" /> instances, detailing status of each segment. </summary>
            public IList<SegmentInfoStatus> segmentInfos = new List<SegmentInfoStatus>();
            
            /// <summary>Directory index is in. </summary>
            public Directory dir;
            
            /// <summary> SegmentInfos instance containing only segments that
            /// had no problems (this is used with the <see cref="CheckIndex.FixIndex" /> 
            /// method to repair the index. 
            /// </summary>
            internal SegmentInfos newSegments;
            
            /// <summary>How many documents will be lost to bad segments. </summary>
            public int totLoseDocCount;
            
            /// <summary>How many bad segments were found. </summary>
            public int numBadSegments;
            
            /// <summary>True if we checked only specific segments (<see cref="CheckIndex.CheckIndex_Renamed_Method(List{string})" />)
            /// was called with non-null
            /// argument). 
            /// </summary>
            public bool partial;
            
            /// <summary>Holds the userData of the last commit in the index </summary>
            public IDictionary<string, string> userData;
            
            /// <summary>Holds the status of each segment in the index.
            /// See <see cref="SegmentInfos" />.
            /// 
            /// <p/><b>WARNING</b>: this API is new and experimental and is
            /// subject to suddenly change in the next release.
            /// </summary>
            public class SegmentInfoStatus
            {
                /// <summary>Name of the segment. </summary>
                public System.String name;
                
                /// <summary>Document count (does not take deletions into account). </summary>
                public int docCount;
                
                /// <summary>True if segment is compound file format. </summary>
                public bool compound;
                
                /// <summary>Number of files referenced by this segment. </summary>
                public int numFiles;
                
                /// <summary>Net size (MB) of the files referenced by this
                /// segment. 
                /// </summary>
                public double sizeMB;
                
                /// <summary>Doc store offset, if this segment shares the doc
                /// store files (stored fields and term vectors) with
                /// other segments.  This is -1 if it does not share. 
                /// </summary>
                public int docStoreOffset = - 1;
                
                /// <summary>String of the shared doc store segment, or null if
                /// this segment does not share the doc store files. 
                /// </summary>
                public System.String docStoreSegment;
                
                /// <summary>True if the shared doc store files are compound file
                /// format. 
                /// </summary>
                public bool docStoreCompoundFile;
                
                /// <summary>True if this segment has pending deletions. </summary>
                public bool hasDeletions;
                
                /// <summary>Name of the current deletions file name. </summary>
                public System.String deletionsFileName;
                
                /// <summary>Number of deleted documents. </summary>
                public int numDeleted;
                
                /// <summary>True if we were able to open a SegmentReader on this
                /// segment. 
                /// </summary>
                public bool openReaderPassed;
                
                /// <summary>Number of fields in this segment. </summary>
                internal int numFields;
                
                /// <summary>True if at least one of the fields in this segment
                /// does not omitTermFreqAndPositions.
                /// </summary>
                /// <seealso cref="AbstractField.OmitTermFreqAndPositions">
                /// </seealso>
                public bool hasProx;

                /// <summary>Map&lt;String, String&gt; that includes certain
                /// debugging details that IndexWriter records into
                /// each segment it creates 
                /// </summary>
                public IDictionary<string, string> diagnostics;
                
                /// <summary>Status for testing of field norms (null if field norms could not be tested). </summary>
                public FieldNormStatus fieldNormStatus;
                
                /// <summary>Status for testing of indexed terms (null if indexed terms could not be tested). </summary>
                public TermIndexStatus termIndexStatus;
                
                /// <summary>Status for testing of stored fields (null if stored fields could not be tested). </summary>
                public StoredFieldStatus storedFieldStatus;
                
                /// <summary>Status for testing of term vectors (null if term vectors could not be tested). </summary>
                public TermVectorStatus termVectorStatus;
            }
            
            /// <summary> Status from testing field norms.</summary>
            public sealed class FieldNormStatus
            {
                /// <summary>Number of fields successfully tested </summary>
                public long totFields = 0L;
                
                /// <summary>Exception thrown during term index test (null on success) </summary>
                public System.Exception error = null;
            }
            
            /// <summary> Status from testing term index.</summary>
            public sealed class TermIndexStatus
            {
                /// <summary>Total term count </summary>
                public long termCount = 0L;
                
                /// <summary>Total frequency across all terms. </summary>
                public long totFreq = 0L;
                
                /// <summary>Total number of positions. </summary>
                public long totPos = 0L;
                
                /// <summary>Exception thrown during term index test (null on success) </summary>
                public System.Exception error = null;
            }
            
            /// <summary> Status from testing stored fields.</summary>
            public sealed class StoredFieldStatus
            {
                
                /// <summary>Number of documents tested. </summary>
                public int docCount = 0;
                
                /// <summary>Total number of stored fields tested. </summary>
                public long totFields = 0;
                
                /// <summary>Exception thrown during stored fields test (null on success) </summary>
                public System.Exception error = null;
            }
            
            /// <summary> Status from testing stored fields.</summary>
            public sealed class TermVectorStatus
            {
                
                /// <summary>Number of documents tested. </summary>
                public int docCount = 0;
                
                /// <summary>Total number of term vectors tested. </summary>
                public long totVectors = 0;
                
                /// <summary>Exception thrown during term vector test (null on success) </summary>
                public System.Exception error = null;
            }
        }
        
        /// <summary>Create a new CheckIndex on the directory. </summary>
        public CheckIndex(Directory dir)
        {
            this.dir = dir;
            infoStream = null;
        }
        
        /// <summary>Set infoStream where messages should go.  If null, no
        /// messages are printed 
        /// </summary>
        public virtual void  SetInfoStream(StreamWriter @out)
        {
            infoStream = @out;
        }
        
        private void  Msg(System.String msg)
        {
            if (infoStream != null)
                infoStream.WriteLine(msg);
        }
        
        private class MySegmentTermDocs:SegmentTermDocs
        {
            
            internal int delCount;
            
            internal MySegmentTermDocs(SegmentReader p):base(p)
            {
            }
            
            public override void  Seek(Term term)
            {
                base.Seek(term);
                delCount = 0;
            }
            
            protected internal override void  SkippingDoc()
            {
                delCount++;
            }
        }
        
        /// <summary>Returns a <see cref="Status" /> instance detailing
        /// the state of the index.
        /// 
        /// <p/>As this method checks every byte in the index, on a large
        /// index it can take quite a long time to run.
        /// 
        /// <p/><b>WARNING</b>: make sure
        /// you only call this when the index is not opened by any
        /// writer. 
        /// </summary>
        public virtual Status CheckIndex_Renamed_Method()
        {
            return CheckIndex_Renamed_Method(null);
        }
        
        /// <summary>Returns a <see cref="Status" /> instance detailing
        /// the state of the index.
        /// 
        /// </summary>
        /// <param name="onlySegments">list of specific segment names to check
        /// 
        /// <p/>As this method checks every byte in the specified
        /// segments, on a large index it can take quite a long
        /// time to run.
        /// 
        /// <p/><b>WARNING</b>: make sure
        /// you only call this when the index is not opened by any
        /// writer. 
        /// </param>
        public virtual Status CheckIndex_Renamed_Method(List<string> onlySegments)
        {
            System.Globalization.NumberFormatInfo nf = System.Globalization.CultureInfo.CurrentCulture.NumberFormat;
            SegmentInfos sis = new SegmentInfos();
            Status result = new Status();
            result.dir = dir;
            try
            {
                sis.Read(dir);
            }
            catch (System.Exception t)
            {
                Msg("ERROR: could not read any segments file in directory");
                result.missingSegments = true;
                if (infoStream != null)
                    infoStream.WriteLine(t.StackTrace);
                return result;
            }
            
            int numSegments = sis.Count;
            var segmentsFileName = sis.GetCurrentSegmentFileName();
            IndexInput input = null;
            try
            {
                input = dir.OpenInput(segmentsFileName);
            }
            catch (System.Exception t)
            {
                Msg("ERROR: could not open segments file in directory");
                if (infoStream != null)
                    infoStream.WriteLine(t.StackTrace);
                result.cantOpenSegments = true;
                return result;
            }
            int format = 0;
            try
            {
                format = input.ReadInt();
            }
            catch (System.Exception t)
            {
                Msg("ERROR: could not read segment file version in directory");
                if (infoStream != null)
                    infoStream.WriteLine(t.StackTrace);
                result.missingSegmentVersion = true;
                return result;
            }
            finally
            {
                if (input != null)
                    input.Close();
            }
            
            System.String sFormat = "";
            bool skip = false;
            
            if (format == SegmentInfos.FORMAT)
                sFormat = "FORMAT [Lucene Pre-2.1]";
            if (format == SegmentInfos.FORMAT_LOCKLESS)
                sFormat = "FORMAT_LOCKLESS [Lucene 2.1]";
            else if (format == SegmentInfos.FORMAT_SINGLE_NORM_FILE)
                sFormat = "FORMAT_SINGLE_NORM_FILE [Lucene 2.2]";
            else if (format == SegmentInfos.FORMAT_SHARED_DOC_STORE)
                sFormat = "FORMAT_SHARED_DOC_STORE [Lucene 2.3]";
            else
            {
                if (format == SegmentInfos.FORMAT_CHECKSUM)
                    sFormat = "FORMAT_CHECKSUM [Lucene 2.4]";
                else if (format == SegmentInfos.FORMAT_DEL_COUNT)
                    sFormat = "FORMAT_DEL_COUNT [Lucene 2.4]";
                else if (format == SegmentInfos.FORMAT_HAS_PROX)
                    sFormat = "FORMAT_HAS_PROX [Lucene 2.4]";
                else if (format == SegmentInfos.FORMAT_USER_DATA)
                    sFormat = "FORMAT_USER_DATA [Lucene 2.9]";
                else if (format == SegmentInfos.FORMAT_DIAGNOSTICS)
                    sFormat = "FORMAT_DIAGNOSTICS [Lucene 2.9]";
                else if (format < SegmentInfos.CURRENT_FORMAT)
                {
                    sFormat = "int=" + format + " [newer version of Lucene than this tool]";
                    skip = true;
                }
                else
                {
                    sFormat = format + " [Lucene 1.3 or prior]";
                }
            }
            
            result.segmentsFileName = segmentsFileName;
            result.numSegments = numSegments;
            result.segmentFormat = sFormat;
            result.userData = sis.UserData;
            System.String userDataString;
            if (sis.UserData.Count > 0)
            {
                userDataString = " userData=" + CollectionsHelper.CollectionToString(sis.UserData);
            }
            else
            {
                userDataString = "";
            }
            
            Msg("Segments file=" + segmentsFileName + " numSegments=" + numSegments + " version=" + sFormat + userDataString);
            
            if (onlySegments != null)
            {
                result.partial = true;
                if (infoStream != null)
                    infoStream.Write("\nChecking only these segments:");
                foreach(string s in onlySegments)
                {
                    if (infoStream != null)
                    {
                        infoStream.Write(" " + s);
                    }
                }
                result.segmentsChecked.AddRange(onlySegments);
                Msg(":");
            }
            
            if (skip)
            {
                Msg("\nERROR: this index appears to be created by a newer version of Lucene than this tool was compiled on; please re-compile this tool on the matching version of Lucene; exiting");
                result.toolOutOfDate = true;
                return result;
            }
            
            
            result.newSegments = (SegmentInfos) sis.Clone();
            result.newSegments.Clear();
            
            for (int i = 0; i < numSegments; i++)
            {
                SegmentInfo info = sis.Info(i);
                if (onlySegments != null && !onlySegments.Contains(info.name))
                    continue;
                var segInfoStat = new Status.SegmentInfoStatus();
                result.segmentInfos.Add(segInfoStat);
                Msg("  " + (1 + i) + " of " + numSegments + ": name=" + info.name + " docCount=" + info.docCount);
                segInfoStat.name = info.name;
                segInfoStat.docCount = info.docCount;
                
                int toLoseDocCount = info.docCount;
                
                SegmentReader reader = null;
                
                try
                {
                    Msg("    compound=" + info.GetUseCompoundFile());
                    segInfoStat.compound = info.GetUseCompoundFile();
                    Msg("    hasProx=" + info.HasProx);
                    segInfoStat.hasProx = info.HasProx;
                    Msg("    numFiles=" + info.Files().Count);
                    segInfoStat.numFiles = info.Files().Count;
                    Msg(System.String.Format(nf, "    size (MB)={0:f}", new System.Object[] { (info.SizeInBytes() / (1024.0 * 1024.0)) }));
                    segInfoStat.sizeMB = info.SizeInBytes() / (1024.0 * 1024.0);
                    IDictionary<string, string> diagnostics = info.Diagnostics;
                    segInfoStat.diagnostics = diagnostics;
                    if (diagnostics.Count > 0)
                    {
                        Msg("    diagnostics = " + CollectionsHelper.CollectionToString(diagnostics));
                    }
                    
                    int docStoreOffset = info.DocStoreOffset;
                    if (docStoreOffset != - 1)
                    {
                        Msg("    docStoreOffset=" + docStoreOffset);
                        segInfoStat.docStoreOffset = docStoreOffset;
                        Msg("    docStoreSegment=" + info.DocStoreSegment);
                        segInfoStat.docStoreSegment = info.DocStoreSegment;
                        Msg("    docStoreIsCompoundFile=" + info.DocStoreIsCompoundFile);
                        segInfoStat.docStoreCompoundFile = info.DocStoreIsCompoundFile;
                    }
                    System.String delFileName = info.GetDelFileName();
                    if (delFileName == null)
                    {
                        Msg("    no deletions");
                        segInfoStat.hasDeletions = false;
                    }
                    else
                    {
                        Msg("    has deletions [delFileName=" + delFileName + "]");
                        segInfoStat.hasDeletions = true;
                        segInfoStat.deletionsFileName = delFileName;
                    }
                    if (infoStream != null)
                        infoStream.Write("    test: open reader.........");
                    reader = SegmentReader.Get(true, info, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR);
                    
                    segInfoStat.openReaderPassed = true;
                    
                    int numDocs = reader.NumDocs();
                    toLoseDocCount = numDocs;
                    if (reader.HasDeletions)
                    {
                        if (reader.deletedDocs.Count() != info.GetDelCount())
                        {
                            throw new System.SystemException("delete count mismatch: info=" + info.GetDelCount() + " vs deletedDocs.count()=" + reader.deletedDocs.Count());
                        }
                        if (reader.deletedDocs.Count() > reader.MaxDoc)
                        {
                            throw new System.SystemException("too many deleted docs: MaxDoc=" + reader.MaxDoc + " vs deletedDocs.count()=" + reader.deletedDocs.Count());
                        }
                        if (info.docCount - numDocs != info.GetDelCount())
                        {
                            throw new System.SystemException("delete count mismatch: info=" + info.GetDelCount() + " vs reader=" + (info.docCount - numDocs));
                        }
                        segInfoStat.numDeleted = info.docCount - numDocs;
                        Msg("OK [" + (segInfoStat.numDeleted) + " deleted docs]");
                    }
                    else
                    {
                        if (info.GetDelCount() != 0)
                        {
                            throw new System.SystemException("delete count mismatch: info=" + info.GetDelCount() + " vs reader=" + (info.docCount - numDocs));
                        }
                        Msg("OK");
                    }
                    if (reader.MaxDoc != info.docCount)
                        throw new System.SystemException("SegmentReader.MaxDoc " + reader.MaxDoc + " != SegmentInfos.docCount " + info.docCount);
                    
                    // Test getFieldNames()
                    if (infoStream != null)
                    {
                        infoStream.Write("    test: fields..............");
                    }
                    ICollection<string> fieldNames = reader.GetFieldNames(IndexReader.FieldOption.ALL);
                    Msg("OK [" + fieldNames.Count + " fields]");
                    segInfoStat.numFields = fieldNames.Count;
                    
                    // Test Field Norms
                    segInfoStat.fieldNormStatus = TestFieldNorms(fieldNames, reader);
                    
                    // Test the Term Index
                    segInfoStat.termIndexStatus = TestTermIndex(info, reader);
                    
                    // Test Stored Fields
                    segInfoStat.storedFieldStatus = TestStoredFields(info, reader, nf);
                    
                    // Test Term Vectors
                    segInfoStat.termVectorStatus = TestTermVectors(info, reader, nf);
                    
                    // Rethrow the first exception we encountered
                    //  This will cause stats for failed segments to be incremented properly
                    if (segInfoStat.fieldNormStatus.error != null)
                    {
                        throw new SystemException("Field Norm test failed");
                    }
                    else if (segInfoStat.termIndexStatus.error != null)
                    {
                        throw new SystemException("Term Index test failed");
                    }
                    else if (segInfoStat.storedFieldStatus.error != null)
                    {
                        throw new SystemException("Stored Field test failed");
                    }
                    else if (segInfoStat.termVectorStatus.error != null)
                    {
                        throw new System.SystemException("Term Vector test failed");
                    }
                    
                    Msg("");
                }
                catch (System.Exception t)
                {
                    Msg("FAILED");
                    const string comment = "fixIndex() would remove reference to this segment";
                    Msg("    WARNING: " + comment + "; full exception:");
                    if (infoStream != null)
                        infoStream.WriteLine(t.StackTrace);
                    Msg("");
                    result.totLoseDocCount += toLoseDocCount;
                    result.numBadSegments++;
                    continue;
                }
                finally
                {
                    if (reader != null)
                        reader.Close();
                }
                
                // Keeper
                result.newSegments.Add((SegmentInfo)info.Clone());
            }
            
            if (0 == result.numBadSegments)
            {
                result.clean = true;
                Msg("No problems were detected with this index.\n");
            }
            else
                Msg("WARNING: " + result.numBadSegments + " broken segments (containing " + result.totLoseDocCount + " documents) detected");
            
            return result;
        }
        
        /// <summary> Test field norms.</summary>
        private Status.FieldNormStatus TestFieldNorms(IEnumerable<string> fieldNames, SegmentReader reader)
        {
            var status = new Status.FieldNormStatus();
            
            try
            {
                // Test Field Norms
                if (infoStream != null)
                {
                    infoStream.Write("    test: field norms.........");
                }

                var b = new byte[reader.MaxDoc];
                foreach(string fieldName in fieldNames)
                {
                    if (reader.HasNorms(fieldName))
                    {
                        reader.Norms(fieldName, b, 0);
                        ++status.totFields;
                    }
                }
                
                Msg("OK [" + status.totFields + " fields]");
            }
            catch (System.Exception e)
            {
                Msg("ERROR [" + System.Convert.ToString(e.Message) + "]");
                status.error = e;
                if (infoStream != null)
                {
                    infoStream.WriteLine(e.StackTrace);
                }
            }
            
            return status;
        }
        
        /// <summary> Test the term index.</summary>
        private Status.TermIndexStatus TestTermIndex(SegmentInfo info, SegmentReader reader)
        {
            var status = new Status.TermIndexStatus();
            
            try
            {
                if (infoStream != null)
                {
                    infoStream.Write("    test: terms, freq, prox...");
                }
                
                TermEnum termEnum = reader.Terms();
                TermPositions termPositions = reader.TermPositions();
                
                // Used only to count up # deleted docs for this term
                var myTermDocs = new MySegmentTermDocs(reader);
                
                int maxDoc = reader.MaxDoc;
                
                while (termEnum.Next())
                {
                    status.termCount++;
                    Term term = termEnum.Term;
                    int docFreq = termEnum.DocFreq();
                    termPositions.Seek(term);
                    int lastDoc = - 1;
                    int freq0 = 0;
                    status.totFreq += docFreq;
                    while (termPositions.Next())
                    {
                        freq0++;
                        int doc = termPositions.Doc;
                        int freq = termPositions.Freq;
                        if (doc <= lastDoc)
                        {
                            throw new System.SystemException("term " + term + ": doc " + doc + " <= lastDoc " + lastDoc);
                        }
                        if (doc >= maxDoc)
                        {
                            throw new System.SystemException("term " + term + ": doc " + doc + " >= maxDoc " + maxDoc);
                        }
                        
                        lastDoc = doc;
                        if (freq <= 0)
                        {
                            throw new System.SystemException("term " + term + ": doc " + doc + ": freq " + freq + " is out of bounds");
                        }
                        
                        int lastPos = - 1;
                        status.totPos += freq;
                        for (int j = 0; j < freq; j++)
                        {
                            int pos = termPositions.NextPosition();
                            if (pos < - 1)
                            {
                                throw new System.SystemException("term " + term + ": doc " + doc + ": pos " + pos + " is out of bounds");
                            }
                            if (pos < lastPos)
                            {
                                throw new System.SystemException("term " + term + ": doc " + doc + ": pos " + pos + " < lastPos " + lastPos);
                            }
                            lastPos = pos;
                        }
                    }
                    
                    // Now count how many deleted docs occurred in
                    // this term:
                    int delCount;
                    if (reader.HasDeletions)
                    {
                        myTermDocs.Seek(term);
                        while (myTermDocs.Next())
                        {
                        }
                        delCount = myTermDocs.delCount;
                    }
                    else
                    {
                        delCount = 0;
                    }
                    
                    if (freq0 + delCount != docFreq)
                    {
                        throw new System.SystemException("term " + term + " docFreq=" + docFreq + " != num docs seen " + freq0 + " + num docs deleted " + delCount);
                    }
                }
                
                Msg("OK [" + status.termCount + " terms; " + status.totFreq + " terms/docs pairs; " + status.totPos + " tokens]");
            }
            catch (System.Exception e)
            {
                Msg("ERROR [" + System.Convert.ToString(e.Message) + "]");
                status.error = e;
                if (infoStream != null)
                {
                    infoStream.WriteLine(e.StackTrace);
                }
            }
            
            return status;
        }
        
        /// <summary> Test stored fields for a segment.</summary>
        private Status.StoredFieldStatus TestStoredFields(SegmentInfo info, SegmentReader reader, System.Globalization.NumberFormatInfo format)
        {
            var status = new Status.StoredFieldStatus();
            
            try
            {
                if (infoStream != null)
                {
                    infoStream.Write("    test: stored fields.......");
                }
                
                // Scan stored fields for all documents
                for (int j = 0; j < info.docCount; ++j)
                {
                    if (!reader.IsDeleted(j))
                    {
                        status.docCount++;
                        Document doc = reader.Document(j);
                        status.totFields += doc.GetFields().Count;
                    }
                }
                
                // Validate docCount
                if (status.docCount != reader.NumDocs())
                {
                    throw new System.SystemException("docCount=" + status.docCount + " but saw " + status.docCount + " undeleted docs");
                }
                
                Msg(string.Format(format, "OK [{0:d} total field count; avg {1:f} fields per doc]", new object[] { status.totFields, (((float) status.totFields) / status.docCount) }));
            }
            catch (System.Exception e)
            {
                Msg("ERROR [" + System.Convert.ToString(e.Message) + "]");
                status.error = e;
                if (infoStream != null)
                {
                    infoStream.WriteLine(e.StackTrace);
                }
            }
            
            return status;
        }
        
        /// <summary> Test term vectors for a segment.</summary>
        private Status.TermVectorStatus TestTermVectors(SegmentInfo info, SegmentReader reader, System.Globalization.NumberFormatInfo format)
        {
            var status = new Status.TermVectorStatus();
            
            try
            {
                if (infoStream != null)
                {
                    infoStream.Write("    test: term vectors........");
                }
                
                for (int j = 0; j < info.docCount; ++j)
                {
                    if (!reader.IsDeleted(j))
                    {
                        status.docCount++;
                        ITermFreqVector[] tfv = reader.GetTermFreqVectors(j);
                        if (tfv != null)
                        {
                            status.totVectors += tfv.Length;
                        }
                    }
                }
                
                Msg(System.String.Format(format, "OK [{0:d} total vector count; avg {1:f} term/freq vector fields per doc]", new object[] { status.totVectors, (((float) status.totVectors) / status.docCount) }));
            }
            catch (System.Exception e)
            {
                Msg("ERROR [" + System.Convert.ToString(e.Message) + "]");
                status.error = e;
                if (infoStream != null)
                {
                    infoStream.WriteLine(e.StackTrace);
                }
            }
            
            return status;
        }
        
        /// <summary>Repairs the index using previously returned result
        /// from <see cref="CheckIndex" />.  Note that this does not
        /// remove any of the unreferenced files after it's done;
        /// you must separately open an <see cref="IndexWriter" />, which
        /// deletes unreferenced files when it's created.
        /// 
        /// <p/><b>WARNING</b>: this writes a
        /// new segments file into the index, effectively removing
        /// all documents in broken segments from the index.
        /// BE CAREFUL.
        /// 
        /// <p/><b>WARNING</b>: Make sure you only call this when the
        /// index is not opened  by any writer. 
        /// </summary>
        public virtual void  FixIndex(Status result)
        {
            if (result.partial)
                throw new System.ArgumentException("can only fix an index that was fully checked (this status checked a subset of segments)");
            result.newSegments.Commit(result.dir);
        }
        
        private static bool assertsOn;
        
        private static bool TestAsserts()
        {
            assertsOn = true;
            return true;
        }
        
        private static bool AssertsOn()
        {
            System.Diagnostics.Debug.Assert(TestAsserts());
            return assertsOn;
        }
        
        /// <summary>Command-line interface to check and fix an index.
        /// <p/>
        /// Run it like this:
        /// <code>
        /// java -ea:Lucene.Net... Lucene.Net.Index.CheckIndex pathToIndex [-fix] [-segment X] [-segment Y]
        /// </code>
        /// <list type="bullet">
        /// <item><c>-fix</c>: actually write a new segments_N file, removing any problematic segments</item>
        /// <item><c>-segment X</c>: only check the specified
        /// segment(s).  This can be specified multiple times,
        /// to check more than one segment, eg <c>-segment _2
        /// -segment _a</c>.  You can't use this with the -fix
        /// option.</item>
        /// </list>
        /// <p/><b>WARNING</b>: <c>-fix</c> should only be used on an emergency basis as it will cause
        /// documents (perhaps many) to be permanently removed from the index.  Always make
        /// a backup copy of your index before running this!  Do not run this tool on an index
        /// that is actively being written to.  You have been warned!
        /// <p/>                Run without -fix, this tool will open the index, report version information
        /// and report any exceptions it hits and what action it would take if -fix were
        /// specified.  With -fix, this tool will remove any segments that have issues and
        /// write a new segments_N file.  This means all documents contained in the affected
        /// segments will be removed.
        /// <p/>
        /// This tool exits with exit code 1 if the index cannot be opened or has any
        /// corruption, else 0.
        /// </summary>
        [STAThread]
        public static void  Main(System.String[] args)
        {
            
            bool doFix = false;
            var onlySegments = new List<string>();
            System.String indexPath = null;
            int i = 0;
            while (i < args.Length)
            {
                if (args[i].Equals("-fix"))
                {
                    doFix = true;
                    i++;
                }
                else if (args[i].Equals("-segment"))
                {
                    if (i == args.Length - 1)
                    {
                        System.Console.Out.WriteLine("ERROR: missing name for -segment option");
                        System.Environment.Exit(1);
                    }
                    onlySegments.Add(args[i + 1]);
                    i += 2;
                }
                else
                {
                    if (indexPath != null)
                    {
                        System.Console.Out.WriteLine("ERROR: unexpected extra argument '" + args[i] + "'");
                        System.Environment.Exit(1);
                    }
                    indexPath = args[i];
                    i++;
                }
            }
            
            if (indexPath == null)
            {
                System.Console.Out.WriteLine("\nERROR: index path not specified");
                System.Console.Out.WriteLine("\nUsage: java Lucene.Net.Index.CheckIndex pathToIndex [-fix] [-segment X] [-segment Y]\n" + "\n" + "  -fix: actually write a new segments_N file, removing any problematic segments\n" + "  -segment X: only check the specified segments.  This can be specified multiple\n" + "              times, to check more than one segment, eg '-segment _2 -segment _a'.\n" + "              You can't use this with the -fix option\n" + "\n" + "**WARNING**: -fix should only be used on an emergency basis as it will cause\n" + "documents (perhaps many) to be permanently removed from the index.  Always make\n" + "a backup copy of your index before running this!  Do not run this tool on an index\n" + "that is actively being written to.  You have been warned!\n" + "\n" + "Run without -fix, this tool will open the index, report version information\n" + "and report any exceptions it hits and what action it would take if -fix were\n" + "specified.  With -fix, this tool will remove any segments that have issues and\n" + "write a new segments_N file.  This means all documents contained in the affected\n" + "segments will be removed.\n" + "\n" + "This tool exits with exit code 1 if the index cannot be opened or has any\n" + "corruption, else 0.\n");
                System.Environment.Exit(1);
            }
            
            if (!AssertsOn())
                System.Console.Out.WriteLine("\nNOTE: testing will be more thorough if you run java with '-ea:Lucene.Net...', so assertions are enabled");
            
            if (onlySegments.Count == 0)
                onlySegments = null;
            else if (doFix)
            {
                System.Console.Out.WriteLine("ERROR: cannot specify both -fix and -segment");
                System.Environment.Exit(1);
            }
            
            System.Console.Out.WriteLine("\nOpening index @ " + indexPath + "\n");
            Directory dir = null;
            try
            {
                dir = FSDirectory.Open(new System.IO.DirectoryInfo(indexPath));
            }
            catch (Exception t)
            {
                Console.Out.WriteLine("ERROR: could not open directory \"" + indexPath + "\"; exiting");
                Console.Out.WriteLine(t.StackTrace);
                Environment.Exit(1);
            }
            
            var checker = new CheckIndex(dir);
            var tempWriter = new System.IO.StreamWriter(System.Console.OpenStandardOutput(), System.Console.Out.Encoding)
                                {AutoFlush = true};
            checker.SetInfoStream(tempWriter);
            
            Status result = checker.CheckIndex_Renamed_Method(onlySegments);
            if (result.missingSegments)
            {
                System.Environment.Exit(1);
            }
            
            if (!result.clean)
            {
                if (!doFix)
                {
                    System.Console.Out.WriteLine("WARNING: would write new segments file, and " + result.totLoseDocCount + " documents would be lost, if -fix were specified\n");
                }
                else
                {
                    Console.Out.WriteLine("WARNING: " + result.totLoseDocCount + " documents will be lost\n");
                    Console.Out.WriteLine("NOTE: will write new segments file in 5 seconds; this will remove " + result.totLoseDocCount + " docs from the index. THIS IS YOUR LAST CHANCE TO CTRL+C!");
                    for (var s = 0; s < 5; s++)
                    {
                        System.Threading.Thread.Sleep(new System.TimeSpan((System.Int64) 10000 * 1000));
                        System.Console.Out.WriteLine("  " + (5 - s) + "...");
                    }
                    Console.Out.WriteLine("Writing...");
                    checker.FixIndex(result);
                    Console.Out.WriteLine("OK");
                    Console.Out.WriteLine("Wrote new segments file \"" + result.newSegments.GetCurrentSegmentFileName() + "\"");
                }
            }
            System.Console.Out.WriteLine("");
            
            int exitCode;
            if (result != null && result.clean == true)
                exitCode = 0;
            else
                exitCode = 1;
            System.Environment.Exit(exitCode);
        }
    }
}