IndexWriter.cs

Go to the documentation of this file.

/* 
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 * 
 * http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

using System;
using System.Collections.Generic;
using System.IO;
using Lucene.Net.Support;
using Analyzer = Lucene.Net.Analysis.Analyzer;
using Document = Lucene.Net.Documents.Document;
using IndexingChain = Lucene.Net.Index.DocumentsWriter.IndexingChain;
using AlreadyClosedException = Lucene.Net.Store.AlreadyClosedException;
using BufferedIndexInput = Lucene.Net.Store.BufferedIndexInput;
using Directory = Lucene.Net.Store.Directory;
using Lock = Lucene.Net.Store.Lock;
using LockObtainFailedException = Lucene.Net.Store.LockObtainFailedException;
using Constants = Lucene.Net.Util.Constants;
using Query = Lucene.Net.Search.Query;
using Similarity = Lucene.Net.Search.Similarity;

namespace Lucene.Net.Index
{
    
    /// <summary>An <c>IndexWriter</c> creates and maintains an index.
    /// <p/>The <c>create</c> argument to the 
    /// <see cref="IndexWriter(Directory, Analyzer, bool, MaxFieldLength)">constructor</see> determines 
    /// whether a new index is created, or whether an existing index is
    /// opened.  Note that you can open an index with <c>create=true</c>
    /// even while readers are using the index.  The old readers will 
    /// continue to search the "point in time" snapshot they had opened, 
    /// and won't see the newly created index until they re-open.  There are
    /// also <see cref="IndexWriter(Directory, Analyzer, MaxFieldLength)">constructors</see>
    /// with no <c>create</c> argument which will create a new index
    /// if there is not already an index at the provided path and otherwise 
    /// open the existing index.<p/>
    /// <p/>In either case, documents are added with <see cref="AddDocument(Document)" />
    /// and removed with <see cref="DeleteDocuments(Term)" /> or
    /// <see cref="DeleteDocuments(Query)" />. A document can be updated with
    /// <see cref="UpdateDocument(Term, Document)" /> (which just deletes
    /// and then adds the entire document). When finished adding, deleting 
    /// and updating documents, <see cref="Close()" /> should be called.<p/>
    /// <a name="flush"></a>
    /// <p/>These changes are buffered in memory and periodically
    /// flushed to the <see cref="Directory" /> (during the above method
    /// calls).  A flush is triggered when there are enough
    /// buffered deletes (see <see cref="SetMaxBufferedDeleteTerms" />)
    /// or enough added documents since the last flush, whichever
    /// is sooner.  For the added documents, flushing is triggered
    /// either by RAM usage of the documents (see 
    /// <see cref="SetRAMBufferSizeMB" />) or the number of added documents.
    /// The default is to flush when RAM usage hits 16 MB.  For
    /// best indexing speed you should flush by RAM usage with a
    /// large RAM buffer.  Note that flushing just moves the
    /// internal buffered state in IndexWriter into the index, but
    /// these changes are not visible to IndexReader until either
    /// <see cref="Commit()" /> or <see cref="Close()" /> is called.  A flush may
    /// also trigger one or more segment merges which by default
    /// run with a background thread so as not to block the
    /// addDocument calls (see <a href="#mergePolicy">below</a>
    /// for changing the <see cref="MergeScheduler" />).
    /// <p/>
    /// If an index will not have more documents added for a while and optimal search
    /// performance is desired, then either the full <see cref="Optimize()" />
    /// method or partial <see cref="Optimize(int)" /> method should be
    /// called before the index is closed.
    /// <p/>
    /// Opening an <c>IndexWriter</c> creates a lock file for the directory in use. Trying to open
    /// another <c>IndexWriter</c> on the same directory will lead to a
    /// <see cref="LockObtainFailedException" />. The <see cref="LockObtainFailedException" />
    /// is also thrown if an IndexReader on the same directory is used to delete documents
    /// from the index.<p/>
    /// </summary>
    /// <summary><a name="deletionPolicy"></a>
    /// <p/>Expert: <c>IndexWriter</c> allows an optional
    /// <see cref="IndexDeletionPolicy" /> implementation to be
    /// specified.  You can use this to control when prior commits
    /// are deleted from the index.  The default policy is <see cref="KeepOnlyLastCommitDeletionPolicy" />
    /// which removes all prior
    /// commits as soon as a new commit is done (this matches
    /// behavior before 2.2).  Creating your own policy can allow
    /// you to explicitly keep previous "point in time" commits
    /// alive in the index for some time, to allow readers to
    /// refresh to the new commit without having the old commit
    /// deleted out from under them.  This is necessary on
    /// filesystems like NFS that do not support "delete on last
    /// close" semantics, which Lucene's "point in time" search
    /// normally relies on. <p/>
    /// <a name="mergePolicy"></a> <p/>Expert:
    /// <c>IndexWriter</c> allows you to separately change
    /// the <see cref="MergePolicy" /> and the <see cref="MergeScheduler" />.
    /// The <see cref="MergePolicy" /> is invoked whenever there are
    /// changes to the segments in the index.  Its role is to
    /// select which merges to do, if any, and return a <see cref="Index.MergePolicy.MergeSpecification" />
    /// describing the merges.  It
    /// also selects merges to do for optimize().  (The default is
    /// <see cref="LogByteSizeMergePolicy" />.  Then, the <see cref="MergeScheduler" />
    /// is invoked with the requested merges and
    /// it decides when and how to run the merges.  The default is
    /// <see cref="ConcurrentMergeScheduler" />. <p/>
    /// <a name="OOME"></a><p/><b>NOTE</b>: if you hit an
    /// OutOfMemoryError then IndexWriter will quietly record this
    /// fact and block all future segment commits.  This is a
    /// defensive measure in case any internal state (buffered
    /// documents and deletions) were corrupted.  Any subsequent
    /// calls to <see cref="Commit()" /> will throw an
    /// IllegalStateException.  The only course of action is to
    /// call <see cref="Close()" />, which internally will call <see cref="Rollback()" />
    ///, to undo any changes to the index since the
    /// last commit.  You can also just call <see cref="Rollback()" />
    /// directly.<p/>
    /// <a name="thread-safety"></a><p/><b>NOTE</b>: 
    /// <see cref="IndexWriter" /> instances are completely thread
    /// safe, meaning multiple threads can call any of its
    /// methods, concurrently.  If your application requires
    /// external synchronization, you should <b>not</b>
    /// synchronize on the <c>IndexWriter</c> instance as
    /// this may cause deadlock; use your own (non-Lucene) objects
    /// instead. <p/>
    /// <b>NOTE:</b> if you call
    /// <c>Thread.Interrupt()</c> on a thread that's within
    /// IndexWriter, IndexWriter will try to catch this (eg, if
    /// it's in a Wait() or Thread.Sleep()), and will then throw
    /// the unchecked exception <see cref="System.Threading.ThreadInterruptedException"/>
    /// and <b>clear</b> the interrupt status on the thread<p/>
    /// </summary>

    /*
    * Clarification: Check Points (and commits)
    * IndexWriter writes new index files to the directory without writing a new segments_N
    * file which references these new files. It also means that the state of 
    * the in memory SegmentInfos object is different than the most recent
    * segments_N file written to the directory.
    * 
    * Each time the SegmentInfos is changed, and matches the (possibly 
    * modified) directory files, we have a new "check point". 
    * If the modified/new SegmentInfos is written to disk - as a new 
    * (generation of) segments_N file - this check point is also an 
    * IndexCommit.
    * 
    * A new checkpoint always replaces the previous checkpoint and 
    * becomes the new "front" of the index. This allows the IndexFileDeleter 
    * to delete files that are referenced only by stale checkpoints.
    * (files that were created since the last commit, but are no longer
    * referenced by the "front" of the index). For this, IndexFileDeleter 
    * keeps track of the last non commit checkpoint.
    */
    public class IndexWriter : System.IDisposable
    {
        private void  InitBlock()
        {
            similarity = Search.Similarity.Default;
            mergePolicy = new LogByteSizeMergePolicy(this);
            readerPool = new ReaderPool(this);
        }
        
        /// <summary> Default value for the write lock timeout (1,000).</summary>
        /// <seealso cref="DefaultWriteLockTimeout">
        /// </seealso>
        public static long WRITE_LOCK_TIMEOUT = 1000;
        
        private long writeLockTimeout = WRITE_LOCK_TIMEOUT;
        
        /// <summary> Name of the write lock in the index.</summary>
        public const System.String WRITE_LOCK_NAME = "write.lock";
        
        /// <summary> Value to denote a flush trigger is disabled</summary>
        public const int DISABLE_AUTO_FLUSH = - 1;
        
        /// <summary> Disabled by default (because IndexWriter flushes by RAM usage
        /// by default). Change using <see cref="SetMaxBufferedDocs(int)" />.
        /// </summary>
        public static readonly int DEFAULT_MAX_BUFFERED_DOCS = DISABLE_AUTO_FLUSH;
        
        /// <summary> Default value is 16 MB (which means flush when buffered
        /// docs consume 16 MB RAM).  Change using <see cref="SetRAMBufferSizeMB" />.
        /// </summary>
        public const double DEFAULT_RAM_BUFFER_SIZE_MB = 16.0;
        
        /// <summary> Disabled by default (because IndexWriter flushes by RAM usage
        /// by default). Change using <see cref="SetMaxBufferedDeleteTerms(int)" />.
        /// </summary>
        public static readonly int DEFAULT_MAX_BUFFERED_DELETE_TERMS = DISABLE_AUTO_FLUSH;
        
        /// <summary> Default value is 10,000. Change using <see cref="SetMaxFieldLength(int)" />.</summary>
        public const int DEFAULT_MAX_FIELD_LENGTH = 10000;
        
        /// <summary> Default value is 128. Change using <see cref="TermIndexInterval" />.</summary>
        public const int DEFAULT_TERM_INDEX_INTERVAL = 128;
        
        /// <summary> Absolute hard maximum length for a term.  If a term
        /// arrives from the analyzer longer than this length, it
        /// is skipped and a message is printed to infoStream, if
        /// set (see <see cref="SetInfoStream" />).
        /// </summary>
        public static readonly int MAX_TERM_LENGTH;
        
        // The normal read buffer size defaults to 1024, but
        // increasing this during merging seems to yield
        // performance gains.  However we don't want to increase
        // it too much because there are quite a few
        // BufferedIndexInputs created during merging.  See
        // LUCENE-888 for details.
        private const int MERGE_READ_BUFFER_SIZE = 4096;
        
        // Used for printing messages
        private static System.Object MESSAGE_ID_LOCK = new System.Object();
        private static int MESSAGE_ID = 0;
        private int messageID = - 1;
        private volatile bool hitOOM;
        
        private Directory directory; // where this index resides
        private Analyzer analyzer; // how to analyze text
        
        private Similarity similarity; // how to normalize
        
        private volatile uint changeCount; // increments every time a change is completed
        private long lastCommitChangeCount; // last changeCount that was committed
        
        private SegmentInfos rollbackSegmentInfos; // segmentInfos we will fallback to if the commit fails
        private HashMap<SegmentInfo, int?> rollbackSegments;
        
        internal volatile SegmentInfos pendingCommit; // set when a commit is pending (after prepareCommit() & before commit())
        internal volatile uint pendingCommitChangeCount;
        
        private SegmentInfos localRollbackSegmentInfos; // segmentInfos we will fallback to if the commit fails
        private int localFlushedDocCount; // saved docWriter.getFlushedDocCount during local transaction
        
        private SegmentInfos segmentInfos = new SegmentInfos(); // the segments
        private int optimizeMaxNumSegments;

        private DocumentsWriter docWriter;
        private IndexFileDeleter deleter;

        private ISet<SegmentInfo> segmentsToOptimize = Lucene.Net.Support.Compatibility.SetFactory.CreateHashSet<SegmentInfo>(); // used by optimize to note those needing optimization
        
        private Lock writeLock;
        
        private int termIndexInterval = DEFAULT_TERM_INDEX_INTERVAL;
        
        private bool closed;
        private bool closing;
        
        // Holds all SegmentInfo instances currently involved in
        // merges
        private HashSet<SegmentInfo> mergingSegments = new HashSet<SegmentInfo>();
        
        private MergePolicy mergePolicy;
        private MergeScheduler mergeScheduler = new ConcurrentMergeScheduler();
        private LinkedList<MergePolicy.OneMerge> pendingMerges = new LinkedList<MergePolicy.OneMerge>();
        private ISet<MergePolicy.OneMerge> runningMerges = Lucene.Net.Support.Compatibility.SetFactory.CreateHashSet<MergePolicy.OneMerge>();
        private IList<MergePolicy.OneMerge> mergeExceptions = new List<MergePolicy.OneMerge>();
        private long mergeGen;
        private bool stopMerges;
        
        private int flushCount;
        private int flushDeletesCount;
        
        // Used to only allow one addIndexes to proceed at once
        // TODO: use ReadWriteLock once we are on 5.0
        private int readCount; // count of how many threads are holding read lock
        private ThreadClass writeThread; // non-null if any thread holds write lock
        internal ReaderPool readerPool;
        private int upgradeCount;

        private int readerTermsIndexDivisor = IndexReader.DEFAULT_TERMS_INDEX_DIVISOR;
        
        // This is a "write once" variable (like the organic dye
        // on a DVD-R that may or may not be heated by a laser and
        // then cooled to permanently record the event): it's
        // false, until getReader() is called for the first time,
        // at which point it's switched to true and never changes
        // back to false.  Once this is true, we hold open and
        // reuse SegmentReader instances internally for applying
        // deletes, doing merges, and reopening near real-time
        // readers.
        private volatile bool poolReaders;
        
        /// <summary> Expert: returns a readonly reader, covering all committed as well as
        /// un-committed changes to the index. This provides "near real-time"
        /// searching, in that changes made during an IndexWriter session can be
        /// quickly made available for searching without closing the writer nor
        /// calling <see cref="Commit()" />.
        /// 
        /// <p/>
        /// Note that this is functionally equivalent to calling {#commit} and then
        /// using <see cref="IndexReader.Open(Lucene.Net.Store.Directory, bool)" /> to open a new reader. But the turarnound
        /// time of this method should be faster since it avoids the potentially
        /// costly <see cref="Commit()" />.
        /// <p/>
        /// 
        /// You must close the <see cref="IndexReader" /> returned by  this method once you are done using it.
        /// 
        /// <p/>
        /// It's <i>near</i> real-time because there is no hard
        /// guarantee on how quickly you can get a new reader after
        /// making changes with IndexWriter.  You'll have to
        /// experiment in your situation to determine if it's
        /// faster enough.  As this is a new and experimental
        /// feature, please report back on your findings so we can
        /// learn, improve and iterate.<p/>
        /// 
        /// <p/>The resulting reader suppports <see cref="IndexReader.Reopen()" />
        ///, but that call will simply forward
        /// back to this method (though this may change in the
        /// future).<p/>
        /// 
        /// <p/>The very first time this method is called, this
        /// writer instance will make every effort to pool the
        /// readers that it opens for doing merges, applying
        /// deletes, etc.  This means additional resources (RAM,
        /// file descriptors, CPU time) will be consumed.<p/>
        /// 
        /// <p/>For lower latency on reopening a reader, you should call <see cref="MergedSegmentWarmer" /> 
        /// to call <see cref="MergedSegmentWarmer" /> to
        /// pre-warm a newly merged segment before it's committed
        /// to the index. This is important for minimizing index-to-search 
        /// delay after a large merge.
        /// 
        /// <p/>If an addIndexes* call is running in another thread,
        /// then this reader will only search those segments from
        /// the foreign index that have been successfully copied
        /// over, so far<p/>.
        /// 
        /// <p/><b>NOTE</b>: Once the writer is closed, any
        /// outstanding readers may continue to be used.  However,
        /// if you attempt to reopen any of those readers, you'll
        /// hit an <see cref="AlreadyClosedException" />.<p/>
        /// 
        /// <p/><b>NOTE:</b> This API is experimental and might
        /// change in incompatible ways in the next release.<p/>
        /// 
        /// </summary>
        /// <returns> IndexReader that covers entire index plus all
        /// changes made so far by this IndexWriter instance
        /// 
        /// </returns>
        /// <throws>  IOException </throws>
        public virtual IndexReader GetReader()
        {
            return GetReader(readerTermsIndexDivisor);
        }
        
        /// <summary>Expert: like <see cref="GetReader()" />, except you can
        /// specify which termInfosIndexDivisor should be used for
        /// any newly opened readers.
        /// </summary>
        /// <param name="termInfosIndexDivisor">Subsambles which indexed
        /// terms are loaded into RAM. This has the same effect as <see cref="IndexWriter.TermIndexInterval" />
        /// except that setting
        /// must be done at indexing time while this setting can be
        /// set per reader.  When set to N, then one in every
        /// N*termIndexInterval terms in the index is loaded into
        /// memory.  By setting this to a value > 1 you can reduce
        /// memory usage, at the expense of higher latency when
        /// loading a TermInfo.  The default value is 1.  Set this
        /// to -1 to skip loading the terms index entirely. 
        /// </param>
        public virtual IndexReader GetReader(int termInfosIndexDivisor)
        {
            EnsureOpen();

            if (infoStream != null)
            {
                Message("flush at getReader");
            }
            
            // Do this up front before flushing so that the readers
            // obtained during this flush are pooled, the first time
            // this method is called:
            poolReaders = true;
            
            // Prevent segmentInfos from changing while opening the
            // reader; in theory we could do similar retry logic,
            // just like we do when loading segments_N
            IndexReader r;
            lock (this)
            {
                Flush(false, true, true);
                r = new ReadOnlyDirectoryReader(this, segmentInfos, termInfosIndexDivisor);
            }
            MaybeMerge();
            return r;
        }
        
        /// <summary>Holds shared SegmentReader instances. IndexWriter uses
        /// SegmentReaders for 1) applying deletes, 2) doing
        /// merges, 3) handing out a real-time reader.  This pool
        /// reuses instances of the SegmentReaders in all these
        /// places if it is in "near real-time mode" (getReader()
        /// has been called on this instance). 
        /// </summary>
        
        internal class ReaderPool : IDisposable
        {
            public ReaderPool(IndexWriter enclosingInstance)
            {
                InitBlock(enclosingInstance);
            }
            private void  InitBlock(IndexWriter enclosingInstance)
            {
                this.enclosingInstance = enclosingInstance;
            }
            private IndexWriter enclosingInstance;
            public IndexWriter Enclosing_Instance
            {
                get
                {
                    return enclosingInstance;
                }
                
            }

            private IDictionary<SegmentInfo, SegmentReader> readerMap = new HashMap<SegmentInfo, SegmentReader>();
            
            /// <summary>Forcefully clear changes for the specifed segments,
            /// and remove from the pool.   This is called on succesful merge. 
            /// </summary>
            internal virtual void  Clear(SegmentInfos infos)
            {
                lock (this)
                {
                    if (infos == null)
                    {
                        foreach(KeyValuePair<SegmentInfo, SegmentReader> ent in readerMap)
                        {
                            ent.Value.hasChanges = false;
                        }
                    }
                    else
                    {
                        foreach(SegmentInfo info in infos)
                        {
                            if (readerMap.ContainsKey(info))
                            {
                                readerMap[info].hasChanges = false;
                            }
                        }
                    }
                }
            }
            
            // used only by asserts
            public virtual bool InfoIsLive(SegmentInfo info)
            {
                lock (this)
                {
                    int idx = Enclosing_Instance.segmentInfos.IndexOf(info);
                    System.Diagnostics.Debug.Assert(idx != -1);
                    System.Diagnostics.Debug.Assert(Enclosing_Instance.segmentInfos[idx] == info);
                    return true;
                }
            }
            
            public virtual SegmentInfo MapToLive(SegmentInfo info)
            {
                lock (this)
                {
                    int idx = Enclosing_Instance.segmentInfos.IndexOf(info);
                    if (idx != - 1)
                    {
                        info = Enclosing_Instance.segmentInfos[idx];
                    }
                    return info;
                }
            }
            
            /// <summary> Release the segment reader (i.e. decRef it and close if there
            /// are no more references.
            /// </summary>
            /// <param name="sr">
            /// </param>
            /// <throws>  IOException </throws>
            public virtual void  Release(SegmentReader sr)
            {
                lock (this)
                {
                    Release(sr, false);
                }
            }

            /// <summary> Release the segment reader (i.e. decRef it and close if there
            /// are no more references.
            /// </summary>
            /// <param name="sr">
            /// </param>
            /// <param name="drop"></param>
            /// <throws>  IOException </throws>
            public virtual void  Release(SegmentReader sr, bool drop)
            {
                lock (this)
                {
                    
                    bool pooled = readerMap.ContainsKey(sr.SegmentInfo);

                    System.Diagnostics.Debug.Assert(!pooled || readerMap[sr.SegmentInfo] == sr);

                    // Drop caller's ref; for an external reader (not
                    // pooled), this decRef will close it
                    sr.DecRef();
                    
                    if (pooled && (drop || (!Enclosing_Instance.poolReaders && sr.RefCount == 1)))
                    {

                        // We invoke deleter.checkpoint below, so we must be
                        // sync'd on IW if there are changes:
                        
                        // TODO: Java 1.5 has this, .NET can't.
                        // System.Diagnostics.Debug.Assert(!sr.hasChanges || Thread.holdsLock(enclosingInstance));

                        // Discard (don't save) changes when we are dropping
                        // the reader; this is used only on the sub-readers
                        // after a successful merge.
                        sr.hasChanges &= !drop;

                        bool hasChanges = sr.hasChanges;
                        
                        // Drop our ref -- this will commit any pending
                        // changes to the dir
                        sr.Close();

                        // We are the last ref to this reader; since we're
                        // not pooling readers, we release it:
                        readerMap.Remove(sr.SegmentInfo);

                        if (hasChanges)
                        {
                            // Must checkpoint w/ deleter, because this
                            // segment reader will have created new _X_N.del
                            // file.
                            enclosingInstance.deleter.Checkpoint(enclosingInstance.segmentInfos, false);
                        }
                    }
                }
            }

            /// <summary>Remove all our references to readers, and commits
            /// any pending changes. 
            /// </summary>
            public void Dispose()
            {
                Dispose(true);
            }

            protected void Dispose(bool disposing)
            {
                if (disposing)
                {
                    // We invoke deleter.checkpoint below, so we must be
                    // sync'd on IW:
                    // TODO: assert Thread.holdsLock(IndexWriter.this);
                    // TODO: Should this class have bool _isDisposed?
                    lock (this)
                    {
                        //var toRemove = new List<SegmentInfo>();
                        foreach (var ent in readerMap)
                        {
                            SegmentReader sr = ent.Value;
                            if (sr.hasChanges)
                            {
                                System.Diagnostics.Debug.Assert(InfoIsLive(sr.SegmentInfo));
                                sr.DoCommit(null);
                                // Must checkpoint w/ deleter, because this
                                // segment reader will have created new _X_N.del
                                // file.
                                enclosingInstance.deleter.Checkpoint(enclosingInstance.segmentInfos, false);
                            }

                            //toRemove.Add(ent.Key);

                            // NOTE: it is allowed that this decRef does not
                            // actually close the SR; this can happen when a
                            // near real-time reader is kept open after the
                            // IndexWriter instance is closed
                            sr.DecRef();
                        }

                        //foreach (var key in toRemove)
                        //    readerMap.Remove(key);
                        readerMap.Clear();
                    }
                }
            }
            
            /// <summary> Commit all segment reader in the pool.</summary>
            /// <throws>  IOException </throws>
            internal virtual void  Commit()
            {
                // We invoke deleter.checkpoint below, so we must be
                // sync'd on IW:
                // TODO: assert Thread.holdsLock(IndexWriter.this);
                lock (this)
                {
                    foreach(KeyValuePair<SegmentInfo,SegmentReader> ent in readerMap)
                    {
                        SegmentReader sr = ent.Value;
                        if (sr.hasChanges)
                        {
                            System.Diagnostics.Debug.Assert(InfoIsLive(sr.SegmentInfo));
                            sr.DoCommit(null);
                            // Must checkpoint w/ deleter, because this
                            // segment reader will have created new _X_N.del
                            // file.
                            enclosingInstance.deleter.Checkpoint(enclosingInstance.segmentInfos, false);
                        }
                    }
                }
            }
            
            /// <summary> Returns a ref to a clone.  NOTE: this clone is not
            /// enrolled in the pool, so you should simply close()
            /// it when you're done (ie, do not call release()).
            /// </summary>
            public virtual SegmentReader GetReadOnlyClone(SegmentInfo info, bool doOpenStores, int termInfosIndexDivisor)
            {
                lock (this)
                {
                    SegmentReader sr = Get(info, doOpenStores, BufferedIndexInput.BUFFER_SIZE, termInfosIndexDivisor);
                    try
                    {
                        return (SegmentReader) sr.Clone(true);
                    }
                    finally
                    {
                        sr.DecRef();
                    }
                }
            }
            
            /// <summary> Obtain a SegmentReader from the readerPool.  The reader
            /// must be returned by calling <see cref="Release(SegmentReader)" />
            /// </summary>
            /// <seealso cref="Release(SegmentReader)">
            /// </seealso>
            /// <param name="info">
            /// </param>
            /// <param name="doOpenStores">
            /// </param>
            /// <throws>  IOException </throws>
            public virtual SegmentReader Get(SegmentInfo info, bool doOpenStores)
            {
                lock (this)
                {
                    return Get(info, doOpenStores, BufferedIndexInput.BUFFER_SIZE, enclosingInstance.readerTermsIndexDivisor);
                }
            }
            /// <summary> Obtain a SegmentReader from the readerPool.  The reader
            /// must be returned by calling <see cref="Release(SegmentReader)" />
            /// 
            /// </summary>
            /// <seealso cref="Release(SegmentReader)">
            /// </seealso>
            /// <param name="info">
            /// </param>
            /// <param name="doOpenStores">
            /// </param>
            /// <param name="readBufferSize">
            /// </param>
            /// <param name="termsIndexDivisor">
            /// </param>
            /// <throws>  IOException </throws>
            public virtual SegmentReader Get(SegmentInfo info, bool doOpenStores, int readBufferSize, int termsIndexDivisor)
            {
                lock (this)
                {
                    if (Enclosing_Instance.poolReaders)
                    {
                        readBufferSize = BufferedIndexInput.BUFFER_SIZE;
                    }
                    
                    SegmentReader sr = readerMap[info];
                    if (sr == null)
                    {
                        // TODO: we may want to avoid doing this while
                        // synchronized
                        // Returns a ref, which we xfer to readerMap:
                        sr = SegmentReader.Get(false, info.dir, info, readBufferSize, doOpenStores, termsIndexDivisor);
                        if (info.dir == enclosingInstance.directory)
                        {
                            // Only pool if reader is not external
                            readerMap[info]=sr;
                        }
                    }
                    else
                    {
                        if (doOpenStores)
                        {
                            sr.OpenDocStores();
                        }
                        if (termsIndexDivisor != - 1 && !sr.TermsIndexLoaded())
                        {
                            // If this reader was originally opened because we
                            // needed to merge it, we didn't load the terms
                            // index.  But now, if the caller wants the terms
                            // index (eg because it's doing deletes, or an NRT
                            // reader is being opened) we ask the reader to
                            // load its terms index.
                            sr.LoadTermsIndex(termsIndexDivisor);
                        }
                    }
                    
                    // Return a ref to our caller
                    if (info.dir == enclosingInstance.directory)
                    {
                        // Only incRef if we pooled (reader is not external)
                        sr.IncRef();
                    }
                    return sr;
                }
            }
            
            // Returns a ref
            public virtual SegmentReader GetIfExists(SegmentInfo info)
            {
                lock (this)
                {
                    SegmentReader sr = readerMap[info];
                    if (sr != null)
                    {
                        sr.IncRef();
                    }
                    return sr;
                }
            }
        }
        
        /// <summary> Obtain the number of deleted docs for a pooled reader.
        /// If the reader isn't being pooled, the segmentInfo's 
        /// delCount is returned.
        /// </summary>
        public virtual int NumDeletedDocs(SegmentInfo info)
        {
            SegmentReader reader = readerPool.GetIfExists(info);
            try
            {
                if (reader != null)
                {
                    return reader.NumDeletedDocs;
                }
                else
                {
                    return info.GetDelCount();
                }
            }
            finally
            {
                if (reader != null)
                {
                    readerPool.Release(reader);
                }
            }
        }
        
        internal virtual void  AcquireWrite()
        {
            lock (this)
            {
                System.Diagnostics.Debug.Assert(writeThread != ThreadClass.Current());
                while (writeThread != null || readCount > 0)
                    DoWait();
                
                // We could have been closed while we were waiting:
                EnsureOpen();
                
                writeThread = ThreadClass.Current();
            }
        }
        
        internal virtual void  ReleaseWrite()
        {
            lock (this)
            {
                System.Diagnostics.Debug.Assert(ThreadClass.Current() == writeThread);
                writeThread = null;
                System.Threading.Monitor.PulseAll(this);
            }
        }
        
        internal virtual void  AcquireRead()
        {
            lock (this)
            {
                ThreadClass current = ThreadClass.Current();
                while (writeThread != null && writeThread != current)
                    DoWait();
                
                readCount++;
            }
        }
        
        // Allows one readLock to upgrade to a writeLock even if
        // there are other readLocks as long as all other
        // readLocks are also blocked in this method:
        internal virtual void  UpgradeReadToWrite()
        {
            lock (this)
            {
                System.Diagnostics.Debug.Assert(readCount > 0);
                upgradeCount++;
                while (readCount > upgradeCount || writeThread != null)
                {
                    DoWait();
                }
                
                writeThread = ThreadClass.Current();
                readCount--;
                upgradeCount--;
            }
        }
        
        internal virtual void  ReleaseRead()
        {
            lock (this)
            {
                readCount--;
                System.Diagnostics.Debug.Assert(readCount >= 0);
                System.Threading.Monitor.PulseAll(this);
            }
        }
        
        internal bool IsOpen(bool includePendingClose)
        {
            lock (this)
            {
                return !(closed || (includePendingClose && closing));
            }
        }
        
        /// <summary> Used internally to throw an <see cref="AlreadyClosedException" />
        /// if this IndexWriter has been
        /// closed.
        /// </summary>
        /// <throws>  AlreadyClosedException if this IndexWriter is </throws>
        protected internal void  EnsureOpen(bool includePendingClose)
        {
            lock (this)
            {
                if (!IsOpen(includePendingClose))
                {
                    throw new AlreadyClosedException("this IndexWriter is closed");
                }
            }
        }
        
        protected internal void  EnsureOpen()
        {
            lock (this)
            {
                EnsureOpen(true);
            }
        }
        
        /// <summary> Prints a message to the infoStream (if non-null),
        /// prefixed with the identifying information for this
        /// writer and the thread that's calling it.
        /// </summary>
        public virtual void  Message(System.String message)
        {
            if (infoStream != null)
                infoStream.WriteLine("IW " + messageID + " [" + DateTime.Now.ToString() + "; " + ThreadClass.Current().Name + "]: " + message);
        }
        
        private void  SetMessageID(System.IO.StreamWriter infoStream)
        {
            lock (this)
            {
                if (infoStream != null && messageID == - 1)
                {
                    lock (MESSAGE_ID_LOCK)
                    {
                        messageID = MESSAGE_ID++;
                    }
                }
                this.infoStream = infoStream;
            }
        }

        /// <summary> Casts current mergePolicy to LogMergePolicy, and throws
        /// an exception if the mergePolicy is not a LogMergePolicy.
        /// </summary>
        private LogMergePolicy LogMergePolicy
        {
            get
            {
                if (mergePolicy is LogMergePolicy)
                    return (LogMergePolicy) mergePolicy;

                throw new System.ArgumentException(
                    "this method can only be called when the merge policy is the default LogMergePolicy");
            }
        }

        /// <summary><p/>Gets or sets the current setting of whether newly flushed
        /// segments will use the compound file format.  Note that
        /// this just returns the value previously set with
        /// setUseCompoundFile(boolean), or the default value
        /// (true).  You cannot use this to query the status of
        /// previously flushed segments.<p/>
        /// 
        /// <p/>Note that this method is a convenience method: it
        /// just calls mergePolicy.getUseCompoundFile as long as
        /// mergePolicy is an instance of <see cref="LogMergePolicy" />.
        /// Otherwise an IllegalArgumentException is thrown.<p/>
        /// 
        /// </summary>
        public virtual bool UseCompoundFile
        {
            get { return LogMergePolicy.GetUseCompoundFile(); }
            set
            {
                LogMergePolicy.SetUseCompoundFile(value);
                LogMergePolicy.SetUseCompoundDocStore(value);
            }
        }

        /// <summary>Expert: Set the Similarity implementation used by this IndexWriter.
        /// </summary>
        public virtual void  SetSimilarity(Similarity similarity)
        {
            EnsureOpen();
            this.similarity = similarity;
            docWriter.SetSimilarity(similarity);
        }

        /// <summary>Expert: Return the Similarity implementation used by this IndexWriter.
        /// 
        /// <p/>This defaults to the current value of <see cref="Search.Similarity.Default" />.
        /// </summary>
        public virtual Similarity Similarity
        {
            get
            {
                EnsureOpen();
                return this.similarity;
            }
        }


        /// <summary>Expert: Gets or sets the interval between indexed terms.  Large values cause less
        /// memory to be used by IndexReader, but slow random-access to terms.  Small
        /// values cause more memory to be used by an IndexReader, and speed
        /// random-access to terms.
        /// 
        /// This parameter determines the amount of computation required per query
        /// term, regardless of the number of documents that contain that term.  In
        /// particular, it is the maximum number of other terms that must be
        /// scanned before a term is located and its frequency and position information
        /// may be processed.  In a large index with user-entered query terms, query
        /// processing time is likely to be dominated not by term lookup but rather
        /// by the processing of frequency and positional data.  In a small index
        /// or when many uncommon query terms are generated (e.g., by wildcard
        /// queries) term lookup may become a dominant cost.
        /// 
        /// In particular, <c>numUniqueTerms/interval</c> terms are read into
        /// memory by an IndexReader, and, on average, <c>interval/2</c> terms
        /// must be scanned for each random term access.
        /// 
        /// </summary>
        /// <seealso cref="DEFAULT_TERM_INDEX_INTERVAL">
        /// </seealso>
        public virtual int TermIndexInterval
        {
            get
            {
                // We pass false because this method is called by SegmentMerger while we are in the process of closing
                EnsureOpen(false);
                return termIndexInterval;
            }
            set
            {
                EnsureOpen();
                this.termIndexInterval = value;
            }
        }

        /// <summary> Constructs an IndexWriter for the index in <c>d</c>.
        /// Text will be analyzed with <c>a</c>.  If <c>create</c>
        /// is true, then a new, empty index will be created in
        /// <c>d</c>, replacing the index already there, if any.
        /// 
        /// </summary>
        /// <param name="d">the index directory
        /// </param>
        /// <param name="a">the analyzer to use
        /// </param>
        /// <param name="create"><c>true</c> to create the index or overwrite
        /// the existing one; <c>false</c> to append to the existing
        /// index
        /// </param>
        /// <param name="mfl">Maximum field length in number of terms/tokens: LIMITED, UNLIMITED, or user-specified
        /// via the MaxFieldLength constructor.
        /// </param>
        /// <throws>  CorruptIndexException if the index is corrupt </throws>
        /// <throws>  LockObtainFailedException if another writer </throws>
        /// <summary>  has this index open (<c>write.lock</c> could not
        /// be obtained)
        /// </summary>
        /// <throws>  IOException if the directory cannot be read/written to, or </throws>
        /// <summary>  if it does not exist and <c>create</c> is
        /// <c>false</c> or if there is any other low-level
        /// IO error
        /// </summary>
        public IndexWriter(Directory d, Analyzer a, bool create, MaxFieldLength mfl)
        {
            InitBlock();
            Init(d, a, create, null, mfl.Limit, null, null);
        }
        
        /// <summary> Constructs an IndexWriter for the index in
        /// <c>d</c>, first creating it if it does not
        /// already exist.  
        /// 
        /// </summary>
        /// <param name="d">the index directory
        /// </param>
        /// <param name="a">the analyzer to use
        /// </param>
        /// <param name="mfl">Maximum field length in number of terms/tokens: LIMITED, UNLIMITED, or user-specified
        /// via the MaxFieldLength constructor.
        /// </param>
        /// <throws>  CorruptIndexException if the index is corrupt </throws>
        /// <throws>  LockObtainFailedException if another writer </throws>
        /// <summary>  has this index open (<c>write.lock</c> could not
        /// be obtained)
        /// </summary>
        /// <throws>  IOException if the directory cannot be </throws>
        /// <summary>  read/written to or if there is any other low-level
        /// IO error
        /// </summary>
        public IndexWriter(Directory d, Analyzer a, MaxFieldLength mfl)
        {
            InitBlock();
            Init(d, a, null, mfl.Limit, null, null);
        }
        
        /// <summary> Expert: constructs an IndexWriter with a custom <see cref="IndexDeletionPolicy" />
        ///, for the index in <c>d</c>,
        /// first creating it if it does not already exist.  Text
        /// will be analyzed with <c>a</c>.
        /// 
        /// </summary>
        /// <param name="d">the index directory
        /// </param>
        /// <param name="a">the analyzer to use
        /// </param>
        /// <param name="deletionPolicy">see <a href="#deletionPolicy">above</a>
        /// </param>
        /// <param name="mfl">whether or not to limit field lengths
        /// </param>
        /// <throws>  CorruptIndexException if the index is corrupt </throws>
        /// <throws>  LockObtainFailedException if another writer </throws>
        /// <summary>  has this index open (<c>write.lock</c> could not
        /// be obtained)
        /// </summary>
        /// <throws>  IOException if the directory cannot be </throws>
        /// <summary>  read/written to or if there is any other low-level
        /// IO error
        /// </summary>
        public IndexWriter(Directory d, Analyzer a, IndexDeletionPolicy deletionPolicy, MaxFieldLength mfl)
        {
            InitBlock();
            Init(d, a, deletionPolicy, mfl.Limit, null, null);
        }
        
        /// <summary> Expert: constructs an IndexWriter with a custom <see cref="IndexDeletionPolicy" />
        ///, for the index in <c>d</c>.
        /// Text will be analyzed with <c>a</c>.  If
        /// <c>create</c> is true, then a new, empty index
        /// will be created in <c>d</c>, replacing the index
        /// already there, if any.
        /// 
        /// </summary>
        /// <param name="d">the index directory
        /// </param>
        /// <param name="a">the analyzer to use
        /// </param>
        /// <param name="create"><c>true</c> to create the index or overwrite
        /// the existing one; <c>false</c> to append to the existing
        /// index
        /// </param>
        /// <param name="deletionPolicy">see <a href="#deletionPolicy">above</a>
        /// </param>
        /// <param name="mfl"><see cref="Lucene.Net.Index.IndexWriter.MaxFieldLength" />, whether or not to limit field lengths.  Value is in number of terms/tokens
        /// </param>
        /// <throws>  CorruptIndexException if the index is corrupt </throws>
        /// <throws>  LockObtainFailedException if another writer </throws>
        /// <summary>  has this index open (<c>write.lock</c> could not
        /// be obtained)
        /// </summary>
        /// <throws>  IOException if the directory cannot be read/written to, or </throws>
        /// <summary>  if it does not exist and <c>create</c> is
        /// <c>false</c> or if there is any other low-level
        /// IO error
        /// </summary>
        public IndexWriter(Directory d, Analyzer a, bool create, IndexDeletionPolicy deletionPolicy, MaxFieldLength mfl)
        {
            InitBlock();
            Init(d, a, create, deletionPolicy, mfl.Limit, null, null);
        }
        
        /// <summary> Expert: constructs an IndexWriter with a custom <see cref="IndexDeletionPolicy" />
        /// and <see cref="IndexingChain" />, 
        /// for the index in <c>d</c>.
        /// Text will be analyzed with <c>a</c>.  If
        /// <c>create</c> is true, then a new, empty index
        /// will be created in <c>d</c>, replacing the index
        /// already there, if any.
        /// 
        /// </summary>
        /// <param name="d">the index directory
        /// </param>
        /// <param name="a">the analyzer to use
        /// </param>
        /// <param name="create"><c>true</c> to create the index or overwrite
        /// the existing one; <c>false</c> to append to the existing
        /// index
        /// </param>
        /// <param name="deletionPolicy">see <a href="#deletionPolicy">above</a>
        /// </param>
        /// <param name="mfl">whether or not to limit field lengths, value is in number of terms/tokens.  See <see cref="Lucene.Net.Index.IndexWriter.MaxFieldLength" />.
        /// </param>
        /// <param name="indexingChain">the <see cref="DocConsumer" /> chain to be used to 
        /// process documents
        /// </param>
        /// <param name="commit">which commit to open
        /// </param>
        /// <throws>  CorruptIndexException if the index is corrupt </throws>
        /// <throws>  LockObtainFailedException if another writer </throws>
        /// <summary>  has this index open (<c>write.lock</c> could not
        /// be obtained)
        /// </summary>
        /// <throws>  IOException if the directory cannot be read/written to, or </throws>
        /// <summary>  if it does not exist and <c>create</c> is
        /// <c>false</c> or if there is any other low-level
        /// IO error
        /// </summary>
        internal IndexWriter(Directory d, Analyzer a, bool create, IndexDeletionPolicy deletionPolicy, MaxFieldLength mfl, IndexingChain indexingChain, IndexCommit commit)
        {
            InitBlock();
            Init(d, a, create, deletionPolicy, mfl.Limit, indexingChain, commit);
        }
        
        /// <summary> Expert: constructs an IndexWriter on specific commit
        /// point, with a custom <see cref="IndexDeletionPolicy" />, for
        /// the index in <c>d</c>.  Text will be analyzed
        /// with <c>a</c>.
        /// 
        /// <p/> This is only meaningful if you've used a <see cref="IndexDeletionPolicy" />
        /// in that past that keeps more than
        /// just the last commit.
        /// 
        /// <p/>This operation is similar to <see cref="Rollback()" />,
        /// except that method can only rollback what's been done
        /// with the current instance of IndexWriter since its last
        /// commit, whereas this method can rollback to an
        /// arbitrary commit point from the past, assuming the
        /// <see cref="IndexDeletionPolicy" /> has preserved past
        /// commits.
        /// 
        /// </summary>
        /// <param name="d">the index directory
        /// </param>
        /// <param name="a">the analyzer to use
        /// </param>
        /// <param name="deletionPolicy">see <a href="#deletionPolicy">above</a>
        /// </param>
        /// <param name="mfl">whether or not to limit field lengths, value is in number of terms/tokens.  See <see cref="Lucene.Net.Index.IndexWriter.MaxFieldLength" />.
        /// </param>
        /// <param name="commit">which commit to open
        /// </param>
        /// <throws>  CorruptIndexException if the index is corrupt </throws>
        /// <throws>  LockObtainFailedException if another writer </throws>
        /// <summary>  has this index open (<c>write.lock</c> could not
        /// be obtained)
        /// </summary>
        /// <throws>  IOException if the directory cannot be read/written to, or </throws>
        /// <summary>  if it does not exist and <c>create</c> is
        /// <c>false</c> or if there is any other low-level
        /// IO error
        /// </summary>
        public IndexWriter(Directory d, Analyzer a, IndexDeletionPolicy deletionPolicy, MaxFieldLength mfl, IndexCommit commit)
        {
            InitBlock();
            Init(d, a, false, deletionPolicy, mfl.Limit, null, commit);
        }
        
        private void  Init(Directory d, Analyzer a, IndexDeletionPolicy deletionPolicy, int maxFieldLength, IndexingChain indexingChain, IndexCommit commit)
        {
            if (IndexReader.IndexExists(d))
            {
                Init(d, a, false, deletionPolicy, maxFieldLength, indexingChain, commit);
            }
            else
            {
                Init(d, a, true, deletionPolicy, maxFieldLength, indexingChain, commit);
            }
        }
        
        private void  Init(Directory d, Analyzer a, bool create, IndexDeletionPolicy deletionPolicy, int maxFieldLength, IndexingChain indexingChain, IndexCommit commit)
        {
            directory = d;
            analyzer = a;
            SetMessageID(defaultInfoStream);
            this.maxFieldLength = maxFieldLength;
            
            if (indexingChain == null)
                indexingChain = DocumentsWriter.DefaultIndexingChain;
            
            if (create)
            {
                // Clear the write lock in case it's leftover:
                directory.ClearLock(WRITE_LOCK_NAME);
            }
            
            Lock writeLock = directory.MakeLock(WRITE_LOCK_NAME);
            if (!writeLock.Obtain(writeLockTimeout))
            // obtain write lock
            {
                throw new LockObtainFailedException("Index locked for write: " + writeLock);
            }
            this.writeLock = writeLock; // save it

            bool success = false;
            try
            {
                if (create)
                {
                    // Try to read first.  This is to allow create
                    // against an index that's currently open for
                    // searching.  In this case we write the next
                    // segments_N file with no segments:
                    bool doCommit;
                    try
                    {
                        segmentInfos.Read(directory);
                        segmentInfos.Clear();
                        doCommit = false;
                    }
                    catch (System.IO.IOException)
                    {
                        // Likely this means it's a fresh directory
                        doCommit = true;
                    }
                    
                    if (doCommit)
                    {
                        // Only commit if there is no segments file 
                        // in this dir already.
                        segmentInfos.Commit(directory);
                        synced.UnionWith(segmentInfos.Files(directory, true));
                    }
                    else
                    {
                        // Record that we have a change (zero out all
                        // segments) pending:
                        changeCount++;
                    }
                }
                else
                {
                    segmentInfos.Read(directory);
                    
                    if (commit != null)
                    {
                        // Swap out all segments, but, keep metadata in
                        // SegmentInfos, like version & generation, to
                        // preserve write-once.  This is important if
                        // readers are open against the future commit
                        // points.
                        if (commit.Directory != directory)
                            throw new System.ArgumentException("IndexCommit's directory doesn't match my directory");
                        SegmentInfos oldInfos = new SegmentInfos();
                        oldInfos.Read(directory, commit.SegmentsFileName);
                        segmentInfos.Replace(oldInfos);
                        changeCount++;
                        if (infoStream != null)
                            Message("init: loaded commit \"" + commit.SegmentsFileName + "\"");
                    }
                    
                    // We assume that this segments_N was previously
                    // properly sync'd:
                    synced.UnionWith(segmentInfos.Files(directory, true));
                }
                
                SetRollbackSegmentInfos(segmentInfos);
                
                docWriter = new DocumentsWriter(directory, this, indexingChain);
                docWriter.SetInfoStream(infoStream);
                docWriter.SetMaxFieldLength(maxFieldLength);
                
                // Default deleter (for backwards compatibility) is
                // KeepOnlyLastCommitDeleter:
                deleter = new IndexFileDeleter(directory, deletionPolicy == null?new KeepOnlyLastCommitDeletionPolicy():deletionPolicy, segmentInfos, infoStream, docWriter, synced);
                
                if (deleter.startingCommitDeleted)
                // Deletion policy deleted the "head" commit point.
                // We have to mark ourself as changed so that if we
                // are closed w/o any further changes we write a new
                // segments_N file.
                    changeCount++;
                
                PushMaxBufferedDocs();
                
                if (infoStream != null)
                {
                    Message("init: create=" + create);
                    MessageState();
                }

                success = true;
            }
            finally
            {
                if (!success)
                {
                    if (infoStream != null)
                    {
                        Message("init: hit exception on init; releasing write lock");
                    }
                    try
                    {
                        writeLock.Release();
                    }
                    catch (Exception)
                    {
                        // don't mask the original exception
                    }
                    writeLock = null;
                }
            }
        }
        
        private void  SetRollbackSegmentInfos(SegmentInfos infos)
        {
            lock (this)
            {
                rollbackSegmentInfos = (SegmentInfos) infos.Clone();
                System.Diagnostics.Debug.Assert(!rollbackSegmentInfos.HasExternalSegments(directory));
                rollbackSegments = new HashMap<SegmentInfo, int?>();
                int size = rollbackSegmentInfos.Count;
                for (int i = 0; i < size; i++)
                    rollbackSegments[rollbackSegmentInfos.Info(i)] = i;
            }
        }
        
        /// <summary> Expert: set the merge policy used by this writer.</summary>
        public virtual void  SetMergePolicy(MergePolicy mp)
        {
            EnsureOpen();
            if (mp == null)
                throw new System.NullReferenceException("MergePolicy must be non-null");
            
            if (mergePolicy != mp)
                mergePolicy.Close();
            mergePolicy = mp;
            PushMaxBufferedDocs();
            if (infoStream != null)
            {
                Message("setMergePolicy " + mp);
            }
        }

        /// <summary> Expert: returns the current MergePolicy in use by this writer.</summary>
        /// <seealso cref="SetMergePolicy">
        /// </seealso>
        public virtual MergePolicy MergePolicy
        {
            get
            {
                EnsureOpen();
                return mergePolicy;
            }
        }

        /// <summary> Expert: set the merge scheduler used by this writer.</summary>
        public virtual void  SetMergeScheduler(MergeScheduler mergeScheduler)
        {
            lock (this)
            {
                EnsureOpen();
                if (mergeScheduler == null)
                    throw new System.NullReferenceException("MergeScheduler must be non-null");
                
                if (this.mergeScheduler != mergeScheduler)
                {
                    FinishMerges(true);
                    this.mergeScheduler.Close();
                }
                this.mergeScheduler = mergeScheduler;
                if (infoStream != null)
                {
                    Message("setMergeScheduler " + mergeScheduler);
                }
            }
        }

        /// <summary> Expert: returns the current MergePolicy in use by this
        /// writer.
        /// </summary>
        /// <seealso cref="SetMergePolicy">
        /// </seealso>
        public virtual MergeScheduler MergeScheduler
        {
            get
            {
                EnsureOpen();
                return mergeScheduler;
            }
        }

        /// <summary> <p/>Gets or sets the largest segment (measured by document
        /// count) that may be merged with other segments.
        /// <p/> 
        /// Small values (e.g., less than 10,000) are best for
        /// interactive indexing, as this limits the length of
        /// pauses while indexing to a few seconds.  Larger values
        /// are best for batched indexing and speedier
        /// searches.
        /// <p/>
        /// The default value is <see cref="int.MaxValue" />.
        /// <p/>
        /// Note that this method is a convenience method: it
        /// just calls mergePolicy.getMaxMergeDocs as long as
        /// mergePolicy is an instance of <see cref="LogMergePolicy" />.
        /// Otherwise an IllegalArgumentException is thrown.<p/>
        /// 
        /// The default merge policy (<see cref="LogByteSizeMergePolicy" />)
        /// also allows you to set this
        /// limit by net size (in MB) of the segment, using 
        /// <see cref="LogByteSizeMergePolicy.MaxMergeMB" />.<p/>
        /// </summary>
        /// <seealso cref="MaxMergeDocs">
        /// </seealso>
        public virtual int MaxMergeDocs
        {
            get { return LogMergePolicy.MaxMergeDocs; }
            set { LogMergePolicy.MaxMergeDocs = value; }
        }

        /// <summary> The maximum number of terms that will be indexed for a single field in a
        /// document.  This limits the amount of memory required for indexing, so that
        /// collections with very large files will not crash the indexing process by
        /// running out of memory.  This setting refers to the number of running terms,
        /// not to the number of different terms.<p/>
        /// <strong>Note:</strong> this silently truncates large documents, excluding from the
        /// index all terms that occur further in the document.  If you know your source
        /// documents are large, be sure to set this value high enough to accomodate
        /// the expected size.  If you set it to Integer.MAX_VALUE, then the only limit
        /// is your memory, but you should anticipate an OutOfMemoryError.<p/>
        /// By default, no more than <see cref="DEFAULT_MAX_FIELD_LENGTH" /> terms
        /// will be indexed for a field.
        /// </summary>
        public virtual void  SetMaxFieldLength(int maxFieldLength)
        {
            EnsureOpen();
            this.maxFieldLength = maxFieldLength;
            docWriter.SetMaxFieldLength(maxFieldLength);
            if (infoStream != null)
                Message("setMaxFieldLength " + maxFieldLength);
        }
        
        /// <summary> Returns the maximum number of terms that will be
        /// indexed for a single field in a document.
        /// </summary>
        /// <seealso cref="SetMaxFieldLength">
        /// </seealso>
        [System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Design", "CA1024:UsePropertiesWhereAppropriate")]
        public virtual int GetMaxFieldLength()
        {
            EnsureOpen();
            return maxFieldLength;
        }

        /// Gets or sets the termsIndexDivisor passed to any readers that
        /// IndexWriter opens, for example when applying deletes
        /// or creating a near-real-time reader in 
        /// <see cref="GetReader()"/>.  Default value is 
        /// <see cref="IndexReader.DEFAULT_TERMS_INDEX_DIVISOR"/>.
        public int ReaderTermsIndexDivisor
        {
            get
            {
                EnsureOpen();
                return readerTermsIndexDivisor;
            }
            set
            {
                EnsureOpen();
                if (value <= 0)
                {
                    throw new ArgumentException("divisor must be >= 1 (got " + value + ")");
                }
                readerTermsIndexDivisor = value;
                if (infoStream != null)
                {
                    Message("setReaderTermsIndexDivisor " + readerTermsIndexDivisor);
                }
            }
        }

        /// <summary>Determines the minimal number of documents required
        /// before the buffered in-memory documents are flushed as
        /// a new Segment.  Large values generally gives faster
        /// indexing.
        /// 
        /// <p/>When this is set, the writer will flush every
        /// maxBufferedDocs added documents.  Pass in <see cref="DISABLE_AUTO_FLUSH" />
        /// to prevent triggering a flush due
        /// to number of buffered documents.  Note that if flushing
        /// by RAM usage is also enabled, then the flush will be
        /// triggered by whichever comes first.<p/>
        /// 
        /// <p/>Disabled by default (writer flushes by RAM usage).<p/>
        /// 
        /// </summary>
        /// <throws>  IllegalArgumentException if maxBufferedDocs is </throws>
        /// <summary> enabled but smaller than 2, or it disables maxBufferedDocs
        /// when ramBufferSize is already disabled
        /// </summary>
        /// <seealso cref="SetRAMBufferSizeMB">
        /// </seealso>
        public virtual void  SetMaxBufferedDocs(int maxBufferedDocs)
        {
            EnsureOpen();
            if (maxBufferedDocs != DISABLE_AUTO_FLUSH && maxBufferedDocs < 2)
                throw new ArgumentException("maxBufferedDocs must at least be 2 when enabled");

            if (maxBufferedDocs == DISABLE_AUTO_FLUSH && (int)GetRAMBufferSizeMB() == DISABLE_AUTO_FLUSH)
                throw new ArgumentException("at least one of ramBufferSize and maxBufferedDocs must be enabled");

            docWriter.MaxBufferedDocs = maxBufferedDocs;
            PushMaxBufferedDocs();
            if (infoStream != null)
                Message("setMaxBufferedDocs " + maxBufferedDocs);
        }
        
        /// <summary> If we are flushing by doc count (not by RAM usage), and
        /// using LogDocMergePolicy then push maxBufferedDocs down
        /// as its minMergeDocs, to keep backwards compatibility.
        /// </summary>
        private void  PushMaxBufferedDocs()
        {
            if (docWriter.MaxBufferedDocs != DISABLE_AUTO_FLUSH)
            {
                MergePolicy mp = mergePolicy;
                if (mp is LogDocMergePolicy)
                {
                    LogDocMergePolicy lmp = (LogDocMergePolicy) mp;
                    int maxBufferedDocs = docWriter.MaxBufferedDocs;
                    if (lmp.MinMergeDocs != maxBufferedDocs)
                    {
                        if (infoStream != null)
                            Message("now push maxBufferedDocs " + maxBufferedDocs + " to LogDocMergePolicy");
                        lmp.MinMergeDocs = maxBufferedDocs;
                    }
                }
            }
        }
        
        /// <summary> Returns the number of buffered added documents that will
        /// trigger a flush if enabled.
        /// </summary>
        /// <seealso cref="SetMaxBufferedDocs">
        /// </seealso>
        [System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Design", "CA1024:UsePropertiesWhereAppropriate")]
        public virtual int GetMaxBufferedDocs()
        {
            EnsureOpen();
            return docWriter.MaxBufferedDocs;
        }
        
        /// <summary>Determines the amount of RAM that may be used for
        /// buffering added documents and deletions before they are
        /// flushed to the Directory.  Generally for faster
        /// indexing performance it's best to flush by RAM usage
        /// instead of document count and use as large a RAM buffer
        /// as you can.
        /// 
        /// <p/>When this is set, the writer will flush whenever
        /// buffered documents and deletions use this much RAM.
        /// Pass in <see cref="DISABLE_AUTO_FLUSH" /> to prevent
        /// triggering a flush due to RAM usage.  Note that if
        /// flushing by document count is also enabled, then the
        /// flush will be triggered by whichever comes first.<p/>
        /// 
        /// <p/> <b>NOTE</b>: the account of RAM usage for pending
        /// deletions is only approximate.  Specifically, if you
        /// delete by Query, Lucene currently has no way to measure
        /// the RAM usage if individual Queries so the accounting
        /// will under-estimate and you should compensate by either
        /// calling commit() periodically yourself, or by using
        /// <see cref="SetMaxBufferedDeleteTerms" /> to flush by count
        /// instead of RAM usage (each buffered delete Query counts
        /// as one).
        /// 
        /// <p/>
        /// <b>NOTE</b>: because IndexWriter uses <c>int</c>s when managing its
        /// internal storage, the absolute maximum value for this setting is somewhat
        /// less than 2048 MB. The precise limit depends on various factors, such as
        /// how large your documents are, how many fields have norms, etc., so it's
        /// best to set this value comfortably under 2048.
        /// <p/>
        /// 
        /// <p/> The default value is <see cref="DEFAULT_RAM_BUFFER_SIZE_MB" />.<p/>
        /// 
        /// </summary>
        /// <throws>  IllegalArgumentException if ramBufferSize is </throws>
        /// <summary> enabled but non-positive, or it disables ramBufferSize
        /// when maxBufferedDocs is already disabled
        /// </summary>
        public virtual void  SetRAMBufferSizeMB(double mb)
        {
            if (mb > 2048.0)
            {
                throw new System.ArgumentException("ramBufferSize " + mb + " is too large; should be comfortably less than 2048");
            }
            if (mb != DISABLE_AUTO_FLUSH && mb <= 0.0)
                throw new System.ArgumentException("ramBufferSize should be > 0.0 MB when enabled");
            if (mb == DISABLE_AUTO_FLUSH && GetMaxBufferedDocs() == DISABLE_AUTO_FLUSH)
                throw new System.ArgumentException("at least one of ramBufferSize and maxBufferedDocs must be enabled");
            docWriter.SetRAMBufferSizeMB(mb);
            if (infoStream != null)
                Message("setRAMBufferSizeMB " + mb);
        }
        
        /// <summary> Returns the value set by <see cref="SetRAMBufferSizeMB" /> if enabled.</summary>
        [System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Design", "CA1024:UsePropertiesWhereAppropriate")]
        public virtual double GetRAMBufferSizeMB()
        {
            return docWriter.GetRAMBufferSizeMB();
        }
        
        /// <summary> <p/>Determines the minimal number of delete terms required before the buffered
        /// in-memory delete terms are applied and flushed. If there are documents
        /// buffered in memory at the time, they are merged and a new segment is
        /// created.<p/>
        /// <p/>Disabled by default (writer flushes by RAM usage).<p/>
        /// 
        /// </summary>
        /// <throws>  IllegalArgumentException if maxBufferedDeleteTerms </throws>
        /// <summary> is enabled but smaller than 1
        /// </summary>
        /// <seealso cref="SetRAMBufferSizeMB">
        /// </seealso>
        public virtual void  SetMaxBufferedDeleteTerms(int maxBufferedDeleteTerms)
        {
            EnsureOpen();
            if (maxBufferedDeleteTerms != DISABLE_AUTO_FLUSH && maxBufferedDeleteTerms < 1)
                throw new System.ArgumentException("maxBufferedDeleteTerms must at least be 1 when enabled");
            docWriter.MaxBufferedDeleteTerms = maxBufferedDeleteTerms;
            if (infoStream != null)
                Message("setMaxBufferedDeleteTerms " + maxBufferedDeleteTerms);
        }
        
        /// <summary> Returns the number of buffered deleted terms that will
        /// trigger a flush if enabled.
        /// </summary>
        /// <seealso cref="SetMaxBufferedDeleteTerms">
        /// </seealso>
        [System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Design", "CA1024:UsePropertiesWhereAppropriate")]
        public virtual int GetMaxBufferedDeleteTerms()
        {
            EnsureOpen();
            return docWriter.MaxBufferedDeleteTerms;
        }

        /// <summary>Gets or sets the number of segments that are merged at
        /// once and also controls the total number of segments
        /// allowed to accumulate in the index.
        /// <p/>Determines how often segment indices are merged by addDocument().  With
        /// smaller values, less RAM is used while indexing, and searches on
        /// unoptimized indices are faster, but indexing speed is slower.  With larger
        /// values, more RAM is used during indexing, and while searches on unoptimized
        /// indices are slower, indexing is faster.  Thus larger values (> 10) are best
        /// for batch index creation, and smaller values (&lt; 10) for indices that are
        /// interactively maintained.
        /// 
        /// <p/>Note that this method is a convenience method: it
        /// just calls mergePolicy.setMergeFactor as long as
        /// mergePolicy is an instance of <see cref="LogMergePolicy" />.
        /// Otherwise an IllegalArgumentException is thrown.<p/>
        /// 
        /// <p/>This must never be less than 2.  The default value is 10.
        /// </summary>
        public virtual int MergeFactor
        {
            set { LogMergePolicy.MergeFactor = value; }
            get { return LogMergePolicy.MergeFactor; }
        }

        /// <summary>Gets or sets the default info stream.
        /// If non-null, this will be the default infoStream used
        /// by a newly instantiated IndexWriter.
        /// </summary>
        /// <seealso cref="SetInfoStream">
        /// </seealso>
        public static StreamWriter DefaultInfoStream
        {
            set { IndexWriter.defaultInfoStream = value; }
            get { return IndexWriter.defaultInfoStream; }
        }

        /// <summary>If non-null, information about merges, deletes and a
        /// message when maxFieldLength is reached will be printed
        /// to this.
        /// </summary>
        public virtual void  SetInfoStream(System.IO.StreamWriter infoStream)
        {
            EnsureOpen();
            SetMessageID(infoStream);
            docWriter.SetInfoStream(infoStream);
            deleter.SetInfoStream(infoStream);
            if (infoStream != null)
                MessageState();
        }
        
        private void  MessageState()
        {
            Message("setInfoStream: dir=" + directory + 
                    " mergePolicy=" + mergePolicy + 
                    " mergeScheduler=" + mergeScheduler +
                    " ramBufferSizeMB=" + docWriter.GetRAMBufferSizeMB() + 
                    " maxBufferedDocs=" +  docWriter.MaxBufferedDocs +
                    " maxBuffereDeleteTerms=" + docWriter.MaxBufferedDeleteTerms +
                    " maxFieldLength=" + maxFieldLength + 
                    " index=" + SegString());
        }

        /// <summary> Returns the current infoStream in use by this writer.</summary>
        /// <seealso cref="SetInfoStream">
        /// </seealso>
        public virtual StreamWriter InfoStream
        {
            get
            {
                EnsureOpen();
                return infoStream;
            }
        }

        /// <summary>Returns true if verbosing is enabled (i.e., infoStream != null). </summary>
        public virtual bool Verbose
        {
            get { return infoStream != null; }
        }

        /// <summary>Gets or sets allowed timeout when acquiring the write lock.</summary>
        public virtual long WriteLockTimeout
        {
            get
            {
                EnsureOpen();
                return writeLockTimeout;
            }
            set
            {
                EnsureOpen();
                this.writeLockTimeout = value;
            }
        }

        /// <summary> Gets or sets the default (for any instance of IndexWriter) maximum time to wait for a write lock (in
        /// milliseconds).
        /// </summary>
        public static long DefaultWriteLockTimeout
        {
            set { IndexWriter.WRITE_LOCK_TIMEOUT = value; }
            get { return IndexWriter.WRITE_LOCK_TIMEOUT; }
        }

        /// <summary> Commits all changes to an index and closes all
        /// associated files.  Note that this may be a costly
        /// operation, so, try to re-use a single writer instead of
        /// closing and opening a new one.  See <see cref="Commit()" /> for
        /// caveats about write caching done by some IO devices.
        /// 
        /// <p/> If an Exception is hit during close, eg due to disk
        /// full or some other reason, then both the on-disk index
        /// and the internal state of the IndexWriter instance will
        /// be consistent.  However, the close will not be complete
        /// even though part of it (flushing buffered documents)
        /// may have succeeded, so the write lock will still be
        /// held.<p/>
        /// 
        /// <p/> If you can correct the underlying cause (eg free up
        /// some disk space) then you can call close() again.
        /// Failing that, if you want to force the write lock to be
        /// released (dangerous, because you may then lose buffered
        /// docs in the IndexWriter instance) then you can do
        /// something like this:<p/>
        /// 
        /// <code>
        /// try {
        ///     writer.close();
        /// } finally {
        ///     if (IndexWriter.isLocked(directory)) {
        ///         IndexWriter.unlock(directory);
        ///     }
        /// }
        /// </code>
        /// 
        /// after which, you must be certain not to use the writer
        /// instance anymore.<p/>
        /// 
        /// <p/><b>NOTE</b>: if this method hits an OutOfMemoryError
        /// you should immediately close the writer, again.  See <a
        /// href="#OOME">above</a> for details.<p/>
        /// 
        /// </summary>
        /// <throws>  CorruptIndexException if the index is corrupt </throws>
        /// <throws>  IOException if there is a low-level IO error </throws>
        [Obsolete("Use Dispose() instead")]
        public void Close()
        {
            Dispose(true);
        }

        /// <summary> Commits all changes to an index and closes all
        /// associated files.  Note that this may be a costly
        /// operation, so, try to re-use a single writer instead of
        /// closing and opening a new one.  See <see cref="Commit()" /> for
        /// caveats about write caching done by some IO devices.
        /// 
        /// <p/> If an Exception is hit during close, eg due to disk
        /// full or some other reason, then both the on-disk index
        /// and the internal state of the IndexWriter instance will
        /// be consistent.  However, the close will not be complete
        /// even though part of it (flushing buffered documents)
        /// may have succeeded, so the write lock will still be
        /// held.<p/>
        /// 
        /// <p/> If you can correct the underlying cause (eg free up
        /// some disk space) then you can call close() again.
        /// Failing that, if you want to force the write lock to be
        /// released (dangerous, because you may then lose buffered
        /// docs in the IndexWriter instance) then you can do
        /// something like this:<p/>
        /// 
        /// <code>
        /// try {
        ///     writer.close();
        /// } finally {
        ///     if (IndexWriter.isLocked(directory)) {
        ///         IndexWriter.unlock(directory);
        ///     }
        /// }
        /// </code>
        /// 
        /// after which, you must be certain not to use the writer
        /// instance anymore.<p/>
        /// 
        /// <p/><b>NOTE</b>: if this method hits an OutOfMemoryError
        /// you should immediately close the writer, again.  See <a
        /// href="#OOME">above</a> for details.<p/>
        /// 
        /// </summary>
        /// <throws>  CorruptIndexException if the index is corrupt </throws>
        /// <throws>  IOException if there is a low-level IO error </throws>
        public virtual void Dispose()
        {
            Dispose(true);
        }

        /// <summary> Closes the index with or without waiting for currently
        /// running merges to finish.  This is only meaningful when
        /// using a MergeScheduler that runs merges in background
        /// threads.
        /// 
        /// <p/><b>NOTE</b>: if this method hits an OutOfMemoryError
        /// you should immediately close the writer, again.  See <a
        /// href="#OOME">above</a> for details.<p/>
        /// 
        /// <p/><b>NOTE</b>: it is dangerous to always call
        /// close(false), especially when IndexWriter is not open
        /// for very long, because this can result in "merge
        /// starvation" whereby long merges will never have a
        /// chance to finish.  This will cause too many segments in
        /// your index over time.<p/>
        /// 
        /// </summary>
        /// <param name="waitForMerges">if true, this call will block
        /// until all merges complete; else, it will ask all
        /// running merges to abort, wait until those merges have
        /// finished (which should be at most a few seconds), and
        /// then return.
        /// </param>
        public virtual void Dispose(bool waitForMerges)
        {
            Dispose(true, waitForMerges);
        }

        protected virtual void Dispose(bool disposing, bool waitForMerges)
        {
            if (disposing)
            {
                // Ensure that only one thread actually gets to do the closing:
                if (ShouldClose())
                {
                    // If any methods have hit OutOfMemoryError, then abort
                    // on close, in case the internal state of IndexWriter
                    // or DocumentsWriter is corrupt
                    if (hitOOM)
                        RollbackInternal();
                    else
                        CloseInternal(waitForMerges);
                }
            }
        }
        
        /// <summary> Closes the index with or without waiting for currently
        /// running merges to finish.  This is only meaningful when
        /// using a MergeScheduler that runs merges in background
        /// threads.
        /// 
        /// <p/><b>NOTE</b>: if this method hits an OutOfMemoryError
        /// you should immediately close the writer, again.  See <a
        /// href="#OOME">above</a> for details.<p/>
        /// 
        /// <p/><b>NOTE</b>: it is dangerous to always call
        /// close(false), especially when IndexWriter is not open
        /// for very long, because this can result in "merge
        /// starvation" whereby long merges will never have a
        /// chance to finish.  This will cause too many segments in
        /// your index over time.<p/>
        /// 
        /// </summary>
        /// <param name="waitForMerges">if true, this call will block
        /// until all merges complete; else, it will ask all
        /// running merges to abort, wait until those merges have
        /// finished (which should be at most a few seconds), and
        /// then return.
        /// </param>
        [Obsolete("Use Dispose(bool) instead")]
        public virtual void Close(bool waitForMerges)
        {
            Dispose(waitForMerges);
        }
        
        // Returns true if this thread should attempt to close, or
        // false if IndexWriter is now closed; else, waits until
        // another thread finishes closing
        private bool ShouldClose()
        {
            lock (this)
            {
                while (true)
                {
                    if (!closed)
                    {
                        if (!closing)
                        {
                            closing = true;
                            return true;
                        }
                        else
                        {
                            // Another thread is presently trying to close;
                            // wait until it finishes one way (closes
                            // successfully) or another (fails to close)
                            DoWait();
                        }
                    }
                    else
                        return false;
                }
            }
        }
        
        private void CloseInternal(bool waitForMerges)
        {
            
            docWriter.PauseAllThreads();
            
            try
            {
                if (infoStream != null)
                    Message("now flush at close");
                
                docWriter.Dispose();
                
                // Only allow a new merge to be triggered if we are
                // going to wait for merges:
                if (!hitOOM)
                {
                    Flush(waitForMerges, true, true);
                }
                
                if (waitForMerges)
                // Give merge scheduler last chance to run, in case
                // any pending merges are waiting:
                    mergeScheduler.Merge(this);
                
                mergePolicy.Close();
                
                FinishMerges(waitForMerges);
                stopMerges = true;
                
                mergeScheduler.Close();
                
                if (infoStream != null)
                    Message("now call final commit()");
                
                if (!hitOOM)
                {
                    Commit(0);
                }
                
                if (infoStream != null)
                    Message("at close: " + SegString());
                
                lock (this)
                {
                    readerPool.Dispose();
                    docWriter = null;
                    deleter.Dispose();
                }
                
                if (writeLock != null)
                {
                    writeLock.Release(); // release write lock
                    writeLock = null;
                }
                lock (this)
                {
                    closed = true;
                }
            }
            catch (System.OutOfMemoryException oom)
            {
                HandleOOM(oom, "closeInternal");
            }
            finally
            {
                lock (this)
                {
                    closing = false;
                    System.Threading.Monitor.PulseAll(this);
                    if (!closed)
                    {
                        if (docWriter != null)
                            docWriter.ResumeAllThreads();
                        if (infoStream != null)
                            Message("hit exception while closing");
                    }
                }
            }
        }
        
        /// <summary>Tells the docWriter to close its currently open shared
        /// doc stores (stored fields &amp; vectors files).
        /// Return value specifices whether new doc store files are compound or not.
        /// </summary>
        private bool FlushDocStores()
        {
            lock (this)
            {
                if (infoStream != null)
                {
                    Message("flushDocStores segment=" + docWriter.DocStoreSegment);
                }

                bool useCompoundDocStore = false;
                if (infoStream != null)
                {
                    Message("closeDocStores segment=" + docWriter.DocStoreSegment);
                }

                System.String docStoreSegment;
                
                bool success = false;
                try
                {
                    docStoreSegment = docWriter.CloseDocStore();
                    success = true;
                }
                finally
                {
                    if (!success && infoStream != null)
                    {
                        Message("hit exception closing doc store segment");
                    }
                }

                if (infoStream != null)
                {
                    Message("flushDocStores files=" + docWriter.ClosedFiles());
                }

                useCompoundDocStore = mergePolicy.UseCompoundDocStore(segmentInfos);
                
                if (useCompoundDocStore && docStoreSegment != null && docWriter.ClosedFiles().Count != 0)
                {
                    // Now build compound doc store file
                    
                    if (infoStream != null)
                    {
                        Message("create compound file " + docStoreSegment + "." + IndexFileNames.COMPOUND_FILE_STORE_EXTENSION);
                    }
                    
                    success = false;
                    
                    int numSegments = segmentInfos.Count;
                    System.String compoundFileName = docStoreSegment + "." + IndexFileNames.COMPOUND_FILE_STORE_EXTENSION;
                    
                    try
                    {
                        CompoundFileWriter cfsWriter = new CompoundFileWriter(directory, compoundFileName);
                        foreach(string file in docWriter.closedFiles)
                        {
                            cfsWriter.AddFile(file);
                        }
                        
                        // Perform the merge
                        cfsWriter.Close();
                        success = true;
                    }
                    finally
                    {
                        if (!success)
                        {
                            if (infoStream != null)
                                Message("hit exception building compound file doc store for segment " + docStoreSegment);
                            deleter.DeleteFile(compoundFileName);
                            docWriter.Abort();
                        }
                    }
                    
                    for (int i = 0; i < numSegments; i++)
                    {
                        SegmentInfo si = segmentInfos.Info(i);
                        if (si.DocStoreOffset != - 1 && si.DocStoreSegment.Equals(docStoreSegment))
                            si.DocStoreIsCompoundFile = true;
                    }
                    
                    Checkpoint();
                    
                    // In case the files we just merged into a CFS were
                    // not previously checkpointed:
                    deleter.DeleteNewFiles(docWriter.ClosedFiles());
                }
                
                return useCompoundDocStore;
            }
        }

        /// <summary>Returns the Directory used by this index. </summary>
        public virtual Directory Directory
        {
            get
            {
                // Pass false because the flush during closing calls getDirectory
                EnsureOpen(false);
                return directory;
            }
        }

        /// <summary>Returns the analyzer used by this index. </summary>
        public virtual Analyzer Analyzer
        {
            get
            {
                EnsureOpen();
                return analyzer;
            }
        }

        /// <summary>Returns total number of docs in this index, including
        /// docs not yet flushed (still in the RAM buffer),
        /// not counting deletions.
        /// </summary>
        /// <seealso cref="NumDocs">
        /// </seealso>
        public virtual int MaxDoc()
        {
            lock (this)
            {
                int count;
                if (docWriter != null)
                    count = docWriter.NumDocsInRAM;
                else
                    count = 0;

                for (int i = 0; i < segmentInfos.Count; i++)
                    count += segmentInfos.Info(i).docCount;
                return count;
            }
        }

        /// <summary>Returns total number of docs in this index, including
        /// docs not yet flushed (still in the RAM buffer), and
        /// including deletions.  <b>NOTE:</b> buffered deletions
        /// are not counted.  If you really need these to be
        /// counted you should call <see cref="Commit()" /> first.
        /// </summary>
        /// <seealso cref="NumDocs">
        /// </seealso>
        public virtual int NumDocs()
        {
            lock (this)
            {
                int count;
                if (docWriter != null)
                    count = docWriter.NumDocsInRAM;
                else
                    count = 0;
                
                for (int i = 0; i < segmentInfos.Count; i++)
                {
                    SegmentInfo info = segmentInfos.Info(i);
                    count += info.docCount - info.GetDelCount();
                }
                return count;
            }
        }
        
        public virtual bool HasDeletions()
        {
            lock (this)
            {
                EnsureOpen();
                if (docWriter.HasDeletes())
                    return true;
                for (int i = 0; i < segmentInfos.Count; i++)
                    if (segmentInfos.Info(i).HasDeletions())
                        return true;
                return false;
            }
        }
        
        /// <summary> The maximum number of terms that will be indexed for a single field in a
        /// document.  This limits the amount of memory required for indexing, so that
        /// collections with very large files will not crash the indexing process by
        /// running out of memory.<p/>
        /// Note that this effectively truncates large documents, excluding from the
        /// index terms that occur further in the document.  If you know your source
        /// documents are large, be sure to set this value high enough to accomodate
        /// the expected size.  If you set it to Integer.MAX_VALUE, then the only limit
        /// is your memory, but you should anticipate an OutOfMemoryError.<p/>
        /// By default, no more than 10,000 terms will be indexed for a field.
        /// 
        /// </summary>
        /// <seealso cref="MaxFieldLength">
        /// </seealso>
        private int maxFieldLength;
        
        /// <summary> Adds a document to this index.  If the document contains more than
        /// <see cref="SetMaxFieldLength(int)" /> terms for a given field, the remainder are
        /// discarded.
        /// 
        /// <p/> Note that if an Exception is hit (for example disk full)
        /// then the index will be consistent, but this document
        /// may not have been added.  Furthermore, it's possible
        /// the index will have one segment in non-compound format
        /// even when using compound files (when a merge has
        /// partially succeeded).<p/>
        /// 
        /// <p/> This method periodically flushes pending documents
        /// to the Directory (see <a href="#flush">above</a>), and
        /// also periodically triggers segment merges in the index
        /// according to the <see cref="MergePolicy" /> in use.<p/>
        /// 
        /// <p/>Merges temporarily consume space in the
        /// directory. The amount of space required is up to 1X the
        /// size of all segments being merged, when no
        /// readers/searchers are open against the index, and up to
        /// 2X the size of all segments being merged when
        /// readers/searchers are open against the index (see
        /// <see cref="Optimize()" /> for details). The sequence of
        /// primitive merge operations performed is governed by the
        /// merge policy.
        /// 
        /// <p/>Note that each term in the document can be no longer
        /// than 16383 characters, otherwise an
        /// IllegalArgumentException will be thrown.<p/>
        /// 
        /// <p/>Note that it's possible to create an invalid Unicode
        /// string in java if a UTF16 surrogate pair is malformed.
        /// In this case, the invalid characters are silently
        /// replaced with the Unicode replacement character
        /// U+FFFD.<p/>
        /// 
        /// <p/><b>NOTE</b>: if this method hits an OutOfMemoryError
        /// you should immediately close the writer.  See <a
        /// href="#OOME">above</a> for details.<p/>
        /// 
        /// </summary>
        /// <throws>  CorruptIndexException if the index is corrupt </throws>
        /// <throws>  IOException if there is a low-level IO error </throws>
        public virtual void  AddDocument(Document doc)
        {
            AddDocument(doc, analyzer);
        }
        
        /// <summary> Adds a document to this index, using the provided analyzer instead of the
        /// value of <see cref="Analyzer" />.  If the document contains more than
        /// <see cref="SetMaxFieldLength(int)" /> terms for a given field, the remainder are
        /// discarded.
        /// 
        /// <p/>See <see cref="AddDocument(Document)" /> for details on
        /// index and IndexWriter state after an Exception, and
        /// flushing/merging temporary free space requirements.<p/>
        /// 
        /// <p/><b>NOTE</b>: if this method hits an OutOfMemoryError
        /// you should immediately close the writer.  See <a
        /// href="#OOME">above</a> for details.<p/>
        /// 
        /// </summary>
        /// <throws>  CorruptIndexException if the index is corrupt </throws>
        /// <throws>  IOException if there is a low-level IO error </throws>
        public virtual void  AddDocument(Document doc, Analyzer analyzer)
        {
            EnsureOpen();
            bool doFlush = false;
            bool success = false;
            try
            {
                try
                {
                    doFlush = docWriter.AddDocument(doc, analyzer);
                    success = true;
                }
                finally
                {
                    if (!success)
                    {
                        
                        if (infoStream != null)
                            Message("hit exception adding document");
                        
                        lock (this)
                        {
                            // If docWriter has some aborted files that were
                            // never incref'd, then we clean them up here
                            if (docWriter != null)
                            {
                                ICollection<string> files = docWriter.AbortedFiles();
                                if (files != null)
                                    deleter.DeleteNewFiles(files);
                            }
                        }
                    }
                }
                if (doFlush)
                    Flush(true, false, false);
            }
            catch (System.OutOfMemoryException oom)
            {
                HandleOOM(oom, "addDocument");
            }
        }
        
        /// <summary> Deletes the document(s) containing <c>term</c>.
        /// 
        /// <p/><b>NOTE</b>: if this method hits an OutOfMemoryError
        /// you should immediately close the writer.  See <a
        /// href="#OOME">above</a> for details.<p/>
        /// 
        /// </summary>
        /// <param name="term">the term to identify the documents to be deleted
        /// </param>
        /// <throws>  CorruptIndexException if the index is corrupt </throws>
        /// <throws>  IOException if there is a low-level IO error </throws>
        public virtual void  DeleteDocuments(Term term)
        {
            EnsureOpen();
            try
            {
                bool doFlush = docWriter.BufferDeleteTerm(term);
                if (doFlush)
                    Flush(true, false, false);
            }
            catch (System.OutOfMemoryException oom)
            {
                HandleOOM(oom, "deleteDocuments(Term)");
            }
        }
        
        /// <summary> Deletes the document(s) containing any of the
        /// terms. All deletes are flushed at the same time.
        /// 
        /// <p/><b>NOTE</b>: if this method hits an OutOfMemoryError
        /// you should immediately close the writer.  See <a
        /// href="#OOME">above</a> for details.<p/>
        /// 
        /// </summary>
        /// <param name="terms">array of terms to identify the documents
        /// to be deleted
        /// </param>
        /// <throws>  CorruptIndexException if the index is corrupt </throws>
        /// <throws>  IOException if there is a low-level IO error </throws>
        public virtual void  DeleteDocuments(params Term[] terms)
        {
            EnsureOpen();
            try
            {
                bool doFlush = docWriter.BufferDeleteTerms(terms);
                if (doFlush)
                    Flush(true, false, false);
            }
            catch (System.OutOfMemoryException oom)
            {
                HandleOOM(oom, "deleteDocuments(params Term[])");
            }
        }
        
        /// <summary> Deletes the document(s) matching the provided query.
        /// 
        /// <p/><b>NOTE</b>: if this method hits an OutOfMemoryError
        /// you should immediately close the writer.  See <a
        /// href="#OOME">above</a> for details.<p/>
        /// 
        /// </summary>
        /// <param name="query">the query to identify the documents to be deleted
        /// </param>
        /// <throws>  CorruptIndexException if the index is corrupt </throws>
        /// <throws>  IOException if there is a low-level IO error </throws>
        public virtual void  DeleteDocuments(Query query)
        {
            EnsureOpen();
            bool doFlush = docWriter.BufferDeleteQuery(query);
            if (doFlush)
                Flush(true, false, false);
        }
        
        /// <summary> Deletes the document(s) matching any of the provided queries.
        /// All deletes are flushed at the same time.
        /// 
        /// <p/><b>NOTE</b>: if this method hits an OutOfMemoryError
        /// you should immediately close the writer.  See <a
        /// href="#OOME">above</a> for details.<p/>
        /// 
        /// </summary>
        /// <param name="queries">array of queries to identify the documents
        /// to be deleted
        /// </param>
        /// <throws>  CorruptIndexException if the index is corrupt </throws>
        /// <throws>  IOException if there is a low-level IO error </throws>
        public virtual void  DeleteDocuments(params Query[] queries)
        {
            EnsureOpen();
            bool doFlush = docWriter.BufferDeleteQueries(queries);
            if (doFlush)
                Flush(true, false, false);
        }
        
        /// <summary> Updates a document by first deleting the document(s)
        /// containing <c>term</c> and then adding the new
        /// document.  The delete and then add are atomic as seen
        /// by a reader on the same index (flush may happen only after
        /// the add).
        /// 
        /// <p/><b>NOTE</b>: if this method hits an OutOfMemoryError
        /// you should immediately close the writer.  See <a
        /// href="#OOME">above</a> for details.<p/>
        /// 
        /// </summary>
        /// <param name="term">the term to identify the document(s) to be
        /// deleted
        /// </param>
        /// <param name="doc">the document to be added
        /// </param>
        /// <throws>  CorruptIndexException if the index is corrupt </throws>
        /// <throws>  IOException if there is a low-level IO error </throws>
        public virtual void  UpdateDocument(Term term, Document doc)
        {
            EnsureOpen();
            UpdateDocument(term, doc, Analyzer);
        }
        
        /// <summary> Updates a document by first deleting the document(s)
        /// containing <c>term</c> and then adding the new
        /// document.  The delete and then add are atomic as seen
        /// by a reader on the same index (flush may happen only after
        /// the add).
        /// 
        /// <p/><b>NOTE</b>: if this method hits an OutOfMemoryError
        /// you should immediately close the writer.  See <a
        /// href="#OOME">above</a> for details.<p/>
        /// 
        /// </summary>
        /// <param name="term">the term to identify the document(s) to be
        /// deleted
        /// </param>
        /// <param name="doc">the document to be added
        /// </param>
        /// <param name="analyzer">the analyzer to use when analyzing the document
        /// </param>
        /// <throws>  CorruptIndexException if the index is corrupt </throws>
        /// <throws>  IOException if there is a low-level IO error </throws>
        public virtual void  UpdateDocument(Term term, Document doc, Analyzer analyzer)
        {
            EnsureOpen();
            try
            {
                bool doFlush = false;
                bool success = false;
                try
                {
                    doFlush = docWriter.UpdateDocument(term, doc, analyzer);
                    success = true;
                }
                finally
                {
                    if (!success)
                    {
                        
                        if (infoStream != null)
                            Message("hit exception updating document");
                        
                        lock (this)
                        {
                            // If docWriter has some aborted files that were
                            // never incref'd, then we clean them up here
                            ICollection<string> files = docWriter.AbortedFiles();
                            if (files != null)
                                deleter.DeleteNewFiles(files);
                        }
                    }
                }
                if (doFlush)
                    Flush(true, false, false);
            }
            catch (System.OutOfMemoryException oom)
            {
                HandleOOM(oom, "updateDocument");
            }
        }
        
        // for test purpose
        internal int GetSegmentCount()
        {
            lock (this)
            {
                return segmentInfos.Count;
            }
        }
        
        // for test purpose
        internal int GetNumBufferedDocuments()
        {
            lock (this)
            {
                return docWriter.NumDocsInRAM;
            }
        }
        
        // for test purpose
        public /*internal*/ int GetDocCount(int i)
        {
            lock (this)
            {
                if (i >= 0 && i < segmentInfos.Count)
                {
                    return segmentInfos.Info(i).docCount;
                }
                else
                {
                    return - 1;
                }
            }
        }
        
        // for test purpose
        internal int GetFlushCount()
        {
            lock (this)
            {
                return flushCount;
            }
        }
        
        // for test purpose
        internal int GetFlushDeletesCount()
        {
            lock (this)
            {
                return flushDeletesCount;
            }
        }
        
        internal System.String NewSegmentName()
        {
            // Cannot synchronize on IndexWriter because that causes
            // deadlock
            lock (segmentInfos)
            {
                // Important to increment changeCount so that the
                // segmentInfos is written on close.  Otherwise we
                // could close, re-open and re-return the same segment
                // name that was previously returned which can cause
                // problems at least with ConcurrentMergeScheduler.
                changeCount++;
                return "_" + Number.ToString(segmentInfos.counter++);
            }
        }
        
        /// <summary>If non-null, information about merges will be printed to this.</summary>
        private System.IO.StreamWriter infoStream = null;
        private static System.IO.StreamWriter defaultInfoStream = null;
        
        /// <summary> Requests an "optimize" operation on an index, priming the index
        /// for the fastest available search. Traditionally this has meant
        /// merging all segments into a single segment as is done in the
        /// default merge policy, but individaul merge policies may implement
        /// optimize in different ways.
        /// 
        /// <p/>It is recommended that this method be called upon completion of indexing.  In
        /// environments with frequent updates, optimize is best done during low volume times, if at all. 
        /// 
        /// <p/>
        /// <p/>See http://www.gossamer-threads.com/lists/lucene/java-dev/47895 for more discussion. <p/>
        /// 
        /// <p/>Note that optimize requires 2X the index size free
        /// space in your Directory (3X if you're using compound
        /// file format).  For example, if your index
        /// size is 10 MB then you need 20 MB free for optimize to
        /// complete (30 MB if you're using compound fiel format).<p/>
        /// 
        /// <p/>If some but not all readers re-open while an
        /// optimize is underway, this will cause > 2X temporary
        /// space to be consumed as those new readers will then
        /// hold open the partially optimized segments at that
        /// time.  It is best not to re-open readers while optimize
        /// is running.<p/>
        /// 
        /// <p/>The actual temporary usage could be much less than
        /// these figures (it depends on many factors).<p/>
        /// 
        /// <p/>In general, once the optimize completes, the total size of the
        /// index will be less than the size of the starting index.
        /// It could be quite a bit smaller (if there were many
        /// pending deletes) or just slightly smaller.<p/>
        /// 
        /// <p/>If an Exception is hit during optimize(), for example
        /// due to disk full, the index will not be corrupt and no
        /// documents will have been lost.  However, it may have
        /// been partially optimized (some segments were merged but
        /// not all), and it's possible that one of the segments in
        /// the index will be in non-compound format even when
        /// using compound file format.  This will occur when the
        /// Exception is hit during conversion of the segment into
        /// compound format.<p/>
        /// 
        /// <p/>This call will optimize those segments present in
        /// the index when the call started.  If other threads are
        /// still adding documents and flushing segments, those
        /// newly created segments will not be optimized unless you
        /// call optimize again.<p/>
        /// 
        /// <p/><b>NOTE</b>: if this method hits an OutOfMemoryError
        /// you should immediately close the writer.  See <a
        /// href="#OOME">above</a> for details.<p/>
        /// 
        /// </summary>
        /// <throws>  CorruptIndexException if the index is corrupt </throws>
        /// <throws>  IOException if there is a low-level IO error </throws>
        /// <seealso cref="Index.LogMergePolicy.FindMergesForOptimize">
        /// </seealso>
        public virtual void  Optimize()
        {
            Optimize(true);
        }

        /// <summary> Optimize the index down to &lt;= maxNumSegments.  If
        /// maxNumSegments==1 then this is the same as <see cref="Optimize()" />
        ///.
        /// 
        /// <p/><b>NOTE</b>: if this method hits an OutOfMemoryError
        /// you should immediately close the writer.  See <a
        /// href="#OOME">above</a> for details.<p/>
        /// 
        /// </summary>
        /// <param name="maxNumSegments">maximum number of segments left
        /// in the index after optimization finishes
        /// </param>
        public virtual void  Optimize(int maxNumSegments)
        {
            Optimize(maxNumSegments, true);
        }
        
        /// <summary>Just like <see cref="Optimize()" />, except you can specify
        /// whether the call should block until the optimize
        /// completes.  This is only meaningful with a
        /// <see cref="MergeScheduler" /> that is able to run merges in
        /// background threads.
        /// 
        /// <p/><b>NOTE</b>: if this method hits an OutOfMemoryError
        /// you should immediately close the writer.  See <a
        /// href="#OOME">above</a> for details.<p/>
        /// </summary>
        public virtual void  Optimize(bool doWait)
        {
            Optimize(1, doWait);
        }
        
        /// <summary>Just like <see cref="Optimize(int)" />, except you can
        /// specify whether the call should block until the
        /// optimize completes.  This is only meaningful with a
        /// <see cref="MergeScheduler" /> that is able to run merges in
        /// background threads.
        /// 
        /// <p/><b>NOTE</b>: if this method hits an OutOfMemoryError
        /// you should immediately close the writer.  See <a
        /// href="#OOME">above</a> for details.<p/>
        /// </summary>
        public virtual void  Optimize(int maxNumSegments, bool doWait)
        {
            EnsureOpen();
            
            if (maxNumSegments < 1)
                throw new System.ArgumentException("maxNumSegments must be >= 1; got " + maxNumSegments);
            
            if (infoStream != null)
                Message("optimize: index now " + SegString());
            
            Flush(true, false, true);
            
            lock (this)
            {
                ResetMergeExceptions();
                segmentsToOptimize = Lucene.Net.Support.Compatibility.SetFactory.CreateHashSet<SegmentInfo>();
                optimizeMaxNumSegments = maxNumSegments;
                int numSegments = segmentInfos.Count;
                for (int i = 0; i < numSegments; i++)
                    segmentsToOptimize.Add(segmentInfos.Info(i));
                
                // Now mark all pending & running merges as optimize
                // merge:
                foreach(MergePolicy.OneMerge merge in pendingMerges)
                {
                    merge.optimize = true;
                    merge.maxNumSegmentsOptimize = maxNumSegments;
                }
                
                foreach(MergePolicy.OneMerge merge in runningMerges)
                {
                    merge.optimize = true;
                    merge.maxNumSegmentsOptimize = maxNumSegments;
                }
            }
            
            MaybeMerge(maxNumSegments, true);
            
            if (doWait)
            {
                lock (this)
                {
                    while (true)
                    {
                        
                        if (hitOOM)
                        {
                            throw new System.SystemException("this writer hit an OutOfMemoryError; cannot complete optimize");
                        }
                        
                        if (mergeExceptions.Count > 0)
                        {
                            // Forward any exceptions in background merge
                            // threads to the current thread:
                            int size = mergeExceptions.Count;
                            for (int i = 0; i < size; i++)
                            {
                                MergePolicy.OneMerge merge = mergeExceptions[i];
                                if (merge.optimize)
                                {
                                    System.IO.IOException err;
                                    System.Exception t = merge.GetException();
                                    if (t != null)
                                        err = new System.IO.IOException("background merge hit exception: " + merge.SegString(directory), t);
                                    else
                                        err = new System.IO.IOException("background merge hit exception: " + merge.SegString(directory));
                                    throw err;
                                }
                            }
                        }
                        
                        if (OptimizeMergesPending())
                            DoWait();
                        else
                            break;
                    }
                }
                
                // If close is called while we are still
                // running, throw an exception so the calling
                // thread will know the optimize did not
                // complete
                EnsureOpen();
            }
            
            // NOTE: in the ConcurrentMergeScheduler case, when
            // doWait is false, we can return immediately while
            // background threads accomplish the optimization
        }
        
        /// <summary>Returns true if any merges in pendingMerges or
        /// runningMerges are optimization merges. 
        /// </summary>
        private bool OptimizeMergesPending()
        {
            lock (this)
            {
                foreach (MergePolicy.OneMerge merge in pendingMerges)
                {
                    if (merge.optimize) return true;
                }

                foreach(MergePolicy.OneMerge merge in runningMerges)
                {
                    if (merge.optimize) return true;
                }
                
                return false;
            }
        }
        
        /// <summary>Just like <see cref="ExpungeDeletes()" />, except you can
        /// specify whether the call should block until the
        /// operation completes.  This is only meaningful with a
        /// <see cref="MergeScheduler" /> that is able to run merges in
        /// background threads.
        /// 
        /// <p/><b>NOTE</b>: if this method hits an OutOfMemoryError
        /// you should immediately close the writer.  See <a
        /// href="#OOME">above</a> for details.<p/>
        /// </summary>
        public virtual void  ExpungeDeletes(bool doWait)
        {
            EnsureOpen();
            
            if (infoStream != null)
                Message("expungeDeletes: index now " + SegString());
            
            MergePolicy.MergeSpecification spec;
            
            lock (this)
            {
                spec = mergePolicy.FindMergesToExpungeDeletes(segmentInfos);
                if (spec != null)
                {
                    int numMerges = spec.merges.Count;
                    for (int i = 0; i < numMerges; i++)
                        RegisterMerge(spec.merges[i]);
                }
            }
            
            mergeScheduler.Merge(this);
            
            if (spec != null && doWait)
            {
                int numMerges = spec.merges.Count;
                lock (this)
                {
                    bool running = true;
                    while (running)
                    {
                        
                        if (hitOOM)
                        {
                            throw new System.SystemException("this writer hit an OutOfMemoryError; cannot complete expungeDeletes");
                        }
                        
                        // Check each merge that MergePolicy asked us to
                        // do, to see if any of them are still running and
                        // if any of them have hit an exception.
                        running = false;
                        for (int i = 0; i < numMerges; i++)
                        {
                            MergePolicy.OneMerge merge = spec.merges[i];
                            if (pendingMerges.Contains(merge) || runningMerges.Contains(merge))
                                running = true;
                            System.Exception t = merge.GetException();
                            if (t != null)
                            {
                                System.IO.IOException ioe = new System.IO.IOException("background merge hit exception: " + merge.SegString(directory), t);
                                throw ioe;
                            }
                        }
                        
                        // If any of our merges are still running, wait:
                        if (running)
                            DoWait();
                    }
                }
            }
            
            // NOTE: in the ConcurrentMergeScheduler case, when
            // doWait is false, we can return immediately while
            // background threads accomplish the optimization
        }
        
        
        /// <summary>Expunges all deletes from the index.  When an index
        /// has many document deletions (or updates to existing
        /// documents), it's best to either call optimize or
        /// expungeDeletes to remove all unused data in the index
        /// associated with the deleted documents.  To see how
        /// many deletions you have pending in your index, call
        /// <see cref="IndexReader.NumDeletedDocs" />
        /// This saves disk space and memory usage while
        /// searching.  expungeDeletes should be somewhat faster
        /// than optimize since it does not insist on reducing the
        /// index to a single segment (though, this depends on the
        /// <see cref="MergePolicy" />; see <see cref="Index.MergePolicy.FindMergesToExpungeDeletes" />.). Note that
        /// this call does not first commit any buffered
        /// documents, so you must do so yourself if necessary.
        /// See also <seealso cref="ExpungeDeletes(bool)" />
        /// 
        /// <p/><b>NOTE</b>: if this method hits an OutOfMemoryError
        /// you should immediately close the writer.  See <a
        /// href="#OOME">above</a> for details.<p/>
        /// </summary>
        public virtual void  ExpungeDeletes()
        {
            ExpungeDeletes(true);
        }
        
        /// <summary> Expert: asks the mergePolicy whether any merges are
        /// necessary now and if so, runs the requested merges and
        /// then iterate (test again if merges are needed) until no
        /// more merges are returned by the mergePolicy.
        /// 
        /// Explicit calls to maybeMerge() are usually not
        /// necessary. The most common case is when merge policy
        /// parameters have changed.
        /// 
        /// <p/><b>NOTE</b>: if this method hits an OutOfMemoryError
        /// you should immediately close the writer.  See <a
        /// href="#OOME">above</a> for details.<p/>
        /// </summary>
        public void  MaybeMerge()
        {
            MaybeMerge(false);
        }
        
        private void  MaybeMerge(bool optimize)
        {
            MaybeMerge(1, optimize);
        }
        
        private void  MaybeMerge(int maxNumSegmentsOptimize, bool optimize)
        {
            UpdatePendingMerges(maxNumSegmentsOptimize, optimize);
            mergeScheduler.Merge(this);
        }
        
        private void  UpdatePendingMerges(int maxNumSegmentsOptimize, bool optimize)
        {
            lock (this)
            {
                System.Diagnostics.Debug.Assert(!optimize || maxNumSegmentsOptimize > 0);

                if (stopMerges)
                {
                    return;
                }
                
                // Do not start new merges if we've hit OOME
                if (hitOOM)
                {
                    return ;
                }
                
                MergePolicy.MergeSpecification spec;
                if (optimize)
                {
                    spec = mergePolicy.FindMergesForOptimize(segmentInfos, maxNumSegmentsOptimize, segmentsToOptimize);

                    if (spec != null)
                    {
                        int numMerges = spec.merges.Count;
                        for (int i = 0; i < numMerges; i++)
                        {
                            MergePolicy.OneMerge merge = spec.merges[i];
                            merge.optimize = true;
                            merge.maxNumSegmentsOptimize = maxNumSegmentsOptimize;
                        }
                    }
                }
                else
                {
                    spec = mergePolicy.FindMerges(segmentInfos);
                }

                if (spec != null)
                {
                    int numMerges = spec.merges.Count;
                    for (int i = 0; i < numMerges; i++)
                        RegisterMerge(spec.merges[i]);
                }
            }
        }
        
        /// <summary>Expert: the <see cref="MergeScheduler" /> calls this method
        /// to retrieve the next merge requested by the
        /// MergePolicy 
        /// </summary>
        internal virtual MergePolicy.OneMerge GetNextMerge()
        {
            lock (this)
            {
                if (pendingMerges.Count == 0)
                    return null;
                else
                {
                    // Advance the merge from pending to running
                    MergePolicy.OneMerge merge = pendingMerges.First.Value;
                    pendingMerges.RemoveFirst();
                    runningMerges.Add(merge);
                    return merge;
                }
            }
        }
        
        /// <summary>Like getNextMerge() except only returns a merge if it's
        /// external. 
        /// </summary>
        private MergePolicy.OneMerge GetNextExternalMerge()
        {
            lock (this)
            {
                if (pendingMerges.Count == 0)
                    return null;
                else
                {
                    var it = pendingMerges.GetEnumerator();
                    while (it.MoveNext())
                    {
                        MergePolicy.OneMerge merge = it.Current;
                        if (merge.isExternal)
                        {
                            // Advance the merge from pending to running
                            pendingMerges.Remove(merge);  // {{Aroush-2.9}} From Mike Garski: this is an O(n) op... is that an issue?
                            runningMerges.Add(merge);
                            return merge;
                        }
                    }
                    
                    // All existing merges do not involve external segments
                    return null;
                }
            }
        }
        
        /*
        * Begin a transaction.  During a transaction, any segment
        * merges that happen (or ram segments flushed) will not
        * write a new segments file and will not remove any files
        * that were present at the start of the transaction.  You
        * must make a matched (try/finally) call to
        * commitTransaction() or rollbackTransaction() to finish
        * the transaction.
        *
        * Note that buffered documents and delete terms are not handled
        * within the transactions, so they must be flushed before the
        * transaction is started.
        */
        private void  StartTransaction(bool haveReadLock)
        {
            lock (this)
            {
                
                bool success = false;
                try
                {
                    if (infoStream != null)
                        Message("now start transaction");
                    
                    System.Diagnostics.Debug.Assert(docWriter.GetNumBufferedDeleteTerms() == 0 , 
                        "calling startTransaction with buffered delete terms not supported: numBufferedDeleteTerms=" + docWriter.GetNumBufferedDeleteTerms());
                    System.Diagnostics.Debug.Assert(docWriter.NumDocsInRAM == 0 , 
                        "calling startTransaction with buffered documents not supported: numDocsInRAM=" + docWriter.NumDocsInRAM);
                    
                    EnsureOpen();
                    
                    // If a transaction is trying to roll back (because
                    // addIndexes hit an exception) then wait here until
                    // that's done:
                    lock (this)
                    {
                        while (stopMerges)
                            DoWait();
                    }
                    success = true;
                }
                finally
                {
                    // Release the write lock if our caller held it, on
                    // hitting an exception
                    if (!success && haveReadLock)
                        ReleaseRead();
                }
                
                if (haveReadLock)
                {
                    UpgradeReadToWrite();
                }
                else
                {
                    AcquireWrite();
                }
                
                success = false;
                try
                {
                    localRollbackSegmentInfos = (SegmentInfos) segmentInfos.Clone();
                    
                    System.Diagnostics.Debug.Assert(!HasExternalSegments());
                    
                    localFlushedDocCount = docWriter.GetFlushedDocCount();

                    // Remove the incRef we did in startTransaction:
                    deleter.IncRef(segmentInfos, false);
                    
                    success = true;
                }
                finally
                {
                    if (!success)
                        FinishAddIndexes();
                }
            }
        }
        
        /*
        * Rolls back the transaction and restores state to where
        * we were at the start.
        */
        private void  RollbackTransaction()
        {
            lock (this)
            {
                
                if (infoStream != null)
                    Message("now rollback transaction");
                
                if (docWriter != null)
                {
                    docWriter.SetFlushedDocCount(localFlushedDocCount);
                }
                
                // Must finish merges before rolling back segmentInfos
                // so merges don't hit exceptions on trying to commit
                // themselves, don't get files deleted out from under
                // them, etc:
                FinishMerges(false);
                
                // Keep the same segmentInfos instance but replace all
                // of its SegmentInfo instances.  This is so the next
                // attempt to commit using this instance of IndexWriter
                // will always write to a new generation ("write once").
                segmentInfos.Clear();
                segmentInfos.AddRange(localRollbackSegmentInfos);
                localRollbackSegmentInfos = null;
                
                // This must come after we rollback segmentInfos, so
                // that if a commit() kicks off it does not see the
                // segmentInfos with external segments
                FinishAddIndexes();
                
                // Ask deleter to locate unreferenced files we had
                // created & remove them:
                deleter.Checkpoint(segmentInfos, false);

                // Remove the incRef we did in startTransaction:
                deleter.DecRef(segmentInfos);
                
                // Also ask deleter to remove any newly created files
                // that were never incref'd; this "garbage" is created
                // when a merge kicks off but aborts part way through
                // before it had a chance to incRef the files it had
                // partially created
                deleter.Refresh();
                
                System.Threading.Monitor.PulseAll(this);
                
                System.Diagnostics.Debug.Assert(!HasExternalSegments());
            }
        }
        
        /*
        * Commits the transaction.  This will write the new
        * segments file and remove and pending deletions we have
        * accumulated during the transaction
        */
        private void  CommitTransaction()
        {
            lock (this)
            {
                
                if (infoStream != null)
                    Message("now commit transaction");
                
                // Give deleter a chance to remove files now:
                Checkpoint();
                
                // Remove the incRef we did in startTransaction.
                deleter.DecRef(localRollbackSegmentInfos);
                
                localRollbackSegmentInfos = null;
                
                System.Diagnostics.Debug.Assert(!HasExternalSegments());
                
                FinishAddIndexes();
            }
        }
        
        /// <summary> Close the <c>IndexWriter</c> without committing
        /// any changes that have occurred since the last commit
        /// (or since it was opened, if commit hasn't been called).
        /// This removes any temporary files that had been created,
        /// after which the state of the index will be the same as
        /// it was when commit() was last called or when this
        /// writer was first opened.  This also clears a previous 
        /// call to <see cref="PrepareCommit()" />.
        /// </summary>
        /// <throws>  IOException if there is a low-level IO error </throws>
        public virtual void  Rollback()
        {
            EnsureOpen();
            
            // Ensure that only one thread actually gets to do the closing:
            if (ShouldClose())
                RollbackInternal();
        }
        
        private void  RollbackInternal()
        {
            
            bool success = false;

            if (infoStream != null)
            {
                Message("rollback");
            }
            
            docWriter.PauseAllThreads();
            
            try
            {
                FinishMerges(false);
                
                // Must pre-close these two, in case they increment
                // changeCount so that we can then set it to false
                // before calling closeInternal
                mergePolicy.Close();
                mergeScheduler.Close();
                
                lock (this)
                {
                    
                    if (pendingCommit != null)
                    {
                        pendingCommit.RollbackCommit(directory);
                        deleter.DecRef(pendingCommit);
                        pendingCommit = null;
                        System.Threading.Monitor.PulseAll(this);
                    }
                    
                    // Keep the same segmentInfos instance but replace all
                    // of its SegmentInfo instances.  This is so the next
                    // attempt to commit using this instance of IndexWriter
                    // will always write to a new generation ("write
                    // once").
                    segmentInfos.Clear();
                    segmentInfos.AddRange(rollbackSegmentInfos);
                    
                    System.Diagnostics.Debug.Assert(!HasExternalSegments());
                    
                    docWriter.Abort();
                    
                    System.Diagnostics.Debug.Assert(TestPoint("rollback before checkpoint"));
                    
                    // Ask deleter to locate unreferenced files & remove
                    // them:
                    deleter.Checkpoint(segmentInfos, false);
                    deleter.Refresh();
                }
                
                // Don't bother saving any changes in our segmentInfos
                readerPool.Clear(null);
                
                lastCommitChangeCount = changeCount;
                
                success = true;
            }
            catch (System.OutOfMemoryException oom)
            {
                HandleOOM(oom, "rollbackInternal");
            }
            finally
            {
                lock (this)
                {
                    if (!success)
                    {
                        docWriter.ResumeAllThreads();
                        closing = false;
                        System.Threading.Monitor.PulseAll(this);
                        if (infoStream != null)
                            Message("hit exception during rollback");
                    }
                }
            }
            
            CloseInternal(false);
        }
        
        /// <summary> Delete all documents in the index.
        /// 
        /// <p/>This method will drop all buffered documents and will 
        /// remove all segments from the index. This change will not be
        /// visible until a <see cref="Commit()" /> has been called. This method
        /// can be rolled back using <see cref="Rollback()" />.<p/>
        /// 
        /// <p/>NOTE: this method is much faster than using deleteDocuments( new MatchAllDocsQuery() ).<p/>
        /// 
        /// <p/>NOTE: this method will forcefully abort all merges
        /// in progress.  If other threads are running <see cref="Optimize()" />
        /// or any of the addIndexes methods, they
        /// will receive <see cref="Index.MergePolicy.MergeAbortedException" />s.
        /// </summary>
        public virtual void  DeleteAll()
        {
            lock (this)
            {
                docWriter.PauseAllThreads();
                try
                {
                    
                    // Abort any running merges
                    FinishMerges(false);
                    
                    // Remove any buffered docs
                    docWriter.Abort();
                    docWriter.SetFlushedDocCount(0);
                    
                    // Remove all segments
                    segmentInfos.Clear();
                    
                    // Ask deleter to locate unreferenced files & remove them:
                    deleter.Checkpoint(segmentInfos, false);
                    deleter.Refresh();
                    
                    // Don't bother saving any changes in our segmentInfos
                    readerPool.Clear(null);
                    
                    // Mark that the index has changed
                    ++changeCount;
                }
                catch (System.OutOfMemoryException oom)
                {
                    HandleOOM(oom, "deleteAll");
                }
                finally
                {
                    docWriter.ResumeAllThreads();
                    if (infoStream != null)
                    {
                        Message("hit exception during deleteAll");
                    }
                }
            }
        }
        
        private void  FinishMerges(bool waitForMerges)
        {
            lock (this)
            {
                if (!waitForMerges)
                {
                    
                    stopMerges = true;
                    
                    // Abort all pending & running merges:
                    foreach(MergePolicy.OneMerge merge in pendingMerges)
                    {
                        if (infoStream != null)
                            Message("now abort pending merge " + merge.SegString(directory));
                        merge.Abort();
                        MergeFinish(merge);
                    }
                    pendingMerges.Clear();
                    
                    foreach(MergePolicy.OneMerge merge in runningMerges)
                    {
                        if (infoStream != null)
                            Message("now abort running merge " + merge.SegString(directory));
                        merge.Abort();
                    }
                    
                    // Ensure any running addIndexes finishes.  It's fine
                    // if a new one attempts to start because its merges
                    // will quickly see the stopMerges == true and abort.
                    AcquireRead();
                    ReleaseRead();
                    
                    // These merges periodically check whether they have
                    // been aborted, and stop if so.  We wait here to make
                    // sure they all stop.  It should not take very long
                    // because the merge threads periodically check if
                    // they are aborted.
                    while (runningMerges.Count > 0)
                    {
                        if (infoStream != null)
                            Message("now wait for " + runningMerges.Count + " running merge to abort");
                        DoWait();
                    }
                    
                    stopMerges = false;
                    System.Threading.Monitor.PulseAll(this);
                    
                    System.Diagnostics.Debug.Assert(0 == mergingSegments.Count);
                    
                    if (infoStream != null)
                        Message("all running merges have aborted");
                }
                else
                {
                    // waitForMerges() will ensure any running addIndexes finishes.  
                    // It's fine if a new one attempts to start because from our
                    // caller above the call will see that we are in the
                    // process of closing, and will throw an
                    // AlreadyClosedException.
                    WaitForMerges();
                }
            }
        }
        
        /// <summary> Wait for any currently outstanding merges to finish.
        /// 
        /// <p/>It is guaranteed that any merges started prior to calling this method 
        /// will have completed once this method completes.<p/>
        /// </summary>
        public virtual void  WaitForMerges()
        {
            lock (this)
            {
                // Ensure any running addIndexes finishes.
                AcquireRead();
                ReleaseRead();
                
                while (pendingMerges.Count > 0 || runningMerges.Count > 0)
                {
                    DoWait();
                }
                
                // sanity check
                System.Diagnostics.Debug.Assert(0 == mergingSegments.Count);
            }
        }
        
        /*
        * Called whenever the SegmentInfos has been updated and
        * the index files referenced exist (correctly) in the
        * index directory.
        */
        private void  Checkpoint()
        {
            lock (this)
            {
                changeCount++;
                deleter.Checkpoint(segmentInfos, false);
            }
        }
        
        private void  FinishAddIndexes()
        {
            ReleaseWrite();
        }
        
        private void  BlockAddIndexes(bool includePendingClose)
        {
            
            AcquireRead();
            
            bool success = false;
            try
            {
                
                // Make sure we are still open since we could have
                // waited quite a while for last addIndexes to finish
                EnsureOpen(includePendingClose);
                success = true;
            }
            finally
            {
                if (!success)
                    ReleaseRead();
            }
        }
        
        private void  ResumeAddIndexes()
        {
            ReleaseRead();
        }
        
        private void  ResetMergeExceptions()
        {
            lock (this)
            {
                mergeExceptions = new List<MergePolicy.OneMerge>();
                mergeGen++;
            }
        }
        
        private void  NoDupDirs(Directory[] dirs)
        {
            HashSet<Directory> dups = new HashSet<Directory>();
            for (int i = 0; i < dirs.Length; i++)
            {
                if (dups.Contains(dirs[i]))
                {
                    throw new System.ArgumentException("Directory " + dirs[i] + " appears more than once");
                }
                if (dirs[i] == directory)
                    throw new System.ArgumentException("Cannot add directory to itself");
                dups.Add(dirs[i]);
            }
        }
        
        /// <summary> Merges all segments from an array of indexes into this
        /// index.
        /// 
        /// <p/>This may be used to parallelize batch indexing.  A large document
        /// collection can be broken into sub-collections.  Each sub-collection can be
        /// indexed in parallel, on a different thread, process or machine.  The
        /// complete index can then be created by merging sub-collection indexes
        /// with this method.
        /// 
        /// <p/><b>NOTE:</b> the index in each Directory must not be
        /// changed (opened by a writer) while this method is
        /// running.  This method does not acquire a write lock in
        /// each input Directory, so it is up to the caller to
        /// enforce this.
        /// 
        /// <p/><b>NOTE:</b> while this is running, any attempts to
        /// add or delete documents (with another thread) will be
        /// paused until this method completes.
        /// 
        /// <p/>This method is transactional in how Exceptions are
        /// handled: it does not commit a new segments_N file until
        /// all indexes are added.  This means if an Exception
        /// occurs (for example disk full), then either no indexes
        /// will have been added or they all will have been.<p/>
        /// 
        /// <p/>Note that this requires temporary free space in the
        /// Directory up to 2X the sum of all input indexes
        /// (including the starting index).  If readers/searchers
        /// are open against the starting index, then temporary
        /// free space required will be higher by the size of the
        /// starting index (see <see cref="Optimize()" /> for details).
        /// <p/>
        /// 
        /// <p/>Once this completes, the final size of the index
        /// will be less than the sum of all input index sizes
        /// (including the starting index).  It could be quite a
        /// bit smaller (if there were many pending deletes) or
        /// just slightly smaller.<p/>
        /// 
        /// <p/>
        /// This requires this index not be among those to be added.
        /// 
        /// <p/><b>NOTE</b>: if this method hits an OutOfMemoryError
        /// you should immediately close the writer.  See <a
        /// href="#OOME">above</a> for details.<p/>
        /// 
        /// </summary>
        /// <throws>  CorruptIndexException if the index is corrupt </throws>
        /// <throws>  IOException if there is a low-level IO error </throws>
        public virtual void  AddIndexesNoOptimize(params Directory[] dirs)
        {
            
            EnsureOpen();
            
            NoDupDirs(dirs);
            
            // Do not allow add docs or deletes while we are running:
            docWriter.PauseAllThreads();
            
            try
            {
                if (infoStream != null)
                    Message("flush at addIndexesNoOptimize");
                Flush(true, false, true);
                
                bool success = false;
                
                StartTransaction(false);
                
                try
                {
                    
                    int docCount = 0;
                    lock (this)
                    {
                        EnsureOpen();
                        
                        for (int i = 0; i < dirs.Length; i++)
                        {
                            if (directory == dirs[i])
                            {
                                // cannot add this index: segments may be deleted in merge before added
                                throw new System.ArgumentException("Cannot add this index to itself");
                            }
                            
                            SegmentInfos sis = new SegmentInfos(); // read infos from dir
                            sis.Read(dirs[i]);
                            for (int j = 0; j < sis.Count; j++)
                            {
                                SegmentInfo info = sis.Info(j);
                                System.Diagnostics.Debug.Assert(!segmentInfos.Contains(info), "dup info dir=" + info.dir + " name=" + info.name);
                                docCount += info.docCount;
                                segmentInfos.Add(info); // add each info
                            }
                        }
                    }
                    
                    // Notify DocumentsWriter that the flushed count just increased
                    docWriter.UpdateFlushedDocCount(docCount);
                    
                    MaybeMerge();
                    
                    EnsureOpen();
                    
                    // If after merging there remain segments in the index
                    // that are in a different directory, just copy these
                    // over into our index.  This is necessary (before
                    // finishing the transaction) to avoid leaving the
                    // index in an unusable (inconsistent) state.
                    ResolveExternalSegments();
                    
                    EnsureOpen();
                    
                    success = true;
                }
                finally
                {
                    if (success)
                    {
                        CommitTransaction();
                    }
                    else
                    {
                        RollbackTransaction();
                    }
                }
            }
            catch (System.OutOfMemoryException oom)
            {
                HandleOOM(oom, "addIndexesNoOptimize");
            }
            finally
            {
                if (docWriter != null)
                {
                    docWriter.ResumeAllThreads();
                }
            }
        }
        
        private bool HasExternalSegments()
        {
            return segmentInfos.HasExternalSegments(directory);
        }
        
        /* If any of our segments are using a directory != ours
        * then we have to either copy them over one by one, merge
        * them (if merge policy has chosen to) or wait until
        * currently running merges (in the background) complete.
        * We don't return until the SegmentInfos has no more
        * external segments.  Currently this is only used by
        * addIndexesNoOptimize(). */
        private void  ResolveExternalSegments()
        {
            
            bool any = false;
            
            bool done = false;
            
            while (!done)
            {
                SegmentInfo info = null;
                MergePolicy.OneMerge merge = null;
                lock (this)
                {
                    
                    if (stopMerges)
                        throw new MergePolicy.MergeAbortedException("rollback() was called or addIndexes* hit an unhandled exception");
                    
                    int numSegments = segmentInfos.Count;
                    
                    done = true;
                    for (int i = 0; i < numSegments; i++)
                    {
                        info = segmentInfos.Info(i);
                        if (info.dir != directory)
                        {
                            done = false;
                            MergePolicy.OneMerge newMerge = new MergePolicy.OneMerge(segmentInfos.Range(i, 1 + i), mergePolicy is LogMergePolicy && UseCompoundFile);
                            
                            // Returns true if no running merge conflicts
                            // with this one (and, records this merge as
                            // pending), ie, this segment is not currently
                            // being merged:
                            if (RegisterMerge(newMerge))
                            {
                                merge = newMerge;
                                
                                // If this segment is not currently being
                                // merged, then advance it to running & run
                                // the merge ourself (below):
                                pendingMerges.Remove(merge);    // {{Aroush-2.9}} From Mike Garski: this is an O(n) op... is that an issue?
                                runningMerges.Add(merge);
                                break;
                            }
                        }
                    }
                    
                    if (!done && merge == null)
                    // We are not yet done (external segments still
                    // exist in segmentInfos), yet, all such segments
                    // are currently "covered" by a pending or running
                    // merge.  We now try to grab any pending merge
                    // that involves external segments:
                        merge = GetNextExternalMerge();
                    
                    if (!done && merge == null)
                    // We are not yet done, and, all external segments
                    // fall under merges that the merge scheduler is
                    // currently running.  So, we now wait and check
                    // back to see if the merge has completed.
                        DoWait();
                }
                
                if (merge != null)
                {
                    any = true;
                    Merge(merge);
                }
            }
            
            if (any)
            // Sometimes, on copying an external segment over,
            // more merges may become necessary:
                mergeScheduler.Merge(this);
        }
        
        /// <summary>Merges the provided indexes into this index.
        /// <p/>After this completes, the index is optimized. <p/>
        /// <p/>The provided IndexReaders are not closed.<p/>
        /// 
        /// <p/><b>NOTE:</b> while this is running, any attempts to
        /// add or delete documents (with another thread) will be
        /// paused until this method completes.
        /// 
        /// <p/>See <see cref="AddIndexesNoOptimize(Directory[])" /> for
        /// details on transactional semantics, temporary free
        /// space required in the Directory, and non-CFS segments
        /// on an Exception.<p/>
        /// 
        /// <p/><b>NOTE</b>: if this method hits an OutOfMemoryError
        /// you should immediately close the writer.  See <a
        /// href="#OOME">above</a> for details.<p/>
        /// 
        /// </summary>
        /// <throws>  CorruptIndexException if the index is corrupt </throws>
        /// <throws>  IOException if there is a low-level IO error </throws>
        public virtual void  AddIndexes(params IndexReader[] readers)
        {
            
            EnsureOpen();
            
            // Do not allow add docs or deletes while we are running:
            docWriter.PauseAllThreads();
            
            // We must pre-acquire a read lock here (and upgrade to
            // write lock in startTransaction below) so that no
            // other addIndexes is allowed to start up after we have
            // flushed & optimized but before we then start our
            // transaction.  This is because the merging below
            // requires that only one segment is present in the
            // index:
            AcquireRead();
            
            try
            {
                
                SegmentInfo info = null;
                System.String mergedName = null;
                SegmentMerger merger = null;
                
                bool success = false;
                
                try
                {
                    Flush(true, false, true);
                    Optimize(); // start with zero or 1 seg
                    success = true;
                }
                finally
                {
                    // Take care to release the read lock if we hit an
                    // exception before starting the transaction
                    if (!success)
                        ReleaseRead();
                }
                
                // true means we already have a read lock; if this
                // call hits an exception it will release the write
                // lock:
                StartTransaction(true);
                
                try
                {
                    mergedName = NewSegmentName();
                    merger = new SegmentMerger(this, mergedName, null);
                    
                    SegmentReader sReader = null;
                    lock (this)
                    {
                        if (segmentInfos.Count == 1)
                        {
                            // add existing index, if any
                            sReader = readerPool.Get(segmentInfos.Info(0), true, BufferedIndexInput.BUFFER_SIZE, - 1);
                        }
                    }
                    
                    success = false;
                    
                    try
                    {
                        if (sReader != null)
                            merger.Add(sReader);
                        
                        for (int i = 0; i < readers.Length; i++)
                        // add new indexes
                            merger.Add(readers[i]);
                        
                        int docCount = merger.Merge(); // merge 'em
                        
                        lock (this)
                        {
                            segmentInfos.Clear(); // pop old infos & add new
                            info = new SegmentInfo(mergedName, docCount, directory, false, true, - 1, null, false, merger.HasProx());
                            SetDiagnostics(info, "addIndexes(params IndexReader[])");
                            segmentInfos.Add(info);
                        }
                        
                        // Notify DocumentsWriter that the flushed count just increased
                        docWriter.UpdateFlushedDocCount(docCount);
                        
                        success = true;
                    }
                    finally
                    {
                        if (sReader != null)
                        {
                            readerPool.Release(sReader);
                        }
                    }
                }
                finally
                {
                    if (!success)
                    {
                        if (infoStream != null)
                            Message("hit exception in addIndexes during merge");
                        RollbackTransaction();
                    }
                    else
                    {
                        CommitTransaction();
                    }
                }
                
                if (mergePolicy is LogMergePolicy && UseCompoundFile)
                {
                    
                    IList<string> files = null;
                    
                    lock (this)
                    {
                        // Must incRef our files so that if another thread
                        // is running merge/optimize, it doesn't delete our
                        // segment's files before we have a change to
                        // finish making the compound file.
                        if (segmentInfos.Contains(info))
                        {
                            files = info.Files();
                            deleter.IncRef(files);
                        }
                    }
                    
                    if (files != null)
                    {
                        
                        success = false;
                        
                        StartTransaction(false);
                        
                        try
                        {
                            merger.CreateCompoundFile(mergedName + ".cfs");
                            lock (this)
                            {
                                info.SetUseCompoundFile(true);
                            }
                            
                            success = true;
                        }
                        finally
                        {
                            lock (this)
                            {
                                deleter.DecRef(files);
                            }
                                                        
                            if (!success)
                            {
                                if (infoStream != null)
                                    Message("hit exception building compound file in addIndexes during merge");
                                
                                RollbackTransaction();
                            }
                            else
                            {
                                CommitTransaction();
                            }
                        }
                    }
                }
            }
            catch (System.OutOfMemoryException oom)
            {
                HandleOOM(oom, "addIndexes(params IndexReader[])");
            }
            finally
            {
                if (docWriter != null)
                {
                    docWriter.ResumeAllThreads();
                }
            }
        }

        ///<summary>
        /// A hook for extending classes to execute operations after pending added and
        /// deleted documents have been flushed to the Directory but before the change
        /// is committed (new segments_N file written).
        ///</summary>   
        protected  virtual void  DoAfterFlush()
        {
        }

        ///<summary>
        /// A hook for extending classes to execute operations before pending added and
        /// deleted documents are flushed to the Directory.
        ///</summary>
        protected virtual void DoBeforeFlush() 
        {
        }
        
        /// <summary>Expert: prepare for commit.
        /// 
        /// <p/><b>NOTE</b>: if this method hits an OutOfMemoryError
        /// you should immediately close the writer.  See <a
        /// href="#OOME">above</a> for details.<p/>
        /// 
        /// </summary>
        /// <seealso cref="PrepareCommit(IDictionary{string,string})">
        /// </seealso>
        public void  PrepareCommit()
        {
            EnsureOpen();
            PrepareCommit(null);
        }
        
        /// <summary><p/>Expert: prepare for commit, specifying
        /// commitUserData Map (String -> String).  This does the
        /// first phase of 2-phase commit. This method does all steps
        /// necessary to commit changes since this writer was
        /// opened: flushes pending added and deleted docs, syncs
        /// the index files, writes most of next segments_N file.
        /// After calling this you must call either <see cref="Commit()" />
        /// to finish the commit, or <see cref="Rollback()" />
        /// to revert the commit and undo all changes
        /// done since the writer was opened.<p/>
        /// 
        /// You can also just call <see cref="Commit(IDictionary{string,string})" /> directly
        /// without prepareCommit first in which case that method
        /// will internally call prepareCommit.
        /// 
        /// <p/><b>NOTE</b>: if this method hits an OutOfMemoryError
        /// you should immediately close the writer.  See <a
        /// href="#OOME">above</a> for details.<p/>
        /// 
        /// </summary>
        /// <param name="commitUserData">Opaque Map (String->String)
        /// that's recorded into the segments file in the index,
        /// and retrievable by <see cref="IndexReader.GetCommitUserData" />.
        /// Note that when IndexWriter commits itself, during <see cref="Close()" />, the
        /// commitUserData is unchanged (just carried over from
        /// the prior commit).  If this is null then the previous
        /// commitUserData is kept.  Also, the commitUserData will
        /// only "stick" if there are actually changes in the
        /// index to commit.
        /// </param>
        private void PrepareCommit(IDictionary<string, string> commitUserData)
        {
            if (hitOOM)
            {
                throw new System.SystemException("this writer hit an OutOfMemoryError; cannot commit");
            }
            
            if (pendingCommit != null)
                throw new System.SystemException("prepareCommit was already called with no corresponding call to commit");
            
            if (infoStream != null)
                Message("prepareCommit: flush");
            
            Flush(true, true, true);
            
            StartCommit(0, commitUserData);
        }
        
        // Used only by commit, below; lock order is commitLock -> IW
        private Object commitLock = new Object();

        private void  Commit(long sizeInBytes)
        {
            lock(commitLock) {
                StartCommit(sizeInBytes, null);
                FinishCommit();
            }
        }
        
        /// <summary> <p/>Commits all pending changes (added &amp; deleted
        /// documents, optimizations, segment merges, added
        /// indexes, etc.) to the index, and syncs all referenced
        /// index files, such that a reader will see the changes
        /// and the index updates will survive an OS or machine
        /// crash or power loss.  Note that this does not wait for
        /// any running background merges to finish.  This may be a
        /// costly operation, so you should test the cost in your
        /// application and do it only when really necessary.<p/>
        /// 
        /// <p/> Note that this operation calls Directory.sync on
        /// the index files.  That call should not return until the
        /// file contents &amp; metadata are on stable storage.  For
        /// FSDirectory, this calls the OS's fsync.  But, beware:
        /// some hardware devices may in fact cache writes even
        /// during fsync, and return before the bits are actually
        /// on stable storage, to give the appearance of faster
        /// performance.  If you have such a device, and it does
        /// not have a battery backup (for example) then on power
        /// loss it may still lose data.  Lucene cannot guarantee
        /// consistency on such devices.  <p/>
        /// 
        /// <p/><b>NOTE</b>: if this method hits an OutOfMemoryError
        /// you should immediately close the writer.  See <a
        /// href="#OOME">above</a> for details.<p/>
        /// 
        /// </summary>
        /// <seealso cref="PrepareCommit()">
        /// </seealso>
        /// <seealso cref="Commit(IDictionary{string,string})">
        /// </seealso>
        public void  Commit()
        {
            Commit(null);
        }
        
        /// <summary>Commits all changes to the index, specifying a
        /// commitUserData Map (String -> String).  This just
        /// calls <see cref="PrepareCommit(IDictionary{string, string})" /> (if you didn't
        /// already call it) and then <see cref="FinishCommit" />.
        /// 
        /// <p/><b>NOTE</b>: if this method hits an OutOfMemoryError
        /// you should immediately close the writer.  See <a
        /// href="#OOME">above</a> for details.<p/>
        /// </summary>
        public void Commit(IDictionary<string, string> commitUserData)
        {
            EnsureOpen();

            if (infoStream != null)
            {
                Message("commit: start");
            }

            lock (commitLock)
            {
                if (infoStream != null)
                {
                    Message("commit: enter lock");
                }
                if (pendingCommit == null)
                {
                    if (infoStream != null)
                    {
                        Message("commit: now prepare");
                    }
                    PrepareCommit(commitUserData);
                }
                else if (infoStream != null)
                {
                    Message("commit: already prepared");
                }

                FinishCommit();
            }
        }
        
        private void  FinishCommit()
        {
            lock (this)
            {
                
                if (pendingCommit != null)
                {
                    try
                    {
                        if (infoStream != null)
                            Message("commit: pendingCommit != null");
                        pendingCommit.FinishCommit(directory);
                        if (infoStream != null)
                            Message("commit: wrote segments file \"" + pendingCommit.GetCurrentSegmentFileName() + "\"");
                        lastCommitChangeCount = pendingCommitChangeCount;
                        segmentInfos.UpdateGeneration(pendingCommit);
                        segmentInfos.UserData = pendingCommit.UserData;
                        SetRollbackSegmentInfos(pendingCommit);
                        deleter.Checkpoint(pendingCommit, true);
                    }
                    finally
                    {
                        deleter.DecRef(pendingCommit);
                        pendingCommit = null;
                        System.Threading.Monitor.PulseAll(this);
                    }
                }
                else if (infoStream != null)
                {
                    Message("commit: pendingCommit == null; skip");
                }

                if (infoStream != null)
                {
                    Message("commit: done");
                }
            }
        }
        
        /// <summary> Flush all in-memory buffered udpates (adds and deletes)
        /// to the Directory.
        /// </summary>
        /// <param name="triggerMerge">if true, we may merge segments (if
        /// deletes or docs were flushed) if necessary
        /// </param>
        /// <param name="flushDocStores">if false we are allowed to keep
        /// doc stores open to share with the next segment
        /// </param>
        /// <param name="flushDeletes">whether pending deletes should also
        /// be flushed
        /// </param>
        public /*protected internal*/ void  Flush(bool triggerMerge, bool flushDocStores, bool flushDeletes)
        {
            // We can be called during close, when closing==true, so we must pass false to ensureOpen:
            EnsureOpen(false);
            if (DoFlush(flushDocStores, flushDeletes) && triggerMerge)
                MaybeMerge();
        }
        
        // TODO: this method should not have to be entirely
        // synchronized, ie, merges should be allowed to commit
        // even while a flush is happening
        private bool DoFlush(bool flushDocStores, bool flushDeletes)
        {
            lock (this)
            {
                try
                {
                    try
                    {
                        return DoFlushInternal(flushDocStores, flushDeletes);
                    }
                    finally
                    {
                        if (docWriter.DoBalanceRAM())
                        {
                            docWriter.BalanceRAM();
                        }
                    }
                }
                finally
                {
                    docWriter.ClearFlushPending();
                }
            }
        }
        
        // TODO: this method should not have to be entirely
        // synchronized, ie, merges should be allowed to commit
        // even while a flush is happening
        private bool DoFlushInternal(bool flushDocStores, bool flushDeletes)
        {
            lock (this)
            {
                if (hitOOM)
                {
                    throw new System.SystemException("this writer hit an OutOfMemoryError; cannot flush");
                }
                
                EnsureOpen(false);
                
                System.Diagnostics.Debug.Assert(TestPoint("startDoFlush"));

                DoBeforeFlush();
                
                flushCount++;
                
                // If we are flushing because too many deletes
                // accumulated, then we should apply the deletes to free
                // RAM:
                flushDeletes |= docWriter.DoApplyDeletes();
                
                // Make sure no threads are actively adding a document.
                // Returns true if docWriter is currently aborting, in
                // which case we skip flushing this segment
                if (infoStream != null)
                {
                    Message("flush: now pause all indexing threads");
                }
                if (docWriter.PauseAllThreads())
                {
                    docWriter.ResumeAllThreads();
                    return false;
                }
                
                try
                {
                    
                    SegmentInfo newSegment = null;
                    
                    int numDocs = docWriter.NumDocsInRAM;
                    
                    // Always flush docs if there are any
                    bool flushDocs = numDocs > 0;
                    
                    System.String docStoreSegment = docWriter.DocStoreSegment;

                    System.Diagnostics.Debug.Assert(docStoreSegment != null || numDocs == 0, "dss=" + docStoreSegment + " numDocs=" + numDocs);
                    
                    if (docStoreSegment == null)
                        flushDocStores = false;
                    
                    int docStoreOffset = docWriter.DocStoreOffset;
                    
                    bool docStoreIsCompoundFile = false;
                    
                    if (infoStream != null)
                    {
                        Message("  flush: segment=" + docWriter.Segment + " docStoreSegment=" + docWriter.DocStoreSegment + " docStoreOffset=" + docStoreOffset + " flushDocs=" + flushDocs + " flushDeletes=" + flushDeletes + " flushDocStores=" + flushDocStores + " numDocs=" + numDocs + " numBufDelTerms=" + docWriter.GetNumBufferedDeleteTerms());
                        Message("  index before flush " + SegString());
                    }
                    
                    // Check if the doc stores must be separately flushed
                    // because other segments, besides the one we are about
                    // to flush, reference it
                    if (flushDocStores && (!flushDocs || !docWriter.Segment.Equals(docWriter.DocStoreSegment)))
                    {
                        // We must separately flush the doc store
                        if (infoStream != null)
                            Message("  flush shared docStore segment " + docStoreSegment);
                        
                        docStoreIsCompoundFile = FlushDocStores();
                        flushDocStores = false;
                    }
                    
                    System.String segment = docWriter.Segment;
                    
                    // If we are flushing docs, segment must not be null:
                    System.Diagnostics.Debug.Assert(segment != null || !flushDocs);
                    
                    if (flushDocs)
                    {
                        
                        bool success = false;
                        int flushedDocCount;
                        
                        try
                        {
                            flushedDocCount = docWriter.Flush(flushDocStores);
                            if (infoStream != null)
                            {
                                Message("flushedFiles=" + docWriter.GetFlushedFiles());
                            }
                            success = true;
                        }
                        finally
                        {
                            if (!success)
                            {
                                if (infoStream != null)
                                    Message("hit exception flushing segment " + segment);
                                deleter.Refresh(segment);
                            }
                        }
                        
                        if (0 == docStoreOffset && flushDocStores)
                        {
                            // This means we are flushing private doc stores
                            // with this segment, so it will not be shared
                            // with other segments
                            System.Diagnostics.Debug.Assert(docStoreSegment != null);
                            System.Diagnostics.Debug.Assert(docStoreSegment.Equals(segment));
                            docStoreOffset = - 1;
                            docStoreIsCompoundFile = false;
                            docStoreSegment = null;
                        }
                        
                        // Create new SegmentInfo, but do not add to our
                        // segmentInfos until deletes are flushed
                        // successfully.
                        newSegment = new SegmentInfo(segment, flushedDocCount, directory, false, true, docStoreOffset, docStoreSegment, docStoreIsCompoundFile, docWriter.HasProx());
                        SetDiagnostics(newSegment, "flush");
                    }
                    
                    docWriter.PushDeletes();
                    
                    if (flushDocs)
                    {
                        segmentInfos.Add(newSegment);
                        Checkpoint();
                    }
                    
                    if (flushDocs && mergePolicy.UseCompoundFile(segmentInfos, newSegment))
                    {
                        // Now build compound file
                        bool success = false;
                        try
                        {
                            docWriter.CreateCompoundFile(segment);
                            success = true;
                        }
                        finally
                        {
                            if (!success)
                            {
                                if (infoStream != null)
                                    Message("hit exception creating compound file for newly flushed segment " + segment);
                                deleter.DeleteFile(segment + "." + IndexFileNames.COMPOUND_FILE_EXTENSION);
                            }
                        }
                        
                        newSegment.SetUseCompoundFile(true);
                        Checkpoint();
                    }
                    
                    if (flushDeletes)
                    {
                        ApplyDeletes();
                    }
                    
                    if (flushDocs)
                        Checkpoint();
                    
                    DoAfterFlush();
                    
                    return flushDocs;
                }
                catch (System.OutOfMemoryException oom)
                {
                    HandleOOM(oom, "doFlush");
                    // never hit
                    return false;
                }
                finally
                {
                    docWriter.ResumeAllThreads();
                }
            }
        }
        
        /// <summary>Expert:  Return the total size of all index files currently cached in memory.
        /// Useful for size management with flushRamDocs()
        /// </summary>
        public long RamSizeInBytes()
        {
            EnsureOpen();
            return docWriter.GetRAMUsed();
        }
        
        /// <summary>Expert:  Return the number of documents currently
        /// buffered in RAM. 
        /// </summary>
        public int NumRamDocs()
        {
            lock (this)
            {
                EnsureOpen();
                return docWriter.NumDocsInRAM;
            }
        }
        
        private int EnsureContiguousMerge(MergePolicy.OneMerge merge)
        {
            
            int first = segmentInfos.IndexOf(merge.segments.Info(0));
            if (first == - 1)
                throw new MergePolicy.MergeException("could not find segment " + merge.segments.Info(0).name + " in current index " + SegString(), directory);
            
            int numSegments = segmentInfos.Count;
            
            int numSegmentsToMerge = merge.segments.Count;
            for (int i = 0; i < numSegmentsToMerge; i++)
            {
                SegmentInfo info = merge.segments.Info(i);
                
                if (first + i >= numSegments || !segmentInfos.Info(first + i).Equals(info))
                {
                    if (segmentInfos.IndexOf(info) == - 1)
                        throw new MergePolicy.MergeException("MergePolicy selected a segment (" + info.name + ") that is not in the current index " + SegString(), directory);
                    else
                        throw new MergePolicy.MergeException("MergePolicy selected non-contiguous segments to merge (" + merge.SegString(directory) + " vs " + SegString() + "), which IndexWriter (currently) cannot handle", directory);
                }
            }
            
            return first;
        }
        
        /// <summary>Carefully merges deletes for the segments we just
        /// merged.  This is tricky because, although merging will
        /// clear all deletes (compacts the documents), new
        /// deletes may have been flushed to the segments since
        /// the merge was started.  This method "carries over"
        /// such new deletes onto the newly merged segment, and
        /// saves the resulting deletes file (incrementing the
        /// delete generation for merge.info).  If no deletes were
        /// flushed, no new deletes file is saved. 
        /// </summary>
        private void  CommitMergedDeletes(MergePolicy.OneMerge merge, SegmentReader mergeReader)
        {
            lock (this)
            {
                
                System.Diagnostics.Debug.Assert(TestPoint("startCommitMergeDeletes"));
                
                SegmentInfos sourceSegments = merge.segments;
                
                if (infoStream != null)
                    Message("commitMergeDeletes " + merge.SegString(directory));
                
                // Carefully merge deletes that occurred after we
                // started merging:
                int docUpto = 0;
                int delCount = 0;
                
                for (int i = 0; i < sourceSegments.Count; i++)
                {
                    SegmentInfo info = sourceSegments.Info(i);
                    int docCount = info.docCount;
                    SegmentReader previousReader = merge.readersClone[i];
                    SegmentReader currentReader = merge.readers[i];
                    if (previousReader.HasDeletions)
                    {
                        
                        // There were deletes on this segment when the merge
                        // started.  The merge has collapsed away those
                        // deletes, but, if new deletes were flushed since
                        // the merge started, we must now carefully keep any
                        // newly flushed deletes but mapping them to the new
                        // docIDs.
                        
                        if (currentReader.NumDeletedDocs > previousReader.NumDeletedDocs)
                        {
                            // This means this segment has had new deletes
                            // committed since we started the merge, so we
                            // must merge them:
                            for (int j = 0; j < docCount; j++)
                            {
                                if (previousReader.IsDeleted(j))
                                {
                                    System.Diagnostics.Debug.Assert(currentReader.IsDeleted(j));
                                }
                                else
                                {
                                    if (currentReader.IsDeleted(j))
                                    {
                                        mergeReader.DoDelete(docUpto);
                                        delCount++;
                                    }
                                    docUpto++;
                                }
                            }
                        }
                        else
                        {
                            docUpto += docCount - previousReader.NumDeletedDocs;
                        }
                    }
                    else if (currentReader.HasDeletions)
                    {
                        // This segment had no deletes before but now it
                        // does:
                        for (int j = 0; j < docCount; j++)
                        {
                            if (currentReader.IsDeleted(j))
                            {
                                mergeReader.DoDelete(docUpto);
                                delCount++;
                            }
                            docUpto++;
                        }
                    }
                    // No deletes before or after
                    else
                        docUpto += info.docCount;
                }
                
                System.Diagnostics.Debug.Assert(mergeReader.NumDeletedDocs == delCount);
                
                mergeReader.hasChanges = delCount > 0;
            }
        }
        
        /* FIXME if we want to support non-contiguous segment merges */
        private bool CommitMerge(MergePolicy.OneMerge merge, SegmentMerger merger, int mergedDocCount, SegmentReader mergedReader)
        {
            lock (this)
            {
                
                System.Diagnostics.Debug.Assert(TestPoint("startCommitMerge"));
                
                if (hitOOM)
                {
                    throw new System.SystemException("this writer hit an OutOfMemoryError; cannot complete merge");
                }
                
                if (infoStream != null)
                    Message("commitMerge: " + merge.SegString(directory) + " index=" + SegString());
                
                System.Diagnostics.Debug.Assert(merge.registerDone);
                
                // If merge was explicitly aborted, or, if rollback() or
                // rollbackTransaction() had been called since our merge
                // started (which results in an unqualified
                // deleter.refresh() call that will remove any index
                // file that current segments does not reference), we
                // abort this merge
                if (merge.IsAborted())
                {
                    if (infoStream != null)
                        Message("commitMerge: skipping merge " + merge.SegString(directory) + ": it was aborted");
                    
                    return false;
                }
                
                int start = EnsureContiguousMerge(merge);
                
                CommitMergedDeletes(merge, mergedReader);
                docWriter.RemapDeletes(segmentInfos, merger.GetDocMaps(), merger.GetDelCounts(), merge, mergedDocCount);

                // If the doc store we are using has been closed and
                // is in now compound format (but wasn't when we
                // started), then we will switch to the compound
                // format as well:
                SetMergeDocStoreIsCompoundFile(merge);
                
                merge.info.HasProx = merger.HasProx();
                
                segmentInfos.RemoveRange(start, start + merge.segments.Count - start);
                System.Diagnostics.Debug.Assert(!segmentInfos.Contains(merge.info));
                segmentInfos.Insert(start, merge.info);

                CloseMergeReaders(merge, false);
                
                // Must note the change to segmentInfos so any commits
                // in-flight don't lose it:
                Checkpoint();
                
                // If the merged segments had pending changes, clear
                // them so that they don't bother writing them to
                // disk, updating SegmentInfo, etc.:
                readerPool.Clear(merge.segments);

                if (merge.optimize)
                {
                    // cascade the optimize:
                    segmentsToOptimize.Add(merge.info);
                }
                return true;
            }
        }
        
        private void  HandleMergeException(System.Exception t, MergePolicy.OneMerge merge)
        {
            
            if (infoStream != null)
            {
                Message("handleMergeException: merge=" + merge.SegString(directory) + " exc=" + t);
            }
            
            // Set the exception on the merge, so if
            // optimize() is waiting on us it sees the root
            // cause exception:
            merge.SetException(t);
            AddMergeException(merge);
            
            if (t is MergePolicy.MergeAbortedException)
            {
                // We can ignore this exception (it happens when
                // close(false) or rollback is called), unless the
                // merge involves segments from external directories,
                // in which case we must throw it so, for example, the
                // rollbackTransaction code in addIndexes* is
                // executed.
                if (merge.isExternal)
                    throw t;
            }
            else if (t is System.IO.IOException || t is System.SystemException || t is System.ApplicationException)
            {
                throw t;
            }
            else
            {
                // Should not get here
                System.Diagnostics.Debug.Fail("Exception is not expected type!");
                throw new System.SystemException(null, t);
            }
        }
        
        public void Merge_ForNUnit(MergePolicy.OneMerge merge)
        {
            Merge(merge);
        }
        /// <summary> Merges the indicated segments, replacing them in the stack with a
        /// single segment.
        /// </summary>
        internal void  Merge(MergePolicy.OneMerge merge)
        {
            
            bool success = false;
            
            try
            {
                try
                {
                    try
                    {
                        MergeInit(merge);
                        
                        if (infoStream != null)
                        {
                            Message("now merge\n  merge=" + merge.SegString(directory) + "\n  merge=" + merge + "\n  index=" + SegString());
                        }
                        
                        MergeMiddle(merge);
                        MergeSuccess(merge);
                        success = true;
                    }
                    catch (System.Exception t)
                    {
                        HandleMergeException(t, merge);
                    }
                }
                finally
                {
                    lock (this)
                    {
                        MergeFinish(merge);
                        
                        if (!success)
                        {
                            if (infoStream != null)
                                Message("hit exception during merge");
                            if (merge.info != null && !segmentInfos.Contains(merge.info))
                                deleter.Refresh(merge.info.name);
                        }
                        
                        // This merge (and, generally, any change to the
                        // segments) may now enable new merges, so we call
                        // merge policy & update pending merges.
                        if (success && !merge.IsAborted() && !closed && !closing)
                            UpdatePendingMerges(merge.maxNumSegmentsOptimize, merge.optimize);
                    }
                }
            }
            catch (System.OutOfMemoryException oom)
            {
                HandleOOM(oom, "merge");
            }
        }
        
        /// <summary>Hook that's called when the specified merge is complete. </summary>
        internal virtual void  MergeSuccess(MergePolicy.OneMerge merge)
        {
        }
        
        /// <summary>Checks whether this merge involves any segments
        /// already participating in a merge.  If not, this merge
        /// is "registered", meaning we record that its segments
        /// are now participating in a merge, and true is
        /// returned.  Else (the merge conflicts) false is
        /// returned. 
        /// </summary>
        internal bool RegisterMerge(MergePolicy.OneMerge merge)
        {
            lock (this)
            {
                
                if (merge.registerDone)
                    return true;
                
                if (stopMerges)
                {
                    merge.Abort();
                    throw new MergePolicy.MergeAbortedException("merge is aborted: " + merge.SegString(directory));
                }
                
                int count = merge.segments.Count;
                bool isExternal = false;
                for (int i = 0; i < count; i++)
                {
                    SegmentInfo info = merge.segments.Info(i);
                    if (mergingSegments.Contains(info))
                    {
                        return false;
                    }
                    if (segmentInfos.IndexOf(info) == -1)
                    {
                        return false;
                    }
                    if (info.dir != directory)
                    {
                        isExternal = true;
                    }
                    if (segmentsToOptimize.Contains(info))
                    {
                        merge.optimize = true;
                        merge.maxNumSegmentsOptimize = optimizeMaxNumSegments;
                    }
                }
                
                EnsureContiguousMerge(merge);
                
                pendingMerges.AddLast(merge);
                
                if (infoStream != null)
                    Message("add merge to pendingMerges: " + merge.SegString(directory) + " [total " + pendingMerges.Count + " pending]");
                
                merge.mergeGen = mergeGen;
                merge.isExternal = isExternal;
                
                // OK it does not conflict; now record that this merge
                // is running (while synchronized) to avoid race
                // condition where two conflicting merges from different
                // threads, start
                for (int i = 0; i < count; i++)
                {
                    SegmentInfo si = merge.segments.Info(i);
                    mergingSegments.Add(si);
                }
                
                // Merge is now registered
                merge.registerDone = true;
                return true;
            }
        }
        
        /// <summary>Does initial setup for a merge, which is fast but holds
        /// the synchronized lock on IndexWriter instance.  
        /// </summary>
        internal void  MergeInit(MergePolicy.OneMerge merge)
        {
            lock (this)
            {
                bool success = false;
                try
                {
                    _MergeInit(merge);
                    success = true;
                }
                finally
                {
                    if (!success)
                    {
                        MergeFinish(merge);
                    }
                }
            }
        }
        
        private void  _MergeInit(MergePolicy.OneMerge merge)
        {
            lock (this)
            {
                
                System.Diagnostics.Debug.Assert(TestPoint("startMergeInit"));
                
                System.Diagnostics.Debug.Assert(merge.registerDone);
                System.Diagnostics.Debug.Assert(!merge.optimize || merge.maxNumSegmentsOptimize > 0);
                
                if (hitOOM)
                {
                    throw new System.SystemException("this writer hit an OutOfMemoryError; cannot merge");
                }
                
                if (merge.info != null)
                // mergeInit already done
                    return ;
                
                if (merge.IsAborted())
                    return ;
                
                ApplyDeletes();
                
                SegmentInfos sourceSegments = merge.segments;
                int end = sourceSegments.Count;
                
                // Check whether this merge will allow us to skip
                // merging the doc stores (stored field & vectors).
                // This is a very substantial optimization (saves tons
                // of IO).
                
                Directory lastDir = directory;
                System.String lastDocStoreSegment = null;
                int next = - 1;
                
                bool mergeDocStores = false;
                bool doFlushDocStore = false;
                System.String currentDocStoreSegment = docWriter.DocStoreSegment;
                
                // Test each segment to be merged: check if we need to
                // flush/merge doc stores
                for (int i = 0; i < end; i++)
                {
                    SegmentInfo si = sourceSegments.Info(i);
                    
                    // If it has deletions we must merge the doc stores
                    if (si.HasDeletions())
                        mergeDocStores = true;
                    
                    // If it has its own (private) doc stores we must
                    // merge the doc stores
                    if (- 1 == si.DocStoreOffset)
                        mergeDocStores = true;
                    
                    // If it has a different doc store segment than
                    // previous segments, we must merge the doc stores
                    System.String docStoreSegment = si.DocStoreSegment;
                    if (docStoreSegment == null)
                        mergeDocStores = true;
                    else if (lastDocStoreSegment == null)
                        lastDocStoreSegment = docStoreSegment;
                    else if (!lastDocStoreSegment.Equals(docStoreSegment))
                        mergeDocStores = true;
                    
                    // Segments' docScoreOffsets must be in-order,
                    // contiguous.  For the default merge policy now
                    // this will always be the case but for an arbitrary
                    // merge policy this may not be the case
                    if (- 1 == next)
                        next = si.DocStoreOffset + si.docCount;
                    else if (next != si.DocStoreOffset)
                        mergeDocStores = true;
                    else
                        next = si.DocStoreOffset + si.docCount;
                    
                    // If the segment comes from a different directory
                    // we must merge
                    if (lastDir != si.dir)
                        mergeDocStores = true;
                    
                    // If the segment is referencing the current "live"
                    // doc store outputs then we must merge
                    if (si.DocStoreOffset != - 1 && currentDocStoreSegment != null && si.DocStoreSegment.Equals(currentDocStoreSegment))
                    {
                        doFlushDocStore = true;
                    }
                }

                // if a mergedSegmentWarmer is installed, we must merge
                // the doc stores because we will open a full
                // SegmentReader on the merged segment:
                if (!mergeDocStores && mergedSegmentWarmer != null && currentDocStoreSegment != null && lastDocStoreSegment != null && lastDocStoreSegment.Equals(currentDocStoreSegment))
                {
                    mergeDocStores = true;
                }

                int docStoreOffset;
                System.String docStoreSegment2;
                bool docStoreIsCompoundFile;
                
                if (mergeDocStores)
                {
                    docStoreOffset = - 1;
                    docStoreSegment2 = null;
                    docStoreIsCompoundFile = false;
                }
                else
                {
                    SegmentInfo si = sourceSegments.Info(0);
                    docStoreOffset = si.DocStoreOffset;
                    docStoreSegment2 = si.DocStoreSegment;
                    docStoreIsCompoundFile = si.DocStoreIsCompoundFile;
                }
                
                if (mergeDocStores && doFlushDocStore)
                {
                    // SegmentMerger intends to merge the doc stores
                    // (stored fields, vectors), and at least one of the
                    // segments to be merged refers to the currently
                    // live doc stores.
                    
                    // TODO: if we know we are about to merge away these
                    // newly flushed doc store files then we should not
                    // make compound file out of them...
                    if (infoStream != null)
                        Message("now flush at merge");
                    DoFlush(true, false);
                }
                
                merge.mergeDocStores = mergeDocStores;
                
                // Bind a new segment name here so even with
                // ConcurrentMergePolicy we keep deterministic segment
                // names.
                merge.info = new SegmentInfo(NewSegmentName(), 0, directory, false, true, docStoreOffset, docStoreSegment2, docStoreIsCompoundFile, false);


                IDictionary<string, string> details = new Dictionary<string, string>();
                details["optimize"] = merge.optimize + "";
                details["mergeFactor"] = end + "";
                details["mergeDocStores"] = mergeDocStores + "";
                SetDiagnostics(merge.info, "merge", details);
                
                // Also enroll the merged segment into mergingSegments;
                // this prevents it from getting selected for a merge
                // after our merge is done but while we are building the
                // CFS:
                mergingSegments.Add(merge.info);
            }
        }
        
        private void  SetDiagnostics(SegmentInfo info, System.String source)
        {
            SetDiagnostics(info, source, null);
        }

        private void SetDiagnostics(SegmentInfo info, System.String source, IDictionary<string, string> details)
        {
            IDictionary<string, string> diagnostics = new Dictionary<string,string>();
            diagnostics["source"] = source;
            diagnostics["lucene.version"] = Constants.LUCENE_VERSION;
            diagnostics["os"] = Constants.OS_NAME + "";
            diagnostics["os.arch"] = Constants.OS_ARCH + "";
            diagnostics["os.version"] = Constants.OS_VERSION + "";
            diagnostics["java.version"] = Constants.JAVA_VERSION + "";
            diagnostics["java.vendor"] = Constants.JAVA_VENDOR + "";
            if (details != null)
            {
                //System.Collections.ArrayList keys = new System.Collections.ArrayList(details.Keys);
                //System.Collections.ArrayList values = new System.Collections.ArrayList(details.Values);
                foreach (string key in details.Keys)
                {
                    diagnostics[key] = details[key];
                }
            }
            info.Diagnostics = diagnostics;
        }

        /// <summary>Does fininishing for a merge, which is fast but holds
        /// the synchronized lock on IndexWriter instance. 
        /// </summary>
        internal void  MergeFinish(MergePolicy.OneMerge merge)
        {
            lock (this)
            {
                
                // Optimize, addIndexes or finishMerges may be waiting
                // on merges to finish.
                System.Threading.Monitor.PulseAll(this);
                
                // It's possible we are called twice, eg if there was an
                // exception inside mergeInit
                if (merge.registerDone)
                {
                    SegmentInfos sourceSegments = merge.segments;
                    int end = sourceSegments.Count;
                    for (int i = 0; i < end; i++)
                        mergingSegments.Remove(sourceSegments.Info(i));
                    if(merge.info != null)
                        mergingSegments.Remove(merge.info);
                    merge.registerDone = false;
                }
                
                runningMerges.Remove(merge);
            }
        }
        
        private void SetMergeDocStoreIsCompoundFile(MergePolicy.OneMerge merge)
        {
            lock (this)
            {
                string mergeDocStoreSegment = merge.info.DocStoreSegment;
                if (mergeDocStoreSegment != null && !merge.info.DocStoreIsCompoundFile)
                {
                    int size = segmentInfos.Count;
                    for (int i = 0; i < size; i++)
                    {
                        SegmentInfo info = segmentInfos.Info(i);
                        string docStoreSegment = info.DocStoreSegment;
                        if (docStoreSegment != null &&
                            docStoreSegment.Equals(mergeDocStoreSegment) &&
                            info.DocStoreIsCompoundFile)
                        {
                            merge.info.DocStoreIsCompoundFile = true;
                            break;
                        }
                    }
                }
            }
        }

        private void CloseMergeReaders(MergePolicy.OneMerge merge, bool suppressExceptions)
        {
            lock (this)
            {
                int numSegments = merge.segments.Count;
                if (suppressExceptions)
                {
                    // Suppress any new exceptions so we throw the
                    // original cause
                    for (int i = 0; i < numSegments; i++)
                    {
                        if (merge.readers[i] != null)
                        {
                            try
                            {
                                readerPool.Release(merge.readers[i], false);
                            }
                            catch (Exception)
                            {
                            }
                            merge.readers[i] = null;
                        }

                        if (merge.readersClone[i] != null)
                        {
                            try
                            {
                                merge.readersClone[i].Close();
                            }
                            catch (Exception)
                            {
                            }
                            // This was a private clone and we had the
                            // only reference
                            System.Diagnostics.Debug.Assert(merge.readersClone[i].RefCount == 0); //: "refCount should be 0 but is " + merge.readersClone[i].getRefCount();
                            merge.readersClone[i] = null;
                        }
                    }
                }
                else
                {
                    for (int i = 0; i < numSegments; i++)
                    {
                        if (merge.readers[i] != null)
                        {
                            readerPool.Release(merge.readers[i], true);
                            merge.readers[i] = null;
                        }

                        if (merge.readersClone[i] != null)
                        {
                            merge.readersClone[i].Close();
                            // This was a private clone and we had the only reference
                            System.Diagnostics.Debug.Assert(merge.readersClone[i].RefCount == 0);
                            merge.readersClone[i] = null;
                        }
                    }
                }
            }
        }


        /// <summary>Does the actual (time-consuming) work of the merge,
        /// but without holding synchronized lock on IndexWriter
        /// instance 
        /// </summary>
        private int MergeMiddle(MergePolicy.OneMerge merge)
        {
            
            merge.CheckAborted(directory);
            
            System.String mergedName = merge.info.name;
            
            SegmentMerger merger = null;
            
            int mergedDocCount = 0;
            
            SegmentInfos sourceSegments = merge.segments;
            int numSegments = sourceSegments.Count;
            
            if (infoStream != null)
                Message("merging " + merge.SegString(directory));
            
            merger = new SegmentMerger(this, mergedName, merge);
            
            merge.readers = new SegmentReader[numSegments];
            merge.readersClone = new SegmentReader[numSegments];
            
            bool mergeDocStores = false;

            String currentDocStoreSegment;
            lock(this) {
                currentDocStoreSegment = docWriter.DocStoreSegment;
            }
            bool currentDSSMerged = false;

            // This is try/finally to make sure merger's readers are
            // closed:
            bool success = false;
            try
            {
                int totDocCount = 0;

                for (int i = 0; i < numSegments; i++)
                {

                    SegmentInfo info = sourceSegments.Info(i);

                    // Hold onto the "live" reader; we will use this to
                    // commit merged deletes
                    SegmentReader reader = merge.readers[i] = readerPool.Get(info, merge.mergeDocStores, MERGE_READ_BUFFER_SIZE, -1);

                    // We clone the segment readers because other
                    // deletes may come in while we're merging so we
                    // need readers that will not change
                    SegmentReader clone = merge.readersClone[i] = (SegmentReader)reader.Clone(true);
                    merger.Add(clone);

                    if (clone.HasDeletions)
                    {
                        mergeDocStores = true;
                    }

                    if (info.DocStoreOffset != -1 && currentDocStoreSegment != null)
                    {
                        currentDSSMerged |= currentDocStoreSegment.Equals(info.DocStoreSegment);
                    }

                    totDocCount += clone.NumDocs();
                }

                if (infoStream != null)
                {
                    Message("merge: total " + totDocCount + " docs");
                }

                merge.CheckAborted(directory);

                // If deletions have arrived and it has now become
                // necessary to merge doc stores, go and open them:
                if (mergeDocStores && !merge.mergeDocStores)
                {
                    merge.mergeDocStores = true;
                    lock (this)
                    {
                        if (currentDSSMerged)
                        {
                            if (infoStream != null)
                            {
                                Message("now flush at mergeMiddle");
                            }
                            DoFlush(true, false);
                        }
                    }

                    for (int i = 0; i < numSegments; i++)
                    {
                        merge.readersClone[i].OpenDocStores();
                    }

                    // Clear DSS
                    merge.info.SetDocStore(-1, null, false);

                }

                // This is where all the work happens:
                mergedDocCount = merge.info.docCount = merger.Merge(merge.mergeDocStores);

                System.Diagnostics.Debug.Assert(mergedDocCount == totDocCount);

                if (merge.useCompoundFile)
                {

                    success = false;
                    string compoundFileName = IndexFileNames.SegmentFileName(mergedName, IndexFileNames.COMPOUND_FILE_EXTENSION);

                    try
                    {
                        if (infoStream != null)
                        {
                            Message("create compound file " + compoundFileName);
                        }
                        merger.CreateCompoundFile(compoundFileName);
                        success = true;
                    }
                    catch (System.IO.IOException ioe)
                    {
                        lock (this)
                        {
                            if (merge.IsAborted())
                            {
                                // This can happen if rollback or close(false)
                                // is called -- fall through to logic below to
                                // remove the partially created CFS:
                            }
                            else
                            {
                                HandleMergeException(ioe, merge);
                            }
                        }
                    }
                    catch (Exception t)
                    {
                        HandleMergeException(t, merge);
                    }
                    finally
                    {
                        if (!success)
                        {
                            if (infoStream != null)
                            {
                                Message("hit exception creating compound file during merge");
                            }

                            lock (this)
                            {
                                deleter.DeleteFile(compoundFileName);
                                deleter.DeleteNewFiles(merger.GetMergedFiles());
                            }
                        }
                    }

                    success = false;

                    lock (this)
                    {

                        // delete new non cfs files directly: they were never
                        // registered with IFD
                        deleter.DeleteNewFiles(merger.GetMergedFiles());

                        if (merge.IsAborted())
                        {
                            if (infoStream != null)
                            {
                                Message("abort merge after building CFS");
                            }
                            deleter.DeleteFile(compoundFileName);
                            return 0;
                        }
                    }

                    merge.info.SetUseCompoundFile(true);
                }

                int termsIndexDivisor;
                bool loadDocStores;

                // if the merged segment warmer was not installed when
                // this merge was started, causing us to not force
                // the docStores to close, we can't warm it now
                bool canWarm = merge.info.DocStoreSegment == null || currentDocStoreSegment == null || !merge.info.DocStoreSegment.Equals(currentDocStoreSegment);

                if (poolReaders && mergedSegmentWarmer != null && canWarm)
                {
                    // Load terms index & doc stores so the segment
                    // warmer can run searches, load documents/term
                    // vectors
                    termsIndexDivisor = readerTermsIndexDivisor;
                    loadDocStores = true;
                }
                else
                {
                    termsIndexDivisor = -1;
                    loadDocStores = false;
                }

                // TODO: in the non-realtime case, we may want to only
                // keep deletes (it's costly to open entire reader
                // when we just need deletes)

                SegmentReader mergedReader = readerPool.Get(merge.info, loadDocStores, BufferedIndexInput.BUFFER_SIZE, termsIndexDivisor);
                try
                {
                    if (poolReaders && mergedSegmentWarmer != null)
                    {
                        mergedSegmentWarmer.Warm(mergedReader);
                    }
                    if (!CommitMerge(merge, merger, mergedDocCount, mergedReader))
                    {
                        // commitMerge will return false if this merge was aborted
                        return 0;
                    }
                }
                finally
                {
                    lock (this)
                    {
                        readerPool.Release(mergedReader);
                    }
                }

                success = true;
            }
            finally
            {
                // Readers are already closed in commitMerge if we didn't hit
                // an exc:
                if (!success)
                {
                    CloseMergeReaders(merge, true);
                }
            }

            return mergedDocCount;
        }
        
        internal virtual void  AddMergeException(MergePolicy.OneMerge merge)
        {
            lock (this)
            {
                System.Diagnostics.Debug.Assert(merge.GetException() != null);
                if (!mergeExceptions.Contains(merge) && mergeGen == merge.mergeGen)
                    mergeExceptions.Add(merge);
            }
        }
        
        // Apply buffered deletes to all segments.
        private bool ApplyDeletes()
        {
            lock (this)
            {
                System.Diagnostics.Debug.Assert(TestPoint("startApplyDeletes"));
                flushDeletesCount++;
                
                bool success = false;
                bool changed;
                try
                {
                    changed = docWriter.ApplyDeletes(segmentInfos);
                    success = true;
                }
                finally
                {
                    if (!success && infoStream != null)
                    {
                        Message("hit exception flushing deletes");
                    }
                }
                
                if (changed)
                    Checkpoint();
                return changed;
            }
        }
        
        // For test purposes.
        internal int GetBufferedDeleteTermsSize()
        {
            lock (this)
            {
                return docWriter.GetBufferedDeleteTerms().Count;
            }
        }
        
        // For test purposes.
        internal int GetNumBufferedDeleteTerms()
        {
            lock (this)
            {
                return docWriter.GetNumBufferedDeleteTerms();
            }
        }
        
        // utility routines for tests
        public /*internal*/ virtual SegmentInfo NewestSegment()
        {
            return segmentInfos.Count > 0 ? segmentInfos.Info(segmentInfos.Count - 1) : null;
        }
        
        public virtual System.String SegString()
        {
            lock (this)
            {
                return SegString(segmentInfos);
            }
        }
        
        private System.String SegString(SegmentInfos infos)
        {
            lock (this)
            {
                System.Text.StringBuilder buffer = new System.Text.StringBuilder();
                int count = infos.Count;
                for (int i = 0; i < count; i++)
                {
                    if (i > 0)
                    {
                        buffer.Append(' ');
                    }
                    SegmentInfo info = infos.Info(i);
                    buffer.Append(info.SegString(directory));
                    if (info.dir != directory)
                        buffer.Append("**");
                }
                return buffer.ToString();
            }
        }
        
        // Files that have been sync'd already
        private HashSet<string> synced = new HashSet<string>();
        
        // Files that are now being sync'd
        private HashSet<string> syncing = new HashSet<string>();
        
        private bool StartSync(System.String fileName, ICollection<string> pending)
        {
            lock (synced)
            {
                if (!synced.Contains(fileName))
                {
                    if (!syncing.Contains(fileName))
                    {
                        syncing.Add(fileName);
                        return true;
                    }
                    else
                    {
                        pending.Add(fileName);
                        return false;
                    }
                }
                else
                    return false;
            }
        }
        
        private void  FinishSync(System.String fileName, bool success)
        {
            lock (synced)
            {
                System.Diagnostics.Debug.Assert(syncing.Contains(fileName));
                syncing.Remove(fileName);
                if (success)
                    synced.Add(fileName);
                System.Threading.Monitor.PulseAll(synced);
            }
        }
        
        /// <summary>Blocks until all files in syncing are sync'd </summary>
        private bool WaitForAllSynced(ICollection<System.String> syncing)
        {
            lock (synced)
            {
                IEnumerator<string> it = syncing.GetEnumerator();
                while (it.MoveNext())
                {
                    System.String fileName = it.Current;
                    while (!synced.Contains(fileName))
                    {
                        if (!syncing.Contains(fileName))
                        // There was an error because a file that was
                        // previously syncing failed to appear in synced
                            return false;
                        else
                            System.Threading.Monitor.Wait(synced);
                            
                    }
                }
                return true;
            }
        }
        
        private void  DoWait()
        {
            lock (this)
            {
                // NOTE: the callers of this method should in theory
                // be able to do simply wait(), but, as a defense
                // against thread timing hazards where notifyAll()
                // falls to be called, we wait for at most 1 second
                // and then return so caller can check if wait
                // conditions are satisified:
                System.Threading.Monitor.Wait(this, TimeSpan.FromMilliseconds(1000));
                
            }
        }
        
        /// <summary>Walk through all files referenced by the current
        /// segmentInfos and ask the Directory to sync each file,
        /// if it wasn't already.  If that succeeds, then we
        /// prepare a new segments_N file but do not fully commit
        /// it. 
        /// </summary>
        private void StartCommit(long sizeInBytes, IDictionary<string, string> commitUserData)
        {
            
            System.Diagnostics.Debug.Assert(TestPoint("startStartCommit"));

            // TODO: as of LUCENE-2095, we can simplify this method,
            // since only 1 thread can be in here at once
            
            if (hitOOM)
            {
                throw new System.SystemException("this writer hit an OutOfMemoryError; cannot commit");
            }
            
            try
            {
                
                if (infoStream != null)
                    Message("startCommit(): start sizeInBytes=" + sizeInBytes);
                
                SegmentInfos toSync = null;
                long myChangeCount;
                
                lock (this)
                {
                    // Wait for any running addIndexes to complete
                    // first, then block any from running until we've
                    // copied the segmentInfos we intend to sync:
                    BlockAddIndexes(false);
                    
                    // On commit the segmentInfos must never
                    // reference a segment in another directory:
                    System.Diagnostics.Debug.Assert(!HasExternalSegments());
                    
                    try
                    {
                        
                        System.Diagnostics.Debug.Assert(lastCommitChangeCount <= changeCount);
                        myChangeCount = changeCount;
                        
                        if (changeCount == lastCommitChangeCount)
                        {
                            if (infoStream != null)
                                Message("  skip startCommit(): no changes pending");
                            return ;
                        }
                        
                        // First, we clone & incref the segmentInfos we intend
                        // to sync, then, without locking, we sync() each file
                        // referenced by toSync, in the background.  Multiple
                        // threads can be doing this at once, if say a large
                        // merge and a small merge finish at the same time:
                        
                        if (infoStream != null)
                            Message("startCommit index=" + SegString(segmentInfos) + " changeCount=" + changeCount);

                        readerPool.Commit();
                        
                        // It's possible another flush (that did not close
                        // the open do stores) snuck in after the flush we
                        // just did, so we remove any tail segments
                        // referencing the open doc store from the
                        // SegmentInfos we are about to sync (the main
                        // SegmentInfos will keep them):
                        toSync = (SegmentInfos) segmentInfos.Clone();
                        string dss = docWriter.DocStoreSegment;
                        if (dss != null)
                        {
                            while (true)
                            {
                                String dss2 = toSync.Info(toSync.Count - 1).DocStoreSegment;
                                if (dss2 == null || !dss2.Equals(dss))
                                {
                                    break;
                                }
                                toSync.RemoveAt(toSync.Count - 1);
                                changeCount++;
                            }
                        }
                        
                        if (commitUserData != null)
                            toSync.UserData = commitUserData;
                        
                        deleter.IncRef(toSync, false);
                                                
                        ICollection<string> files = toSync.Files(directory, false);
                        foreach(string fileName in files)
                        {
                            System.Diagnostics.Debug.Assert(directory.FileExists(fileName), "file " + fileName + " does not exist");
                            // If this trips it means we are missing a call to
                            // .checkpoint somewhere, because by the time we
                            // are called, deleter should know about every
                            // file referenced by the current head
                            // segmentInfos:
                            System.Diagnostics.Debug.Assert(deleter.Exists(fileName));
                        }
                    }
                    finally
                    {
                        ResumeAddIndexes();
                    }
                }
                
                System.Diagnostics.Debug.Assert(TestPoint("midStartCommit"));
                
                bool setPending = false;
                
                try
                {
                    // Loop until all files toSync references are sync'd:
                    while (true)
                    {
                        ICollection<string> pending = new List<string>();
                        
                        IEnumerator<string> it = toSync.Files(directory, false).GetEnumerator();
                        while (it.MoveNext())
                        {
                            string fileName = it.Current;
                            if (StartSync(fileName, pending))
                            {
                                bool success = false;
                                try
                                {
                                    // Because we incRef'd this commit point, above,
                                    // the file had better exist:
                                    System.Diagnostics.Debug.Assert(directory.FileExists(fileName), "file '" + fileName + "' does not exist dir=" + directory);
                                    if (infoStream != null)
                                        Message("now sync " + fileName);
                                    directory.Sync(fileName);
                                    success = true;
                                }
                                finally
                                {
                                    FinishSync(fileName, success);
                                }
                            }
                        }
                        
                        // All files that I require are either synced or being
                        // synced by other threads.  If they are being synced,
                        // we must at this point block until they are done.
                        // If this returns false, that means an error in
                        // another thread resulted in failing to actually
                        // sync one of our files, so we repeat:
                        if (WaitForAllSynced(pending))
                            break;
                    }
                    
                    System.Diagnostics.Debug.Assert(TestPoint("midStartCommit2"));
                    
                    lock (this)
                    {
                        // If someone saved a newer version of segments file
                        // since I first started syncing my version, I can
                        // safely skip saving myself since I've been
                        // superseded:
                        
                        while (true)
                        {
                            if (myChangeCount <= lastCommitChangeCount)
                            {
                                if (infoStream != null)
                                {
                                    Message("sync superseded by newer infos");
                                }
                                break;
                            }
                            else if (pendingCommit == null)
                            {
                                // My turn to commit
                                
                                if (segmentInfos.Generation > toSync.Generation)
                                    toSync.UpdateGeneration(segmentInfos);
                                
                                bool success = false;
                                try
                                {
                                    
                                    // Exception here means nothing is prepared
                                    // (this method unwinds everything it did on
                                    // an exception)
                                    try
                                    {
                                        toSync.PrepareCommit(directory);
                                    }
                                    finally
                                    {
                                        // Have our master segmentInfos record the
                                        // generations we just prepared.  We do this
                                        // on error or success so we don't
                                        // double-write a segments_N file.
                                        segmentInfos.UpdateGeneration(toSync);
                                    }
                                    
                                    System.Diagnostics.Debug.Assert(pendingCommit == null);
                                    setPending = true;
                                    pendingCommit = toSync;
                                    pendingCommitChangeCount = (uint) myChangeCount;
                                    success = true;
                                }
                                finally
                                {
                                    if (!success && infoStream != null)
                                        Message("hit exception committing segments file");
                                }
                                break;
                            }
                            else
                            {
                                // Must wait for other commit to complete
                                DoWait();
                            }
                        }
                    }
                    
                    if (infoStream != null)
                        Message("done all syncs");
                    
                    System.Diagnostics.Debug.Assert(TestPoint("midStartCommitSuccess"));
                }
                finally
                {
                    lock (this)
                    {
                        if (!setPending)
                            deleter.DecRef(toSync);
                    }
                }
            }
            catch (System.OutOfMemoryException oom)
            {
                HandleOOM(oom, "startCommit");
            }
            System.Diagnostics.Debug.Assert(TestPoint("finishStartCommit"));
        }
        
        /// <summary> Returns <c>true</c> iff the index in the named directory is
        /// currently locked.
        /// </summary>
        /// <param name="directory">the directory to check for a lock
        /// </param>
        /// <throws>  IOException if there is a low-level IO error </throws>
        public static bool IsLocked(Directory directory)
        {
            return directory.MakeLock(WRITE_LOCK_NAME).IsLocked();
        }
        
        /// <summary> Forcibly unlocks the index in the named directory.
        /// <p/>
        /// Caution: this should only be used by failure recovery code,
        /// when it is known that no other process nor thread is in fact
        /// currently accessing this index.
        /// </summary>
        public static void  Unlock(Directory directory)
        {
            directory.MakeLock(IndexWriter.WRITE_LOCK_NAME).Release();
        }
        
        /// <summary> Specifies maximum field length (in number of tokens/terms) in <see cref="IndexWriter" /> constructors.
        /// <see cref="SetMaxFieldLength(int)" /> overrides the value set by
        /// the constructor.
        /// </summary>
        public sealed class MaxFieldLength
        {
            
            private int limit;
            private System.String name;
            
            /// <summary> Private type-safe-enum-pattern constructor.
            /// 
            /// </summary>
            /// <param name="name">instance name
            /// </param>
            /// <param name="limit">maximum field length
            /// </param>
            internal MaxFieldLength(System.String name, int limit)
            {
                this.name = name;
                this.limit = limit;
            }
            
            /// <summary> Public constructor to allow users to specify the maximum field size limit.
            /// 
            /// </summary>
            /// <param name="limit">The maximum field length
            /// </param>
            public MaxFieldLength(int limit):this("User-specified", limit)
            {
            }

            public int Limit
            {
                get { return limit; }
            }

            public override System.String ToString()
            {
                return name + ":" + limit;
            }
            
            /// <summary>Sets the maximum field length to <see cref="int.MaxValue" />. </summary>
            public static readonly MaxFieldLength UNLIMITED = new MaxFieldLength("UNLIMITED", System.Int32.MaxValue);
            
            /// <summary>  Sets the maximum field length to 
            /// <see cref="DEFAULT_MAX_FIELD_LENGTH" /> 
            /// 
            /// </summary>
            public static readonly MaxFieldLength LIMITED;
            static MaxFieldLength()
            {
                LIMITED = new MaxFieldLength("LIMITED", Lucene.Net.Index.IndexWriter.DEFAULT_MAX_FIELD_LENGTH);
            }
        }
        
        /// <summary>If <see cref="GetReader()" /> has been called (ie, this writer
        /// is in near real-time mode), then after a merge
        /// completes, this class can be invoked to warm the
        /// reader on the newly merged segment, before the merge
        /// commits.  This is not required for near real-time
        /// search, but will reduce search latency on opening a
        /// new near real-time reader after a merge completes.
        /// 
        /// <p/><b>NOTE:</b> This API is experimental and might
        /// change in incompatible ways in the next release.<p/>
        /// 
        /// <p/><b>NOTE</b>: warm is called before any deletes have
        /// been carried over to the merged segment. 
        /// </summary>
        public abstract class IndexReaderWarmer
        {
            public abstract void  Warm(IndexReader reader);
        }
        
        private IndexReaderWarmer mergedSegmentWarmer;

        /// <summary>Gets or sets the merged segment warmer.  See <see cref="IndexReaderWarmer" />
        ///. 
        /// </summary>
        public virtual IndexReaderWarmer MergedSegmentWarmer
        {
            set { mergedSegmentWarmer = value; }
            get { return mergedSegmentWarmer; }
        }

        private void  HandleOOM(System.OutOfMemoryException oom, System.String location)
        {
            if (infoStream != null)
            {
                Message("hit OutOfMemoryError inside " + location);
            }
            hitOOM = true;
            throw oom;
        }
        
        // Used only by assert for testing.  Current points:
        //   startDoFlush
        //   startCommitMerge
        //   startStartCommit
        //   midStartCommit
        //   midStartCommit2
        //   midStartCommitSuccess
        //   finishStartCommit
        //   startCommitMergeDeletes
        //   startMergeInit
        //   startApplyDeletes
        //   DocumentsWriter.ThreadState.init start
        public /*internal*/ virtual bool TestPoint(System.String name)
        {
            return true;
        }
        
        internal virtual bool NrtIsCurrent(SegmentInfos infos)
        {
            lock (this)
            {
                if (!infos.Equals(segmentInfos))
                {
                    // if any structural changes (new segments), we are
                    // stale
                    return false;
                }
                else if (infos.Generation != segmentInfos.Generation)
                {
                    // if any commit took place since we were opened, we
                    // are stale
                    return false;
                }
                else
                {
                    return !docWriter.AnyChanges;
                }
            }
        }
        
        internal virtual bool IsClosed()
        {
            lock (this)
            {
                return closed;
            }
        }

        static IndexWriter()
        {
            MAX_TERM_LENGTH = DocumentsWriter.MAX_TERM_LENGTH;
        }
    }
}