Lucene.Net  3.0.3
Lucene.Net is a port of the Lucene search engine library, written in C# and targeted at .NET runtime users.
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Properties Pages
CurrentIndex.cs
Go to the documentation of this file.
1 /*
2  * Licensed to the Apache Software Foundation (ASF) under one or more
3  * contributor license agreements. See the NOTICE file distributed with
4  * this work for additional information regarding copyright ownership.
5  * The ASF licenses this file to You under the Apache License, Version 2.0
6  * (the "License"); you may not use this file except in compliance with
7  * the License. You may obtain a copy of the License at
8  *
9  * http://www.apache.org/licenses/LICENSE-2.0
10  *
11  * Unless required by applicable law or agreed to in writing, software
12  * distributed under the License is distributed on an "AS IS" BASIS,
13  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14  * See the License for the specific language governing permissions and
15  * limitations under the License.
16  */
17 
18 using System;
19 using System.Collections;
20 using System.Collections.Specialized;
21 using System.Configuration;
22 using System.Diagnostics;
23 using System.IO;
24 using System.Xml;
25 using Lucene.Net.Analysis;
26 using Lucene.Net.Analysis.Standard;
27 using Lucene.Net.Documents;
28 using Lucene.Net.Index;
29 using Lucene.Net.Store;
30 using Lucene.Net.Distributed;
31 
32 namespace Lucene.Net.Distributed.Configuration
33 {
34  public enum IndexSetting
35  {
36  NoSetting = 0,
37  IndexA = 1,
38  IndexB = 2
39  }
40 
41  /// <summary>
42  /// Definition of current index information managed by the
43  /// LuceneUpdater windows service. The <copy> node within the
44  /// <indexset> node represents the information needed to load
45  /// a CurrentIndex object for a given IndexSet.
46  ///
47  /// An example configuration would look like the following:
48  /// <code>
49  /// <indexsets RAMAllocation="100000" CompoundFile="true" DeltaDirectory="c:\indexes\indexdocuments">
50  /// <indexSet id="1" action="1" analyzer="1">
51  /// <add key="localpath" value="c:\lucene\masterindex\index1" />
52  /// <copy>
53  /// <targetPath indexA="\\LuceneServer\lucene\indexA\index1" indexB="\\LuceneServer\lucene\indexA\index1" />
54  /// <statusDir value="\\LuceneServer\lucene\statusfile\" />
55  /// </copy>
56  /// <add key="bottomid" value="1"/>
57  /// <add key="topid" value="1000"/>
58  /// <add key="idcolumn" value="pkId"/>
59  /// </indexSet>
60  /// </indexsets>
61  /// </code>
62  /// </summary>
63  public class CurrentIndex
64  {
65  #region Variables
66  private static readonly string CURRENTINDEX = "currentIndex";
67  private static readonly string INDEX_A = "indexA";
68  private static readonly string INDEX_B = "indexB";
69  private static readonly string TOGGLE = "toggle";
70  private string _strLocalPath;
71  private string _strStatusDir;
72  private string _strIndexAPath;
73  private string _strIndexBPath;
74  private bool _bIndexChanged=false;
75 
76  private int _mergeFactor = (ConfigurationManager.AppSettings["IndexMergeFactor"] != null ? Convert.ToInt32(ConfigurationManager.AppSettings["IndexMergeFactor"]) : 5);
77  private int _maxMergeDocs = (ConfigurationManager.AppSettings["IndexMaxMergeDocs"] != null ? Convert.ToInt32(ConfigurationManager.AppSettings["IndexMaxMergeDocs"]) : 9999999);
78 
79  #endregion
80 
81  #region Constructors
82  /// <summary>
83  /// Constructs a new CurrentIndex using the XmlNode value (from IndexSetConfigurationHandler configuration)
84  /// </summary>
85  /// <param name="node">XmlNode containing configuration information</param>
86  /// <param name="strLocalPath">Local filesystem path to source index</param>
87  public CurrentIndex(XmlNode node, string strLocalPath)
88  {
89  this._strLocalPath=strLocalPath;
90  this.LoadValues(node);
91  }
92 
93  /// <summary>
94  /// Constructs a shell CurrentIndex. Use this constructor to interact
95  /// with the underlying status and toggle files ONLY.
96  /// </summary>
97  /// <param name="sStatusDir">Filesystem path to the status and toggle files for an index</param>
98  public CurrentIndex(string sStatusDir)
99  {
100  this._strStatusDir=sStatusDir;
101  }
102  #endregion
103 
104  #region Internals
105  /// <summary>
106  /// Internal routine for use by constructor that accepts a configuration
107  /// entry structured as XmlNode.
108  /// </summary>
109  /// <param name="node">XmlNode containing configuration information</param>
110  internal void LoadValues(XmlNode node)
111  {
112  foreach (XmlNode c in node.ChildNodes)
113  {
114  if (c.Name.ToLower()=="targetpath")
115  {
116  this._strIndexAPath = c.Attributes["indexA"].Value;
117  this._strIndexBPath = c.Attributes["indexB"].Value;
118  }
119  else if (c.Name.ToLower()=="statusdir")
120  {
121  this._strStatusDir = c.Attributes["value"].Value;
122  }
123  }
124  this.CheckValidConfiguration(node);
125  }
126  #endregion
127 
128  #region Public properties
129  /// <summary>
130  /// Filesystem path to the local source for an index; this is the path to the master index.
131  /// </summary>
132  public string LocalPath
133  {
134  get {return this._strLocalPath;}
135  }
136  /// <summary>
137  /// Filesystem path to a LuceneServer's status and toggle file for a given IndexSet
138  /// </summary>
139  public string StatusDirectory
140  {
141  get {return this._strStatusDir;}
142  }
143 
144  /// <summary>
145  /// Indicates the current index directory (IndexSetting enum) in use (the online set)
146  /// </summary>
148  {
149  get
150  {
151  string input=(this.GetCurrentIndex());
152  return (input==CurrentIndex.INDEX_A ? IndexSetting.IndexA : (input==CurrentIndex.INDEX_B ? IndexSetting.IndexB : IndexSetting.IndexA));
153  }
154  }
155 
156  /// <summary>
157  /// Indicates the index directory to be used in any index searcher refresh
158  /// by determining if any updates have been applied
159  /// </summary>
160  public IndexSetting IndexSettingRefresh
161  {
162  get
163  {
164  if (this.HasChanged)
165  {
166  return (this.IndexSetting==IndexSetting.IndexA ? IndexSetting.IndexB : (this.IndexSetting==IndexSetting.IndexB ? IndexSetting.IndexA : IndexSetting.IndexB ));
167  }
168  else
169  {
170  return this.IndexSetting;
171  }
172  }
173  }
174 
175  /// <summary>
176  /// Indicates if the current index permits updated indexes to be copied to CopyTargetPath
177  /// </summary>
178  public bool CanCopy
179  {
180  get {return (!this.GetToggle() && (this.LocalIndexVersion!=this.TargetIndexVersion));}
181  }
182 
183  /// <summary>
184  /// Indicates if the current index has pending updates (in the offline directory) to be used by an index searcher
185  /// in a refresh evaluation
186  /// </summary>
187  public bool HasChanged
188  {
189  get {return this.GetToggle();}
190  }
191 
192  /// <summary>
193  /// The target directory path to be used when updating the offline index
194  /// </summary>
195  public string CopyTargetPath
196  {
197  get {return (this.IndexSetting==IndexSetting.IndexA ? this._strIndexBPath : (this.IndexSetting==IndexSetting.IndexB ? this._strIndexAPath : ""));}
198  }
199  #endregion
200 
201  #region Public methods
202  /// <summary>
203  /// Method that executes a filesystem copy of all directory files from a local path to
204  /// the proper offline index. This method ensures no conflicts occur with the online index.
205  /// </summary>
206  /// <returns>bool</returns>
207  public bool Copy()
208  {
209  try
210  {
211  if (this.CanCopy && this.CopyTargetPath!="")
212  {
213  this.DeleteDirectoryFiles(this.CopyTargetPath);
214  this.CopyDirectory(this._strLocalPath, this.CopyTargetPath);
215  return true;
216  }
217  else
218  {
219  return false;
220  }
221  }
222  catch (Exception e)
223  {
224  //Do something with e
225  return false;
226  }
227  }
228 
229  /// <summary>
230  /// Method that executes a filesystem copy of updated or new files from a local path to
231  /// the proper offline index. This method ensures no conflicts occur with the online index.
232  /// </summary>
233  /// <returns></returns>
234  public bool CopyIncremental()
235  {
236  try
237  {
238  if (this.CanCopy && this.CopyTargetPath!="")
239  {
240  this.CopyDirectoryIncremental(this._strLocalPath, this.CopyTargetPath);
241  return true;
242  }
243  else
244  {
245  return false;
246  }
247  }
248  catch (Exception e)
249  {
250  //Do something with e
251  return false;
252  }
253  }
254 
255  /// <summary>
256  /// Takes a name/value pair collection to be used in updating an index.
257  /// Deletes are necessary to ensure no duplication occurs within the index.
258  /// </summary>
259  /// <param name="nvcDeleteCollection">Set of record IDs (with underlying field name) to be applied for index updating</param>
260  public void ProcessLocalIndexDeletes(NameValueCollection nvcDeleteCollection)
261  {
262  if (IndexReader.IndexExists(this._strLocalPath) && nvcDeleteCollection.Count>0)
263  {
264  IndexReader idxDeleter = IndexReader.Open(this._strLocalPath);
265  string[] arKeys = nvcDeleteCollection.AllKeys;
266  int xDelete=0;
267  for (int k=0;k<arKeys.Length;k++)
268  {
269  string[] arKeyValues=nvcDeleteCollection.GetValues(arKeys[k]);
270  for (int v=0;v<arKeyValues.Length;v++)
271  xDelete=idxDeleter.DeleteDocuments(new Term(arKeys[k].ToString(),arKeyValues[v].ToString()));
272  }
273  idxDeleter.Close();
274  }
275  }
276 
277  /// <summary>
278  /// Executes a loop on the Documents arraylist, adding each one to the index with the associated analyzer.
279  /// </summary>
280  /// <param name="oAnalyzer">Analyzer to be used in index document addition</param>
281  /// <param name="alAddDocuments">Arraylist of Lucene Document objects to be inserted in the index</param>
282  /// <param name="bCompoundFile">Setting to dictate if the index should use compound format</param>
283  public void ProcessLocalIndexAdditions(Analyzer oAnalyzer, Hashtable htAddDocuments, bool bCompoundFile)
284  {
285  IndexWriter idxWriter = this.GetIndexWriter(this._strLocalPath, oAnalyzer, bCompoundFile);
286  idxWriter.SetMergeFactor(5);
287  idxWriter.SetMaxMergeDocs(9999999);
288 
289  foreach (DictionaryEntry de in htAddDocuments)
290  {
291  Document d = (Document)de.Key;
292  Analyzer a = (Analyzer)de.Value;
293  idxWriter.AddDocument(d,a);
294  }
295  idxWriter.Close();
296  }
297 
298  /// <summary>
299  /// Single method to be used by a searchhost to indicate an index refresh has completed.
300  /// </summary>
301  public void IndexRefresh()
302  {
303  if (this.HasChanged)
304  {
305  this.SetCurrentIndex(this.IndexSettingRefresh);
306  this.SetToggle(false);
307  }
308  }
309 
310  /// <summary>
311  /// Single method to be used by an index updater to indicate an index update has completed.
312  /// </summary>
313  public void UpdateRefresh()
314  {
315  this.SetToggle(true);
316  }
317  #endregion
318 
319  #region Private properties
320  /// <summary>
321  /// The filesystem path to the underlying index status file
322  /// </summary>
323  private string CurrentIndexFile
324  {
325  get {return (this._strStatusDir+(this._strStatusDir.EndsWith(@"\") ? "" : @"\")+CURRENTINDEX);}
326  }
327  /// <summary>
328  /// The filesystem path to the underlying index toggle file
329  /// </summary>
330  private string ToggleFile
331  {
332  get {return (this._strStatusDir+(this._strStatusDir.EndsWith(@"\") ? "" : @"\")+TOGGLE);}
333  }
334 
335  #endregion
336 
337  #region Private methods
338 
339  /// <summary>
340  /// Validation routine to ensure all required values were present within xml configuration node
341  /// used in constructor.
342  /// </summary>
343  /// <param name="node">XmlNode containing configuration information</param>
344  private void CheckValidConfiguration(XmlNode node)
345  {
346  if (this._strLocalPath == null) throw new ConfigurationErrorsException("CurrentIndex local path invalid: "+Environment.NewLine+node.OuterXml);
347  if (this._strStatusDir == null) throw new ConfigurationErrorsException("CurrentIndex statusDir invalid: " + Environment.NewLine + node.OuterXml);
348  if (this._strIndexAPath == null) throw new ConfigurationErrorsException("CurrentIndex indexA invalid: " + Environment.NewLine + node.OuterXml);
349  if (this._strIndexBPath == null) throw new ConfigurationErrorsException("CurrentIndex indexB invalid: " + Environment.NewLine + node.OuterXml);
350  }
351 
352  /// <summary>
353  /// Returns the current toggle file setting
354  /// </summary>
355  /// <returns>bool</returns>
356  private bool GetToggle()
357  {
358  bool bValue=false;
359  string input="";
360  try
361  {
362  if (!File.Exists(this.ToggleFile))
363  {
364  this.SetToggle(false);
365  }
366  else
367  {
368  StreamReader sr = File.OpenText(this.ToggleFile);
369  input = sr.ReadLine();
370  sr.Close();
371  bValue = (input.ToLower()=="true" ? true : false);
372  }
373  }
374  catch (Exception ex)
375  {
376  //Do something with ex
377  }
378  return bValue;
379  }
380 
381  /// <summary>
382  /// Returns the current status file setting
383  /// </summary>
384  /// <returns>string</returns>
385  private string GetCurrentIndex()
386  {
387  string input="";
388  try
389  {
390  if (!File.Exists(this.CurrentIndexFile))
391  {
392  this.SetCurrentIndex(IndexSetting.IndexA);
393  input=IndexSetting.IndexA.ToString();
394  }
395  else
396  {
397  StreamReader sr = File.OpenText(this.CurrentIndexFile);
398  input = sr.ReadLine();
399  sr.Close();
400  }
401  }
402  catch (Exception ex)
403  {
404  //Do something with ex
405  }
406  return input;
407  }
408 
409  /// <summary>
410  /// Updates the status file with the IndexSetting value parameter
411  /// </summary>
412  /// <param name="eIndexSetting">Setting to be applied to the status file</param>
413  private void SetCurrentIndex(IndexSetting eIndexSetting)
414  {
415  try
416  {
417  StreamWriter sw = File.CreateText(this.CurrentIndexFile);
418  sw.WriteLine((eIndexSetting==IndexSetting.IndexA ? CurrentIndex.INDEX_A : CurrentIndex.INDEX_B));
419  sw.Close();
420  }
421  catch (Exception ex)
422  {
423  //Do something with ex
424  }
425  }
426 
427  /// <summary>
428  /// IndexWriter that can be used to apply updates to an index
429  /// </summary>
430  /// <param name="indexPath">File system path to the target index</param>
431  /// <param name="oAnalyzer">Lucene Analyzer to be used by the underlying IndexWriter</param>
432  /// <param name="bCompoundFile">Setting to dictate if the index should use compound format</param>
433  /// <returns></returns>
434  private IndexWriter GetIndexWriter(string indexPath, Analyzer oAnalyzer, bool bCompoundFile)
435  {
436  bool bExists = System.IO.Directory.Exists(indexPath);
437  if (bExists==false)
438  System.IO.Directory.CreateDirectory(indexPath);
439  bExists=IndexReader.IndexExists(FSDirectory.GetDirectory(indexPath, false));
440  IndexWriter idxWriter = new IndexWriter(indexPath, oAnalyzer, !bExists);
441  idxWriter.SetUseCompoundFile(bCompoundFile);
442  return idxWriter;
443  }
444 
445  /// <summary>
446  /// Updates the toggle file with the bool value parameter
447  /// </summary>
448  /// <param name="bValue">Bool to be applied to the toggle file</param>
449  private void SetToggle(bool bValue)
450  {
451  try
452  {
453  StreamWriter sw = File.CreateText(this.ToggleFile);
454  sw.WriteLine(bValue.ToString());
455  sw.Close();
456  this._bIndexChanged=bValue;
457  }
458  catch (Exception ex)
459  {
460  //Do something with ex
461  }
462  }
463 
464  /// <summary>
465  /// Returns the numeric index version (using Lucene objects) for the index located at LocalPath
466  /// </summary>
467  private long LocalIndexVersion
468  {
469  get {return IndexReader.GetCurrentVersion(this.LocalPath);}
470  }
471  /// <summary>
472  /// Returns the numeric index version (using Lucene objects) for the index located at CopyTargetPath
473  /// </summary>
474  private long TargetIndexVersion
475  {
476  get {return (IndexReader.IndexExists(this.CopyTargetPath) ? IndexReader.GetCurrentVersion(this.CopyTargetPath) : 0);}
477  }
478 
479  /// <summary>
480  /// Deletes index files at the filesystem directoryPath location
481  /// </summary>
482  /// <param name="directoryPath">Filesystem path</param>
483  private void DeleteDirectoryFiles(string directoryPath)
484  {
485  try
486  {
487  if(!System.IO.Directory.Exists(directoryPath))
488  return;
489  DirectoryInfo di = new DirectoryInfo(directoryPath);
490  FileInfo[] arFi = di.GetFiles();
491  foreach(FileInfo fi in arFi)
492  fi.Delete();
493  }
494  catch(Exception e)
495  {
496  //Do something with e
497  }
498  }
499 
500  /// <summary>
501  /// Copy all index files from the sourceDirPath to the destDirPath
502  /// </summary>
503  /// <param name="sourceDirPath">Filesystem path</param>
504  /// <param name="destDirPath">Filesystem path</param>
505  private void CopyDirectory(string sourceDirPath, string destDirPath)
506  {
507  string[] Files;
508 
509  if(destDirPath[destDirPath.Length-1]!=Path.DirectorySeparatorChar)
510  destDirPath+=Path.DirectorySeparatorChar;
511  if(!System.IO.Directory.Exists(destDirPath)) System.IO.Directory.CreateDirectory(destDirPath);
512  Files=System.IO.Directory.GetFileSystemEntries(sourceDirPath);
513  foreach(string Element in Files)
514  {
515  // Sub directories
516  if(System.IO.Directory.Exists(Element))
517  CopyDirectory(Element,destDirPath+Path.GetFileName(Element));
518  // Files in directory
519  else
520  File.Copy(Element,destDirPath+Path.GetFileName(Element),true);
521  }
522 
523  }
524 
525  /// <summary>
526  /// Copy only new and updated index files from the sourceDirPath to the destDirPath
527  /// </summary>
528  /// <param name="sourceDirPath">Filesystem path</param>
529  /// <param name="destDirPath">Filesystem path</param>
530  private void CopyDirectoryIncremental(string sourceDirPath, string destDirPath)
531  {
532  string[] Files;
533 
534  if(destDirPath[destDirPath.Length-1]!=Path.DirectorySeparatorChar)
535  destDirPath+=Path.DirectorySeparatorChar;
536  Files=System.IO.Directory.GetFileSystemEntries(sourceDirPath);
537  if(!System.IO.Directory.Exists(destDirPath))
538  {
539  System.IO.Directory.CreateDirectory(destDirPath);
540  foreach(string Element in Files)
541  {
542  // Sub directories
543  if(System.IO.Directory.Exists(Element))
544  CopyDirectory(Element,destDirPath+Path.GetFileName(Element));
545  // Files in directory
546  else
547  File.Copy(Element,destDirPath+Path.GetFileName(Element),true);
548  }
549  }
550  else
551  {
552  foreach(string Element in Files)
553  {
554  if(System.IO.Directory.Exists(Element))
555  {
556  CopyDirectoryIncremental(Element,destDirPath+Path.GetFileName(Element));
557  }
558  else
559  {
560  if (System.IO.File.Exists(destDirPath+Path.GetFileName(Element)))
561  this.CopyFileIncremental(Element, destDirPath+Path.GetFileName(Element));
562  else
563  File.Copy(Element,destDirPath+Path.GetFileName(Element),true);
564  }
565  }
566  }
567  }
568 
569  /// <summary>
570  /// Evaluates the LastWriteTime and Length properties of two files to determine
571  /// if a file should be copied.
572  /// </summary>
573  /// <param name="filepath1">Filesystem path</param>
574  /// <param name="filepath2">Filesystem path</param>
575  private void CopyFileIncremental(string filepath1, string filepath2)
576  {
577  FileInfo fi1 = new FileInfo(filepath1);
578  FileInfo fi2 = new FileInfo(filepath2);
579  if ((fi1.LastWriteTime!=fi2.LastWriteTime)||(fi1.Length!=fi2.Length))
580  File.Copy(filepath1,filepath2,true);
581  }
582  #endregion
583 
584  #region Static methods
585  /// <summary>
586  /// Returns an Analyzer for the given AnalyzerType
587  /// </summary>
588  /// <param name="oAnalyzerType">Enumeration value</param>
589  /// <returns>Analyzer</returns>
590  public static Analyzer GetAnalyzer(AnalyzerType oAnalyzerType)
591  {
592  Analyzer oAnalyzer = null;
593  switch (oAnalyzerType)
594  {
595  case AnalyzerType.SimpleAnalyzer:
596  oAnalyzer = new SimpleAnalyzer();
597  break;
598  case AnalyzerType.StopAnalyzer:
599  oAnalyzer = new StopAnalyzer();
600  break;
601  case AnalyzerType.WhitespaceAnalyzer:
602  oAnalyzer = new WhitespaceAnalyzer();
603  break;
604  default:
605  case AnalyzerType.StandardAnalyzer:
606  oAnalyzer = new StandardAnalyzer();
607  break;
608  }
609  return oAnalyzer;
610  }
611  #endregion
612 
613  }
614 }