Lucene.Net  3.0.3
Lucene.Net is a port of the Lucene search engine library, written in C# and targeted at .NET runtime users.
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Properties Pages
FSDirectory.cs
Go to the documentation of this file.
1 /*
2  * Licensed to the Apache Software Foundation (ASF) under one or more
3  * contributor license agreements. See the NOTICE file distributed with
4  * this work for additional information regarding copyright ownership.
5  * The ASF licenses this file to You under the Apache License, Version 2.0
6  * (the "License"); you may not use this file except in compliance with
7  * the License. You may obtain a copy of the License at
8  *
9  * http://www.apache.org/licenses/LICENSE-2.0
10  *
11  * Unless required by applicable law or agreed to in writing, software
12  * distributed under the License is distributed on an "AS IS" BASIS,
13  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14  * See the License for the specific language governing permissions and
15  * limitations under the License.
16  */
17 
18 using System;
19 using System.Collections.Generic;
20 
21 // Used only for WRITE_LOCK_NAME in deprecated create=true case:
22 using System.IO;
23 using Lucene.Net.Support;
24 using IndexFileNameFilter = Lucene.Net.Index.IndexFileNameFilter;
25 using IndexWriter = Lucene.Net.Index.IndexWriter;
26 using Constants = Lucene.Net.Util.Constants;
27 
28 namespace Lucene.Net.Store
29 {
30 
31  /// <summary> <a name="subclasses"/>
32  /// Base class for Directory implementations that store index
33  /// files in the file system. There are currently three core
34  /// subclasses:
35  ///
36  /// <list type="bullet">
37  ///
38  /// <item> <see cref="SimpleFSDirectory" /> is a straightforward
39  /// implementation using java.io.RandomAccessFile.
40  /// However, it has poor concurrent performance
41  /// (multiple threads will bottleneck) as it
42  /// synchronizes when multiple threads read from the
43  /// same file.</item>
44  ///
45  /// <item> <see cref="NIOFSDirectory" /> uses java.nio's
46  /// FileChannel's positional io when reading to avoid
47  /// synchronization when reading from the same file.
48  /// Unfortunately, due to a Windows-only <a
49  /// href="http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=6265734">Sun
50  /// JRE bug</a> this is a poor choice for Windows, but
51  /// on all other platforms this is the preferred
52  /// choice. Applications using <see cref="System.Threading.Thread.Interrupt()" /> or
53  /// <c>Future#cancel(boolean)</c> (on Java 1.5) should use
54  /// <see cref="SimpleFSDirectory" /> instead. See <see cref="NIOFSDirectory" /> java doc
55  /// for details.
56  ///
57  ///
58  ///
59  /// <item> <see cref="MMapDirectory" /> uses memory-mapped IO when
60  /// reading. This is a good choice if you have plenty
61  /// of virtual memory relative to your index size, eg
62  /// if you are running on a 64 bit JRE, or you are
63  /// running on a 32 bit JRE but your index sizes are
64  /// small enough to fit into the virtual memory space.
65  /// Java has currently the limitation of not being able to
66  /// unmap files from user code. The files are unmapped, when GC
67  /// releases the byte buffers. Due to
68  /// <a href="http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=4724038">
69  /// this bug</a> in Sun's JRE, MMapDirectory's <see cref="IndexInput.Close" />
70  /// is unable to close the underlying OS file handle. Only when
71  /// GC finally collects the underlying objects, which could be
72  /// quite some time later, will the file handle be closed.
73  /// This will consume additional transient disk usage: on Windows,
74  /// attempts to delete or overwrite the files will result in an
75  /// exception; on other platforms, which typically have a &quot;delete on
76  /// last close&quot; semantics, while such operations will succeed, the bytes
77  /// are still consuming space on disk. For many applications this
78  /// limitation is not a problem (e.g. if you have plenty of disk space,
79  /// and you don't rely on overwriting files on Windows) but it's still
80  /// an important limitation to be aware of. This class supplies a
81  /// (possibly dangerous) workaround mentioned in the bug report,
82  /// which may fail on non-Sun JVMs.</item>
83  ///
84  /// Applications using <see cref="System.Threading.Thread.Interrupt()" /> or
85  /// <c>Future#cancel(boolean)</c> (on Java 1.5) should use
86  /// <see cref="SimpleFSDirectory" /> instead. See <see cref="MMapDirectory" />
87  /// java doc for details.</item>
88  /// </list>
89  ///
90  /// Unfortunately, because of system peculiarities, there is
91  /// no single overall best implementation. Therefore, we've
92  /// added the <see cref="Open(System.IO.DirectoryInfo)" /> method, to allow Lucene to choose
93  /// the best FSDirectory implementation given your
94  /// environment, and the known limitations of each
95  /// implementation. For users who have no reason to prefer a
96  /// specific implementation, it's best to simply use <see cref="FSDirectory.Open(System.IO.DirectoryInfo)" />
97  ///. For all others, you should instantiate the
98  /// desired implementation directly.
99  ///
100  /// <p/>The locking implementation is by default <see cref="NativeFSLockFactory" />
101  ///, but can be changed by
102  /// passing in a custom <see cref="LockFactory" /> instance.
103  /// </summary>
104  public abstract class FSDirectory : Directory
105  {
106  private static System.Security.Cryptography.HashAlgorithm DIGESTER;
107 
108  static FSDirectory()
109  {
110  try
111  {
112  DIGESTER = Cryptography.HashAlgorithm;
113  }
114  catch (System.Exception e)
115  {
116  throw new System.SystemException(e.ToString(), e);
117  }
118  }
119 
120  private bool checked_Renamed;
121 
122  internal void CreateDir()
123  {
124  if (!checked_Renamed)
125  {
126  if (!this.internalDirectory.Exists)
127  {
128  try
129  {
130  this.internalDirectory.Create();
131  }
132  catch (Exception)
133  {
134  throw new System.IO.IOException("Cannot create directory: " + internalDirectory);
135  }
136  this.internalDirectory.Refresh(); // need to see the creation
137  }
138 
139  checked_Renamed = true;
140  }
141  }
142 
143  /// <summary>Initializes the directory to create a new file with the given name.
144  /// This method should be used in <see cref="Lucene.Net.Store.Directory.CreateOutput(string)" />.
145  /// </summary>
146  protected internal void InitOutput(System.String name)
147  {
148  EnsureOpen();
149  CreateDir();
150  System.IO.FileInfo file = new System.IO.FileInfo(System.IO.Path.Combine(internalDirectory.FullName, name));
151  if (file.Exists) // delete existing, if any
152  {
153  try
154  {
155  file.Delete();
156  }
157  catch (Exception)
158  {
159  throw new System.IO.IOException("Cannot overwrite: " + file);
160  }
161  }
162  }
163 
164  /// <summary>The underlying filesystem directory </summary>
165  protected internal System.IO.DirectoryInfo internalDirectory = null;
166 
167  /// <summary>Create a new FSDirectory for the named location (ctor for subclasses).</summary>
168  /// <param name="path">the path of the directory
169  /// </param>
170  /// <param name="lockFactory">the lock factory to use, or null for the default
171  /// (<see cref="NativeFSLockFactory" />);
172  /// </param>
173  /// <throws> IOException </throws>
174  protected internal FSDirectory(System.IO.DirectoryInfo path, LockFactory lockFactory)
175  {
176  // new ctors use always NativeFSLockFactory as default:
177  if (lockFactory == null)
178  {
179  lockFactory = new NativeFSLockFactory();
180  }
181  // Set up lockFactory with cascaded defaults: if an instance was passed in,
182  // use that; else if locks are disabled, use NoLockFactory; else if the
183  // system property Lucene.Net.Store.FSDirectoryLockFactoryClass is set,
184  // instantiate that; else, use SimpleFSLockFactory:
185 
186  internalDirectory = path;
187 
188  // due to differences in how Java & .NET refer to files, the checks are a bit different
189  if (!internalDirectory.Exists && System.IO.File.Exists(internalDirectory.FullName))
190  {
191  throw new NoSuchDirectoryException("file '" + internalDirectory.FullName + "' exists but is not a directory");
192  }
193  SetLockFactory(lockFactory);
194 
195  // for filesystem based LockFactory, delete the lockPrefix, if the locks are placed
196  // in index dir. If no index dir is given, set ourselves
197  if (lockFactory is FSLockFactory)
198  {
199  FSLockFactory lf = (FSLockFactory)lockFactory;
200  System.IO.DirectoryInfo dir = lf.LockDir;
201  // if the lock factory has no lockDir set, use the this directory as lockDir
202  if (dir == null)
203  {
204  lf.LockDir = this.internalDirectory;
205  lf.LockPrefix = null;
206  }
207  else if (dir.FullName.Equals(this.internalDirectory.FullName))
208  {
209  lf.LockPrefix = null;
210  }
211  }
212  }
213 
214  /// <summary>Creates an FSDirectory instance, trying to pick the
215  /// best implementation given the current environment.
216  /// The directory returned uses the <see cref="NativeFSLockFactory" />.
217  ///
218  /// <p/>Currently this returns <see cref="SimpleFSDirectory" /> as
219  /// NIOFSDirectory is currently not supported.
220  ///
221  /// <p/><b>NOTE</b>: this method may suddenly change which
222  /// implementation is returned from release to release, in
223  /// the event that higher performance defaults become
224  /// possible; if the precise implementation is important to
225  /// your application, please instantiate it directly,
226  /// instead. On 64 bit systems, it may also good to
227  /// return <see cref="MMapDirectory" />, but this is disabled
228  /// because of officially missing unmap support in Java.
229  /// For optimal performance you should consider using
230  /// this implementation on 64 bit JVMs.
231  ///
232  /// <p/>See <a href="#subclasses">above</a>
233  /// </summary>
234  public static FSDirectory Open(string path)
235  {
236  return Open(new DirectoryInfo(path), null);
237  }
238 
239  /// <summary>Creates an FSDirectory instance, trying to pick the
240  /// best implementation given the current environment.
241  /// The directory returned uses the <see cref="NativeFSLockFactory" />.
242  ///
243  /// <p/>Currently this returns <see cref="SimpleFSDirectory" /> as
244  /// NIOFSDirectory is currently not supported.
245  ///
246  /// <p/><b>NOTE</b>: this method may suddenly change which
247  /// implementation is returned from release to release, in
248  /// the event that higher performance defaults become
249  /// possible; if the precise implementation is important to
250  /// your application, please instantiate it directly,
251  /// instead. On 64 bit systems, it may also good to
252  /// return <see cref="MMapDirectory" />, but this is disabled
253  /// because of officially missing unmap support in Java.
254  /// For optimal performance you should consider using
255  /// this implementation on 64 bit JVMs.
256  ///
257  /// <p/>See <a href="#subclasses">above</a>
258  /// </summary>
259  public static FSDirectory Open(System.IO.DirectoryInfo path)
260  {
261  return Open(path, null);
262  }
263 
264  /// <summary>Just like <see cref="Open(System.IO.DirectoryInfo)" />, but allows you to
265  /// also specify a custom <see cref="LockFactory" />.
266  /// </summary>
267  public static FSDirectory Open(System.IO.DirectoryInfo path, LockFactory lockFactory)
268  {
269  /* For testing:
270  MMapDirectory dir=new MMapDirectory(path, lockFactory);
271  dir.setUseUnmap(true);
272  return dir;
273  */
274 
275  if (Constants.WINDOWS)
276  {
277  return new SimpleFSDirectory(path, lockFactory);
278  }
279  else
280  {
281  //NIOFSDirectory is not implemented in Lucene.Net
282  //return new NIOFSDirectory(path, lockFactory);
283  return new SimpleFSDirectory(path, lockFactory);
284  }
285  }
286 
287  /// <summary>Lists all files (not subdirectories) in the
288  /// directory. This method never returns null (throws
289  /// <see cref="System.IO.IOException" /> instead).
290  ///
291  /// </summary>
292  /// <throws> NoSuchDirectoryException if the directory </throws>
293  /// <summary> does not exist, or does exist but is not a
294  /// directory.
295  /// </summary>
296  /// <throws> IOException if list() returns null </throws>
297  public static System.String[] ListAll(System.IO.DirectoryInfo dir)
298  {
299  if (!dir.Exists)
300  {
301  throw new NoSuchDirectoryException("directory '" + dir.FullName + "' does not exist");
302  }
303  else if (System.IO.File.Exists(dir.FullName))
304  {
305  throw new NoSuchDirectoryException("File '" + dir.FullName + "' does not exist");
306  }
307 
308  // Exclude subdirs, only the file names, not the paths
309  System.IO.FileInfo[] files = dir.GetFiles();
310  System.String[] result = new System.String[files.Length];
311  for (int i = 0; i < files.Length; i++)
312  {
313  result[i] = files[i].Name;
314  }
315 
316  // no reason to return null, if the directory cannot be listed, an exception
317  // will be thrown on the above call to dir.GetFiles()
318  // use of LINQ to create the return value array may be a bit more efficient
319 
320  return result;
321  }
322 
323  /// <summary>Lists all files (not subdirectories) in the
324  /// directory.
325  /// </summary>
326  /// <seealso cref="ListAll(System.IO.DirectoryInfo)">
327  /// </seealso>
328  public override System.String[] ListAll()
329  {
330  EnsureOpen();
331  return ListAll(internalDirectory);
332  }
333 
334  /// <summary>Returns true iff a file with the given name exists. </summary>
335  public override bool FileExists(System.String name)
336  {
337  EnsureOpen();
338  System.IO.FileInfo file = new System.IO.FileInfo(System.IO.Path.Combine(internalDirectory.FullName, name));
339  return file.Exists;
340  }
341 
342  /// <summary>Returns the time the named file was last modified. </summary>
343  public override long FileModified(System.String name)
344  {
345  EnsureOpen();
346  System.IO.FileInfo file = new System.IO.FileInfo(System.IO.Path.Combine(internalDirectory.FullName, name));
347  return (long)file.LastWriteTime.ToUniversalTime().Subtract(new DateTime(1970, 1, 1, 0, 0, 0)).TotalMilliseconds; //{{LUCENENET-353}}
348  }
349 
350  /// <summary>Returns the time the named file was last modified. </summary>
351  public static long FileModified(System.IO.FileInfo directory, System.String name)
352  {
353  System.IO.FileInfo file = new System.IO.FileInfo(System.IO.Path.Combine(directory.FullName, name));
354  return (long)file.LastWriteTime.ToUniversalTime().Subtract(new DateTime(1970, 1, 1, 0, 0, 0)).TotalMilliseconds; //{{LUCENENET-353}}
355  }
356 
357  /// <summary>Set the modified time of an existing file to now. </summary>
358  public override void TouchFile(System.String name)
359  {
360  EnsureOpen();
361  System.IO.FileInfo file = new System.IO.FileInfo(System.IO.Path.Combine(internalDirectory.FullName, name));
362  file.LastWriteTime = System.DateTime.Now;
363  }
364 
365  /// <summary>Returns the length in bytes of a file in the directory. </summary>
366  public override long FileLength(System.String name)
367  {
368  EnsureOpen();
369  System.IO.FileInfo file = new System.IO.FileInfo(System.IO.Path.Combine(internalDirectory.FullName, name));
370  return file.Exists ? file.Length : 0;
371  }
372 
373  /// <summary>Removes an existing file in the directory. </summary>
374  public override void DeleteFile(System.String name)
375  {
376  EnsureOpen();
377  System.IO.FileInfo file = new System.IO.FileInfo(System.IO.Path.Combine(internalDirectory.FullName, name));
378  try
379  {
380  file.Delete();
381  }
382  catch (Exception)
383  {
384  throw new System.IO.IOException("Cannot delete " + file);
385  }
386  }
387 
388  public override void Sync(System.String name)
389  {
390  EnsureOpen();
391  System.IO.FileInfo fullFile = new System.IO.FileInfo(System.IO.Path.Combine(internalDirectory.FullName, name));
392  bool success = false;
393  int retryCount = 0;
394  System.IO.IOException exc = null;
395  while (!success && retryCount < 5)
396  {
397  retryCount++;
398  System.IO.FileStream file = null;
399  try
400  {
401  try
402  {
403  file = new System.IO.FileStream(fullFile.FullName, System.IO.FileMode.OpenOrCreate, System.IO.FileAccess.Write, System.IO.FileShare.ReadWrite);
404  FileSupport.Sync(file);
405  success = true;
406  }
407  finally
408  {
409  if (file != null)
410  file.Close();
411  }
412  }
413  catch (System.IO.IOException ioe)
414  {
415  if (exc == null)
416  exc = ioe;
417 
418  // Pause 5 msec
419  System.Threading.Thread.Sleep(5);
420 
421  }
422  }
423 
424  if (!success && exc != null)
425  // Throw original exception
426  throw exc;
427  }
428 
429  // Inherit javadoc
430  public override IndexInput OpenInput(System.String name)
431  {
432  EnsureOpen();
433  return OpenInput(name, BufferedIndexInput.BUFFER_SIZE);
434  }
435 
436  /// <summary> So we can do some byte-to-hexchar conversion below</summary>
437  private static readonly char[] HEX_DIGITS = new char[]{'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'};
438 
439 
440  public override string GetLockId()
441  {
442  EnsureOpen();
443  System.String dirName; // name to be hashed
444  try
445  {
446  dirName = internalDirectory.FullName;
447  }
448  catch (System.IO.IOException e)
449  {
450  throw new System.SystemException(e.ToString(), e);
451  }
452 
453  byte[] digest;
454  lock (DIGESTER)
455  {
456  digest = DIGESTER.ComputeHash(System.Text.Encoding.UTF8.GetBytes(dirName));
457  }
458  System.Text.StringBuilder buf = new System.Text.StringBuilder();
459  buf.Append("lucene-");
460  for (int i = 0; i < digest.Length; i++)
461  {
462  int b = digest[i];
463  buf.Append(HEX_DIGITS[(b >> 4) & 0xf]);
464  buf.Append(HEX_DIGITS[b & 0xf]);
465  }
466 
467  return buf.ToString();
468  }
469 
470  protected override void Dispose(bool disposing)
471  {
472  lock (this)
473  {
474  isOpen = false;
475  }
476  }
477 
478  // Java Lucene implements GetFile() which returns a FileInfo.
479  // For Lucene.Net, GetDirectory() is more appropriate
480 
481  public virtual DirectoryInfo Directory
482  {
483  get
484  {
485  EnsureOpen();
486  return internalDirectory;
487  }
488  }
489 
490  /// <summary>For debug output. </summary>
491  public override System.String ToString()
492  {
493  return this.GetType().FullName + "@" + internalDirectory + " lockFactory=" + LockFactory;
494  }
495 
496  /// <summary> Default read chunk size. This is a conditional
497  /// default: on 32bit JVMs, it defaults to 100 MB. On
498  /// 64bit JVMs, it's <c>Integer.MAX_VALUE</c>.
499  /// </summary>
500  /// <seealso cref="ReadChunkSize">
501  /// </seealso>
502  public static readonly int DEFAULT_READ_CHUNK_SIZE = Constants.JRE_IS_64BIT ? int.MaxValue: 100 * 1024 * 1024;
503 
504  // LUCENE-1566
505  private int chunkSize = DEFAULT_READ_CHUNK_SIZE;
506 
507  /// <summary> The maximum number of bytes to read at once from the
508  /// underlying file during <see cref="IndexInput.ReadBytes(byte[],int,int)" />.
509  /// </summary>
510  /// <seealso cref="ReadChunkSize">
511  /// </seealso>
512  public int ReadChunkSize
513  {
514  get
515  {
516  // LUCENE-1566
517  return chunkSize;
518  }
519  set
520  {
521  // LUCENE-1566
522  if (value <= 0)
523  {
524  throw new System.ArgumentException("chunkSize must be positive");
525  }
526  if (!Constants.JRE_IS_64BIT)
527  {
528  this.chunkSize = value;
529  }
530  }
531  }
532  }
533 }