Lucene.Net  3.0.3
Lucene.Net is a port of the Lucene search engine library, written in C# and targeted at .NET runtime users.
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Properties Pages
NormsWriter.cs
Go to the documentation of this file.
1 /*
2  * Licensed to the Apache Software Foundation (ASF) under one or more
3  * contributor license agreements. See the NOTICE file distributed with
4  * this work for additional information regarding copyright ownership.
5  * The ASF licenses this file to You under the Apache License, Version 2.0
6  * (the "License"); you may not use this file except in compliance with
7  * the License. You may obtain a copy of the License at
8  *
9  * http://www.apache.org/licenses/LICENSE-2.0
10  *
11  * Unless required by applicable law or agreed to in writing, software
12  * distributed under the License is distributed on an "AS IS" BASIS,
13  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14  * See the License for the specific language governing permissions and
15  * limitations under the License.
16  */
17 
18 using System;
19 using System.Collections.Generic;
20 using Lucene.Net.Support;
21 using IndexOutput = Lucene.Net.Store.IndexOutput;
22 using Similarity = Lucene.Net.Search.Similarity;
23 
24 namespace Lucene.Net.Index
25 {
26 
27  // TODO FI: norms could actually be stored as doc store
28 
29  /// <summary>Writes norms. Each thread X field accumulates the norms
30  /// for the doc/fields it saw, then the flush method below
31  /// merges all of these together into a single _X.nrm file.
32  /// </summary>
33 
35  {
36 
37  private static readonly byte defaultNorm;
38  private FieldInfos fieldInfos;
39  public override InvertedDocEndConsumerPerThread AddThread(DocInverterPerThread docInverterPerThread)
40  {
41  return new NormsWriterPerThread(docInverterPerThread, this);
42  }
43 
44  public override void Abort()
45  {
46  }
47 
48  // We only write the _X.nrm file at flush
49  internal void Files(ICollection<string> files)
50  {
51  }
52 
53  internal override void SetFieldInfos(FieldInfos fieldInfos)
54  {
55  this.fieldInfos = fieldInfos;
56  }
57 
58  /// <summary>Produce _X.nrm if any document had a field with norms
59  /// not disabled
60  /// </summary>
61  public override void Flush(IDictionary<InvertedDocEndConsumerPerThread,ICollection<InvertedDocEndConsumerPerField>> threadsAndFields, SegmentWriteState state)
62  {
63 
64  IDictionary<FieldInfo, IList<NormsWriterPerField>> byField = new HashMap<FieldInfo, IList<NormsWriterPerField>>();
65 
66  // Typically, each thread will have encountered the same
67  // field. So first we collate by field, ie, all
68  // per-thread field instances that correspond to the
69  // same FieldInfo
70  foreach(var entry in threadsAndFields)
71  {
72  ICollection<InvertedDocEndConsumerPerField> fields = entry.Value;
73  IEnumerator<InvertedDocEndConsumerPerField> fieldsIt = fields.GetEnumerator();
74  var fieldsToRemove = new HashSet<NormsWriterPerField>();
75  while (fieldsIt.MoveNext())
76  {
77  NormsWriterPerField perField = (NormsWriterPerField) fieldsIt.Current;
78 
79  if (perField.upto > 0)
80  {
81  // It has some norms
82  IList<NormsWriterPerField> l = byField[perField.fieldInfo];
83  if (l == null)
84  {
85  l = new List<NormsWriterPerField>();
86  byField[perField.fieldInfo] = l;
87  }
88  l.Add(perField);
89  }
90  // Remove this field since we haven't seen it
91  // since the previous flush
92  else
93  {
94  fieldsToRemove.Add(perField);
95  }
96  }
97  foreach (var field in fieldsToRemove)
98  {
99  fields.Remove(field);
100  }
101  }
102 
103  System.String normsFileName = state.segmentName + "." + IndexFileNames.NORMS_EXTENSION;
104  state.flushedFiles.Add(normsFileName);
105  IndexOutput normsOut = state.directory.CreateOutput(normsFileName);
106 
107  try
108  {
109  normsOut.WriteBytes(SegmentMerger.NORMS_HEADER, 0, SegmentMerger.NORMS_HEADER.Length);
110 
111  int numField = fieldInfos.Size();
112 
113  int normCount = 0;
114 
115  for (int fieldNumber = 0; fieldNumber < numField; fieldNumber++)
116  {
117 
118  FieldInfo fieldInfo = fieldInfos.FieldInfo(fieldNumber);
119 
120  IList<NormsWriterPerField> toMerge = byField[fieldInfo];
121  int upto = 0;
122  if (toMerge != null)
123  {
124 
125  int numFields = toMerge.Count;
126 
127  normCount++;
128 
129  NormsWriterPerField[] fields = new NormsWriterPerField[numFields];
130  int[] uptos = new int[numFields];
131 
132  for (int j = 0; j < numFields; j++)
133  fields[j] = toMerge[j];
134 
135  int numLeft = numFields;
136 
137  while (numLeft > 0)
138  {
139 
140  System.Diagnostics.Debug.Assert(uptos [0] < fields [0].docIDs.Length, " uptos[0]=" + uptos [0] + " len=" +(fields [0].docIDs.Length));
141 
142  int minLoc = 0;
143  int minDocID = fields[0].docIDs[uptos[0]];
144 
145  for (int j = 1; j < numLeft; j++)
146  {
147  int docID = fields[j].docIDs[uptos[j]];
148  if (docID < minDocID)
149  {
150  minDocID = docID;
151  minLoc = j;
152  }
153  }
154 
155  System.Diagnostics.Debug.Assert(minDocID < state.numDocs);
156 
157  // Fill hole
158  for (; upto < minDocID; upto++)
159  normsOut.WriteByte(defaultNorm);
160 
161  normsOut.WriteByte(fields[minLoc].norms[uptos[minLoc]]);
162  (uptos[minLoc])++;
163  upto++;
164 
165  if (uptos[minLoc] == fields[minLoc].upto)
166  {
167  fields[minLoc].Reset();
168  if (minLoc != numLeft - 1)
169  {
170  fields[minLoc] = fields[numLeft - 1];
171  uptos[minLoc] = uptos[numLeft - 1];
172  }
173  numLeft--;
174  }
175  }
176 
177  // Fill final hole with defaultNorm
178  for (; upto < state.numDocs; upto++)
179  normsOut.WriteByte(defaultNorm);
180  }
181  else if (fieldInfo.isIndexed && !fieldInfo.omitNorms)
182  {
183  normCount++;
184  // Fill entire field with default norm:
185  for (; upto < state.numDocs; upto++)
186  normsOut.WriteByte(defaultNorm);
187  }
188 
189  System.Diagnostics.Debug.Assert(4 + normCount * state.numDocs == normsOut.FilePointer, ".nrm file size mismatch: expected=" +(4 + normCount * state.numDocs) + " actual=" + normsOut.FilePointer);
190  }
191  }
192  finally
193  {
194  normsOut.Close();
195  }
196  }
197 
198  internal override void CloseDocStore(SegmentWriteState state)
199  {
200  }
201  static NormsWriter()
202  {
203  defaultNorm = Similarity.EncodeNorm(1.0f);
204  }
205  }
206 }