Lucene.Net  3.0.3
Lucene.Net is a port of the Lucene search engine library, written in C# and targeted at .NET runtime users.
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Properties Pages
DefaultSkipListWriter.cs
Go to the documentation of this file.
1 /*
2  * Licensed to the Apache Software Foundation (ASF) under one or more
3  * contributor license agreements. See the NOTICE file distributed with
4  * this work for additional information regarding copyright ownership.
5  * The ASF licenses this file to You under the Apache License, Version 2.0
6  * (the "License"); you may not use this file except in compliance with
7  * the License. You may obtain a copy of the License at
8  *
9  * http://www.apache.org/licenses/LICENSE-2.0
10  *
11  * Unless required by applicable law or agreed to in writing, software
12  * distributed under the License is distributed on an "AS IS" BASIS,
13  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14  * See the License for the specific language governing permissions and
15  * limitations under the License.
16  */
17 
18 using System;
19 
20 using IndexOutput = Lucene.Net.Store.IndexOutput;
21 
22 namespace Lucene.Net.Index
23 {
24 
25 
26  /// <summary> Implements the skip list writer for the default posting list format
27  /// that stores positions and payloads.
28  ///
29  /// </summary>
31  {
32  private int[] lastSkipDoc;
33  private int[] lastSkipPayloadLength;
34  private long[] lastSkipFreqPointer;
35  private long[] lastSkipProxPointer;
36 
37  private IndexOutput freqOutput;
38  private IndexOutput proxOutput;
39 
40  private int curDoc;
41  private bool curStorePayloads;
42  private int curPayloadLength;
43  private long curFreqPointer;
44  private long curProxPointer;
45 
46  internal DefaultSkipListWriter(int skipInterval, int numberOfSkipLevels, int docCount, IndexOutput freqOutput, IndexOutput proxOutput):base(skipInterval, numberOfSkipLevels, docCount)
47  {
48  this.freqOutput = freqOutput;
49  this.proxOutput = proxOutput;
50 
51  lastSkipDoc = new int[numberOfSkipLevels];
52  lastSkipPayloadLength = new int[numberOfSkipLevels];
53  lastSkipFreqPointer = new long[numberOfSkipLevels];
54  lastSkipProxPointer = new long[numberOfSkipLevels];
55  }
56 
57  internal virtual void SetFreqOutput(IndexOutput freqOutput)
58  {
59  this.freqOutput = freqOutput;
60  }
61 
62  internal virtual void SetProxOutput(IndexOutput proxOutput)
63  {
64  this.proxOutput = proxOutput;
65  }
66 
67  /// <summary> Sets the values for the current skip data. </summary>
68  internal virtual void SetSkipData(int doc, bool storePayloads, int payloadLength)
69  {
70  this.curDoc = doc;
71  this.curStorePayloads = storePayloads;
72  this.curPayloadLength = payloadLength;
73  this.curFreqPointer = freqOutput.FilePointer;
74  if (proxOutput != null)
75  this.curProxPointer = proxOutput.FilePointer;
76  }
77 
78  protected internal override void ResetSkip()
79  {
80  base.ResetSkip();
81  for (int i = 0; i < lastSkipDoc.Length; i++) lastSkipDoc[i] = 0;
82  for (int i = 0; i < lastSkipPayloadLength.Length; i++) lastSkipPayloadLength[i] = -1; // we don't have to write the first length in the skip list
83  for (int i = 0; i < lastSkipFreqPointer.Length; i++) lastSkipFreqPointer[i] = freqOutput.FilePointer;
84  if (proxOutput != null)
85  for (int i = 0; i < lastSkipProxPointer.Length; i++) lastSkipProxPointer[i] = proxOutput.FilePointer;
86  }
87 
88  protected internal override void WriteSkipData(int level, IndexOutput skipBuffer)
89  {
90  // To efficiently store payloads in the posting lists we do not store the length of
91  // every payload. Instead we omit the length for a payload if the previous payload had
92  // the same length.
93  // However, in order to support skipping the payload length at every skip point must be known.
94  // So we use the same length encoding that we use for the posting lists for the skip data as well:
95  // Case 1: current field does not store payloads
96  // SkipDatum --> DocSkip, FreqSkip, ProxSkip
97  // DocSkip,FreqSkip,ProxSkip --> VInt
98  // DocSkip records the document number before every SkipInterval th document in TermFreqs.
99  // Document numbers are represented as differences from the previous value in the sequence.
100  // Case 2: current field stores payloads
101  // SkipDatum --> DocSkip, PayloadLength?, FreqSkip,ProxSkip
102  // DocSkip,FreqSkip,ProxSkip --> VInt
103  // PayloadLength --> VInt
104  // In this case DocSkip/2 is the difference between
105  // the current and the previous value. If DocSkip
106  // is odd, then a PayloadLength encoded as VInt follows,
107  // if DocSkip is even, then it is assumed that the
108  // current payload length equals the length at the previous
109  // skip point
110  if (curStorePayloads)
111  {
112  int delta = curDoc - lastSkipDoc[level];
113  if (curPayloadLength == lastSkipPayloadLength[level])
114  {
115  // the current payload length equals the length at the previous skip point,
116  // so we don't store the length again
117  skipBuffer.WriteVInt(delta * 2);
118  }
119  else
120  {
121  // the payload length is different from the previous one. We shift the DocSkip,
122  // set the lowest bit and store the current payload length as VInt.
123  skipBuffer.WriteVInt(delta * 2 + 1);
124  skipBuffer.WriteVInt(curPayloadLength);
125  lastSkipPayloadLength[level] = curPayloadLength;
126  }
127  }
128  else
129  {
130  // current field does not store payloads
131  skipBuffer.WriteVInt(curDoc - lastSkipDoc[level]);
132  }
133  skipBuffer.WriteVInt((int) (curFreqPointer - lastSkipFreqPointer[level]));
134  skipBuffer.WriteVInt((int) (curProxPointer - lastSkipProxPointer[level]));
135 
136  lastSkipDoc[level] = curDoc;
137  //System.out.println("write doc at level " + level + ": " + curDoc);
138 
139  lastSkipFreqPointer[level] = curFreqPointer;
140  lastSkipProxPointer[level] = curProxPointer;
141  }
142  }
143 }