19 using Lucene.Net.Analysis.Tokenattributes;
20 using Lucene.Net.Documents;
21 using Lucene.Net.Support;
24 namespace Lucene.Net.Index
29 private void InitBlock()
31 postingsHashHalfSize = postingsHashSize / 2;
32 postingsHashMask = postingsHashSize - 1;
48 internal int streamCount;
49 internal int numPostingInt;
53 internal bool postingsCompacted;
54 internal int numPostings;
55 private int postingsHashSize = 4;
56 private int postingsHashHalfSize;
57 private int postingsHashMask;
64 this.perThread = perThread;
65 intPool = perThread.intPool;
66 charPool = perThread.charPool;
67 bytePool = perThread.bytePool;
68 docState = perThread.docState;
69 fieldState = docInverterPerField.fieldState;
70 this.consumer = perThread.consumer.
AddField(
this, fieldInfo);
71 streamCount = consumer.GetStreamCount();
72 numPostingInt = 2 * streamCount;
73 this.fieldInfo = fieldInfo;
74 if (nextPerThread != null)
75 nextPerField = (
TermsHashPerField) nextPerThread.AddField(docInverterPerField, fieldInfo);
80 internal void ShrinkHash(
int targetSize)
82 System.Diagnostics.Debug.Assert(postingsCompacted || numPostings == 0);
86 if (newSize != postingsHash.Length)
89 postingsHashSize = newSize;
90 postingsHashHalfSize = newSize / 2;
91 postingsHashMask = newSize - 1;
93 System.Array.Clear(postingsHash,0,postingsHash.Length);
98 if (!postingsCompacted)
100 System.Diagnostics.Debug.Assert(numPostings <= postingsHash.Length);
103 perThread.termsHash.RecyclePostings(postingsHash, numPostings);
104 Array.Clear(postingsHash, 0, numPostings);
107 postingsCompacted =
false;
108 if (nextPerField != null)
109 nextPerField.Reset();
112 public override void Abort()
117 if (nextPerField != null)
118 nextPerField.Abort();
124 System.Diagnostics.Debug.Assert(stream < streamCount);
125 int[] ints = intPool.buffers[p.intStart >>
DocumentsWriter.INT_BLOCK_SHIFT];
127 reader.
Init(bytePool, p.byteStart + stream *
ByteBlockPool.FIRST_LEVEL_SIZE, ints[upto + stream]);
130 private void CompactPostings()
135 for (
int i = 0; i < postingsHashSize; i++)
137 if (postingsHash[i] != null)
141 postingsHash[upto] = postingsHash[i];
142 postingsHash[i] = null;
148 System.Diagnostics.Debug.Assert(upto == numPostings);
149 postingsCompacted =
true;
157 QuickSort(postingsHash, 0, numPostings - 1);
161 internal void QuickSort(
RawPostingList[] postings,
int lo,
int hi)
165 else if (hi == 1 + lo)
167 if (ComparePostings(postings[lo], postings[hi]) > 0)
170 postings[lo] = postings[hi];
178 if (ComparePostings(postings[lo], postings[mid]) > 0)
180 RawPostingList tmp = postings[lo];
181 postings[lo] = postings[mid];
185 if (ComparePostings(postings[mid], postings[hi]) > 0)
187 RawPostingList tmp = postings[mid];
188 postings[mid] = postings[hi];
191 if (ComparePostings(postings[lo], postings[mid]) > 0)
193 RawPostingList tmp2 = postings[lo];
194 postings[lo] = postings[mid];
195 postings[mid] = tmp2;
205 RawPostingList partition = postings[mid];
209 while (ComparePostings(postings[right], partition) > 0)
212 while (left < right && ComparePostings(postings[left], partition) <= 0)
217 RawPostingList tmp = postings[left];
218 postings[left] = postings[right];
219 postings[right] = tmp;
228 QuickSort(postings, lo, left);
229 QuickSort(postings, left + 1, hi);
235 internal int ComparePostings(RawPostingList p1, RawPostingList p2)
241 char[] text1 = charPool.buffers[p1.textStart >> DocumentsWriter.CHAR_BLOCK_SHIFT];
242 int pos1 = p1.textStart & DocumentsWriter.CHAR_BLOCK_MASK;
243 char[] text2 = charPool.buffers[p2.textStart >> DocumentsWriter.CHAR_BLOCK_SHIFT];
244 int pos2 = p2.textStart & DocumentsWriter.CHAR_BLOCK_MASK;
246 System.Diagnostics.Debug.Assert(text1 != text2 || pos1 != pos2);
250 char c1 = text1[pos1++];
251 char c2 = text2[pos2++];
256 else if (0xffff == c1)
264 System.Diagnostics.Debug.Assert(c1 != 0xffff);
271 private bool PostingEquals(
char[] tokenText,
int tokenTextLen)
274 char[] text = perThread.charPool.buffers[p.textStart >> DocumentsWriter.CHAR_BLOCK_SHIFT];
275 System.Diagnostics.Debug.Assert(text != null);
276 int pos = p.textStart & DocumentsWriter.CHAR_BLOCK_MASK;
279 for (; tokenPos < tokenTextLen; pos++, tokenPos++)
280 if (tokenText[tokenPos] != text[pos])
282 return 0xffff == text[pos];
286 private bool doNextCall;
290 termAtt = fieldState.attributeSource.AddAttribute<
ITermAttribute>();
292 if (nextPerField != null)
294 nextPerField.Start(f);
298 internal override bool Start(
IFieldable[] fields,
int count)
300 doCall = consumer.Start(fields, count);
301 if (nextPerField != null)
302 doNextCall = nextPerField.Start(fields, count);
303 return doCall || doNextCall;
309 public void Add(
int textStart)
312 int code = textStart;
314 int hashPos = code & postingsHashMask;
316 System.Diagnostics.Debug.Assert(!postingsCompacted);
319 p = postingsHash[hashPos];
321 if (p != null && p.textStart != textStart)
325 int inc = ((code >> 8) + code) | 1;
329 hashPos = code & postingsHashMask;
330 p = postingsHash[hashPos];
332 while (p != null && p.textStart != textStart);
342 if (0 == perThread.freePostingsCount)
343 perThread.MorePostings();
346 p = perThread.freePostings[--perThread.freePostingsCount];
347 System.Diagnostics.Debug.Assert(p != null);
349 p.textStart = textStart;
351 System.Diagnostics.Debug.Assert(postingsHash [hashPos] == null);
352 postingsHash[hashPos] = p;
355 if (numPostings == postingsHashHalfSize)
356 RehashPostings(2 * postingsHashSize);
360 intPool.NextBuffer();
363 bytePool.NextBuffer();
365 intUptos = intPool.buffer;
366 intUptoStart = intPool.intUpto;
367 intPool.intUpto += streamCount;
369 p.intStart = intUptoStart + intPool.intOffset;
371 for (
int i = 0; i < streamCount; i++)
373 int upto = bytePool.NewSlice(
ByteBlockPool.FIRST_LEVEL_SIZE);
374 intUptos[intUptoStart + i] = upto + bytePool.byteOffset;
376 p.byteStart = intUptos[intUptoStart];
382 intUptos = intPool.buffers[p.intStart >>
DocumentsWriter.INT_BLOCK_SHIFT];
389 internal override void Add()
392 System.Diagnostics.Debug.Assert(!postingsCompacted);
398 char[] tokenText = termAtt.TermBuffer();
400 int tokenTextLen = termAtt.TermLength();
403 int downto = tokenTextLen;
407 char ch = tokenText[--downto];
414 ch = tokenText[downto] = (char) (
UnicodeUtil.UNI_REPLACEMENT_CHAR);
418 char ch2 = tokenText[downto - 1];
423 code = ((code * 31) + ch) * 31 + ch2;
430 ch = tokenText[downto] = (char) (
UnicodeUtil.UNI_REPLACEMENT_CHAR);
437 ch = tokenText[downto] = (char) (
UnicodeUtil.UNI_REPLACEMENT_CHAR);
440 code = (code * 31) + ch;
443 int hashPos = code & postingsHashMask;
446 p = postingsHash[hashPos];
448 if (p != null && !PostingEquals(tokenText, tokenTextLen))
452 int inc = ((code >> 8) + code) | 1;
456 hashPos = code & postingsHashMask;
457 p = postingsHash[hashPos];
459 while (p != null && !PostingEquals(tokenText, tokenTextLen));
467 int textLen1 = 1 + tokenTextLen;
468 if (textLen1 + charPool.charUpto > DocumentsWriter.CHAR_BLOCK_SIZE)
470 if (textLen1 > DocumentsWriter.CHAR_BLOCK_SIZE)
478 if (docState.maxTermPrefix == null)
479 docState.maxTermPrefix =
new System.String(tokenText, 0, 30);
481 consumer.SkippingLongTerm();
484 charPool.NextBuffer();
488 if (0 == perThread.freePostingsCount)
489 perThread.MorePostings();
492 p = perThread.freePostings[--perThread.freePostingsCount];
493 System.Diagnostics.Debug.Assert(p != null);
495 char[] text = charPool.buffer;
496 int textUpto = charPool.charUpto;
497 p.textStart = textUpto + charPool.charOffset;
498 charPool.charUpto += textLen1;
499 Array.Copy(tokenText, 0, text, textUpto, tokenTextLen);
500 text[textUpto + tokenTextLen] = (char) (0xffff);
502 System.Diagnostics.Debug.Assert(postingsHash [hashPos] == null);
503 postingsHash[hashPos] = p;
506 if (numPostings == postingsHashHalfSize)
507 RehashPostings(2 * postingsHashSize);
510 if (numPostingInt + intPool.intUpto > DocumentsWriter.INT_BLOCK_SIZE)
511 intPool.NextBuffer();
513 if (DocumentsWriter.BYTE_BLOCK_SIZE - bytePool.byteUpto < numPostingInt * ByteBlockPool.FIRST_LEVEL_SIZE)
514 bytePool.NextBuffer();
516 intUptos = intPool.buffer;
517 intUptoStart = intPool.intUpto;
518 intPool.intUpto += streamCount;
520 p.intStart = intUptoStart + intPool.intOffset;
522 for (
int i = 0; i < streamCount; i++)
524 int upto = bytePool.NewSlice(ByteBlockPool.FIRST_LEVEL_SIZE);
525 intUptos[intUptoStart + i] = upto + bytePool.byteOffset;
527 p.byteStart = intUptos[intUptoStart];
533 intUptos = intPool.buffers[p.intStart >> DocumentsWriter.INT_BLOCK_SHIFT];
534 intUptoStart = p.intStart & DocumentsWriter.INT_BLOCK_MASK;
539 nextPerField.Add(p.textStart);
542 internal int[] intUptos;
543 internal int intUptoStart;
545 internal void WriteByte(
int stream, byte b)
547 int upto = intUptos[intUptoStart + stream];
548 byte[] bytes = bytePool.buffers[upto >> DocumentsWriter.BYTE_BLOCK_SHIFT];
549 System.Diagnostics.Debug.Assert(bytes != null);
550 int offset = upto & DocumentsWriter.BYTE_BLOCK_MASK;
551 if (bytes[offset] != 0)
554 offset = bytePool.AllocSlice(bytes, offset);
555 bytes = bytePool.buffer;
556 intUptos[intUptoStart + stream] = offset + bytePool.byteOffset;
559 (intUptos[intUptoStart + stream])++;
562 public void WriteBytes(
int stream, byte[] b,
int offset,
int len)
565 int end = offset + len;
566 for (
int i = offset; i < end; i++)
567 WriteByte(stream, b[i]);
570 internal void WriteVInt(
int stream,
int i)
572 System.Diagnostics.Debug.Assert(stream < streamCount);
573 while ((i & ~ 0x7F) != 0)
575 WriteByte(stream, (byte) ((i & 0x7f) | 0x80));
578 WriteByte(stream, (byte) i);
581 internal override void Finish()
584 if (nextPerField != null)
585 nextPerField.Finish();
591 internal void RehashPostings(
int newSize)
594 int newMask = newSize - 1;
596 RawPostingList[] newHash =
new RawPostingList[newSize];
597 for (
int i = 0; i < postingsHashSize; i++)
599 RawPostingList p0 = postingsHash[i];
603 if (perThread.primary)
605 int start = p0.textStart & DocumentsWriter.CHAR_BLOCK_MASK;
606 char[] text = charPool.buffers[p0.textStart >> DocumentsWriter.CHAR_BLOCK_SHIFT];
608 while (text[pos] != 0xffff)
612 code = (code * 31) + text[--pos];
617 int hashPos = code & newMask;
618 System.Diagnostics.Debug.Assert(hashPos >= 0);
619 if (newHash[hashPos] != null)
621 int inc = ((code >> 8) + code) | 1;
625 hashPos = code & newMask;
627 while (newHash[hashPos] != null);
629 newHash[hashPos] = p0;
633 postingsHashMask = newMask;
634 postingsHash = newHash;
635 postingsHashSize = newSize;
636 postingsHashHalfSize = newSize >> 1;