diff --git a/build.go b/build.go index ef0e1e70..9d754aa0 100644 --- a/build.go +++ b/build.go @@ -159,8 +159,8 @@ func persistStoredFieldValues(fieldID int, func InitSegmentBase(mem []byte, memCRC uint32, chunkMode uint32, fieldsMap map[string]uint16, fieldsInv []string, numDocs uint64, - storedIndexOffset uint64, fieldsIndexOffset uint64, docValueOffset uint64, - dictLocs []uint64, sectionsIndexOffset uint64) (*SegmentBase, error) { + storedIndexOffset uint64, dictLocs []uint64, + sectionsIndexOffset uint64) (*SegmentBase, error) { sb := &SegmentBase{ mem: mem, memCRC: memCRC, @@ -169,10 +169,10 @@ func InitSegmentBase(mem []byte, memCRC uint32, chunkMode uint32, fieldsInv: fieldsInv, numDocs: numDocs, storedIndexOffset: storedIndexOffset, - fieldsIndexOffset: fieldsIndexOffset, + fieldsIndexOffset: sectionsIndexOffset, sectionsIndexOffset: sectionsIndexOffset, fieldDvReaders: make([]map[uint16]*docValueReader, len(segmentSections)), - docValueOffset: docValueOffset, + docValueOffset: 0, // docvalueOffsets identified automicatically by each section dictLocs: dictLocs, fieldFSTs: make(map[uint16]*vellum.FST), } diff --git a/merge.go b/merge.go index 32028c26..21a4cb6c 100644 --- a/merge.go +++ b/merge.go @@ -159,7 +159,7 @@ func MergeToWriter(segments []*SegmentBase, drops []*roaring.Bitmap, // we can persist the fields section index now, this will point // to the various indexes (each in different section) available for a field. - sectionsIndexOffset, err = persistFieldsSectionsIndex(fieldsInv, cr, dictLocs, mergeOpaque) + sectionsIndexOffset, err = persistFieldsSection(fieldsInv, cr, dictLocs, mergeOpaque) if err != nil { return nil, 0, 0, 0, 0, nil, nil, nil, 0, err } diff --git a/new.go b/new.go index 26727b7f..6cbc16f3 100644 --- a/new.go +++ b/new.go @@ -66,15 +66,14 @@ func (*ZapPlugin) newWithChunkMode(results []index.Document, s.chunkMode = chunkMode s.w = NewCountHashWriter(&br) - storedIndexOffset, fieldsIndexOffset, fdvIndexOffset, dictOffsets, sectionsIndexOffset, - err := s.convert() + storedIndexOffset, dictOffsets, sectionsIndexOffset, err := s.convert() if err != nil { return nil, uint64(0), err } sb, err := InitSegmentBase(br.Bytes(), s.w.Sum32(), chunkMode, s.FieldsMap, s.FieldsInv, uint64(len(results)), - storedIndexOffset, fieldsIndexOffset, fdvIndexOffset, dictOffsets, sectionsIndexOffset) + storedIndexOffset, dictOffsets, sectionsIndexOffset) // get the bytes written before the interim's reset() call // write it to the newly formed segment base. @@ -169,7 +168,7 @@ type interimLoc struct { arrayposs []uint64 } -func (s *interim) convert() (uint64, uint64, uint64, []uint64, uint64, error) { +func (s *interim) convert() (uint64, []uint64, uint64, error) { s.FieldsMap = map[string]uint16{} s.opaque = map[int]resetable{} @@ -202,7 +201,7 @@ func (s *interim) convert() (uint64, uint64, uint64, []uint64, uint64, error) { storedIndexOffset, err := s.writeStoredFields() if err != nil { - return 0, 0, 0, nil, 0, err + return 0, nil, 0, err } var dictOffsets []uint64 @@ -212,7 +211,7 @@ func (s *interim) convert() (uint64, uint64, uint64, []uint64, uint64, error) { for _, x := range segmentSections { _, err = x.Persist(s.opaque, s.w) if err != nil { - return 0, 0, 0, nil, 0, err + return 0, nil, 0, err } } @@ -222,12 +221,12 @@ func (s *interim) convert() (uint64, uint64, uint64, []uint64, uint64, error) { // we can persist a new fields section here // this new fields section will point to the various indexes available - sectionsIndexOffset, err := persistFieldsSectionsIndex(s.FieldsInv, s.w, dictOffsets, s.opaque) + sectionsIndexOffset, err := persistFieldsSection(s.FieldsInv, s.w, dictOffsets, s.opaque) if err != nil { - return 0, 0, 0, nil, 0, err + return 0, nil, 0, err } - return storedIndexOffset, sectionsIndexOffset, 0, dictOffsets, sectionsIndexOffset, nil + return storedIndexOffset, dictOffsets, sectionsIndexOffset, nil } func (s *interim) getOrDefineField(fieldName string) int { diff --git a/section.go b/section.go index bafe833e..1031cca5 100644 --- a/section.go +++ b/section.go @@ -42,5 +42,5 @@ const ( ) var segmentSections = map[uint16]section{ - sectionInvertedIndex: &invertedIndexSection{}, + sectionInvertedIndex: &invertedTextIndexSection{}, } diff --git a/section_inverted_index.go b/section_inverted_text_index.go similarity index 97% rename from section_inverted_index.go rename to section_inverted_text_index.go index 43bfb689..6744c94b 100644 --- a/section_inverted_index.go +++ b/section_inverted_text_index.go @@ -12,21 +12,21 @@ import ( "github.com/blevesearch/vellum" ) -type invertedIndexSection struct { +type invertedTextIndexSection struct { } -func (i *invertedIndexSection) Process(opaque map[int]resetable, docNum uint64, field index.Field, fieldID uint16) { +func (i *invertedTextIndexSection) Process(opaque map[int]resetable, docNum uint64, field index.Field, fieldID uint16) { invIndexOpaque := i.getInvertedIndexOpaque(opaque) invIndexOpaque.process(field, fieldID, docNum) } -func (i *invertedIndexSection) Persist(opaque map[int]resetable, w *CountHashWriter) (n int64, err error) { +func (i *invertedTextIndexSection) Persist(opaque map[int]resetable, w *CountHashWriter) (n int64, err error) { invIndexOpaque := i.getInvertedIndexOpaque(opaque) _, err = invIndexOpaque.writeDicts(w) return 0, err } -func (i *invertedIndexSection) AddrForField(opaque map[int]resetable, fieldID int) int { +func (i *invertedTextIndexSection) AddrForField(opaque map[int]resetable, fieldID int) int { invIndexOpaque := i.getInvertedIndexOpaque(opaque) return invIndexOpaque.fieldAddrs[fieldID] } @@ -338,7 +338,7 @@ func mergeAndPersistInvertedSection(segments []*SegmentBase, dropsIn []*roaring. return fieldAddrs, fieldDvLocsOffset, nil } -func (i *invertedIndexSection) Merge(opaque map[int]resetable, segments []*SegmentBase, +func (i *invertedTextIndexSection) Merge(opaque map[int]resetable, segments []*SegmentBase, drops []*roaring.Bitmap, fieldsInv []string, newDocNumsIn [][]uint64, w *CountHashWriter, closeCh chan struct{}) error { io := i.getInvertedIndexOpaque(opaque) @@ -801,7 +801,7 @@ func (i *invertedIndexOpaque) allocateSpace() { } } -func (i *invertedIndexSection) getInvertedIndexOpaque(opaque map[int]resetable) *invertedIndexOpaque { +func (i *invertedTextIndexSection) getInvertedIndexOpaque(opaque map[int]resetable) *invertedIndexOpaque { if _, ok := opaque[sectionInvertedIndex]; !ok { opaque[sectionInvertedIndex] = i.InitOpaque(nil) } @@ -829,7 +829,7 @@ func (i *invertedIndexOpaque) getOrDefineField(fieldName string) int { return int(fieldIDPlus1 - 1) } -func (i *invertedIndexSection) InitOpaque(args map[string]interface{}) resetable { +func (i *invertedTextIndexSection) InitOpaque(args map[string]interface{}) resetable { rv := &invertedIndexOpaque{ fieldAddrs: map[int]int{}, } diff --git a/write.go b/write.go index a090c3b9..ff4afb70 100644 --- a/write.go +++ b/write.go @@ -50,7 +50,7 @@ func writeRoaringWithLen(r *roaring.Bitmap, w io.Writer, return tw, nil } -func persistFieldsSectionsIndex(fieldsInv []string, w *CountHashWriter, dictLocs []uint64, opaque map[int]resetable) (uint64, error) { +func persistFieldsSection(fieldsInv []string, w *CountHashWriter, dictLocs []uint64, opaque map[int]resetable) (uint64, error) { var rv uint64 var fieldsOffsets []uint64