renaming vars, files and some refactoring of code

build.go

@@ -159,8 +159,8 @@ func persistStoredFieldValues(fieldID int,
 
 func InitSegmentBase(mem []byte, memCRC uint32, chunkMode uint32,
 	fieldsMap map[string]uint16, fieldsInv []string, numDocs uint64,
-	storedIndexOffset uint64, fieldsIndexOffset uint64, docValueOffset uint64,
-	dictLocs []uint64, sectionsIndexOffset uint64) (*SegmentBase, error) {
+	storedIndexOffset uint64, dictLocs []uint64,
+	sectionsIndexOffset uint64) (*SegmentBase, error) {
 	sb := &SegmentBase{
 		mem:                 mem,
 		memCRC:              memCRC,
@@ -169,17 +169,17 @@ func InitSegmentBase(mem []byte, memCRC uint32, chunkMode uint32,
 		fieldsInv:           fieldsInv,
 		numDocs:             numDocs,
 		storedIndexOffset:   storedIndexOffset,
-		fieldsIndexOffset:   fieldsIndexOffset,
+		fieldsIndexOffset:   sectionsIndexOffset,
 		sectionsIndexOffset: sectionsIndexOffset,
 		fieldDvReaders:      make([]map[uint16]*docValueReader, len(segmentSections)),
-		docValueOffset:      docValueOffset,
+		docValueOffset:      0, // docvalueOffsets identified automicatically by the section
 		dictLocs:            dictLocs,
 		fieldFSTs:           make(map[uint16]*vellum.FST),
 	}
 	sb.updateSize()
 
-	// FIXME load map based on new fields index offset data
-	// (maybe this works now)
+	// load the data/section starting offsets for each field
+	// by via the sectionsIndexOffset as starting point.
 	err := sb.loadFieldsNew()
 	if err != nil {
 		return nil, err

merge.go

@@ -159,7 +159,7 @@ func MergeToWriter(segments []*SegmentBase, drops []*roaring.Bitmap,
 
 	// we can persist the fields section index now, this will point
 	// to the various indexes (each in different section) available for a field.
-	sectionsIndexOffset, err = persistFieldsSectionsIndex(fieldsInv, cr, dictLocs, mergeOpaque)
+	sectionsIndexOffset, err = persistFieldsSection(fieldsInv, cr, dictLocs, mergeOpaque)
 	if err != nil {
 		return nil, 0, 0, 0, 0, nil, nil, nil, 0, err
 	}

new.go

@@ -66,15 +66,14 @@ func (*ZapPlugin) newWithChunkMode(results []index.Document,
 	s.chunkMode = chunkMode
 	s.w = NewCountHashWriter(&br)
 
-	storedIndexOffset, fieldsIndexOffset, fdvIndexOffset, dictOffsets, sectionsIndexOffset,
-		err := s.convert()
+	storedIndexOffset, dictOffsets, sectionsIndexOffset, err := s.convert()
 	if err != nil {
 		return nil, uint64(0), err
 	}
 
 	sb, err := InitSegmentBase(br.Bytes(), s.w.Sum32(), chunkMode,
 		s.FieldsMap, s.FieldsInv, uint64(len(results)),
-		storedIndexOffset, fieldsIndexOffset, fdvIndexOffset, dictOffsets, sectionsIndexOffset)
+		storedIndexOffset, dictOffsets, sectionsIndexOffset)
 
 	// get the bytes written before the interim's reset() call
 	// write it to the newly formed segment base.
@@ -169,7 +168,7 @@ type interimLoc struct {
 	arrayposs []uint64
 }
 
-func (s *interim) convert() (uint64, uint64, uint64, []uint64, uint64, error) {
+func (s *interim) convert() (uint64, []uint64, uint64, error) {
 	s.FieldsMap = map[string]uint16{}
 	s.opaque = map[int]resetable{}
 
@@ -202,7 +201,7 @@ func (s *interim) convert() (uint64, uint64, uint64, []uint64, uint64, error) {
 
 	storedIndexOffset, err := s.writeStoredFields()
 	if err != nil {
-		return 0, 0, 0, nil, 0, err
+		return 0, nil, 0, err
 	}
 
 	var dictOffsets []uint64
@@ -212,7 +211,7 @@ func (s *interim) convert() (uint64, uint64, uint64, []uint64, uint64, error) {
 	for _, x := range segmentSections {
 		_, err = x.Persist(s.opaque, s.w)
 		if err != nil {
-			return 0, 0, 0, nil, 0, err
+			return 0, nil, 0, err
 		}
 	}
 
@@ -222,12 +221,12 @@ func (s *interim) convert() (uint64, uint64, uint64, []uint64, uint64, error) {
 
 	// we can persist a new fields section here
 	// this new fields section will point to the various indexes available
-	sectionsIndexOffset, err := persistFieldsSectionsIndex(s.FieldsInv, s.w, dictOffsets, s.opaque)
+	sectionsIndexOffset, err := persistFieldsSection(s.FieldsInv, s.w, dictOffsets, s.opaque)
 	if err != nil {
-		return 0, 0, 0, nil, 0, err
+		return 0, nil, 0, err
 	}
 
-	return storedIndexOffset, sectionsIndexOffset, 0, dictOffsets, sectionsIndexOffset, nil
+	return storedIndexOffset, dictOffsets, sectionsIndexOffset, nil
 }
 
 func (s *interim) getOrDefineField(fieldName string) int {

section.go

@@ -42,5 +42,5 @@ const (
 )
 
 var segmentSections = map[uint16]section{
-	sectionInvertedIndex: &invertedIndexSection{},
+	sectionInvertedIndex: &invertedTextIndexSection{},
 }

section_inverted_index.go → section_inverted_text_index.go

@@ -12,21 +12,21 @@ import (
 	"github.com/blevesearch/vellum"
 )
 
-type invertedIndexSection struct {
+type invertedTextIndexSection struct {
 }
 
-func (i *invertedIndexSection) Process(opaque map[int]resetable, docNum uint64, field index.Field, fieldID uint16) {
+func (i *invertedTextIndexSection) Process(opaque map[int]resetable, docNum uint64, field index.Field, fieldID uint16) {
 	invIndexOpaque := i.getInvertedIndexOpaque(opaque)
 	invIndexOpaque.process(field, fieldID, docNum)
 }
 
-func (i *invertedIndexSection) Persist(opaque map[int]resetable, w *CountHashWriter) (n int64, err error) {
+func (i *invertedTextIndexSection) Persist(opaque map[int]resetable, w *CountHashWriter) (n int64, err error) {
 	invIndexOpaque := i.getInvertedIndexOpaque(opaque)
 	_, err = invIndexOpaque.writeDicts(w)
 	return 0, err
 }
 
-func (i *invertedIndexSection) AddrForField(opaque map[int]resetable, fieldID int) int {
+func (i *invertedTextIndexSection) AddrForField(opaque map[int]resetable, fieldID int) int {
 	invIndexOpaque := i.getInvertedIndexOpaque(opaque)
 	return invIndexOpaque.fieldAddrs[fieldID]
 }
@@ -338,7 +338,7 @@ func mergeAndPersistInvertedSection(segments []*SegmentBase, dropsIn []*roaring.
 	return fieldAddrs, fieldDvLocsOffset, nil
 }
 
-func (i *invertedIndexSection) Merge(opaque map[int]resetable, segments []*SegmentBase,
+func (i *invertedTextIndexSection) Merge(opaque map[int]resetable, segments []*SegmentBase,
 	drops []*roaring.Bitmap, fieldsInv []string, newDocNumsIn [][]uint64,
 	w *CountHashWriter, closeCh chan struct{}) error {
 	io := i.getInvertedIndexOpaque(opaque)
@@ -801,7 +801,7 @@ func (i *invertedIndexOpaque) allocateSpace() {
 	}
 }
 
-func (i *invertedIndexSection) getInvertedIndexOpaque(opaque map[int]resetable) *invertedIndexOpaque {
+func (i *invertedTextIndexSection) getInvertedIndexOpaque(opaque map[int]resetable) *invertedIndexOpaque {
 	if _, ok := opaque[sectionInvertedIndex]; !ok {
 		opaque[sectionInvertedIndex] = i.InitOpaque(nil)
 	}
@@ -829,7 +829,7 @@ func (i *invertedIndexOpaque) getOrDefineField(fieldName string) int {
 	return int(fieldIDPlus1 - 1)
 }
 
-func (i *invertedIndexSection) InitOpaque(args map[string]interface{}) resetable {
+func (i *invertedTextIndexSection) InitOpaque(args map[string]interface{}) resetable {
 	rv := &invertedIndexOpaque{
 		fieldAddrs: map[int]int{},
 	}

write.go

@@ -50,7 +50,7 @@ func writeRoaringWithLen(r *roaring.Bitmap, w io.Writer,
 	return tw, nil
 }
 
-func persistFieldsSectionsIndex(fieldsInv []string, w *CountHashWriter, dictLocs []uint64, opaque map[int]resetable) (uint64, error) {
+func persistFieldsSection(fieldsInv []string, w *CountHashWriter, dictLocs []uint64, opaque map[int]resetable) (uint64, error) {
 	var rv uint64
 	var fieldsOffsets []uint64