internal/runtime/maps: small maps point directly to a group

If the map contains 8 or fewer entries, it is wasteful to have a
directory that points to a table that points to a group.

Add a special case that replaces the directory with a direct pointer to
a group.

We could theoretically do similar for single table maps (no directory,
just point directly to a table), but that is left for later.

For golang#54766.

Cq-Include-Trybots: luci.golang.try:gotip-linux-amd64-longtest-swissmap
Change-Id: I6fc04dfc11c31dadfe5b5d6481b4c4abd43d48ed
Reviewed-on: https://go-review.googlesource.com/c/go/+/611188
Reviewed-by: Keith Randall <[email protected]>
LUCI-TryBot-Result: Go LUCI <[email protected]>
Auto-Submit: Michael Pratt <[email protected]>
Reviewed-by: Keith Randall <[email protected]>

src/cmd/compile/internal/reflectdata/map_swiss.go

@@ -144,7 +144,8 @@ func SwissMapType() *types.Type {
 	//     typ  unsafe.Pointer // *abi.SwissMapType
 	//     seed uintptr
 	//
-	//     directory []*table
+	//     dirPtr unsafe.Pointer
+	//     dirLen int
 	//
 	//     globalDepth uint8
 	//     // N.B Padding
@@ -156,7 +157,8 @@ func SwissMapType() *types.Type {
 		makefield("used", types.Types[types.TUINT64]),
 		makefield("typ", types.Types[types.TUNSAFEPTR]),
 		makefield("seed", types.Types[types.TUINTPTR]),
-		makefield("directory", types.NewSlice(types.NewPtr(swissTableType()))),
+		makefield("dirPtr", types.Types[types.TUNSAFEPTR]),
+		makefield("dirLen", types.Types[types.TINT]),
 		makefield("globalDepth", types.Types[types.TUINT8]),
 		makefield("clearSeq", types.Types[types.TUINT64]),
 	}
@@ -169,9 +171,9 @@ func SwissMapType() *types.Type {
 	m.SetUnderlying(types.NewStruct(fields))
 	types.CalcSize(m)
 
-	// The size of Map should be 64 bytes on 64 bit
-	// and 40 bytes on 32 bit platforms.
-	if size := int64(2*8 + 6*types.PtrSize); m.Size() != size {
+	// The size of Map should be 56 bytes on 64 bit
+	// and 36 bytes on 32 bit platforms.
+	if size := int64(2*8 + 5*types.PtrSize /* one extra for globalDepth + padding */); m.Size() != size {
 		base.Fatalf("internal/runtime/maps.Map size not correct: got %d, want %d", m.Size(), size)
 	}
 
@@ -204,7 +206,9 @@ func SwissMapIterType() *types.Type {
 	//
 	//    dirIdx int
 	//
-	//    tab *table
+	//    tab             *table
+	//    groupSmall_typ  unsafe.Pointer // *SwissMapType
+	//    groupSmall_data unsafe.Pointer
 	//
 	//    entryIdx uint64
 	// }
@@ -220,10 +224,12 @@ func SwissMapIterType() *types.Type {
 		makefield("globalDepth", types.Types[types.TUINT8]),
 		makefield("dirIdx", types.Types[types.TINT]),
 		makefield("tab", types.NewPtr(swissTableType())),
+		makefield("groupSmall_typ", types.Types[types.TUNSAFEPTR]),
+		makefield("groupSmall_data", types.Types[types.TUNSAFEPTR]),
 		makefield("entryIdx", types.Types[types.TUINT64]),
 	}
 
-	// build iterator struct hswissing the above fields
+	// build iterator struct holding the above fields
 	n := ir.NewDeclNameAt(src.NoXPos, ir.OTYPE, ir.Pkgs.InternalMaps.Lookup("Iter"))
 	iter := types.NewNamed(n)
 	n.SetType(iter)
@@ -232,9 +238,9 @@ func SwissMapIterType() *types.Type {
 	iter.SetUnderlying(types.NewStruct(fields))
 	types.CalcSize(iter)
 
-	// The size of Iter should be 88 bytes on 64 bit
-	// and 60 bytes on 32 bit platforms.
-	if size := 7*types.PtrSize /* one extra for globalDepth + padding */ + 4*8; iter.Size() != int64(size) {
+	// The size of Iter should be 104 bytes on 64 bit
+	// and 68 bytes on 32 bit platforms.
+	if size := 9*types.PtrSize /* one extra for globalDepth + padding */ + 4*8; iter.Size() != int64(size) {
 		base.Fatalf("internal/runtime/maps.Iter size not correct: got %d, want %d", iter.Size(), size)
 	}
 

src/cmd/link/internal/ld/dwarf.go

@@ -875,7 +875,6 @@ func (d *dwctxt) synthesizemaptypes(ctxt *Link, die *dwarf.DWDie) {
 func (d *dwctxt) synthesizemaptypesSwiss(ctxt *Link, die *dwarf.DWDie) {
 	mapType := walktypedef(d.findprotodie(ctxt, "type:internal/runtime/maps.Map"))
 	tableType := walktypedef(d.findprotodie(ctxt, "type:internal/runtime/maps.table"))
-	tableSliceType := walktypedef(d.findprotodie(ctxt, "type:[]*internal/runtime/maps.table"))
 	groupsReferenceType := walktypedef(d.findprotodie(ctxt, "type:internal/runtime/maps.groupsReference"))
 
 	for ; die != nil; die = die.Link {
@@ -916,19 +915,16 @@ func (d *dwctxt) synthesizemaptypesSwiss(ctxt *Link, die *dwarf.DWDie) {
 			newattr(dwh, dwarf.DW_AT_go_kind, dwarf.DW_CLS_CONSTANT, int64(abi.Struct), 0)
 		})
 
-		// Construct type to represent []*table[K,V].
-		dwTableSlice := d.mkinternaltype(ctxt, dwarf.DW_ABRV_SLICETYPE, "[]*table", keyName, valName, func(dwh *dwarf.DWDie) {
-			d.copychildren(ctxt, dwh, tableSliceType)
-			d.substitutetype(dwh, "array", d.defptrto(d.defptrto(dwTable)))
-			d.newrefattr(dwh, dwarf.DW_AT_go_elem, d.defptrto(dwTable))
-			newattr(dwh, dwarf.DW_AT_byte_size, dwarf.DW_CLS_CONSTANT, getattr(tableSliceType, dwarf.DW_AT_byte_size).Value, nil)
-			newattr(dwh, dwarf.DW_AT_go_kind, dwarf.DW_CLS_CONSTANT, int64(abi.Slice), 0)
-		})
-
 		// Construct map[K,V]
 		dwMap := d.mkinternaltype(ctxt, dwarf.DW_ABRV_STRUCTTYPE, "map", keyName, valName, func(dwh *dwarf.DWDie) {
 			d.copychildren(ctxt, dwh, mapType)
-			d.substitutetype(dwh, "directory", dwTableSlice)
+			// dirPtr is a pointer to a variable-length array of
+			// *table[K,V], of length dirLen.
+			//
+			// Since we can't directly define a variable-length
+			// array, store this as **table[K,V]. i.e., pointer to
+			// the first entry in the array.
+			d.substitutetype(dwh, "dirPtr", d.defptrto(d.defptrto(dwTable)))
 			newattr(dwh, dwarf.DW_AT_byte_size, dwarf.DW_CLS_CONSTANT, getattr(mapType, dwarf.DW_AT_byte_size).Value, nil)
 			newattr(dwh, dwarf.DW_AT_go_kind, dwarf.DW_CLS_CONSTANT, int64(abi.Struct), 0)
 		})
@@ -1851,7 +1847,6 @@ func dwarfGenerateDebugInfo(ctxt *Link) {
 	if buildcfg.Experiment.SwissMap {
 		prototypedies["type:internal/runtime/maps.Map"] = nil
 		prototypedies["type:internal/runtime/maps.table"] = nil
-		prototypedies["type:[]*internal/runtime/maps.table"] = nil
 		prototypedies["type:internal/runtime/maps.groupsReference"] = nil
 	} else {
 		prototypedies["type:runtime.hmap"] = nil

src/internal/runtime/maps/export_test.go

@@ -24,25 +24,47 @@ func NewTestMap[K comparable, V any](length uint64) (*Map, *abi.SwissMapType) {
 }
 
 func (m *Map) TableCount() int {
-	return len(m.directory)
+	if m.dirLen <= 0 {
+		return 0
+	}
+	return m.dirLen
 }
 
 // Total group count, summed across all tables.
 func (m *Map) GroupCount() uint64 {
+	if m.dirLen <= 0 {
+		if m.dirPtr == nil {
+			return 0
+		}
+		return 1
+	}
+
 	var n uint64
-	for _, t := range m.directory {
+	var lastTab *table
+	for i := range m.dirLen {
+		t := m.directoryAt(uintptr(i))
+		if t == lastTab {
+			continue
+		}
+		lastTab = t
 		n += t.groups.lengthMask + 1
 	}
 	return n
 }
 
-// Return a key from a group containing no empty slots, or nil if there are no
-// full groups.
+// Return a key from a group containing no empty slots.
 //
-// Also returns nil if a group is full but contains entirely deleted slots.
+// Returns nil if there are no full groups.
+// Returns nil if a group is full but contains entirely deleted slots.
+// Returns nil if the map is small.
 func (m *Map) KeyFromFullGroup() unsafe.Pointer {
+	if m.dirLen <= 0 {
+		return nil
+	}
+
 	var lastTab *table
-	for _, t := range m.directory {
+	for i := range m.dirLen {
+		t := m.directoryAt(uintptr(i))
 		if t == lastTab {
 			continue
 		}
@@ -68,10 +90,15 @@ func (m *Map) KeyFromFullGroup() unsafe.Pointer {
 	return nil
 }
 
+// Returns nil if the map is small.
 func (m *Map) TableFor(key unsafe.Pointer) *table {
+	if m.dirLen <= 0 {
+		return nil
+	}
+
 	hash := m.typ.Hasher(key, m.seed)
 	idx := m.directoryIndex(hash)
-	return m.directory[idx]
+	return m.directoryAt(idx)
 }
 
 func (t *table) GrowthLeft() uint64 {