feat(addressbook): prune stale entries by last-seen time

pkg/addressbook/addressbook.go

@@ -9,6 +9,7 @@ import (
 	"errors"
 	"fmt"
 	"strings"
+	"time"
 
 	"github.com/ethersphere/bee/v2/pkg/bzz"
 	"github.com/ethersphere/bee/v2/pkg/storage"
@@ -22,22 +23,30 @@ var _ Interface = (*store)(nil)
 var ErrNotFound = errors.New("addressbook: not found")
 
 // verifiedAddress pairs a bzz.Address with a flag indicating whether the peer
-// has been verified.
+// has been verified, and the last time the overlay was seen.
 type verifiedAddress struct {
 	Address  *bzz.Address `json:"address"`
 	Verified bool         `json:"verified"`
+	// LastSeen is the Unix timestamp (seconds) of the last time the overlay
+	// was seen over hive or in kademlia. Used to prune stale entries.
+	LastSeen int64 `json:"last_seen"`
 }
 
 // Interface is the AddressBook interface.
 type Interface interface {
 	GetPutter
 	Remover
+	// UpdateLastSeen marks the overlay as seen at the current time.
+	UpdateLastSeen(overlay swarm.Address) error
 	// Overlays returns a list of all overlay addresses saved in addressbook.
 	Overlays() ([]swarm.Address, error)
 	// IterateOverlays exposes overlays in a form of an iterator.
 	IterateOverlays(func(swarm.Address) (bool, error)) error
 	// Addresses returns a list of all bzz.Address-es saved in addressbook.
 	Addresses() ([]bzz.Address, error)
+	// Prune removes all entries whose overlay has not been seen since the
+	// given time.
+	Prune(before time.Time) error
 }
 
 type GetPutter interface {
@@ -63,12 +72,14 @@ type Remover interface {
 
 type store struct {
 	store storage.StateStorer
+	now   func() time.Time
 }
 
 // New creates new addressbook for state storer.
 func New(storer storage.StateStorer) Interface {
 	return &store{
 		store: storer,
+		now:   time.Now,
 	}
 }
 
@@ -90,13 +101,63 @@ func (s *store) Put(overlay swarm.Address, addr bzz.Address, verified bool) (err
 	return s.store.Put(key, &verifiedAddress{
 		Address:  &addr,
 		Verified: verified,
+		LastSeen: s.now().Unix(),
 	})
 }
 
+// UpdateLastSeen marks the overlay as seen at the current time. It is a no-op
+// if the overlay is not present in the addressbook.
+func (s *store) UpdateLastSeen(overlay swarm.Address) error {
+	key := keyPrefix + overlay.String()
+	v := &verifiedAddress{}
+	if err := s.store.Get(key, v); err != nil {
+		if errors.Is(err, storage.ErrNotFound) {
+			return nil
+		}
+		return err
+	}
+	v.LastSeen = s.now().Unix()
+	return s.store.Put(key, v)
+}
+
 func (s *store) Remove(overlay swarm.Address) error {
 	return s.store.Delete(keyPrefix + overlay.String())
 }
 
+// Prune removes all entries whose overlay has not been seen since before.
+// Entries without a recorded last-seen time (LastSeen == 0) are kept, leaving
+// pruning to a later run once they have been observed.
+func (s *store) Prune(before time.Time) error {
+	cutoff := before.Unix()
+
+	var stale []swarm.Address
+	err := s.store.Iterate(keyPrefix, func(key, value []byte) (stop bool, err error) {
+		entry := &verifiedAddress{}
+		if err := json.Unmarshal(value, entry); err != nil {
+			return true, err
+		}
+		if entry.LastSeen != 0 && entry.LastSeen < cutoff {
+			addr, err := swarm.ParseHexAddress(strings.TrimPrefix(string(key), keyPrefix))
+			if err != nil {
+				return true, err
+			}
+			stale = append(stale, addr)
+		}
+		return false, nil
+	})
+	if err != nil {
+		return err
+	}
+
+	for _, addr := range stale {
+		if err := s.Remove(addr); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
 func (s *store) IterateOverlays(cb func(swarm.Address) (bool, error)) error {
 	return s.store.Iterate(keyPrefix, func(key, _ []byte) (stop bool, err error) {
 		k := string(key)

pkg/addressbook/addressbook_test.go

@@ -7,6 +7,7 @@ package addressbook_test
 import (
 	"errors"
 	"testing"
+	"time"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethersphere/bee/v2/pkg/addressbook"
@@ -17,6 +18,24 @@ import (
 	ma "github.com/multiformats/go-multiaddr"
 )
 
+func newTestAddr(t *testing.T, overlay swarm.Address) bzz.Address {
+	t.Helper()
+
+	multiaddr, err := ma.NewMultiaddr("/ip4/1.1.1.1")
+	if err != nil {
+		t.Fatal(err)
+	}
+	pk, err := crypto.GenerateSecp256k1Key()
+	if err != nil {
+		t.Fatal(err)
+	}
+	bzzAddr, err := bzz.NewAddress(crypto.NewDefaultSigner(pk), []ma.Multiaddr{multiaddr}, overlay, 1, common.HexToHash("0x1").Bytes(), 1, common.Address{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	return *bzzAddr
+}
+
 type bookFunc func() (book addressbook.Interface)
 
 func TestInMem(t *testing.T) {
@@ -96,3 +115,96 @@ func run(t *testing.T, f bookFunc) {
 		t.Fatalf("expected addresses len %v, got %v", 1, len(addresses))
 	}
 }
+
+func TestUpdateLastSeen(t *testing.T) {
+	t.Parallel()
+
+	now := time.Unix(1000, 0)
+	store := addressbook.NewWithClock(mock.NewStateStore(), func() time.Time { return now })
+
+	overlay := swarm.NewAddress([]byte{0, 1, 2, 3})
+
+	// UpdateLastSeen on a missing overlay is a no-op and must not create an entry.
+	if err := store.UpdateLastSeen(overlay); err != nil {
+		t.Fatal(err)
+	}
+	if _, _, err := store.Get(overlay); !errors.Is(err, addressbook.ErrNotFound) {
+		t.Fatalf("expected ErrNotFound, got %v", err)
+	}
+
+	if err := store.Put(overlay, newTestAddr(t, overlay), true); err != nil {
+		t.Fatal(err)
+	}
+
+	// advance the clock and bump last-seen; the entry must survive a prune at
+	// the original time.
+	now = time.Unix(5000, 0)
+	if err := store.UpdateLastSeen(overlay); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := store.Prune(time.Unix(4000, 0)); err != nil {
+		t.Fatal(err)
+	}
+	if _, _, err := store.Get(overlay); err != nil {
+		t.Fatalf("entry pruned despite recent last-seen: %v", err)
+	}
+}
+
+func TestPrune(t *testing.T) {
+	t.Parallel()
+
+	now := time.Unix(0, 0)
+	store := addressbook.NewWithClock(mock.NewStateStore(), func() time.Time { return now })
+
+	stale := swarm.NewAddress([]byte{0, 1, 2, 3})
+	fresh := swarm.NewAddress([]byte{0, 1, 2, 4})
+
+	now = time.Unix(1000, 0)
+	if err := store.Put(stale, newTestAddr(t, stale), true); err != nil {
+		t.Fatal(err)
+	}
+
+	now = time.Unix(9000, 0)
+	if err := store.Put(fresh, newTestAddr(t, fresh), true); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := store.Prune(time.Unix(5000, 0)); err != nil {
+		t.Fatal(err)
+	}
+
+	if _, _, err := store.Get(stale); !errors.Is(err, addressbook.ErrNotFound) {
+		t.Fatalf("stale entry should have been pruned, got err=%v", err)
+	}
+	if _, _, err := store.Get(fresh); err != nil {
+		t.Fatalf("fresh entry should survive: %v", err)
+	}
+}
+
+func TestPruneKeepsEntriesWithoutLastSeen(t *testing.T) {
+	t.Parallel()
+
+	mockStore := mock.NewStateStore()
+	overlay := swarm.NewAddress([]byte{0, 1, 2, 3})
+
+	// Seed an entry without a last_seen field, mirroring records that predate
+	// pruning before the stamping migration runs.
+	if err := mockStore.Put("addressbook_entry_"+overlay.String(), &addressbook.VerifiedAddress{
+		Address:  addrPtr(newTestAddr(t, overlay)),
+		Verified: true,
+	}); err != nil {
+		t.Fatal(err)
+	}
+
+	store := addressbook.New(mockStore)
+	if err := store.Prune(time.Unix(5000, 0)); err != nil {
+		t.Fatal(err)
+	}
+
+	if _, _, err := store.Get(overlay); err != nil {
+		t.Fatalf("entry without last_seen must not be pruned: %v", err)
+	}
+}
+
+func addrPtr(a bzz.Address) *bzz.Address { return &a }

pkg/addressbook/export_test.go

@@ -4,4 +4,18 @@
 
 package addressbook
 
+import (
+	"time"
+
+	"github.com/ethersphere/bee/v2/pkg/storage"
+)
+
 type VerifiedAddress = verifiedAddress
+
+// NewWithClock creates an addressbook with an overridable clock, for testing.
+func NewWithClock(storer storage.StateStorer, now func() time.Time) Interface {
+	return &store{
+		store: storer,
+		now:   now,
+	}
+}

pkg/node/node.go

@@ -216,6 +216,7 @@ const (
 	reserveMinEvictCount          = 1_000
 	cacheMinEvictCount            = 10_000
 	maxAllowedDoubling            = 1
+	addressbookPruneAfter         = 30 * 24 * time.Hour // remove addressbook entries not seen for this long
 )
 
 func NewBee(
@@ -382,6 +383,12 @@ func NewBee(
 
 	addressbook := addressbook.New(stateStore)
 
+	// Prune addressbook entries whose overlays have not been seen recently, so
+	// the address book does not accumulate stale peers indefinitely.
+	if err := addressbook.Prune(time.Now().Add(-addressbookPruneAfter)); err != nil {
+		logger.Warning("addressbook prune failed", "error", err)
+	}
+
 	logger.Info("using overlay address", "address", swarmAddress)
 
 	// this will set overlay if it was not set before

pkg/statestore/storeadapter/export_test.go

@@ -4,8 +4,13 @@
 
 package storeadapter
 
-var RewriteAddressbookEnvelope = rewriteAddressbookEnvelope
+var (
+	RewriteAddressbookEnvelope = rewriteAddressbookEnvelope
+	StampAddressbookLastSeen   = stampAddressbookLastSeen
+)
 
-type LegacyEntry = legacyEntry
-type MigratedEntry = migratedEntry
-type MigratedAddress = migratedAddress
+type (
+	LegacyEntry     = legacyEntry
+	MigratedEntry   = migratedEntry
+	MigratedAddress = migratedAddress
+)