Merge branch 'master' into feat/support-pebble

cmd/ipfs/kubo/daemon.go

@@ -600,8 +600,25 @@ take effect.
 		fmt.Println("(Hit ctrl-c again to force-shutdown the daemon.)")
 	}()
 
-	// Give the user heads up if daemon running in online mode has no peers after 1 minute
 	if !offline {
+		// Warn users who were victims of 'lowprofile' footgun (https://github.com/ipfs/kubo/pull/10524)
+		if cfg.Experimental.StrategicProviding {
+			fmt.Print(`
+⚠️ Reprovide system is disabled due to 'Experimental.StrategicProviding=true'
+⚠️ Local CIDs will not be announced to Amino DHT, making them impossible to retrieve without manual peering
+⚠️ If this is not intentional, call 'ipfs config profile apply announce-on'
+
+`)
+		} else if cfg.Reprovider.Interval.WithDefault(config.DefaultReproviderInterval) == 0 {
+			fmt.Print(`
+⚠️ Reprovider system is disabled due to 'Reprovider.Interval=0'
+⚠️ Local CIDs will not be announced to Amino DHT, making them impossible to retrieve without manual peering
+⚠️ If this is not intentional, call 'ipfs config profile apply announce-on', or set 'Reprovider.Interval=22h'
+
+`)
+		}
+
+		// Give the user heads up if daemon running in online mode has no peers after 1 minute
 		time.AfterFunc(1*time.Minute, func() {
 			cfg, err := cctx.GetConfig()
 			if err != nil {

config/profile.go

@@ -209,10 +209,12 @@ functionality - performance of content discovery and data
 fetching may be degraded.
 `,
 		Transform: func(c *Config) error {
+			// Disable "server" services (dht, autonat, limited relay)
 			c.Routing.Type = NewOptionalString("autoclient")
 			c.AutoNAT.ServiceMode = AutoNATServiceDisabled
-			c.Reprovider.Interval = NewOptionalDuration(0)
+			c.Swarm.RelayService.Enabled = False
 
+			// Keep bare minimum connections around
 			lowWater := int64(20)
 			highWater := int64(40)
 			gracePeriod := time.Minute
@@ -223,6 +225,29 @@ fetching may be degraded.
 			return nil
 		},
 	},
+	"announce-off": {
+		Description: `Disables Reprovide system (and announcing to Amino DHT).
+
+		USE WITH CAUTION:
+		The main use case for this is setups with manual Peering.Peers config.
+		Data from this node will not be announced on the DHT. This will make
+		DHT-based routing an data retrieval impossible if this node is the only
+		one hosting it, and other peers are not already connected to it.
+`,
+		Transform: func(c *Config) error {
+			c.Reprovider.Interval = NewOptionalDuration(0) // 0 disables periodic reprovide
+			c.Experimental.StrategicProviding = true       // this is not a typo (the name is counter-intuitive)
+			return nil
+		},
+	},
+	"announce-on": {
+		Description: `Re-enables Reprovide system (reverts announce-off profile).`,
+		Transform: func(c *Config) error {
+			c.Reprovider.Interval = NewOptionalDuration(DefaultReproviderInterval) // have to apply explicit default because nil would be ignored
+			c.Experimental.StrategicProviding = false                              // this is not a typo (the name is counter-intuitive)
+			return nil
+		},
+	},
 	"randomports": {
 		Description: `Use a random port number for swarm.`,
 

core/node/core.go

@@ -146,7 +146,7 @@ func Dag(bs blockservice.BlockService) format.DAGService {
 }
 
 // Files loads persisted MFS root
-func Files(mctx helpers.MetricsCtx, lc fx.Lifecycle, repo repo.Repo, dag format.DAGService) (*mfs.Root, error) {
+func Files(mctx helpers.MetricsCtx, lc fx.Lifecycle, repo repo.Repo, dag format.DAGService, bs blockstore.Blockstore) (*mfs.Root, error) {
 	dsk := datastore.NewKey("/local/filesroot")
 	pf := func(ctx context.Context, c cid.Cid) error {
 		rootDS := repo.Datastore()
@@ -172,17 +172,18 @@ func Files(mctx helpers.MetricsCtx, lc fx.Lifecycle, repo repo.Repo, dag format.
 		nd = unixfs.EmptyDirNode()
 		err := dag.Add(ctx, nd)
 		if err != nil {
-			return nil, fmt.Errorf("failure writing to dagstore: %s", err)
+			return nil, fmt.Errorf("failure writing filesroot to dagstore: %s", err)
 		}
 	case err == nil:
 		c, err := cid.Cast(val)
 		if err != nil {
 			return nil, err
 		}
 
-		rnd, err := dag.Get(ctx, c)
+		offineDag := merkledag.NewDAGService(blockservice.New(bs, offline.Exchange(bs)))
+		rnd, err := offineDag.Get(ctx, c)
 		if err != nil {
-			return nil, fmt.Errorf("error loading filesroot from DAG: %s", err)
+			return nil, fmt.Errorf("error loading filesroot from dagservice: %s", err)
 		}
 
 		pbnd, ok := rnd.(*merkledag.ProtoNode)

docs/changelogs/v0.31.md

@@ -7,6 +7,8 @@
 - [Overview](#overview)
 - [🔦 Highlights](#-highlights)
   - [Experimental Pebble Datastore](#experimental-pebble-datastore)
+  - [New metrics](#new-metrics)
+  - [`lowpower` profile no longer breaks DHT announcements](#lowpower-profile-no-longer-breaks-dht-announcements)
 - [📝 Changelog](#-changelog)
 - [👨‍👩‍👧‍👦 Contributors](#-contributors)
 
@@ -23,6 +25,24 @@ A fresh Kubo node can be initialized with [`pebbleds` profile](https://github.co
 There are a number of parameters available for tuning pebble's performance to your specific needs. Default values are used for any parameters that are not configured or are set to their zero-value.
 For a description of the available tuning parameters, see [kubo/docs/datastores.md#pebbleds](https://github.com/ipfs/kubo/blob/master/docs/datastores.md#pebbleds).
 
+#### New metrics
+
+- Added 3 new go metrics: `go_gc_gogc_percent`, `go_gc_gomemlimit_bytes` and `go_sched_gomaxprocs_threads` as those are [recommended by the Go team](https://github.com/prometheus/client_golang/pull/1559)
+- Added [network usage metrics](https://github.com/prometheus/client_golang/pull/1555): `process_network_receive_bytes_total` and `process_network_transmit_bytes_total`
+- Removed `go_memstat_lookups_total` metric [which was always 0](https://github.com/prometheus/client_golang/pull/1577)
+
+#### `lowpower` profile no longer breaks DHT announcements
+
+We've notices users were applying `lowpower` profile, and then reporting content routing issues. This was because `lowpower` disabled reprovider system and locally hosted data was no longer announced on Amino DHT.
+
+This release changes [`lowpower` profile](https://github.com/ipfs/kubo/blob/master/docs/config.md#lowpower-profile) to not change reprovider settings, ensuring the new users are not sabotaging themselves. It also adds [`annouce-on`](https://github.com/ipfs/kubo/blob/master/docs/config.md#announce-on-profile) and [`announce-off`](https://github.com/ipfs/kubo/blob/master/docs/config.md#announce-off-profile) profiles for controlling announcement settings separately.
+
+> [!IMPORTANT]
+> If you've ever applied the `lowpower` profile before, there is a high chance your node is not announcing to DHT anymore.
+> If you have `Reprovider.Interval` set to `0` you may want to wet it to `22h` (or run `ipfs config profile apply announce-on`) to fix your system.
+>
+> As a convenience, `ipfs daemon` will warn if reprovide system is disabled, creating oportinity to fix configuration if it was not intentional.
+
 ### 📝 Changelog
 
 ### 👨‍👩‍👧‍👦 Contributors

docs/config.md

@@ -184,6 +184,8 @@ config file at runtime.
     - [`pebbleds` profile](#pebbleds-profile)
     - [`badgerds` profile](#badgerds-profile)
     - [`lowpower` profile](#lowpower-profile)
+    - [`announce-off` profile](#announce-off-profile)
+    - [`announce-on` profile](#announce-on-profile)
     - [`legacy-cid-v0` profile](#legacy-cid-v0-profile)
     - [`test-cid-v1` profile](#test-cid-v1-profile)
   - [Types](#types)
@@ -300,7 +302,7 @@ Map of HTTP headers to set on responses from the RPC (`/api/v0`) HTTP server.
 Example:
 ```json
 {
-	"Foo": ["bar"]
+  "Foo": ["bar"]
 }
 ```
 
@@ -530,27 +532,27 @@ Default:
 ```
 {
   "mounts": [
-	{
-	  "child": {
-		"path": "blocks",
-		"shardFunc": "/repo/flatfs/shard/v1/next-to-last/2",
-		"sync": true,
-		"type": "flatfs"
-	  },
-	  "mountpoint": "/blocks",
-	  "prefix": "flatfs.datastore",
-	  "type": "measure"
-	},
-	{
-	  "child": {
-		"compression": "none",
-		"path": "datastore",
-		"type": "levelds"
-	  },
-	  "mountpoint": "/",
-	  "prefix": "leveldb.datastore",
-	  "type": "measure"
-	}
+  {
+    "child": {
+    "path": "blocks",
+    "shardFunc": "/repo/flatfs/shard/v1/next-to-last/2",
+    "sync": true,
+    "type": "flatfs"
+    },
+    "mountpoint": "/blocks",
+    "prefix": "flatfs.datastore",
+    "type": "measure"
+  },
+  {
+    "child": {
+    "compression": "none",
+    "path": "datastore",
+    "type": "levelds"
+    },
+    "mountpoint": "/",
+    "prefix": "leveldb.datastore",
+    "type": "measure"
+  }
   ],
   "type": "mount"
 }
@@ -1141,7 +1143,7 @@ Example:
         "API" : {
           "Endpoint" : "https://pinningservice.tld:1234/my/api/path",
           "Key" : "someOpaqueKey"
-				}
+        }
       }
     }
   }
@@ -2462,18 +2464,30 @@ Also, be aware that:
 
 ### `lowpower` profile
 
-Reduces daemon overhead on the system. Affects node
-functionality - performance of content discovery and data
-fetching may be degraded.
-
-> [!CAUTION]
-> Local data won't be announced on routing systems like Amino DHT.
+Reduces daemon overhead on the system by disabling optional swarm services.
 
+- [`Routing.Type`](#routingtype) set to `autoclient` (no DHT server, only client).
 - `Swarm.ConnMgr` set to maintain minimum number of p2p connections at a time.
-- Disables [`Reprovider`](#reprovider) service → no CID will be announced on Amino DHT and other routing systems(!)
 - Disables [`AutoNAT`](#autonat).
+- Disables [`Swam.RelayService`](#swarmrelayservice).
+
+> [!NOTE]
+> This profile is provided for legacy reasons.
+> With modern Kubo setting the above should not be necessary.
+
+### `announce-off` profile
+
+Disables [Reprovider](#reprovider) system (and announcing to Amino DHT).
+
+> [!CAUTION]
+> The main use case for this is setups with manual Peering.Peers config.
+> Data from this node will not be announced on the DHT. This will make
+> DHT-based routing an data retrieval impossible if this node is the only
+> one hosting it, and other peers are not already connected to it.
+
+### `announce-on` profile
 
-Use this profile with caution.
+(Re-)enables [Reprovider](#reprovider) system (reverts [`announce-off` profile](#annouce-off-profile).
 
 ### `legacy-cid-v0` profile
 

docs/examples/kubo-as-a-library/go.mod

@@ -7,9 +7,9 @@ go 1.22
 replace github.com/ipfs/kubo => ./../../..
 
 require (
-	github.com/ipfs/boxo v0.23.1-0.20240927234853-19a402b7dc34
+	github.com/ipfs/boxo v0.24.0
 	github.com/ipfs/kubo v0.0.0-00010101000000-000000000000
-	github.com/libp2p/go-libp2p v0.36.3
+	github.com/libp2p/go-libp2p v0.36.4
 	github.com/multiformats/go-multiaddr v0.13.0
 )
 
@@ -91,7 +91,7 @@ require (
 	github.com/ipfs/go-ipfs-pq v0.0.3 // indirect
 	github.com/ipfs/go-ipfs-redirects-file v0.1.1 // indirect
 	github.com/ipfs/go-ipfs-util v0.0.3 // indirect
-	github.com/ipfs/go-ipld-cbor v0.1.0 // indirect
+	github.com/ipfs/go-ipld-cbor v0.2.0 // indirect
 	github.com/ipfs/go-ipld-format v0.6.0 // indirect
 	github.com/ipfs/go-ipld-git v0.1.1 // indirect
 	github.com/ipfs/go-ipld-legacy v0.2.1 // indirect
@@ -103,7 +103,7 @@ require (
 	github.com/ipfs/go-unixfsnode v1.9.1 // indirect
 	github.com/ipfs/go-verifcid v0.0.3 // indirect
 	github.com/ipld/go-car v0.6.2 // indirect
-	github.com/ipld/go-car/v2 v2.13.1 // indirect
+	github.com/ipld/go-car/v2 v2.14.2 // indirect
 	github.com/ipld/go-codec-dagpb v1.6.0 // indirect
 	github.com/ipld/go-ipld-prime v0.21.0 // indirect
 	github.com/jackpal/go-nat-pmp v1.0.2 // indirect
@@ -120,7 +120,7 @@ require (
 	github.com/libp2p/go-flow-metrics v0.1.0 // indirect
 	github.com/libp2p/go-libp2p-asn-util v0.4.1 // indirect
 	github.com/libp2p/go-libp2p-kad-dht v0.26.1 // indirect
-	github.com/libp2p/go-libp2p-kbucket v0.6.3 // indirect
+	github.com/libp2p/go-libp2p-kbucket v0.6.4 // indirect
 	github.com/libp2p/go-libp2p-pubsub v0.11.0 // indirect
 	github.com/libp2p/go-libp2p-pubsub-router v0.6.0 // indirect
 	github.com/libp2p/go-libp2p-record v0.2.0 // indirect
@@ -142,7 +142,7 @@ require (
 	github.com/mr-tron/base58 v1.2.0 // indirect
 	github.com/multiformats/go-base32 v0.1.0 // indirect
 	github.com/multiformats/go-base36 v0.2.0 // indirect
-	github.com/multiformats/go-multiaddr-dns v0.3.1 // indirect
+	github.com/multiformats/go-multiaddr-dns v0.4.0 // indirect
 	github.com/multiformats/go-multiaddr-fmt v0.1.0 // indirect
 	github.com/multiformats/go-multibase v0.2.0 // indirect
 	github.com/multiformats/go-multicodec v0.9.0 // indirect
@@ -175,7 +175,7 @@ require (
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/polydawn/refmt v0.89.0 // indirect
-	github.com/prometheus/client_golang v1.19.1 // indirect
+	github.com/prometheus/client_golang v1.20.4 // indirect
 	github.com/prometheus/client_model v0.6.1 // indirect
 	github.com/prometheus/common v0.55.0 // indirect
 	github.com/prometheus/procfs v0.15.1 // indirect
@@ -215,15 +215,15 @@ require (
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.27.0 // indirect
 	go4.org v0.0.0-20230225012048-214862532bf5 // indirect
-	golang.org/x/crypto v0.25.0 // indirect
+	golang.org/x/crypto v0.27.0 // indirect
 	golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect
 	golang.org/x/mod v0.19.0 // indirect
 	golang.org/x/net v0.27.0 // indirect
-	golang.org/x/sync v0.7.0 // indirect
-	golang.org/x/sys v0.24.0 // indirect
-	golang.org/x/text v0.16.0 // indirect
+	golang.org/x/sync v0.8.0 // indirect
+	golang.org/x/sys v0.25.0 // indirect
+	golang.org/x/text v0.18.0 // indirect
 	golang.org/x/tools v0.23.0 // indirect
-	golang.org/x/xerrors v0.0.0-20240716161551-93cc26a95ae9 // indirect
+	golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
 	gonum.org/v1/gonum v0.15.0 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20240617180043-68d350f18fd4 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20240617180043-68d350f18fd4 // indirect