Merge pull request #12 from SiaFoundation/nate/store-blocks-directly

Store blocks directly

Dockerfile

@@ -13,7 +13,7 @@ COPY . .
 
 # Build the Go application
 RUN go generate ./...
-RUN CGO_ENABLED=0 go build -o bin/ -tags='netgo timetzdata' -trimpath -a -ldflags '-s -w' ./cmd/fsd
+RUN CGO_ENABLED=1 go build -o bin/ -tags='netgo timetzdata' -trimpath -a -ldflags '-s -w -linkmode external -extldflags "-static"' ./cmd/fsd
 
 FROM scratch
 

cmd/fsd/main.go

@@ -19,9 +19,12 @@ import (
  "go.sia.tech/fsd/config"
  shttp "go.sia.tech/fsd/http"
  "go.sia.tech/fsd/ipfs"
- "go.sia.tech/fsd/persist/badger"
- "go.sia.tech/fsd/sia"
+ "go.sia.tech/fsd/persist/sqlite"
+ "go.sia.tech/fsd/renterd"
+ "go.sia.tech/fsd/renterd/downloader"
  "go.sia.tech/jape"
+ "go.sia.tech/renterd/bus"
+ "go.sia.tech/renterd/worker"
  "go.uber.org/zap"
  "go.uber.org/zap/zapcore"
  "gopkg.in/yaml.v3"
@@ -36,6 +39,10 @@ var (
  BusAddress: "http://localhost:9980/api/bus",
  Bucket: "ipfs",
  },
+ BlockStore: config.BlockStore{
+ CacheSize: 10000,
+ MaxConcurrent: 1000,
+ },
  IPFS: config.IPFS{
  Gateway: config.HTTPGateway{
  ListenAddress: ":8080",
@@ -119,13 +126,8 @@ func main() {
  ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM, syscall.SIGINT)
  defer cancel()
 
- db, err := badger.OpenDatabase(filepath.Join(dir, "fsd.badgerdb"), log.Named("badger"))
- if err != nil {
- log.Fatal("failed to open badger database", zap.Error(err))
- }
- defer db.Close()
-
  var privateKey crypto.PrivKey
+ var err error
  if cfg.IPFS.PrivateKey != "" {
  buf, err := hex.DecodeString(strings.TrimPrefix(cfg.IPFS.PrivateKey, "ed25519:"))
  if err != nil {
@@ -150,15 +152,36 @@ func main() {
  }
  defer ds.Close()
 
- bs := sia.NewBlockStore(db, cfg.Renterd, log.Named("blockstore"))
+ workerClient := worker.NewClient(cfg.Renterd.WorkerAddress, cfg.Renterd.WorkerPassword)
+ busClient := bus.NewClient(cfg.Renterd.BusAddress, cfg.Renterd.BusPassword)
 
- inode, err := ipfs.NewNode(ctx, privateKey, cfg.IPFS, ds, bs)
+ metadata, err := sqlite.OpenDatabase(filepath.Join(dir, "fsd.sqlite3"), log.Named("sqlite"))
  if err != nil {
- log.Fatal("failed to start ipfs node", zap.Error(err))
+ log.Fatal("failed to open sqlite database", zap.Error(err))
+ }
+ defer metadata.Close()
+
+ bd, err := downloader.NewBlockDownloader(metadata, cfg.Renterd.Bucket, cfg.BlockStore.CacheSize, cfg.BlockStore.MaxConcurrent, workerClient, log.Named("downloader"))
+ if err != nil {
+ log.Fatal("failed to create block downloader", zap.Error(err))
+ }
+
+ bs, err := renterd.NewBlockStore(
+ renterd.WithBucket(cfg.Renterd.Bucket),
+ renterd.WithMetadataStore(metadata),
+ renterd.WithWorker(workerClient),
+ renterd.WithBus(busClient),
+ renterd.WithDownloader(bd),
+ renterd.WithLog(log.Named("blockstore")))
+ if err != nil {
+ log.Fatal("failed to create blockstore", zap.Error(err))
  }
- defer inode.Close()
 
- snode := sia.New(db, inode, cfg.Renterd, log.Named("sia"))
+ ipfs, err := ipfs.NewNode(ctx, privateKey, cfg.IPFS, ds, bs, log.Named("ipfs"))
+ if err != nil {
+ log.Fatal("failed to start ipfs node", zap.Error(err))
+ }
+ defer ipfs.Close()
 
  apiListener, err := net.Listen("tcp", cfg.API.Address)
  if err != nil {
@@ -173,12 +196,12 @@ func main() {
  defer gatewayListener.Close()
 
  apiServer := &http.Server{
- Handler: jape.BasicAuth(cfg.API.Password)(shttp.NewAPIHandler(inode, snode, cfg, log.Named("api"))),
+ Handler: jape.BasicAuth(cfg.API.Password)(shttp.NewAPIHandler(ipfs, cfg, log.Named("api"))),
  }
  defer apiServer.Close()
 
  gatewayServer := &http.Server{
- Handler: shttp.NewIPFSGatewayHandler(inode, snode, cfg, log.Named("gateway")),
+ Handler: shttp.NewIPFSGatewayHandler(ipfs, cfg, log.Named("gateway")),
  }
  defer gatewayServer.Close()
 
@@ -201,7 +224,7 @@ func main() {
  prettyKey := "ed25519:" + hex.EncodeToString(buf)
 
  log.Info("fsd started",
- zap.Stringer("peerID", inode.PeerID()),
+ zap.Stringer("peerID", ipfs.PeerID()),
  zap.String("privateKey", prettyKey),
  zap.String("apiAddress", apiListener.Addr().String()),
  zap.String("gatewayAddress", gatewayListener.Addr().String()),

config/config.go

@@ -15,6 +15,14 @@ type (
  Bucket string `yaml:"bucket"`
  }
 
+ // BlockStore configures the blockstore.
+ BlockStore struct {
+ // MaxConcurrent is the maximum number of concurrent block fetches.
+ MaxConcurrent int `yaml:"maxConcurrent"`
+ // CacheSize is the maximum number of blocks to cache in memory.
+ CacheSize int `yaml:"cacheSize"`
+ }
+
  // RemoteFetch contains settings for enabling/disabling remote IPFS block
  // fetching.
  RemoteFetch struct {
@@ -61,9 +69,10 @@ type (
 
  // Config contains the configuration for fsd
  Config struct {
- Renterd Renterd `yaml:"renterd"`
- IPFS IPFS `yaml:"ipfs"`
- API API `yaml:"api"`
- Log Log `yaml:"log"`
+ Renterd Renterd `yaml:"renterd"`
+ BlockStore BlockStore `yaml:"blockstore"`
+ IPFS IPFS `yaml:"ipfs"`
+ API API `yaml:"api"`
+ Log Log `yaml:"log"`
  }
 )

go.mod

@@ -3,76 +3,68 @@ module go.sia.tech/fsd
 go 1.21
 
 require (
- github.com/dgraph-io/badger/v4 v4.2.0
- github.com/gogo/protobuf v1.3.2
+ github.com/hashicorp/golang-lru/v2 v2.0.7
  github.com/ipfs/boxo v0.15.0
- github.com/ipfs/go-block-format v0.1.2
+ github.com/ipfs/go-block-format v0.2.0
  github.com/ipfs/go-cid v0.4.1
  github.com/ipfs/go-datastore v0.6.0
  github.com/ipfs/go-ds-leveldb v0.5.0
- github.com/ipfs/go-ipld-format v0.5.0
- github.com/libp2p/go-libp2p v0.32.0
+ github.com/ipfs/go-ipld-format v0.6.0
+ github.com/ipld/go-car/v2 v2.10.2-0.20230622090957-499d0c909d33
+ github.com/libp2p/go-libp2p v0.32.2
  github.com/libp2p/go-libp2p-kad-dht v0.24.4
- github.com/multiformats/go-multiaddr v0.12.0
+ github.com/mattn/go-sqlite3 v1.14.18
+ github.com/multiformats/go-multiaddr v0.12.1
  github.com/multiformats/go-multicodec v0.9.0
  github.com/multiformats/go-multihash v0.2.3
  go.sia.tech/jape v0.11.1
- go.sia.tech/renterd v0.6.1-0.20231129193254-22f903d6143a
+ go.sia.tech/renterd v1.0.6-0.20240228125621-9a935c3b8f30
  go.uber.org/zap v1.26.0
  gopkg.in/yaml.v3 v3.0.1
  lukechampine.com/frand v1.4.2
 )
 
 require (
  github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da // indirect
- github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect
- github.com/aws/aws-sdk-go v1.45.16 // indirect
+ github.com/alecthomas/units v0.0.0-20231202071711-9a357b53e9c9 // indirect
+ github.com/aws/aws-sdk-go v1.49.1 // indirect
  github.com/benbjohnson/clock v1.3.5 // indirect
  github.com/beorn7/perks v1.0.1 // indirect
- github.com/cenkalti/backoff/v4 v4.2.1 // indirect
  github.com/cespare/xxhash/v2 v2.2.0 // indirect
  github.com/containerd/cgroups v1.1.0 // indirect
  github.com/coreos/go-systemd/v22 v22.5.0 // indirect
- github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 // indirect
+ github.com/crackcomm/go-gitignore v0.0.0-20231225121904-e25f5bc08668 // indirect
  github.com/cskr/pubsub v1.0.2 // indirect
  github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect
  github.com/dchest/threefish v0.0.0-20120919164726-3ecf4c494abf // indirect
  github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect
- github.com/dgraph-io/ristretto v0.1.1 // indirect
- github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect
  github.com/docker/go-units v0.5.0 // indirect
- github.com/dustin/go-humanize v1.0.1 // indirect
  github.com/elastic/gosigar v0.14.2 // indirect
- github.com/felixge/httpsnoop v1.0.3 // indirect
- github.com/flynn/noise v1.0.0 // indirect
+ github.com/flynn/noise v1.0.1 // indirect
  github.com/francoispqt/gojay v1.2.13 // indirect
- github.com/gabriel-vasile/mimetype v1.4.2 // indirect
- github.com/go-logr/logr v1.2.4 // indirect
+ github.com/gabriel-vasile/mimetype v1.4.3 // indirect
+ github.com/go-logr/logr v1.4.1 // indirect
  github.com/go-logr/stdr v1.2.2 // indirect
  github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
  github.com/godbus/dbus/v5 v5.1.0 // indirect
- github.com/golang/glog v1.1.0 // indirect
- github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
- github.com/golang/protobuf v1.5.3 // indirect
+ github.com/gogo/protobuf v1.3.2 // indirect
  github.com/golang/snappy v0.0.4 // indirect
- github.com/google/flatbuffers v1.12.1 // indirect
  github.com/google/gopacket v1.1.19 // indirect
- github.com/google/pprof v0.0.0-20231023181126-ff6d637d2a7b // indirect
- github.com/google/uuid v1.3.1 // indirect
- github.com/gorilla/websocket v1.5.0 // indirect
+ github.com/google/pprof v0.0.0-20231229205709-960ae82b1e42 // indirect
+ github.com/google/uuid v1.5.0 // indirect
+ github.com/gorilla/websocket v1.5.1 // indirect
  github.com/gotd/contrib v0.19.0 // indirect
- github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.2 // indirect
  github.com/hashicorp/errwrap v1.1.0 // indirect
  github.com/hashicorp/go-multierror v1.1.1 // indirect
- github.com/hashicorp/golang-lru v0.5.4 // indirect
- github.com/hashicorp/golang-lru/v2 v2.0.5 // indirect
+ github.com/hashicorp/golang-lru v1.0.2 // indirect
  github.com/huin/goupnp v1.3.0 // indirect
  github.com/ipfs/bbloom v0.0.4 // indirect
  github.com/ipfs/go-bitfield v1.1.0 // indirect
  github.com/ipfs/go-cidutil v0.1.0 // indirect
  github.com/ipfs/go-ipfs-delay v0.0.1 // indirect
  github.com/ipfs/go-ipfs-pq v0.0.3 // indirect
  github.com/ipfs/go-ipfs-util v0.0.3 // indirect
+ github.com/ipfs/go-ipld-cbor v0.0.6 // indirect
  github.com/ipfs/go-ipld-legacy v0.2.1 // indirect
  github.com/ipfs/go-log v1.0.5 // indirect
  github.com/ipfs/go-log/v2 v2.5.1 // indirect
@@ -86,14 +78,14 @@ require (
  github.com/jinzhu/inflection v1.0.0 // indirect
  github.com/jinzhu/now v1.1.5 // indirect
  github.com/julienschmidt/httprouter v1.3.0 // indirect
- github.com/klauspost/compress v1.17.2 // indirect
- github.com/klauspost/cpuid/v2 v2.2.5 // indirect
- github.com/klauspost/reedsolomon v1.11.8 // indirect
+ github.com/klauspost/compress v1.17.4 // indirect
+ github.com/klauspost/cpuid/v2 v2.2.6 // indirect
+ github.com/klauspost/reedsolomon v1.12.1 // indirect
  github.com/koron/go-ssdp v0.0.4 // indirect
  github.com/libp2p/go-buffer-pool v0.1.0 // indirect
  github.com/libp2p/go-cidranger v1.1.0 // indirect
  github.com/libp2p/go-flow-metrics v0.1.0 // indirect
- github.com/libp2p/go-libp2p-asn-util v0.3.0 // indirect
+ github.com/libp2p/go-libp2p-asn-util v0.4.1 // indirect
  github.com/libp2p/go-libp2p-kbucket v0.6.3 // indirect
  github.com/libp2p/go-libp2p-record v0.2.0 // indirect
  github.com/libp2p/go-libp2p-routing-helpers v0.7.3 // indirect
@@ -105,8 +97,8 @@ require (
  github.com/libp2p/go-yamux/v4 v4.0.1 // indirect
  github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd // indirect
  github.com/mattn/go-isatty v0.0.20 // indirect
- github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
- github.com/miekg/dns v1.1.56 // indirect
+ github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect
+ github.com/miekg/dns v1.1.57 // indirect
  github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b // indirect
  github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc // indirect
  github.com/minio/sha256-simd v1.0.1 // indirect
@@ -119,25 +111,28 @@ require (
  github.com/multiformats/go-multibase v0.2.0 // indirect
  github.com/multiformats/go-multistream v0.5.0 // indirect
  github.com/multiformats/go-varint v0.0.7 // indirect
- github.com/onsi/ginkgo/v2 v2.13.0 // indirect
+ github.com/onsi/ginkgo/v2 v2.13.2 // indirect
  github.com/opencontainers/runtime-spec v1.1.0 // indirect
  github.com/opentracing/opentracing-go v1.2.0 // indirect
  github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect
+ github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 // indirect
  github.com/pkg/errors v0.9.1 // indirect
  github.com/polydawn/refmt v0.89.0 // indirect
- github.com/prometheus/client_golang v1.16.0 // indirect
- github.com/prometheus/client_model v0.4.0 // indirect
- github.com/prometheus/common v0.44.0 // indirect
- github.com/prometheus/procfs v0.11.1 // indirect
+ github.com/prometheus/client_golang v1.18.0 // indirect
+ github.com/prometheus/client_model v0.5.0 // indirect
+ github.com/prometheus/common v0.45.0 // indirect
+ github.com/prometheus/procfs v0.12.0 // indirect
  github.com/quic-go/qpack v0.4.0 // indirect
- github.com/quic-go/qtls-go1-20 v0.3.4 // indirect
- github.com/quic-go/quic-go v0.39.3 // indirect
+ github.com/quic-go/qtls-go1-20 v0.4.1 // indirect
+ github.com/quic-go/quic-go v0.40.1 // indirect
  github.com/quic-go/webtransport-go v0.6.0 // indirect
  github.com/raulk/go-watchdog v1.3.0 // indirect
  github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 // indirect
  github.com/shabbyrobe/gocovmerge v0.0.0-20230507112040-c3350d9342df // indirect
  github.com/spaolacci/murmur3 v1.1.0 // indirect
  github.com/syndtr/goleveldb v1.0.0 // indirect
+ github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11 // indirect
+ github.com/whyrusleeping/cbor-gen v0.0.0-20230126041949-52956bd4c9aa // indirect
  github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f // indirect
  github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 // indirect
  gitlab.com/NebulousLabs/bolt v1.4.4 // indirect
@@ -153,37 +148,29 @@ require (
  gitlab.com/NebulousLabs/siamux v0.0.2-0.20220630142132-142a1443a259 // indirect
  gitlab.com/NebulousLabs/threadgroup v0.0.0-20200608151952-38921fbef213 // indirect
  go.opencensus.io v0.24.0 // indirect
- go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.42.0 // indirect
- go.opentelemetry.io/otel v1.16.0 // indirect
- go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0 // indirect
- go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0 // indirect
- go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.16.0 // indirect
- go.opentelemetry.io/otel/metric v1.16.0 // indirect
- go.opentelemetry.io/otel/sdk v1.16.0 // indirect
- go.opentelemetry.io/otel/trace v1.16.0 // indirect
- go.opentelemetry.io/proto/otlp v1.0.0 // indirect
- go.sia.tech/core v0.1.12-0.20231011172826-6ca0ac7b3b6b // indirect
+ go.opentelemetry.io/otel v1.21.0 // indirect
+ go.opentelemetry.io/otel/metric v1.21.0 // indirect
+ go.opentelemetry.io/otel/trace v1.21.0 // indirect
+ go.sia.tech/core v0.1.12-0.20231211182757-77190f04f90b // indirect
  go.sia.tech/gofakes3 v0.0.0-20231109151325-e0d47c10dce2 // indirect
  go.sia.tech/mux v1.2.0 // indirect
  go.sia.tech/siad v1.5.10-0.20230228235644-3059c0b930ca // indirect
  go.uber.org/dig v1.17.1 // indirect
  go.uber.org/fx v1.20.1 // indirect
  go.uber.org/goleak v1.2.1 // indirect
- go.uber.org/mock v0.3.0 // indirect
+ go.uber.org/mock v0.4.0 // indirect
  go.uber.org/multierr v1.11.0 // indirect
- golang.org/x/crypto v0.15.0 // indirect
- golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect
+ golang.org/x/crypto v0.18.0 // indirect
+ golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc // indirect
  golang.org/x/mod v0.14.0 // indirect
- golang.org/x/net v0.18.0 // indirect
- golang.org/x/sync v0.5.0 // indirect
- golang.org/x/sys v0.14.0 // indirect
+ golang.org/x/net v0.20.0 // indirect
+ golang.org/x/sync v0.6.0 // indirect
+ golang.org/x/sys v0.16.0 // indirect
  golang.org/x/text v0.14.0 // indirect
- golang.org/x/tools v0.15.0 // indirect
- gonum.org/v1/gonum v0.13.0 // indirect
- google.golang.org/genproto/googleapis/api v0.0.0-20230815205213-6bfd019c3878 // indirect
- google.golang.org/genproto/googleapis/rpc v0.0.0-20230920204549-e6e6cdab5c13 // indirect
- google.golang.org/grpc v1.58.2 // indirect
- google.golang.org/protobuf v1.31.0 // indirect
- gorm.io/gorm v1.25.4 // indirect
+ golang.org/x/tools v0.16.1 // indirect
+ golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
+ gonum.org/v1/gonum v0.14.0 // indirect
+ google.golang.org/protobuf v1.32.0 // indirect
+ gorm.io/gorm v1.25.7 // indirect
  lukechampine.com/blake3 v1.2.1 // indirect
 )