feat(storage): FS storage driver (#597)

.github/workflows/tests.yml

@@ -128,7 +128,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        storage-driver: [memory, redis]
+        storage-driver: [memory, redis, fs]
         pubsub-driver: [memory, redis]
     services:
       redis:
@@ -139,11 +139,13 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - {uses: actions/download-artifact@v4, with: {name: webhook-tester-linux-amd64}}
+      - run: mkdir ./data
       - run: | # start the server in the background
           chmod +x ./webhook-tester-linux-amd64
           ./webhook-tester-linux-amd64 start --port 8081 \
             --storage-driver "${{ matrix.storage-driver }}" \
             --pubsub-driver "${{ matrix.pubsub-driver }}" \
+            --fs-storage-dir ./data \
             --redis-dsn "redis://127.0.0.1:6379/0" &
       - uses: grafana/setup-k6-action@v1
       - uses: grafana/run-k6-action@v1

.gitignore

@@ -1,4 +1,4 @@
-# IDEs
+## IDEs
 /nbproject
 /.idea
 
@@ -11,7 +11,7 @@
 *_enum.go
 *.pb.go
 
-# Temp dirs & trash
+## Temp dirs & trash
 *.env
 /__old__
 /temp

Dockerfile

@@ -79,10 +79,10 @@ RUN --mount=type=bind,from=frontend,source=/src/web/dist,target=/src/web/dist \
     # prepare rootfs for runtime
     && mkdir -p /tmp/rootfs \
     && cd /tmp/rootfs \
-    && mkdir -p ./etc/ssl/certs ./bin ./tmp \
+    && mkdir -p ./etc/ssl/certs ./bin ./tmp ./data \
     && echo 'appuser:x:10001:10001::/nonexistent:/sbin/nologin' > ./etc/passwd \
     && echo 'appuser:x:10001:' > ./etc/group \
-    && chmod 777 ./tmp \
+    && chmod 777 ./tmp ./data \
     && cp /etc/ssl/certs/ca-certificates.crt ./etc/ssl/certs/ \
     && mv /src/app ./bin/app
 
@@ -111,9 +111,11 @@ ENV \
   # logging format
   LOG_FORMAT=json \
   # logging level
-  LOG_LEVEL=info
+  LOG_LEVEL=info \
+  # default fs storage directory
+  FS_STORAGE_DIR=/data
 
-#EXPOSE "80/tcp" "443/tcp"
+#EXPOSE "8080/tcp"
 
 HEALTHCHECK --interval=10s --start-interval=1s --start-period=5s --timeout=1s CMD ["/bin/app", "start", "healthcheck"]
 ENTRYPOINT ["/bin/app"]

README.md

@@ -47,6 +47,7 @@ The app supports two storage options: **memory** and **Redis** (configured with
   upon app shutdown
 - **Redis** driver: Retains data across app restarts, suitable for environments where data persistence is required.
   Redis is also necessary when running multiple instances behind a load balancer
+- **FS** driver: Keep all the data in the local filesystem, useful when you need to store data between app restarts
 
 ### 📢 Pub/Sub
 
@@ -151,9 +152,10 @@ The following flags are supported:
 | `--read-timeout="…"`          | maximum duration for reading the entire request, including the body (zero = no timeout)                                   |           `1m0s`           |     `HTTP_READ_TIMEOUT`      |
 | `--write-timeout="…"`         | maximum duration before timing out writes of the response (zero = no timeout)                                             |           `1m0s`           |     `HTTP_WRITE_TIMEOUT`     |
 | `--idle-timeout="…"`          | maximum amount of time to wait for the next request (keep-alive, zero = no timeout)                                       |           `1m0s`           |     `HTTP_IDLE_TIMEOUT`      |
-| `--storage-driver="…"`        | storage driver (memory/redis)                                                                                             |          `memory`          |       `STORAGE_DRIVER`       |
+| `--storage-driver="…"`        | storage driver (memory/redis/fs)                                                                                          |          `memory`          |       `STORAGE_DRIVER`       |
 | `--session-ttl="…"`           | session TTL (time-to-live, lifetime)                                                                                      |         `168h0m0s`         |        `SESSION_TTL`         |
 | `--max-requests="…"`          | maximal number of requests to store in the storage (zero means unlimited)                                                 |           `128`            |        `MAX_REQUESTS`        |
+| `--fs-storage-dir="…"`        | path to the directory for local fs storage (directory must exist)                                                         |                            |       `FS_STORAGE_DIR`       |
 | `--max-request-body-size="…"` | maximal webhook request body size (in bytes), zero means unlimited                                                        |            `0`             |   `MAX_REQUEST_BODY_SIZE`    |
 | `--auto-create-sessions`      | automatically create sessions for incoming requests                                                                       |          `false`           |    `AUTO_CREATE_SESSIONS`    |
 | `--pubsub-driver="…"`         | pub/sub driver (memory/redis)                                                                                             |          `memory`          |       `PUBSUB_DRIVER`        |

go.sum

@@ -41,6 +41,8 @@ github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovk
 github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
 github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
 github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/mattn/go-sqlite3 v1.14.24 h1:tpSp2G2KyMnnQu99ngJ47EIkWVmliIizyZBfPrBWDRM=
+github.com/mattn/go-sqlite3 v1.14.24/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
 github.com/oapi-codegen/runtime v1.1.1 h1:EXLHh0DXIJnWhdRPN2w4MXAzFyE4CskzhNLUmtpMYro=
 github.com/oapi-codegen/runtime v1.1.1/go.mod h1:SK9X900oXmPWilYR5/WKPzt3Kqxn/uS/+lbpREv+eCg=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=

internal/cli/start/command.go

@@ -7,6 +7,7 @@ import (
 	"math"
 	"net"
 	"net/url"
+	"os"
 	"strings"
 	"time"
 
@@ -42,6 +43,7 @@ type (
 				driver      string        // storage driver
 				sessionTTL  time.Duration // session TTL
 				maxRequests uint16        // maximal number of requests
+				fsDir       string        // path to the directory for local fs storage
 			}
 			pubSub struct {
 				driver string // Pub/Sub driver
@@ -65,9 +67,9 @@ type (
 )
 
 const (
-	pubSubDriverMemory, pubSubDriverRedis   = "memory", "redis"
-	storageDriverMemory, storageDriverRedis = "memory", "redis"
-	tunnelDriverNgrok                       = "ngrok"
+	pubSubDriverMemory, pubSubDriverRedis                    = "memory", "redis"
+	storageDriverMemory, storageDriverRedis, storageDriverFS = "memory", "redis", "fs"
+	tunnelDriverNgrok                                        = "ngrok"
 )
 
 // NewCommand creates new `start` command.
@@ -140,15 +142,19 @@ func NewCommand(log *zap.Logger, defaultHttpPort uint16) *cli.Command { //nolint
 			Validator: validateDuration("idle timeout", time.Millisecond, time.Hour),
 		}
 		storageDriverFlag = cli.StringFlag{
-			Name:     "storage-driver",
-			Value:    storageDriverMemory,
-			Usage:    "storage driver (" + strings.Join([]string{storageDriverMemory, storageDriverRedis}, "/") + ")",
+			Name:  "storage-driver",
+			Value: storageDriverMemory,
+			Usage: "storage driver (" + strings.Join([]string{
+				storageDriverMemory,
+				storageDriverRedis,
+				storageDriverFS,
+			}, "/") + ")",
 			Sources:  cli.EnvVars("STORAGE_DRIVER"),
 			OnlyOnce: true,
 			Config:   cli.StringConfig{TrimSpace: true},
 			Validator: func(s string) error {
 				switch s {
-				case storageDriverMemory, storageDriverRedis:
+				case storageDriverMemory, storageDriverRedis, storageDriverFS:
 					return nil
 				default:
 					return fmt.Errorf("wrong storage driver [%s]", s)
@@ -177,6 +183,19 @@ func NewCommand(log *zap.Logger, defaultHttpPort uint16) *cli.Command { //nolint
 				return nil
 			},
 		}
+		storageFsDirFlag = cli.StringFlag{
+			Name:     "fs-storage-dir",
+			Usage:    "path to the directory for local fs storage (directory must exist)",
+			Sources:  cli.EnvVars("FS_STORAGE_DIR"),
+			OnlyOnce: true,
+			Validator: func(s string) error {
+				if stat, err := os.Stat(s); err == nil && !stat.IsDir() {
+					return fmt.Errorf("not a directory [%s]", s)
+				}
+
+				return nil
+			},
+		}
 		maxRequestPayloadSizeFlag = cli.UintFlag{
 			Name:     "max-request-body-size",
 			Usage:    "maximal webhook request body size (in bytes), zero means unlimited",
@@ -282,7 +301,8 @@ func NewCommand(log *zap.Logger, defaultHttpPort uint16) *cli.Command { //nolint
 			opt.timeouts.httpIdle = c.Duration(httpIdleTimeoutFlag.Name)
 			opt.storage.driver = c.String(storageDriverFlag.Name)
 			opt.storage.sessionTTL = c.Duration(storageSessionTTLFlag.Name)
-			opt.storage.maxRequests = uint16(c.Uint(storageMaxRequestsFlag.Name))      //nolint:gosec
+			opt.storage.maxRequests = uint16(c.Uint(storageMaxRequestsFlag.Name)) //nolint:gosec
+			opt.storage.fsDir = c.String(storageFsDirFlag.Name)
 			opt.maxRequestPayloadSize = uint32(c.Uint(maxRequestPayloadSizeFlag.Name)) //nolint:gosec
 			opt.autoCreateSessions = c.Bool(autoCreateSessionsFlag.Name)
 			opt.pubSub.driver = c.String(pubSubDriverFlag.Name)
@@ -309,6 +329,7 @@ func NewCommand(log *zap.Logger, defaultHttpPort uint16) *cli.Command { //nolint
 			&storageDriverFlag,
 			&storageSessionTTLFlag,
 			&storageMaxRequestsFlag,
+			&storageFsDirFlag,
 			&maxRequestPayloadSizeFlag,
 			&autoCreateSessionsFlag,
 			&pubSubDriverFlag,
@@ -343,7 +364,7 @@ func validateDuration(name string, minValue, maxValue time.Duration) func(d time
 }
 
 // Run current command.
-func (cmd *command) Run(parentCtx context.Context, log *zap.Logger) error { //nolint:funlen,gocyclo
+func (cmd *command) Run(parentCtx context.Context, log *zap.Logger) error { //nolint:funlen,gocyclo,gocognit
 	ctx, cancel := context.WithCancel(parentCtx)
 	defer cancel()
 
@@ -376,6 +397,22 @@ func (cmd *command) Run(parentCtx context.Context, log *zap.Logger) error { //no
 		db = inMemory //nolint:wsl
 	case storageDriverRedis:
 		db = storage.NewRedis(rdc, cmd.options.storage.sessionTTL, uint32(cmd.options.storage.maxRequests))
+	case storageDriverFS:
+		if stat, err := os.Stat(cmd.options.storage.fsDir); err != nil {
+			return fmt.Errorf("failed to get the storage directory [%s]: %w", cmd.options.storage.fsDir, err)
+		} else if !stat.IsDir() {
+			return fmt.Errorf("not a directory [%s]", cmd.options.storage.fsDir)
+		}
+
+		var fs = storage.NewFS( //nolint:contextcheck
+			cmd.options.storage.fsDir,
+			cmd.options.storage.sessionTTL,
+			uint32(cmd.options.storage.maxRequests),
+		)
+
+		defer func() { _ = fs.Close() }()
+
+		db = fs
 	default:
 		return fmt.Errorf("unknown storage driver [%s]", cmd.options.storage.driver)
 	}