Merge branch 'main' into stable

.github/actions/genprotos/action.yml

@@ -0,0 +1,36 @@
+name: 'PeerDB genprotos'
+description: 'Install buf with local plugins, generate protos and cache'
+runs:
+ using: "composite"
+ steps:
+ - uses: actions/checkout@v4
+ - name: check cache
+ id: cache
+ uses: actions/cache@v4
+ with:
+ path: |
+ ./flow/generated/protos
+ ./nexus/pt/src/gen
+ ./ui/grpc_generated
+ key: ${{ runner.os }}-build-genprotos-${{ hashFiles('./protos/peers.proto', './protos/flow.proto', './protos/route.proto') }}
+
+ - if: steps.cache.outputs.cache-hit != 'true'
+ uses: actions/setup-go@v5
+ with:
+ go-version: '1.22.3'
+ cache: false
+ - if: steps.cache.outputs.cache-hit != 'true'
+ uses: bufbuild/[email protected]
+ - if: steps.cache.outputs.cache-hit != 'true'
+ uses: dtolnay/rust-toolchain@stable
+ - if: steps.cache.outputs.cache-hit != 'true'
+ name: install protobuf plugins and generate protos
+ shell: bash
+ run: |
+ go install google.golang.org/protobuf/cmd/[email protected]
+ go install google.golang.org/grpc/cmd/[email protected]
+ go install github.com/grpc-ecosystem/grpc-gateway/v2/[email protected]
+ npm install ts-proto
+ cargo install protoc-gen-prost protoc-gen-tonic protoc-gen-prost-serde
+ PATH="$(go env GOPATH)/bin:$HOME/.cargo/bin:$PATH" buf generate --template buf.gen-local.yaml --debug protos
+

.github/workflows/ci.yml

@@ -30,10 +30,9 @@ jobs:
  --health-retries 5
  steps:
  - uses: actions/checkout@v4
- with:
- submodules: recursive
 
- - uses: dtolnay/rust-toolchain@stable
+ - name: generate or hydrate protos
+ uses: ./.github/actions/genprotos
 
  - name: install system dependencies
  run: |
@@ -60,11 +59,6 @@ jobs:
  with:
  workspaces: nexus
 
- - uses: bufbuild/[email protected]
- with:
- github_token: ${{ secrets.GITHUB_TOKEN }}
-
- - run: buf generate protos
 
  - name: cargo check
  run: cargo check

.github/workflows/cleanup.yml

@@ -14,7 +14,7 @@ jobs:
 
  - uses: actions/setup-go@v5
  with:
- go-version: "1.22"
+ go-version: '1.22.3'
  cache-dependency-path: e2e_cleanup/go.sum
 
  - name: download go modules

.github/workflows/codeql-analysis.yml

@@ -36,11 +36,8 @@ jobs:
  - name: Checkout repository
  uses: actions/checkout@v4
 
- - uses: bufbuild/[email protected]
- with:
- github_token: ${{ secrets.GITHUB_TOKEN }}
-
- - run: buf generate protos
+ - name: generate or hydrate protos
+ uses: ./.github/actions/genprotos
 
  - name: install lib-geos
  run: |

.github/workflows/customer-docker.yml

@@ -18,16 +18,10 @@ jobs:
  contents: read
  packages: write
  steps:
- - name: checkout
- uses: actions/checkout@v4
- with:
- submodules: recursive
-
- - uses: bufbuild/[email protected]
- with:
- github_token: ${{ secrets.GITHUB_TOKEN }}
+ - uses: actions/checkout@v4
 
- - run: buf generate protos
+ - name: generate or hydrate protos
+ uses: ./.github/actions/genprotos
 
  - uses: depot/setup-action@v1
 

.github/workflows/dev-docker.yml

@@ -17,16 +17,10 @@ jobs:
  contents: read
  packages: write
  steps:
- - name: checkout
- uses: actions/checkout@v4
- with:
- submodules: recursive
-
- - uses: bufbuild/[email protected]
- with:
- github_token: ${{ secrets.GITHUB_TOKEN }}
+ - uses: actions/checkout@v4
 
- - run: buf generate protos
+ - name: generate or hydrate protos
+ uses: ./.github/actions/genprotos
 
  - uses: depot/setup-action@v1
 

.github/workflows/flow.yml

@@ -32,18 +32,14 @@ jobs:
  xpack.security.enrollment.enabled: false
 
  steps:
- - name: checkout sources
- uses: actions/checkout@v4
+ - uses: actions/checkout@v4
 
- - uses: bufbuild/[email protected]
- with:
- github_token: ${{ secrets.GITHUB_TOKEN }}
-
- - run: buf generate protos
+ - name: generate or hydrate protos
+ uses: ./.github/actions/genprotos
 
  - uses: actions/setup-go@v5
  with:
- go-version: "1.22"
+ go-version: '1.22.3'
  cache-dependency-path: flow/go.sum
 
  - name: install lib-geos

.github/workflows/golang-lint.yml

@@ -14,21 +14,21 @@ jobs:
  runs-on: [ubicloud-standard-4-ubuntu-2204-arm]
  steps:
  - uses: actions/checkout@v4
- - uses: bufbuild/[email protected]
-  with:
-  github_token: ${{ secrets.GITHUB_TOKEN }}
- - run: buf generate protos
+
+ - name: generate or hydrate protos
+ uses: ./.github/actions/genprotos
+
  - name: install lib-geos
  run: |
  sudo apt-get update
  sudo apt-get install libgeos-dev
  - uses: actions/setup-go@v5
  with:
- go-version: "1.22"
+ go-version: '1.22.3'
  cache: false
  - name: golangci-lint
- uses: golangci/golangci-lint-action@v4
+ uses: golangci/golangci-lint-action@v6
  with:
- version: v1.57
+ version: v1.58
  working-directory: ./flow
  args: --timeout=10m

.github/workflows/rust-lint.yml

@@ -16,20 +16,10 @@ jobs:
  runner: [ubicloud-standard-4-ubuntu-2204-arm]
  runs-on: ${{ matrix.runner }}
  steps:
- - name: checkout
- uses: actions/checkout@v4
- with:
- submodules: recursive
+ - uses: actions/checkout@v4
 
- - uses: bufbuild/[email protected]
- with:
- github_token: ${{ secrets.GITHUB_TOKEN }}
-
- - run: buf generate protos
-
- - uses: dtolnay/rust-toolchain@stable
- with:
- components: clippy
+ - name: generate or hydrate protos
+ uses: ./.github/actions/genprotos
 
  - name: clippy
  run: cargo clippy -- -D warnings

.github/workflows/stable-docker.yml

@@ -15,16 +15,10 @@ jobs:
  contents: read
  packages: write
  steps:
- - name: checkout
- uses: actions/checkout@v4
- with:
- submodules: recursive
-
- - uses: bufbuild/[email protected]
- with:
- github_token: ${{ secrets.GITHUB_TOKEN }}
+ - uses: actions/checkout@v4
 
- - run: buf generate protos
+ - name: generate or hydrate protos
+ uses: ./.github/actions/genprotos
 
  - uses: depot/setup-action@v1
 

.github/workflows/ui-build.yml

@@ -18,11 +18,8 @@ jobs:
  - name: checkout
  uses: actions/checkout@v4
 
- - uses: bufbuild/[email protected]
- with:
- github_token: ${{ secrets.GITHUB_TOKEN }}
-
- - run: buf generate protos
+ - name: generate or hydrate protos
+ uses: ./.github/actions/genprotos
 
  - name: Install Node.js dependencies
  working-directory: ui

.github/workflows/ui-lint.yml

@@ -22,11 +22,8 @@ jobs:
  - name: checkout
  uses: actions/checkout@v4
 
- - uses: bufbuild/[email protected]
- with:
- github_token: ${{ secrets.GITHUB_TOKEN }}
-
- - run: buf generate protos
+ - name: generate or hydrate protos
+ uses: ./.github/actions/genprotos
 
  - name: Install Node.js dependencies
  working-directory: ui

README.md

@@ -20,7 +20,7 @@ We support different modes of streaming - log based (CDC), cursor based (timesta
 ## Get started
 
 ```bash
-git clone --recursive [email protected]:PeerDB-io/peerdb.git
+git clone [email protected]:PeerDB-io/peerdb.git
 cd peerdb
 
 # Run docker containers: postgres as catalog, temporal, PeerDB server, PeerDB flow API + workers, PeerDB UI

buf.gen-local.yaml

@@ -0,0 +1,37 @@
+version: v1
+managed:
+ enabled: true
+ go_package_prefix:
+ default: generated/protos
+ except:
+ - buf.build/googleapis/googleapis
+plugins:
+ - plugin: go
+ out: flow/generated/protos
+ opt: paths=source_relative
+ - plugin: go-grpc
+ out: flow/generated/protos
+ opt:
+ - paths=source_relative
+ - plugin: prost
+ out: nexus/pt/src/gen
+ opt:
+ - compile_well_known_types
+ - extern_path=.google.protobuf=::pbjson_types
+ - plugin: tonic
+ out: nexus/pt/src/gen
+ - plugin: prost-serde
+ out: nexus/pt/src/gen
+ opt:
+ - ignore_unknown_fields=true
+ - name: ts
+ out: ui/grpc_generated
+ strategy: all
+ path: ./node_modules/ts-proto/protoc-gen-ts_proto
+ opt:
+ - esModuleInterop=true
+ - outputServices=grpc-js
+ - plugin: grpc-gateway
+ out: flow/generated/protos
+ opt:
+ - paths=source_relative

docker-compose.yml

@@ -35,6 +35,7 @@ services:
  catalog:
  container_name: catalog
  image: debezium/postgres:16-alpine
+ restart: unless-stopped
  ports:
  - 9901:5432
  environment:
@@ -54,6 +55,7 @@ services:
 
  temporal:
  container_name: temporal
+ restart: unless-stopped
  depends_on:
  catalog:
  condition: service_healthy
@@ -74,6 +76,7 @@ services:
 
  temporal-admin-tools:
  container_name: temporal-admin-tools
+ restart: unless-stopped
  depends_on:
  - temporal
  environment:
@@ -83,7 +86,6 @@ services:
  image: temporalio/admin-tools:1.22
  stdin_open: true
  tty: true
- restart: on-failure
  entrypoint: /etc/temporal/entrypoint.sh
  healthcheck:
  test: ["CMD", "tctl", "workflow", "list"]
@@ -95,6 +97,7 @@ services:
 
  temporal-ui:
  container_name: temporal-ui
+ restart: unless-stopped
  depends_on:
  - temporal
  environment:
@@ -108,6 +111,7 @@ services:
  flow-api:
  container_name: flow_api
  image: ghcr.io/peerdb-io/flow-api:latest-dev
+ restart: unless-stopped
  ports:
  - 8112:8112
  - 8113:8113
@@ -120,6 +124,7 @@ services:
  flow-snapshot-worker:
  container_name: flow-snapshot-worker
  image: ghcr.io/peerdb-io/flow-snapshot-worker:latest-dev
+ restart: unless-stopped
  environment:
  <<: [*catalog-config, *flow-worker-env, *minio-config]
  depends_on:
@@ -129,6 +134,7 @@ services:
  flow-worker:
  container_name: flow-worker
  image: ghcr.io/peerdb-io/flow-worker:latest-dev
+ restart: unless-stopped
  environment:
  <<: [*catalog-config, *flow-worker-env, *minio-config]
  depends_on:
@@ -139,6 +145,7 @@ services:
  container_name: peerdb-server
  stop_signal: SIGINT
  image: ghcr.io/peerdb-io/peerdb-server:latest-dev
+ restart: unless-stopped
  environment:
  <<: *catalog-config
  PEERDB_LOG_DIR: /var/log/peerdb
@@ -155,6 +162,7 @@ services:
  peerdb-ui:
  container_name: peerdb-ui
  image: ghcr.io/peerdb-io/peerdb-ui:latest-dev
+ restart: unless-stopped
  ports:
  - 3000:3000
  environment:
@@ -168,6 +176,7 @@ services:
 
  minio:
  image: minio/minio
+ restart: unless-stopped
  volumes:
  - minio-data:/data
  ports:

flow/.golangci.yml

@@ -1,6 +1,7 @@
 issues:
  exclude-dirs:
  - generated
+ - connectors/postgres/sanitize
 linters:
  enable:
  - containedctx