-
Notifications
You must be signed in to change notification settings - Fork 59
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
16 changed files
with
286 additions
and
135 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,83 @@ | ||
| package connclickhouse | ||
|
|
||
| import ( | ||
| "context" | ||
| "encoding/json" | ||
| "fmt" | ||
|
|
||
| "github.com/jackc/pgx/v5" | ||
| "github.com/jackc/pgx/v5/pgxpool" | ||
|
|
||
| utils "github.com/PeerDB-io/peer-flow/connectors/utils/avro" | ||
| "github.com/PeerDB-io/peer-flow/peerdbenv" | ||
| ) | ||
|
|
||
| type ClickHouseS3Stage struct{} | ||
|
|
||
| func NewClickHouseS3Stage() *ClickHouseS3Stage { | ||
| return &ClickHouseS3Stage{} | ||
| } | ||
|
|
||
| func (c *ClickHouseS3Stage) SetAvroStage( | ||
| ctx context.Context, | ||
| flowJobName string, | ||
| syncBatchID int64, | ||
| avroFile *utils.AvroFile, | ||
| ) error { | ||
| avroFileJSON, err := json.Marshal(avroFile) | ||
| if err != nil { | ||
| return fmt.Errorf("failed to marshal avro file: %w", err) | ||
| } | ||
|
|
||
| conn, err := c.getConn(ctx) | ||
| if err != nil { | ||
| return fmt.Errorf("failed to get connection: %w", err) | ||
| } | ||
|
|
||
| _, err = conn.Exec(ctx, ` | ||
| INSERT INTO ch_s3_stage (flow_job_name, sync_batch_id, avro_file) | ||
| VALUES ($1, $2, $3) | ||
| ON CONFLICT (flow_job_name, sync_batch_id) | ||
| DO UPDATE SET avro_file = $3, created_at = CURRENT_TIMESTAMP | ||
| `, flowJobName, syncBatchID, avroFileJSON) | ||
| if err != nil { | ||
| return fmt.Errorf("failed to set avro stage: %w", err) | ||
| } | ||
|
|
||
| return nil | ||
| } | ||
|
|
||
| func (c *ClickHouseS3Stage) GetAvroStage(ctx context.Context, flowJobName string, syncBatchID int64) (*utils.AvroFile, error) { | ||
| conn, err := c.getConn(ctx) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("failed to get connection: %w", err) | ||
| } | ||
|
|
||
| var avroFileJSON []byte | ||
| err = conn.QueryRow(ctx, ` | ||
| SELECT avro_file FROM ch_s3_stage | ||
| WHERE flow_job_name = $1 AND sync_batch_id = $2 | ||
| `, flowJobName, syncBatchID).Scan(&avroFileJSON) | ||
| if err != nil { | ||
| if err == pgx.ErrNoRows { | ||
| return nil, fmt.Errorf("no avro stage found for flow job %s and sync batch %d", flowJobName, syncBatchID) | ||
| } | ||
| return nil, fmt.Errorf("failed to get avro stage: %w", err) | ||
| } | ||
|
|
||
| var avroFile utils.AvroFile | ||
| if err := json.Unmarshal(avroFileJSON, &avroFile); err != nil { | ||
| return nil, fmt.Errorf("failed to unmarshal avro file: %w", err) | ||
| } | ||
|
|
||
| return &avroFile, nil | ||
| } | ||
|
|
||
| func (c *ClickHouseS3Stage) getConn(ctx context.Context) (*pgxpool.Pool, error) { | ||
| conn, err := peerdbenv.GetCatalogConnectionPoolFromEnv(ctx) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("unable to create catalog connection pool: %w", err) | ||
| } | ||
|
|
||
| return conn, nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.