-
Notifications
You must be signed in to change notification settings - Fork 13
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #19 from Nordstrom/configurable_payload_formatter
Can specify a payload formatter
- Loading branch information
Showing
19 changed files
with
837 additions
and
731 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
9 changes: 9 additions & 0 deletions
9
src/main/java/com/nordstrom/kafka/connect/formatters/PayloadFormatter.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,9 @@ | ||
| package com.nordstrom.kafka.connect.formatters; | ||
|
|
||
| import java.util.Collection; | ||
| import org.apache.kafka.connect.sink.SinkRecord; | ||
|
|
||
| public interface PayloadFormatter { | ||
| String format(final SinkRecord record) throws PayloadFormattingException; | ||
| String formatBatch(final Collection<SinkRecord> records) throws PayloadFormattingException; | ||
| } |
7 changes: 7 additions & 0 deletions
7
src/main/java/com/nordstrom/kafka/connect/formatters/PayloadFormattingException.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,7 @@ | ||
| package com.nordstrom.kafka.connect.formatters; | ||
|
|
||
| public class PayloadFormattingException extends RuntimeException { | ||
| public PayloadFormattingException (final Throwable e) { | ||
| super(e); | ||
| } | ||
| } |
64 changes: 64 additions & 0 deletions
64
src/main/java/com/nordstrom/kafka/connect/formatters/PlainPayload.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,64 @@ | ||
| package com.nordstrom.kafka.connect.formatters; | ||
|
|
||
| import org.apache.kafka.connect.sink.SinkRecord; | ||
|
|
||
| public class PlainPayload { | ||
| private String key; | ||
| private String keySchemaName; | ||
| private String value; | ||
| private String valueSchemaName; | ||
| private String topic; | ||
| private int partition; | ||
| private long offset; | ||
| private long timestamp; | ||
| private String timestampTypeName; | ||
|
|
||
| protected PlainPayload() { | ||
| } | ||
|
|
||
| public PlainPayload(final SinkRecord record) { | ||
| this.key = record.key() == null ? "" : record.key().toString(); | ||
| if (record.keySchema() != null) | ||
| this.keySchemaName = record.keySchema().name(); | ||
|
|
||
| this.value = record.value() == null ? "" : record.value().toString(); | ||
| if (record.valueSchema() != null) | ||
| this.valueSchemaName = record.valueSchema().name(); | ||
|
|
||
| this.topic = record.topic(); | ||
| this.partition = record.kafkaPartition(); | ||
| this.offset = record.kafkaOffset(); | ||
|
|
||
| if (record.timestamp() != null) | ||
| this.timestamp = record.timestamp(); | ||
| if (record.timestampType() != null) | ||
| this.timestampTypeName = record.timestampType().name; | ||
| } | ||
|
|
||
| public String getValue() { return this.value; } | ||
| public void setValue(final String value) { this.value = value; } | ||
|
|
||
| public long getOffset() { return this.offset; } | ||
| public void setOffset(final long offset) { this.offset = offset; } | ||
|
|
||
| public Long getTimestamp() { return this.timestamp; } | ||
| public void setTimestamp(final long timestamp) { this.timestamp = timestamp; } | ||
|
|
||
| public String getTimestampTypeName() { return this.timestampTypeName; } | ||
| public void setTimestampTypeName(final String timestampTypeName) { this.timestampTypeName = timestampTypeName; } | ||
|
|
||
| public int getPartition() { return this.partition; } | ||
| public void setPartition(final int partition) { this.partition = partition; } | ||
|
|
||
| public String getKey() { return this.key; } | ||
| public void setKey(final String key) { this.key = key; } | ||
|
|
||
| public String getKeySchemaName() { return this.keySchemaName; } | ||
| public void setKeySchemaName(final String keySchemaName) { this.keySchemaName = keySchemaName; } | ||
|
|
||
| public String getValueSchemaName() { return this.valueSchemaName; } | ||
| public void setValueSchemaName(final String valueSchemaName) { this.valueSchemaName = valueSchemaName; } | ||
|
|
||
| public String getTopic() { return this.topic; } | ||
| public void setTopic(final String topic) { this.topic = topic; } | ||
| } |
43 changes: 43 additions & 0 deletions
43
src/main/java/com/nordstrom/kafka/connect/formatters/PlainPayloadFormatter.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,43 @@ | ||
| package com.nordstrom.kafka.connect.formatters; | ||
|
|
||
| import com.fasterxml.jackson.core.JsonProcessingException; | ||
| import com.fasterxml.jackson.databind.ObjectMapper; | ||
| import com.fasterxml.jackson.databind.ObjectWriter; | ||
|
|
||
| import org.apache.kafka.connect.sink.SinkRecord; | ||
|
|
||
| import java.util.Collection; | ||
|
|
||
| import org.slf4j.Logger; | ||
| import org.slf4j.LoggerFactory; | ||
|
|
||
| public class PlainPayloadFormatter implements PayloadFormatter { | ||
| private static final Logger LOGGER = LoggerFactory.getLogger(PlainPayloadFormatter.class); | ||
| private final ObjectWriter recordWriter = new ObjectMapper().writerFor(PlainPayload.class); | ||
| private final ObjectWriter recordsWriter = new ObjectMapper().writerFor(PlainPayload[].class); | ||
|
|
||
| public String format(final SinkRecord record) { | ||
| PlainPayload payload = new PlainPayload(record); | ||
|
|
||
| try { | ||
| return this.recordWriter.writeValueAsString(payload); | ||
| } catch (final JsonProcessingException e) { | ||
| LOGGER.error(e.getLocalizedMessage(), e); | ||
| throw new PayloadFormattingException(e); | ||
| } | ||
| } | ||
|
|
||
| public String formatBatch(final Collection<SinkRecord> records) { | ||
| final PlainPayload[] payloads = records | ||
| .stream() | ||
| .map(record -> new PlainPayload(record)) | ||
| .toArray(PlainPayload[]::new); | ||
|
|
||
| try { | ||
| return this.recordsWriter.writeValueAsString(payloads); | ||
| } catch (final JsonProcessingException e) { | ||
| LOGGER.error(e.getLocalizedMessage(), e); | ||
| throw new PayloadFormattingException(e); | ||
| } | ||
| } | ||
| } |
Oops, something went wrong.