use of com.thinkbiganalytics.kylo.catalog.spark.sources.jdbc.JdbcHighWaterMarkVisitor in project kylo by Teradata.
the class AbstractJdbcDataSetProvider method updateHighWaterMark.
/**
* Scans the specified field and updates the specified high water mark.
*/
@Nonnull
@VisibleForTesting
T updateHighWaterMark(@Nonnull final T dataSet, @Nonnull final String fieldName, @Nonnull final JdbcHighWaterMark highWaterMark, @Nonnull final KyloCatalogClient<T> client) {
// Determine function to convert column to Long
final DataType fieldType = schema(dataSet).apply(fieldName).dataType();
final Function1<?, Long> toLong;
if (fieldType == DataTypes.DateType) {
toLong = new DateToLong();
} else if (fieldType == DataTypes.TimestampType) {
toLong = new TimestampToLong();
} else {
throw new KyloCatalogException("Unsupported column type for high water mark: " + fieldType);
}
// Create UDF and apply to field
final String accumulableId = (highWaterMark.getName() != null) ? highWaterMark.getName() : UUID.randomUUID().toString();
final Accumulable<JdbcHighWaterMark, Long> accumulable = accumulable(highWaterMark, accumulableId, new JdbcHighWaterMarkAccumulableParam(), client);
final JdbcHighWaterMarkVisitor<?> visitor = new JdbcHighWaterMarkVisitor<>(accumulable, toLong);
return map(dataSet, fieldName, visitor, fieldType);
}
Aggregations