use of com.thinkbiganalytics.util.TableRegisterConfiguration in project kylo by Teradata.
the class RegisterFeedTables method onTrigger.
@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
// Verify flow file exists
final FlowFile flowFile = session.get();
if (flowFile == null) {
return;
}
// Verify properties and attributes
final String feedFormatOptions = Optional.ofNullable(context.getProperty(FEED_FORMAT_SPECS).evaluateAttributeExpressions(flowFile).getValue()).filter(StringUtils::isNotEmpty).orElse(DEFAULT_FEED_FORMAT_OPTIONS);
final String targetFormatOptions = Optional.ofNullable(context.getProperty(TARGET_FORMAT_SPECS).evaluateAttributeExpressions(flowFile).getValue()).filter(StringUtils::isNotEmpty).orElse(DEFAULT_STORAGE_FORMAT);
final String targetTableProperties = context.getProperty(TARGET_TBLPROPERTIES).evaluateAttributeExpressions(flowFile).getValue();
final ColumnSpec[] partitions = Optional.ofNullable(context.getProperty(PARTITION_SPECS).evaluateAttributeExpressions(flowFile).getValue()).filter(StringUtils::isNotEmpty).map(ColumnSpec::createFromString).orElse(new ColumnSpec[0]);
final String tableType = context.getProperty(TABLE_TYPE).getValue();
final ColumnSpec[] columnSpecs = Optional.ofNullable(context.getProperty(FIELD_SPECIFICATION).evaluateAttributeExpressions(flowFile).getValue()).filter(StringUtils::isNotEmpty).map(ColumnSpec::createFromString).orElse(new ColumnSpec[0]);
if (columnSpecs == null || columnSpecs.length == 0) {
getLog().error("Missing field specification");
session.transfer(flowFile, IngestProperties.REL_FAILURE);
return;
}
ColumnSpec[] feedColumnSpecs = Optional.ofNullable(context.getProperty(FEED_FIELD_SPECIFICATION).evaluateAttributeExpressions(flowFile).getValue()).filter(StringUtils::isNotEmpty).map(ColumnSpec::createFromString).orElse(new ColumnSpec[0]);
if (feedColumnSpecs == null || feedColumnSpecs.length == 0) {
// Backwards compatibility with older templates we set the source and target to the same
feedColumnSpecs = columnSpecs;
}
final String entity = context.getProperty(IngestProperties.FEED_NAME).evaluateAttributeExpressions(flowFile).getValue();
if (entity == null || entity.isEmpty()) {
getLog().error("Missing feed name");
session.transfer(flowFile, IngestProperties.REL_FAILURE);
return;
}
final String source = context.getProperty(IngestProperties.FEED_CATEGORY).evaluateAttributeExpressions(flowFile).getValue();
if (source == null || source.isEmpty()) {
getLog().error("Missing category name");
session.transfer(flowFile, IngestProperties.REL_FAILURE);
return;
}
final String feedRoot = context.getProperty(FEED_ROOT).evaluateAttributeExpressions(flowFile).getValue();
final String profileRoot = context.getProperty(PROFILE_ROOT).evaluateAttributeExpressions(flowFile).getValue();
final String masterRoot = context.getProperty(MASTER_ROOT).evaluateAttributeExpressions(flowFile).getValue();
final TableRegisterConfiguration config = new TableRegisterConfiguration(feedRoot, profileRoot, masterRoot);
// Register the tables
final ThriftService thriftService = context.getProperty(THRIFT_SERVICE).asControllerService(ThriftService.class);
try (final Connection conn = thriftService.getConnection()) {
final TableRegisterSupport register = new TableRegisterSupport(conn, config);
final boolean result;
if (ALL_TABLES.equals(tableType)) {
result = register.registerStandardTables(source, entity, feedColumnSpecs, feedFormatOptions, targetFormatOptions, partitions, columnSpecs, targetTableProperties);
} else {
result = register.registerTable(source, entity, feedColumnSpecs, feedFormatOptions, targetFormatOptions, partitions, columnSpecs, targetTableProperties, TableType.valueOf(tableType), true);
}
final Relationship relnResult = (result ? REL_SUCCESS : REL_FAILURE);
session.transfer(flowFile, relnResult);
} catch (final ProcessException | SQLException e) {
getLog().error("Unable to obtain connection for {} due to {}; routing to failure", new Object[] { flowFile, e });
session.transfer(flowFile, REL_FAILURE);
}
}
use of com.thinkbiganalytics.util.TableRegisterConfiguration in project kylo by Teradata.
the class TableRegisterSupportTest method testTableCreateS3.
@Test
public void testTableCreateS3() {
ColumnSpec[] specs = ColumnSpec.createFromString("id|bigint|my comment\nname|string\ncompany|string|some description\nzip|string\nphone|string\nemail|string\ncountry|string\nhired|date");
ColumnSpec[] parts = ColumnSpec.createFromString("year|int\ncountry|string");
TableRegisterConfiguration conf = new TableRegisterConfiguration("s3a://testBucket/model.db/", "s3a://testBucket/model.db/", "s3a://testBucket/app/warehouse/");
TableRegisterSupport support = new TableRegisterSupport(connection, conf);
TableType[] tableTypes = new TableType[] { TableType.FEED, TableType.INVALID, TableType.VALID, TableType.MASTER };
for (TableType tableType : tableTypes) {
String ddl = support.createDDL("bar", "employee", specs, parts, "ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.OpenCSVSerde'", "stored as orc", "tblproperties (\"orc.compress\"=\"SNAPPY\")", tableType);
String location = StringUtils.substringBetween(ddl, "LOCATION '", "'");
if (tableType == TableType.MASTER) {
assertEquals("Master location does not match", "s3a://testBucket/app/warehouse/bar/employee", location);
} else {
assertEquals("Locations do not match", "s3a://testBucket/model.db/bar/employee/" + tableType.toString().toLowerCase(), location);
}
}
}
use of com.thinkbiganalytics.util.TableRegisterConfiguration in project kylo by Teradata.
the class TableRegisterSupportTest method testRemovingColumns.
@Test
public void testRemovingColumns() {
ColumnSpec[] feedSpecs = ColumnSpec.createFromString("id|string|my comment|0|0|0|id\n" + "name|string||0|0|0|name\n" + "company|string|some description|0|0|0|change_company\n" + "zip|string||0|0|0|zip_code\n" + "phone|string\n" + "email|string\n" + "country|string\n" + "hired|string");
ColumnSpec[] targetSpecs = ColumnSpec.createFromString("id|bigint|my comment|0|0|0|id\n" + "name|string||0|0|0|name\n" + "change_company|string|some description|0|0|0|company\n" + "zip_code|string||0|0|0|zip\n" + "email|string\n" + "hired|date||0|0|0|hired");
ColumnSpec[] parts = ColumnSpec.createFromString("year|int\ncountry|string");
TableRegisterConfiguration conf = new TableRegisterConfiguration();
TableRegisterSupport support = new TableRegisterSupport(connection, conf);
ColumnSpec[] invalidColumnSpecs = support.adjustInvalidColumnSpec(feedSpecs, targetSpecs);
assertEquals(targetSpecs.length, invalidColumnSpecs.length);
Map<String, ColumnSpec> feedColumnSpecMap = Arrays.asList(feedSpecs).stream().collect(Collectors.toMap(ColumnSpec::getName, Function.identity()));
for (ColumnSpec invalid : invalidColumnSpecs) {
if (StringUtils.isNotBlank(invalid.getOtherColumnName())) {
assertEquals(invalid.getDataType(), feedColumnSpecMap.get(invalid.getOtherColumnName()).getDataType());
}
}
TableType[] tableTypes = new TableType[] { TableType.FEED, TableType.INVALID, TableType.VALID, TableType.MASTER };
for (TableType tableType : tableTypes) {
ColumnSpec[] useColumnSpecs = targetSpecs;
if (tableType == TableType.INVALID) {
useColumnSpecs = invalidColumnSpecs;
} else if (tableType == TableType.FEED) {
useColumnSpecs = feedSpecs;
}
String ddl = support.createDDL("source_table", "target_table", useColumnSpecs, parts, "ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.OpenCSVSerde'", "stored as orc", "tblproperties (\"orc.compress\"=\"SNAPPY\")", tableType);
int i = 0;
}
}
Aggregations