Examples with ColumnSpec com.thinkbiganalytics.util.ColumnSpec used on opensource projects

Example 11 with ColumnSpec

use of com.thinkbiganalytics.util.ColumnSpec in project kylo by Teradata.

the class TableMergeSyncSupportTest method testMergePartitionPKWithEmptyTargetTable.

@Test
public /**
 * Tests the merge partition with empty target table
 */
void testMergePartitionPKWithEmptyTargetTable() throws Exception {
    List<String> results = fetchEmployees(targetSchema, targetTable);
    assertEquals(0, results.size());
    ColumnSpec columnSpec1 = new ColumnSpec("id", "String", "", true, false, false);
    ColumnSpec columnSpec2 = new ColumnSpec("name", "String", "", false, false, false);
    ColumnSpec[] columnSpecs = Arrays.asList(columnSpec1, columnSpec2).toArray(new ColumnSpec[0]);
    // Call merge
    mergeSyncSupport.doPKMerge(sourceSchema, sourceTable, targetSchema, targetTable, spec, processingPartition, columnSpecs);
    // We should have 4 records
    results = fetchEmployees(targetSchema, targetTable);
    assertEquals(4, results.size());
}

Example 12 with ColumnSpec

use of com.thinkbiganalytics.util.ColumnSpec in project kylo by Teradata.

the class TableRegisterSupportTest method testProfileTableCreateS3.

@Test
public void testProfileTableCreateS3() {
    ColumnSpec[] specs = ColumnSpec.createFromString("id|bigint|my comment\nname|string\ncompany|string|some description\nzip|string\nphone|string\nemail|string\ncountry|string\nhired|date");
    ColumnSpec[] parts = ColumnSpec.createFromString("year|int\ncountry|string");
    TableRegisterConfiguration conf = new TableRegisterConfiguration("s3a://testBucket/model.db/", "s3a://testBucket/model.db/", "s3a://testBucket/app/warehouse/");
    TableRegisterSupport support = new TableRegisterSupport(connection, conf);
    String ddl = support.createDDL("bar", "employee", specs, parts, "ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.OpenCSVSerde'", "stored as orc", "tblproperties (\"orc.compress\"=\"SNAPPY\")", TableType.PROFILE);
    String expectedDDL = "CREATE TABLE IF NOT EXISTS `bar`.`employee_" + TableType.PROFILE.toString().toLowerCase() + "` ( `columnname` string,`metrictype` string,`metricvalue` string)   PARTITIONED BY (`processing_dttm` string)  stored as orc LOCATION 's3a://testBucket/model.db/bar/employee/" + TableType.PROFILE.toString().toLowerCase() + "'";
    assertEquals("DDL do not match", expectedDDL, ddl);
}

Example 13 with ColumnSpec

use of com.thinkbiganalytics.util.ColumnSpec in project kylo by Teradata.

the class TableRegisterSupportTest method testTableCreate.

@Test
public void testTableCreate() {
    ColumnSpec[] specs = ColumnSpec.createFromString("id|bigint|my comment\nname|string\ncompany|string|some description\nzip|string\nphone|string\nemail|string\ncountry|string\nhired|date");
    ColumnSpec[] parts = ColumnSpec.createFromString("year|int\ncountry|string");
    TableRegisterSupport support = new TableRegisterSupport(connection);
    TableType[] tableTypes = new TableType[] { TableType.FEED, TableType.INVALID, TableType.VALID, TableType.MASTER };
    for (TableType tableType : tableTypes) {
        String ddl = support.createDDL("bar", "employee", specs, parts, "ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.OpenCSVSerde'", "stored as orc", "tblproperties (\"orc.compress\"=\"SNAPPY\")", tableType);
        // Hack to make a legal file root
        ddl = ddl.replace("LOCATION '", "LOCATION '${hiveconf:MY.HDFS.DIR}");
        hiveShell.execute(ddl);
    }
}

Example 14 with ColumnSpec

use of com.thinkbiganalytics.util.ColumnSpec in project kylo by Teradata.

the class TableRegisterSupportTest method testRemovingColumns.

@Test
public void testRemovingColumns() {
    ColumnSpec[] feedSpecs = ColumnSpec.createFromString("id|string|my comment|0|0|0|id\n" + "name|string||0|0|0|name\n" + "company|string|some description|0|0|0|change_company\n" + "zip|string||0|0|0|zip_code\n" + "phone|string\n" + "email|string\n" + "country|string\n" + "hired|string");
    ColumnSpec[] targetSpecs = ColumnSpec.createFromString("id|bigint|my comment|0|0|0|id\n" + "name|string||0|0|0|name\n" + "change_company|string|some description|0|0|0|company\n" + "zip_code|string||0|0|0|zip\n" + "email|string\n" + "hired|date||0|0|0|hired");
    ColumnSpec[] parts = ColumnSpec.createFromString("year|int\ncountry|string");
    TableRegisterConfiguration conf = new TableRegisterConfiguration();
    TableRegisterSupport support = new TableRegisterSupport(connection, conf);
    ColumnSpec[] invalidColumnSpecs = support.adjustInvalidColumnSpec(feedSpecs, targetSpecs);
    assertEquals(targetSpecs.length, invalidColumnSpecs.length);
    Map<String, ColumnSpec> feedColumnSpecMap = Arrays.asList(feedSpecs).stream().collect(Collectors.toMap(ColumnSpec::getName, Function.identity()));
    for (ColumnSpec invalid : invalidColumnSpecs) {
        if (StringUtils.isNotBlank(invalid.getOtherColumnName())) {
            assertEquals(invalid.getDataType(), feedColumnSpecMap.get(invalid.getOtherColumnName()).getDataType());
        }
    }
    TableType[] tableTypes = new TableType[] { TableType.FEED, TableType.INVALID, TableType.VALID, TableType.MASTER };
    for (TableType tableType : tableTypes) {
        ColumnSpec[] useColumnSpecs = targetSpecs;
        if (tableType == TableType.INVALID) {
            useColumnSpecs = invalidColumnSpecs;
        } else if (tableType == TableType.FEED) {
            useColumnSpecs = feedSpecs;
        }
        String ddl = support.createDDL("source_table", "target_table", useColumnSpecs, parts, "ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.OpenCSVSerde'", "stored as orc", "tblproperties (\"orc.compress\"=\"SNAPPY\")", tableType);
        int i = 0;
    }
}