Examples with DefaultField com.thinkbiganalytics.discovery.model.DefaultField used on opensource projects

Example 6 with DefaultField

use of com.thinkbiganalytics.discovery.model.DefaultField in project kylo by Teradata.

the class DomainTypeIT method testCreateAndUpdateDatasource.

@Test
public void testCreateAndUpdateDatasource() {
    FieldStandardizationRule toUpperCase = new FieldStandardizationRule();
    FieldValidationRule email = new FieldValidationRule();
    toUpperCase.setName("Uppercase");
    toUpperCase.setDisplayName("Uppercase");
    toUpperCase.setDescription("Convert string to uppercase");
    toUpperCase.setObjectClassType("com.thinkbiganalytics.policy.standardization.UppercaseStandardizer");
    toUpperCase.setObjectShortClassType("UppercaseStandardizer");
    email.setName("email");
    email.setDisplayName("Email");
    email.setDescription("Valid email address");
    email.setObjectClassType("com.thinkbiganalytics.policy.validation.EmailValidator");
    email.setObjectShortClassType("EmailValidator");
    DomainType[] initialDomainTypes = getDomainTypes();
    // create new domain type
    DomainType dt = new DomainType();
    dt.setTitle("Domain Type 1");
    dt.setDescription("domain type created by integration tests");
    DomainType response = createDomainType(dt);
    assertEquals(dt.getTitle(), response.getTitle());
    assertEquals(dt.getDescription(), response.getDescription());
    assertEquals(null, response.getIcon());
    assertEquals(null, response.getIconColor());
    // assert new domain type was added
    DomainType[] currentDomainTypes = getDomainTypes();
    assertEquals(initialDomainTypes.length + 1, currentDomainTypes.length);
    // update existing domain type
    dt = getDomainType(response.getId());
    dt.setTitle("Domain Type 1 with updated title");
    dt.setDescription("domain type description updated by integration tests");
    dt.setIcon("stars");
    dt.setIconColor("green");
    DefaultField field = new DefaultField();
    Tag tag1 = new DefaultTag("tag1");
    Tag tag2 = new DefaultTag("tag2");
    field.setTags(Arrays.asList(tag1, tag2));
    field.setName("field-name");
    field.setDerivedDataType("decimal");
    field.setPrecisionScale("9, 3");
    dt.setField(field);
    dt.setFieldNamePattern("field-name-pattern");
    dt.setFieldPolicy(newPolicyBuilder(null).withStandardisation(toUpperCase).withValidation(email).toPolicy());
    dt.setRegexPattern("regex-pattern");
    DomainType updated = createDomainType(dt);
    assertEquals(dt.getTitle(), updated.getTitle());
    assertEquals(dt.getDescription(), updated.getDescription());
    assertEquals("stars", updated.getIcon());
    assertEquals("green", updated.getIconColor());
    Field updatedField = updated.getField();
    assertEquals(field.getName(), updatedField.getName());
    assertEquals(field.getDerivedDataType(), updatedField.getDerivedDataType());
    assertEquals(field.getPrecisionScale(), updatedField.getPrecisionScale());
    assertEquals(field.getTags().size(), updatedField.getTags().size());
    assertEquals(tag1.getName(), updatedField.getTags().get(0).getName());
    assertEquals(tag2.getName(), updatedField.getTags().get(1).getName());
    assertEquals(dt.getFieldNamePattern(), updated.getFieldNamePattern());
    assertEquals(dt.getRegexPattern(), updated.getRegexPattern());
    FieldStandardizationRule updatedStandardisation = updated.getFieldPolicy().getStandardization().get(0);
    assertEquals(toUpperCase.getName(), updatedStandardisation.getName());
    assertEquals(toUpperCase.getObjectShortClassType(), updatedStandardisation.getObjectShortClassType());
    FieldValidationRule updatedValidation = updated.getFieldPolicy().getValidation().get(0);
    assertEquals(email.getName(), updatedValidation.getName());
    assertEquals(email.getObjectShortClassType(), updatedValidation.getObjectShortClassType());
    // assert domain type was updated, rather than added
    currentDomainTypes = getDomainTypes();
    assertEquals(initialDomainTypes.length + 1, currentDomainTypes.length);
    // delete domain type
    deleteDomainType(dt.getId());
    currentDomainTypes = getDomainTypes();
    assertEquals(initialDomainTypes.length, currentDomainTypes.length);
    // assert domain type was removed
    getDomainTypeExpectingStatus(dt.getId(), HTTP_NOT_FOUND);
}

Example 7 with DefaultField

use of com.thinkbiganalytics.discovery.model.DefaultField in project kylo by Teradata.

the class IntegrationTestBase method newNamedField.

protected DefaultField newNamedField(String name, DefaultDataTypeDescriptor typeDescriptor, String type) {
    DefaultField field = new DefaultField();
    field.setName(name);
    field.setDerivedDataType(type);
    field.setDataTypeDescriptor(typeDescriptor);
    return field;
}

Example 8 with DefaultField

use of com.thinkbiganalytics.discovery.model.DefaultField in project kylo by Teradata.

the class SparkFileSchemaParserService method toHiveSchema.

private DefaultHiveSchema toHiveSchema(TransformQueryResult result, SparkFileType fileType) {
    DefaultHiveSchema schema = new DefaultHiveSchema();
    schema.setHiveFormat("STORED AS " + fileType);
    schema.setStructured(true);
    ArrayList<Field> fields = new ArrayList<>();
    List<? extends QueryResultColumn> columns = result.getColumns();
    for (int i = 0; i < columns.size(); ++i) {
        QueryResultColumn column = columns.get(i);
        DefaultField field = new DefaultField();
        field.setName(column.getDisplayName());
        field.setNativeDataType(column.getDataType());
        field.setDerivedDataType(column.getDataType());
        field.setDataTypeDescriptor(ParserHelper.hiveTypeToDescriptor(column.getDataType()));
        // strip the precisionScale and assign to the field property
        setPrecisionAndScale(field);
        // Add sample values
        List<List<Object>> values = result.getRows();
        for (List<Object> colMap : values) {
            Object oVal = colMap.get(i);
            if (oVal != null) {
                field.getSampleValues().add(oVal.toString());
            }
        }
        fields.add(field);
    }
    schema.setFields(fields);
    return schema;
}

Example 9 with DefaultField

use of com.thinkbiganalytics.discovery.model.DefaultField in project kylo by Teradata.

the class HiveMetastoreService method getTable.

public TableSchema getTable(String schema, String table) throws DataAccessException {
    // Must use JDBC metadata for user impersonation
    if (userImpersonationEnabled) {
        return hiveService.getTableSchema(schema, table);
    }
    String query = "SELECT d.NAME as \"DATABASE_NAME\", t.TBL_NAME, c.COLUMN_NAME, c.TYPE_NAME " + "FROM COLUMNS_V2 c " + "JOIN  SDS s on s.CD_ID = c.CD_ID " + "JOIN  TBLS t ON s.SD_ID = t.SD_ID " + "JOIN  DBS d on d.DB_ID = t.DB_ID " + "WHERE t.TBL_NAME='" + table + "' and d.NAME='" + schema + "' " + "ORDER BY d.NAME, t.TBL_NAME, c.INTEGER_IDX";
    if (DatabaseType.POSTGRES.equals(getMetastoreDatabaseType())) {
        query = "SELECT d.\"NAME\" as \"DATABASE_NAME\", t.\"TBL_NAME\", c.\"COLUMN_NAME\",c.\"TYPE_NAME\" " + "FROM \"COLUMNS_V2\" c " + "JOIN  \"SDS\" s on s.\"CD_ID\" = c.\"CD_ID\" " + "JOIN  \"TBLS\" t ON s.\"SD_ID\" = t.\"SD_ID\" " + "JOIN  \"DBS\" d on d.\"DB_ID\" = t.\"DB_ID\" " + "WHERE t.\"TBL_NAME\"='" + table + "' and d.\"NAME\"='" + schema + "' " + "ORDER BY d.\"NAME\", t.\"TBL_NAME\", c.\"INTEGER_IDX\"";
    }
    final DefaultTableSchema metadata = new DefaultTableSchema();
    hiveMetatoreJdbcTemplate.query(query, new RowMapper<Object>() {

        @Override
        public TableSchema mapRow(ResultSet rs, int i) throws SQLException {
            String dbName = rs.getString("DATABASE_NAME");
            String columnName = rs.getString("COLUMN_NAME");
            String tableName = rs.getString("TBL_NAME");
            String columnType = rs.getString("TYPE_NAME");
            metadata.setName(tableName);
            metadata.setSchemaName(dbName);
            DefaultField field = new DefaultField();
            field.setName(columnName);
            field.setNativeDataType(columnType);
            field.setDerivedDataType(columnType);
            metadata.getFields().add(field);
            return metadata;
        }
    });
    return metadata;
}