Examples with DefaultField com.thinkbiganalytics.discovery.model.DefaultField used on opensource projects

Example 1 with DefaultField

use of com.thinkbiganalytics.discovery.model.DefaultField in project kylo by Teradata.

the class DBSchemaParser method columnsResultSetToField.

private List<Field> columnsResultSetToField(ResultSet columns, Set<String> pkSet, String schema) throws SQLException {
    List<Field> fields = new Vector<>();
    if (columns != null) {
        while (columns.next()) {
            String cat = columns.getString("TABLE_CAT");
            if (StringUtils.isNotBlank(cat) && StringUtils.isNotBlank(schema)) {
                // this db supports Catalogs.  Ensure the cat matches the supplied schema
                if (!schema.equalsIgnoreCase(cat)) {
                    continue;
                }
            }
            DefaultField field = new DefaultField();
            field.setName(columns.getString("COLUMN_NAME"));
            Integer dataType = columns.getInt("DATA_TYPE");
            field.setNativeDataType(ParserHelper.toNativeType(dataType));
            field.setDerivedDataType(ParserHelper.sqlTypeToHiveType(dataType));
            field.setDescription(columns.getString("REMARKS"));
            String isNullableString = columns.getString("IS_NULLABLE");
            if ("NO".equals(isNullableString)) {
                field.setNullable(false);
            }
            if (pkSet.contains(field.getName())) {
                field.setPrimaryKey(true);
            }
            fields.add(field);
        }
    }
    return fields;
}

Example 2 with DefaultField

use of com.thinkbiganalytics.discovery.model.DefaultField in project kylo by Teradata.

the class DBSchemaParser method columnsResultSetToField.

@Nonnull
private List<Field> columnsResultSetToField(@Nullable ResultSet columns, @Nonnull Set<String> pkSet, @Nullable String schema, boolean schemaIsCat) throws SQLException {
    List<Field> fields = new ArrayList<>();
    if (columns != null) {
        while (columns.next()) {
            String cat = columns.getString(CATALOG_COLUMN);
            if (StringUtils.isNotBlank(cat) && StringUtils.isNotBlank(schema)) {
                // this db supports Catalogs.  Ensure the cat matches the supplied schema
                if (schemaIsCat && !schema.equalsIgnoreCase(cat)) {
                    continue;
                }
            }
            DefaultField field = new DefaultField();
            field.setName(columns.getString("COLUMN_NAME"));
            Integer dataType = columns.getInt("DATA_TYPE");
            field.setNativeDataType(ParserHelper.toNativeType(dataType));
            field.setDerivedDataType(ParserHelper.sqlTypeToHiveType(dataType));
            field.setDescription(columns.getString("REMARKS"));
            String isNullableString = columns.getString("IS_NULLABLE");
            if ("NO".equals(isNullableString)) {
                field.setNullable(false);
            }
            if (pkSet.contains(field.getName())) {
                field.setPrimaryKey(true);
            }
            fields.add(field);
        }
    }
    return fields;
}

Example 3 with DefaultField

use of com.thinkbiganalytics.discovery.model.DefaultField in project kylo by Teradata.

the class TableSetupTest method test.

@Test
public void test() throws Exception {
    ObjectMapper mapper = new ObjectMapper();
    FeedMetadata feedMetadata = new FeedMetadata();
    feedMetadata.setCategory(new FeedCategory());
    feedMetadata.setTable(new TableSetup());
    feedMetadata.getTable().setTableSchema(new DefaultTableSchema());
    feedMetadata.getTable().getTableSchema().setName("test");
    DefaultField f1 = new DefaultField();
    f1.setName("field1");
    feedMetadata.getTable().getTableSchema().getFields().add(f1);
    String json = mapper.writeValueAsString(feedMetadata);
    FeedMetadata feedMetadata2 = mapper.readValue(json, FeedMetadata.class);
    assertEquals(feedMetadata2.getTable().getTableSchema().getName(), feedMetadata.getTable().getTableSchema().getName());
}

Example 4 with DefaultField

use of com.thinkbiganalytics.discovery.model.DefaultField in project kylo by Teradata.

the class HiveMetastoreService method getTableSchemas.

public List<TableSchema> getTableSchemas() throws DataAccessException {
    String query = "SELECT d.NAME as \"DATABASE_NAME\", t.TBL_NAME, c.COLUMN_NAME, c.TYPE_NAME " + "FROM COLUMNS_V2 c " + "JOIN  SDS s on s.CD_ID = c.CD_ID " + "JOIN  TBLS t ON s.SD_ID = t.SD_ID " + "JOIN  DBS d on d.DB_ID = t.DB_ID " + "ORDER BY d.NAME, t.TBL_NAME";
    if (DatabaseType.POSTGRES.equals(getMetastoreDatabaseType())) {
        query = "SELECT d.\"NAME\" as \"DATABASE_NAME\", t.\"TBL_NAME\", c.\"COLUMN_NAME\",c.\"TYPE_NAME\" " + "FROM \"COLUMNS_V2\" c " + "JOIN  \"SDS\" s on s.\"CD_ID\" = c.\"CD_ID\" " + "JOIN  \"TBLS\" t ON s.\"SD_ID\" = t.\"SD_ID\" " + "JOIN  \"DBS\" d on d.\"DB_ID\" = t.\"DB_ID\" ";
    }
    final List<TableSchema> metadata = new ArrayList<>();
    final Map<String, Map<String, TableSchema>> databaseTables = new HashMap<>();
    hiveMetatoreJdbcTemplate.query(query, new RowMapper<Object>() {

        @Override
        public TableSchema mapRow(ResultSet rs, int i) throws SQLException {
            String dbName = rs.getString("DATABASE_NAME");
            String columnName = rs.getString("COLUMN_NAME");
            String tableName = rs.getString("TBL_NAME");
            String columnType = rs.getString("TYPE_NAME");
            if (!databaseTables.containsKey(dbName)) {
                databaseTables.put(dbName, new HashMap<String, TableSchema>());
            }
            Map<String, TableSchema> tables = databaseTables.get(dbName);
            if (!tables.containsKey(tableName)) {
                DefaultTableSchema schema = new DefaultTableSchema();
                schema.setName(tableName);
                schema.setSchemaName(dbName);
                schema.setFields(new ArrayList<Field>());
                tables.put(tableName, schema);
                metadata.add(schema);
            }
            TableSchema schema = tables.get(tableName);
            DefaultField field = new DefaultField();
            field.setName(columnName);
            field.setNativeDataType(columnType);
            field.setDerivedDataType(columnType);
            schema.getFields().add(field);
            return schema;
        }
    });
    return metadata;
}

Example 5 with DefaultField

use of com.thinkbiganalytics.discovery.model.DefaultField in project kylo by Teradata.

the class CSVFileSchemaParser method populateSchema.

private DefaultFileSchema populateSchema(CSVParser parser) {
    DefaultFileSchema fileSchema = new DefaultFileSchema();
    int i = 0;
    ArrayList<Field> fields = new ArrayList<>();
    for (CSVRecord record : parser) {
        if (i > 9) {
            break;
        }
        int size = record.size();
        for (int j = 0; j < size; j++) {
            DefaultField field = null;
            if (i == 0) {
                field = new DefaultField();
                if (headerRow) {
                    field.setName(record.get(j));
                } else {
                    field.setName("Col_" + (j + 1));
                }
                fields.add(field);
            } else {
                try {
                    field = (DefaultField) fields.get(j);
                    field.getSampleValues().add(StringUtils.defaultString(record.get(j), ""));
                } catch (IndexOutOfBoundsException e) {
                    LOG.warn("Sample file has potential sparse column problem at row [?] field [?]", i + 1, j + 1);
                }
            }
        }
        i++;
    }
    fileSchema.setFields(fields);
    return fileSchema;
}