Examples with ColumnMetadata io.prestosql.spi.connector.ColumnMetadata used on opensource projects

Example 96 with ColumnMetadata

use of io.prestosql.spi.connector.ColumnMetadata in project pulsar by apache.

the class PulsarProtobufNativeRowDecoderFactory method extractColumnMetadata.

@Override
public List<ColumnMetadata> extractColumnMetadata(TopicName topicName, SchemaInfo schemaInfo, PulsarColumnHandle.HandleKeyValueType handleKeyValueType) {
    List<ColumnMetadata> columnMetadata;
    String schemaJson = new String(schemaInfo.getSchema());
    if (StringUtils.isBlank(schemaJson)) {
        throw new PrestoException(NOT_SUPPORTED, "Topic " + topicName.toString() + " does not have a valid schema");
    }
    Descriptors.Descriptor schema;
    try {
        schema = ((GenericProtobufNativeSchema) GenericProtobufNativeSchema.of(schemaInfo)).getProtobufNativeSchema();
    } catch (Exception ex) {
        log.error(ex);
        throw new PrestoException(NOT_SUPPORTED, "Topic " + topicName.toString() + " does not have a valid schema");
    }
    // Protobuf have not yet supported Cyclic Objects.
    columnMetadata = schema.getFields().stream().map(field -> new PulsarColumnMetadata(PulsarColumnMetadata.getColumnName(handleKeyValueType, field.getName()), parseProtobufPrestoType(field), field.getType().toString(), null, false, false, handleKeyValueType, new PulsarColumnMetadata.DecoderExtraInfo(field.getName(), null, null))).collect(toList());
    return columnMetadata;
}

Example 97 with ColumnMetadata

use of io.prestosql.spi.connector.ColumnMetadata in project hetu-core by openlookeng.

the class CarbondataMetadata method updateSchemaInfoAddColumn.

private SchemaEvolutionEntry updateSchemaInfoAddColumn(ColumnMetadata column) {
    HiveColumnHandle columnHandle = new HiveColumnHandle(column.getName(), HiveType.toHiveType(typeTranslator, column.getType()), column.getType().getTypeSignature(), tableInfo.getFactTable().getListOfColumns().size(), HiveColumnHandle.ColumnType.REGULAR, Optional.empty());
    TableSchema tableSchema = tableInfo.getFactTable();
    List<ColumnSchema> tableColumns = tableSchema.getListOfColumns();
    int currentSchemaOrdinal = tableColumns.stream().max(Comparator.comparing(ColumnSchema::getSchemaOrdinal)).orElseThrow(NoSuchElementException::new).getSchemaOrdinal() + 1;
    List<ColumnSchema> longStringColumns = new ArrayList<>();
    List<ColumnSchema> allColumns = tableColumns.stream().filter(cols -> cols.isDimensionColumn() && !cols.getDataType().isComplexType() && cols.getSchemaOrdinal() != -1 && (cols.getDataType() != DataTypes.VARCHAR)).collect(toList());
    TableSchemaBuilder schemaBuilder = new TableSchemaBuilder();
    List<ColumnSchema> columnSchemas = new ArrayList<ColumnSchema>();
    ColumnSchema newColumn = schemaBuilder.addColumn(new StructField(columnHandle.getName(), CarbondataHetuFilterUtil.spi2CarbondataTypeMapper(columnHandle)), null, false, false);
    newColumn.setSchemaOrdinal(currentSchemaOrdinal);
    columnSchemas.add(newColumn);
    if (newColumn.getDataType() == DataTypes.VARCHAR) {
        longStringColumns.add(newColumn);
    } else if (newColumn.isDimensionColumn()) {
        // add the column which is not long string
        allColumns.add(newColumn);
    }
    // put the old long string columns
    allColumns.addAll(tableColumns.stream().filter(cols -> cols.isDimensionColumn() && (cols.getDataType() == DataTypes.VARCHAR)).collect(toList()));
    // and the new long string column after old long string columns
    allColumns.addAll(longStringColumns);
    // put complex type columns at the end of dimension columns
    allColumns.addAll(tableColumns.stream().filter(cols -> cols.isDimensionColumn() && (cols.isComplexColumn() || cols.getSchemaOrdinal() == -1)).collect(toList()));
    // original measure columns
    allColumns.addAll(tableColumns.stream().filter(cols -> !cols.isDimensionColumn()).collect(toList()));
    // add new measure column
    if (!newColumn.isDimensionColumn()) {
        allColumns.add(newColumn);
    }
    allColumns.stream().filter(cols -> !cols.isInvisible()).collect(Collectors.groupingBy(ColumnSchema::getColumnName)).forEach((columnName, schemaList) -> {
        if (schemaList.size() > 2) {
            throw new PrestoException(GENERIC_INTERNAL_ERROR, format("Duplicate columns found"));
        }
    });
    if (newColumn.isComplexColumn()) {
        throw new PrestoException(GENERIC_INTERNAL_ERROR, format("Complex column cannot be added"));
    }
    List<ColumnSchema> finalAllColumns = allColumns;
    allColumns.stream().forEach(columnSchema -> {
        List<ColumnSchema> colWithSameId = finalAllColumns.stream().filter(x -> x.getColumnUniqueId().equals(columnSchema.getColumnUniqueId())).collect(toList());
        if (colWithSameId.size() > 1) {
            throw new PrestoException(GENERIC_INTERNAL_ERROR, format("Two columns can not have same columnId"));
        }
    });
    if (tableInfo.getFactTable().getPartitionInfo() != null) {
        List<ColumnSchema> par = tableInfo.getFactTable().getPartitionInfo().getColumnSchemaList();
        allColumns = allColumns.stream().filter(cols -> !par.contains(cols)).collect(toList());
        allColumns.addAll(par);
    }
    tableSchema.setListOfColumns(allColumns);
    tableInfo.setLastUpdatedTime(timeStamp);
    tableInfo.setFactTable(tableSchema);
    SchemaEvolutionEntry schemaEvolutionEntry = new SchemaEvolutionEntry();
    schemaEvolutionEntry.setTimeStamp(timeStamp);
    schemaEvolutionEntry.setAdded(columnSchemas);
    return schemaEvolutionEntry;
}

Example 98 with ColumnMetadata

use of io.prestosql.spi.connector.ColumnMetadata in project hetu-core by openlookeng.

the class ClickHouseClient method beginInsertTable.

@Override
public JdbcOutputTableHandle beginInsertTable(ConnectorSession session, ConnectorTableMetadata tableMetadata) {
    SchemaTableName schemaTableName = tableMetadata.getTable();
    JdbcIdentity identity = JdbcIdentity.from(session);
    if (!getSchemaNames(identity).contains(schemaTableName.getSchemaName())) {
        throw new PrestoException(NOT_FOUND, "Schema not found: " + schemaTableName.getSchemaName());
    }
    try (Connection connection = connectionFactory.openConnection(identity)) {
        boolean uppercase = connection.getMetaData().storesUpperCaseIdentifiers();
        String remoteSchema = toRemoteSchemaName(identity, connection, schemaTableName.getSchemaName());
        String remoteTable = toRemoteTableName(identity, connection, remoteSchema, schemaTableName.getTableName());
        String temporaryTableName = generateTemporaryTableName();
        String catalog = connection.getCatalog();
        ImmutableList.Builder<String> columnNames = ImmutableList.builder();
        ImmutableList.Builder<Type> columnTypes = ImmutableList.builder();
        ImmutableList.Builder<String> columnList = ImmutableList.builder();
        for (ColumnMetadata column : tableMetadata.getColumns()) {
            String columnName = column.getName();
            if (uppercase) {
                columnName = columnName.toUpperCase(ENGLISH);
            }
            columnNames.add(columnName);
            columnTypes.add(column.getType());
            columnList.add(getColumnSql(session, column, columnName));
        }
        String schema = checkCatalog(catalog, remoteSchema);
        String sql = format("CREATE TABLE %s engine=Log AS SELECT * FROM %s WHERE 0 = 1", quoted(null, schema, temporaryTableName), quoted(null, schema, remoteTable));
        execute(connection, sql);
        return new JdbcOutputTableHandle(catalog, remoteSchema, remoteTable, columnNames.build(), columnTypes.build(), temporaryTableName);
    } catch (SQLException e) {
        throw new PrestoException(JDBC_ERROR, e);
    }
}

Example 99 with ColumnMetadata

use of io.prestosql.spi.connector.ColumnMetadata in project hetu-core by openlookeng.

the class TestHBaseConnector method testAddColumnHTableNull.

/**
 * testAddColumnHtableNull
 */
@Test
public void testAddColumnHTableNull() {
    try {
        HBaseTableHandle tableHandle = TestUtils.createHBaseTableHandle("hbase", "test");
        ColumnMetadata column = new ColumnMetadata("name", VARCHAR);
        hcm.addColumn(session, tableHandle, column);
        throw new PrestoException(HBaseErrorCode.HBASE_TABLE_DNE, "testAddColumnHtableNull : failed");
    } catch (PrestoException e) {
        assertEquals(e.getMessage(), format("addColumn fail, cause table[test] not exists."));
    }
}

Example 100 with ColumnMetadata

use of io.prestosql.spi.connector.ColumnMetadata in project hetu-core by openlookeng.

the class TestHBaseConnector method testHBaseConnectionGetColumnHandles.

/**
 * testHBaseConnectionGetColumnHandles
 *
 * @throws InvocationTargetException
 */
@Test
public void testHBaseConnectionGetColumnHandles() throws Exception {
    Method method = HBaseConnection.class.getDeclaredMethod("getColumnHandles", ConnectorTableMetadata.class, String.class);
    method.setAccessible(true);
    List<ColumnMetadata> columns = new ArrayList<>();
    columns.add(new ColumnMetadata("rowkey", VARCHAR));
    columns.add(new ColumnMetadata("name", VARCHAR));
    ConnectorTableMetadata connectorTableMetadata = new ConnectorTableMetadata(schemaTableName, columns);
    try {
        method.invoke(hconn, connectorTableMetadata, "rowkey");
    } catch (InvocationTargetException e) {
        assertEquals(e.getMessage(), null);
    }
}