Examples with ColumnMetadata com.facebook.presto.spi.ColumnMetadata used on opensource projects

Example 41 with ColumnMetadata

use of com.facebook.presto.spi.ColumnMetadata in project presto by prestodb.

the class MetadataManager method listTableColumns.

@Override
public Map<QualifiedObjectName, List<ColumnMetadata>> listTableColumns(Session session, QualifiedTablePrefix prefix) {
    requireNonNull(prefix, "prefix is null");
    Optional<CatalogMetadata> catalog = getOptionalCatalogMetadata(session, prefix.getCatalogName());
    Map<QualifiedObjectName, List<ColumnMetadata>> tableColumns = new HashMap<>();
    if (catalog.isPresent()) {
        CatalogMetadata catalogMetadata = catalog.get();
        SchemaTablePrefix tablePrefix = prefix.asSchemaTablePrefix();
        for (ConnectorId connectorId : catalogMetadata.listConnectorIds()) {
            ConnectorMetadata metadata = catalogMetadata.getMetadataFor(connectorId);
            ConnectorSession connectorSession = session.toConnectorSession(connectorId);
            for (Entry<SchemaTableName, List<ColumnMetadata>> entry : metadata.listTableColumns(connectorSession, tablePrefix).entrySet()) {
                QualifiedObjectName tableName = new QualifiedObjectName(prefix.getCatalogName(), entry.getKey().getSchemaName(), entry.getKey().getTableName());
                tableColumns.put(tableName, entry.getValue());
            }
            // if table and view names overlap, the view wins
            for (Entry<SchemaTableName, ConnectorViewDefinition> entry : metadata.getViews(connectorSession, tablePrefix).entrySet()) {
                QualifiedObjectName tableName = new QualifiedObjectName(prefix.getCatalogName(), entry.getKey().getSchemaName(), entry.getKey().getTableName());
                ImmutableList.Builder<ColumnMetadata> columns = ImmutableList.builder();
                for (ViewColumn column : deserializeView(entry.getValue().getViewData()).getColumns()) {
                    columns.add(new ColumnMetadata(column.getName(), column.getType()));
                }
                tableColumns.put(tableName, columns.build());
            }
        }
    }
    return ImmutableMap.copyOf(tableColumns);
}

Example 42 with ColumnMetadata

use of com.facebook.presto.spi.ColumnMetadata in project presto by prestodb.

the class TestHiveIntegrationSmokeTest method doTestPathHiddenColumn.

private void doTestPathHiddenColumn(Session session, HiveStorageFormat storageFormat) {
    @Language("SQL") String createTable = "CREATE TABLE test_path " + "WITH (" + "format = '" + storageFormat + "'," + "partitioned_by = ARRAY['col1']" + ") AS " + "SELECT * FROM (VALUES " + "(0, 0), (3, 0), (6, 0), " + "(1, 1), (4, 1), (7, 1), " + "(2, 2), (5, 2) " + " ) t(col0, col1) ";
    assertUpdate(session, createTable, 8);
    assertTrue(getQueryRunner().tableExists(getSession(), "test_path"));
    TableMetadata tableMetadata = getTableMetadata(catalog, TPCH_SCHEMA, "test_path");
    assertEquals(tableMetadata.getMetadata().getProperties().get(STORAGE_FORMAT_PROPERTY), storageFormat);
    List<String> columnNames = ImmutableList.of("col0", "col1", PATH_COLUMN_NAME);
    List<ColumnMetadata> columnMetadatas = tableMetadata.getColumns();
    assertEquals(columnMetadatas.size(), columnNames.size());
    for (int i = 0; i < columnMetadatas.size(); i++) {
        ColumnMetadata columnMetadata = columnMetadatas.get(i);
        assertEquals(columnMetadata.getName(), columnNames.get(i));
        if (columnMetadata.getName().equals(PATH_COLUMN_NAME)) {
            // $path should be hidden column
            assertTrue(columnMetadata.isHidden());
        }
    }
    assertEquals(getPartitions("test_path").size(), 3);
    MaterializedResult results = computeActual(session, format("SELECT *, \"%s\" FROM test_path", PATH_COLUMN_NAME));
    Map<Integer, String> partitionPathMap = new HashMap<>();
    for (int i = 0; i < results.getRowCount(); i++) {
        MaterializedRow row = results.getMaterializedRows().get(i);
        int col0 = (int) row.getField(0);
        int col1 = (int) row.getField(1);
        String pathName = (String) row.getField(2);
        String parentDirectory = new Path(pathName).getParent().toString();
        assertTrue(pathName.length() > 0);
        assertEquals((int) (col0 % 3), col1);
        if (partitionPathMap.containsKey(col1)) {
            // the rows in the same partition should be in the same partition directory
            assertEquals(partitionPathMap.get(col1), parentDirectory);
        } else {
            partitionPathMap.put(col1, parentDirectory);
        }
    }
    assertEquals(partitionPathMap.size(), 3);
    assertUpdate(session, "DROP TABLE test_path");
    assertFalse(getQueryRunner().tableExists(session, "test_path"));
}

Example 43 with ColumnMetadata

use of com.facebook.presto.spi.ColumnMetadata in project presto by prestodb.

the class TestHiveIntegrationSmokeTest method createPartitionedTable.

public void createPartitionedTable(Session session, HiveStorageFormat storageFormat) throws Exception {
    @Language("SQL") String createTable = "" + "CREATE TABLE test_partitioned_table (" + "  _string VARCHAR" + ",  _varchar VARCHAR(65535)" + ", _char CHAR(10)" + ", _bigint BIGINT" + ", _integer INTEGER" + ", _smallint SMALLINT" + ", _tinyint TINYINT" + ", _real REAL" + ", _double DOUBLE" + ", _boolean BOOLEAN" + ", _decimal_short DECIMAL(3,2)" + ", _decimal_long DECIMAL(30,10)" + ", _partition_string VARCHAR" + ", _partition_varchar VARCHAR(65535)" + ", _partition_char CHAR(10)" + ", _partition_tinyint TINYINT" + ", _partition_smallint SMALLINT" + ", _partition_integer INTEGER" + ", _partition_bigint BIGINT" + ", _partition_decimal_short DECIMAL(3,2)" + ", _partition_decimal_long DECIMAL(30,10)" + ") " + "WITH (" + "format = '" + storageFormat + "', " + "partitioned_by = ARRAY[ '_partition_string', '_partition_varchar', '_partition_char', '_partition_tinyint', '_partition_smallint', '_partition_integer', '_partition_bigint', '_partition_decimal_short', '_partition_decimal_long' ]" + ") ";
    if (storageFormat == HiveStorageFormat.AVRO) {
        createTable = createTable.replace(" _smallint SMALLINT,", " _smallint INTEGER,");
        createTable = createTable.replace(" _tinyint TINYINT,", " _tinyint INTEGER,");
    }
    assertUpdate(session, createTable);
    TableMetadata tableMetadata = getTableMetadata(catalog, TPCH_SCHEMA, "test_partitioned_table");
    assertEquals(tableMetadata.getMetadata().getProperties().get(STORAGE_FORMAT_PROPERTY), storageFormat);
    List<String> partitionedBy = ImmutableList.of("_partition_string", "_partition_varchar", "_partition_char", "_partition_tinyint", "_partition_smallint", "_partition_integer", "_partition_bigint", "_partition_decimal_short", "_partition_decimal_long");
    assertEquals(tableMetadata.getMetadata().getProperties().get(PARTITIONED_BY_PROPERTY), partitionedBy);
    for (ColumnMetadata columnMetadata : tableMetadata.getColumns()) {
        boolean partitionKey = partitionedBy.contains(columnMetadata.getName());
        assertEquals(columnMetadata.getExtraInfo(), columnExtraInfo(partitionKey));
    }
    assertColumnType(tableMetadata, "_string", createUnboundedVarcharType());
    assertColumnType(tableMetadata, "_varchar", createVarcharType(65535));
    assertColumnType(tableMetadata, "_char", createCharType(10));
    assertColumnType(tableMetadata, "_partition_string", createUnboundedVarcharType());
    assertColumnType(tableMetadata, "_partition_varchar", createVarcharType(65535));
    MaterializedResult result = computeActual("SELECT * from test_partitioned_table");
    assertEquals(result.getRowCount(), 0);
    @Language("SQL") String select = "" + "SELECT" + " 'foo' _string" + ", 'bar' _varchar" + ", CAST('boo' AS CHAR(10)) _char" + ", CAST(1 AS BIGINT) _bigint" + ", 2 _integer" + ", CAST (3 AS SMALLINT) _smallint" + ", CAST (4 AS TINYINT) _tinyint" + ", CAST('123.45' AS REAL) _real" + ", CAST('3.14' AS DOUBLE) _double" + ", true _boolean" + ", CAST('3.14' AS DECIMAL(3,2)) _decimal_short" + ", CAST('12345678901234567890.0123456789' AS DECIMAL(30,10)) _decimal_long" + ", 'foo' _partition_string" + ", 'bar' _partition_varchar" + ", CAST('boo' AS CHAR(10)) _partition_char" + ", CAST(1 AS TINYINT) _partition_tinyint" + ", CAST(1 AS SMALLINT) _partition_smallint" + ", 1 _partition_integer" + ", CAST (1 AS BIGINT) _partition_bigint" + ", CAST('3.14' AS DECIMAL(3,2)) _partition_decimal_short" + ", CAST('12345678901234567890.0123456789' AS DECIMAL(30,10)) _partition_decimal_long";
    if (storageFormat == HiveStorageFormat.AVRO) {
        select = select.replace(" CAST (3 AS SMALLINT) _smallint,", " 3 _smallint,");
        select = select.replace(" CAST (4 AS TINYINT) _tinyint,", " 4 _tinyint,");
    }
    assertUpdate(session, "INSERT INTO test_partitioned_table " + select, 1);
    assertQuery(session, "SELECT * from test_partitioned_table", select);
    assertUpdate(session, "DROP TABLE test_partitioned_table");
    assertFalse(getQueryRunner().tableExists(session, "test_partitioned_table"));
}

Example 44 with ColumnMetadata

use of com.facebook.presto.spi.ColumnMetadata in project presto by prestodb.

the class TestHiveIntegrationSmokeTest method verifyPartition.

private void verifyPartition(boolean hasPartition, TableMetadata tableMetadata, List<String> partitionKeys) {
    Object partitionByProperty = tableMetadata.getMetadata().getProperties().get(PARTITIONED_BY_PROPERTY);
    if (hasPartition) {
        assertEquals(partitionByProperty, partitionKeys);
        for (ColumnMetadata columnMetadata : tableMetadata.getColumns()) {
            boolean partitionKey = partitionKeys.contains(columnMetadata.getName());
            assertEquals(columnMetadata.getExtraInfo(), columnExtraInfo(partitionKey));
        }
    } else {
        assertNull(partitionByProperty);
    }
}

Example 45 with ColumnMetadata

use of com.facebook.presto.spi.ColumnMetadata in project presto by prestodb.

the class H2QueryRunner method insertRows.

private static void insertRows(ConnectorTableMetadata tableMetadata, Handle handle, RecordSet data) {
    List<ColumnMetadata> columns = tableMetadata.getColumns().stream().filter(columnMetadata -> !columnMetadata.isHidden()).collect(toImmutableList());
    String vars = Joiner.on(',').join(nCopies(columns.size(), "?"));
    String sql = format("INSERT INTO %s VALUES (%s)", tableMetadata.getTable().getTableName(), vars);
    RecordCursor cursor = data.cursor();
    while (true) {
        // insert 1000 rows at a time
        PreparedBatch batch = handle.prepareBatch(sql);
        for (int row = 0; row < 1000; row++) {
            if (!cursor.advanceNextPosition()) {
                batch.execute();
                return;
            }
            PreparedBatchPart part = batch.add();
            for (int column = 0; column < columns.size(); column++) {
                Type type = columns.get(column).getType();
                if (BOOLEAN.equals(type)) {
                    part.bind(column, cursor.getBoolean(column));
                } else if (BIGINT.equals(type)) {
                    part.bind(column, cursor.getLong(column));
                } else if (INTEGER.equals(type)) {
                    part.bind(column, (int) cursor.getLong(column));
                } else if (DOUBLE.equals(type)) {
                    part.bind(column, cursor.getDouble(column));
                } else if (type instanceof VarcharType) {
                    part.bind(column, cursor.getSlice(column).toStringUtf8());
                } else if (DATE.equals(type)) {
                    long millisUtc = TimeUnit.DAYS.toMillis(cursor.getLong(column));
                    // H2 expects dates in to be millis at midnight in the JVM timezone
                    long localMillis = DateTimeZone.UTC.getMillisKeepLocal(DateTimeZone.getDefault(), millisUtc);
                    part.bind(column, new Date(localMillis));
                } else {
                    throw new IllegalArgumentException("Unsupported type " + type);
                }
            }
        }
        batch.execute();
    }
}