Examples with TableMetadata io.prestosql.metadata.TableMetadata used on opensource projects

Example 11 with TableMetadata

use of io.prestosql.metadata.TableMetadata in project hetu-core by openlookeng.

the class TestHiveIntegrationSmokeTest method getTableMetadata.

private TableMetadata getTableMetadata(String catalog, String schema, String tableName) {
    Session session = getSession();
    Metadata metadata = ((DistributedQueryRunner) getQueryRunner()).getCoordinator().getMetadata();
    return transaction(getQueryRunner().getTransactionManager(), getQueryRunner().getAccessControl()).readOnly().execute(session, transactionSession -> {
        Optional<TableHandle> tableHandle = metadata.getTableHandle(transactionSession, new QualifiedObjectName(catalog, schema, tableName));
        assertTrue(tableHandle.isPresent());
        return metadata.getTableMetadata(transactionSession, tableHandle.get());
    });
}

Example 12 with TableMetadata

use of io.prestosql.metadata.TableMetadata in project hetu-core by openlookeng.

the class TestHiveIntegrationSmokeTest method testVacuumOnDeleteDelta.

@Test
public void testVacuumOnDeleteDelta() {
    String table = "tab4";
    String schema = "default";
    assertUpdate(String.format("CREATE SCHEMA IF NOT EXISTS %s", schema));
    assertUpdate(String.format("CREATE TABLE %s.%s (a int, b int) with (transactional=true, format='orc')", schema, table));
    assertUpdate(String.format("INSERT INTO %s.%s VALUES (1, 2)", schema, table), 1);
    assertUpdate(String.format("INSERT INTO %s.%s VALUES (3, 4)", schema, table), 1);
    assertUpdate(String.format("INSERT INTO %s.%s VALUES (5, 6)", schema, table), 1);
    assertUpdate(String.format("UPDATE %s.%s SET b = -1 WHERE a > 2", schema, table), 2);
    assertUpdate(String.format("VACUUM TABLE %s.%s AND WAIT", schema, table), 7);
    TableMetadata tableMetadata = getTableMetadata("hive", schema, table);
    String tablePath = ((String) tableMetadata.getMetadata().getProperties().get("location"));
    assertFilesAfterCleanup(tablePath, 2);
    assertUpdate(String.format("VACUUM TABLE %s.%s FULL AND WAIT", schema, table), 3);
    assertFilesAfterCleanup(tablePath, 1);
    assertUpdate(String.format("DROP TABLE %s.%s", schema, table));
}

Example 13 with TableMetadata

use of io.prestosql.metadata.TableMetadata in project hetu-core by openlookeng.

the class TestHiveIntegrationSmokeTest method testInsertPartitionedTableWithPartitionedRedistribute.

// Presto: test case for partitioned redistribute writes type
private void testInsertPartitionedTableWithPartitionedRedistribute(Session session, HiveStorageFormat storageFormat) {
    @Language("SQL") String createTable = "" + "CREATE TABLE test_insert_partitioned_table " + "(" + "  ORDER_KEY BIGINT," + "  SHIP_PRIORITY INTEGER," + "  ORDER_STATUS VARCHAR" + ") " + "WITH (" + "format = '" + storageFormat + "', " + "partitioned_by = ARRAY[ 'SHIP_PRIORITY', 'ORDER_STATUS' ]" + ") ";
    assertUpdate(session, createTable);
    TableMetadata tableMetadata = getTableMetadata(catalog, TPCH_SCHEMA, "test_insert_partitioned_table");
    assertEquals(tableMetadata.getMetadata().getProperties().get(STORAGE_FORMAT_PROPERTY), storageFormat);
    assertEquals(tableMetadata.getMetadata().getProperties().get(PARTITIONED_BY_PROPERTY), ImmutableList.of("ship_priority", "order_status"));
    String partitionsTable = "\"test_insert_partitioned_table$partitions\"";
    assertQuery(session, "SELECT * FROM " + partitionsTable, "SELECT shippriority, orderstatus FROM orders LIMIT 0");
    // Hive will reorder the partition keys, so we must insert into the table assuming the partition keys have been moved to the end
    Long count = (Long) computeActual("SELECT count(*) from orders").getOnlyValue();
    assertUpdate(Session.builder(getSession()).setSystemProperty("redistribute_writes_type", "PARTITIONED").build(), "" + "INSERT INTO test_insert_partitioned_table " + "SELECT orderkey, shippriority, orderstatus " + "FROM tpch.tiny.orders", count, assertRemotePartitionedExchange("orderstatus"));
    // verify the partitions
    List<?> partitions = getPartitions("test_insert_partitioned_table");
    assertEquals(partitions.size(), 3);
    assertQuery(session, "SELECT * from test_insert_partitioned_table", "SELECT orderkey, shippriority, orderstatus FROM orders");
    assertQuery(session, "SELECT * FROM " + partitionsTable, "SELECT DISTINCT shippriority, orderstatus FROM orders");
    assertQuery(session, "SELECT * FROM " + partitionsTable + " ORDER BY order_status LIMIT 2", "SELECT DISTINCT shippriority, orderstatus FROM orders ORDER BY orderstatus LIMIT 2");
    assertQuery(session, "SELECT * FROM " + partitionsTable + " WHERE order_status = 'O'", "SELECT DISTINCT shippriority, orderstatus FROM orders WHERE orderstatus = 'O'");
    assertUpdate(session, "DROP TABLE test_insert_partitioned_table");
    assertFalse(getQueryRunner().tableExists(session, "test_insert_partitioned_table"));
}

Example 14 with TableMetadata

use of io.prestosql.metadata.TableMetadata in project hetu-core by openlookeng.

the class TestHiveIntegrationSmokeTest method testVacuumOnPartitionedTable.

@Test
public void testVacuumOnPartitionedTable() {
    String table = "tab7_partitioned";
    String schema = "default";
    assertUpdate(String.format("CREATE SCHEMA IF NOT EXISTS %s", schema));
    String partitionedColumn = "b";
    assertUpdate(String.format("CREATE TABLE %s.%s (a int, b int) with (transactional=true, format='orc', partitioned_by=Array['%s'])", schema, table, partitionedColumn));
    TableMetadata tableMetadata = getTableMetadata("hive", schema, table);
    String tablePath = (String) tableMetadata.getMetadata().getProperties().get("location");
    assertUpdate(String.format("INSERT INTO %s.%s VALUES (1,1),(1,2)", schema, table), 2);
    assertUpdate(String.format("INSERT INTO %s.%s VALUES (2,1),(2,2)", schema, table), 2);
    assertUpdate(String.format("VACUUM TABLE %s.%s FULL AND WAIT", schema, table), 4);
    assertFilesAfterCleanupOnPartitionTable(tablePath, partitionedColumn, ImmutableList.of("2"), 1);
    // INSERT ONLY to partition b=1 and CALL VACUUM FULL, should compact only partition b=1 with 4 rows.
    assertUpdate(String.format("INSERT INTO %s.%s VALUES (3, 1)", schema, table), 1);
    assertUpdate(String.format("INSERT INTO %s.%s VALUES (4, 1)", schema, table), 1);
    String[] part2Dirs = listPartition(tablePath, "b=2");
    assertUpdate(String.format("VACUUM TABLE %s.%s FULL AND WAIT", schema, table), 4);
    verifyPartitionDirs(tablePath, "b=2", part2Dirs.length, part2Dirs);
    assertFilesAfterCleanupOnPartitionTable(tablePath, partitionedColumn, ImmutableList.of("1"), 1);
    // INSERT ONLY to partition b=3 and CALL VACUUM FULL, should compact only partition b=3 with 2 rows.
    String[] part1Dirs = listPartition(tablePath, "b=1");
    assertUpdate(String.format("INSERT INTO %s.%s VALUES (3, 3)", schema, table), 1);
    assertUpdate(String.format("INSERT INTO %s.%s VALUES (4, 3)", schema, table), 1);
    assertUpdate(String.format("VACUUM TABLE %s.%s FULL AND WAIT", schema, table), 2);
    assertUpdate(String.format("INSERT INTO %s.%s VALUES (5, 3)", schema, table), 1);
    assertUpdate(String.format("INSERT INTO %s.%s VALUES (6, 3)", schema, table), 1);
    // partition 3 should now have baseDir along with 2 delta dirs.
    String[] part3Dirs = listPartition(tablePath, "b=3");
    long minId = Long.MAX_VALUE;
    long maxId = Long.MIN_VALUE;
    for (String delta : part3Dirs) {
        Matcher matcher = DELTA_PATTERN.matcher(delta);
        if (matcher.matches()) {
            minId = Math.min(Long.parseLong(matcher.group(2)), minId);
            maxId = Math.max(Long.parseLong(matcher.group(3)), maxId);
        }
    }
    assertUpdate(String.format("VACUUM TABLE %s.%s AND WAIT", schema, table), 2);
    verifyPartitionDirs(tablePath, "b=2", part2Dirs.length, part2Dirs);
    verifyPartitionDirs(tablePath, "b=1", part1Dirs.length, part1Dirs);
    assertFilesAfterCleanupOnPartitionTable(tablePath, partitionedColumn, ImmutableList.of("3"), 2);
    verifyPartitionDirs(tablePath, "b=3", 2, part3Dirs[0], String.format("delta_%07d_%07d", minId, maxId));
}

Example 15 with TableMetadata

use of io.prestosql.metadata.TableMetadata in project hetu-core by openlookeng.

the class TestHiveIntegrationSmokeTest method testCreateTableAs.

private void testCreateTableAs(Session session, HiveStorageFormat storageFormat) {
    @Language("SQL") String select = "SELECT" + " 'foo' _varchar" + ", CAST('bar' AS CHAR(10)) _char" + ", CAST (1 AS BIGINT) _bigint" + ", 2 _integer" + ", CAST (3 AS SMALLINT) _smallint" + ", CAST (4 AS TINYINT) _tinyint" + ", CAST ('123.45' as REAL) _real" + ", CAST('3.14' AS DOUBLE) _double" + ", true _boolean" + ", CAST('3.14' AS DECIMAL(3,2)) _decimal_short" + ", CAST('12345678901234567890.0123456789' AS DECIMAL(30,10)) _decimal_long";
    if (storageFormat == HiveStorageFormat.AVRO) {
        select = select.replace(" CAST (3 AS SMALLINT) _smallint,", " 3 _smallint,");
        select = select.replace(" CAST (4 AS TINYINT) _tinyint,", " 4 _tinyint,");
    }
    String createTableAs = format("CREATE TABLE test_format_table WITH (format = '%s') AS %s", storageFormat, select);
    assertUpdate(session, createTableAs, 1);
    TableMetadata tableMetadata = getTableMetadata(catalog, TPCH_SCHEMA, "test_format_table");
    assertEquals(tableMetadata.getMetadata().getProperties().get(STORAGE_FORMAT_PROPERTY), storageFormat);
    assertColumnType(tableMetadata, "_varchar", createVarcharType(3));
    assertColumnType(tableMetadata, "_char", createCharType(10));
    // assure reader supports basic column reordering and pruning
    assertQuery(session, "SELECT _integer, _varchar, _integer from test_format_table", "SELECT 2, 'foo', 2");
    assertQuery(session, "SELECT * from test_format_table", select);
    assertUpdate(session, "DROP TABLE test_format_table");
    assertFalse(getQueryRunner().tableExists(session, "test_format_table"));
}