Examples with TableMetadata io.prestosql.metadata.TableMetadata used on opensource projects

Example 6 with TableMetadata

use of io.prestosql.metadata.TableMetadata in project hetu-core by openlookeng.

the class TestHiveIntegrationSmokeTest method testFullVacuum1.

@Test
public void testFullVacuum1() {
    String table = "tab2";
    String schema = "default";
    assertUpdate(String.format("CREATE SCHEMA IF NOT EXISTS %s", schema));
    assertUpdate(String.format("CREATE TABLE %s.%s (a int, b int) with (transactional=true, format='orc')", schema, table));
    assertUpdate(String.format("INSERT INTO %s.%s VALUES (1, 2)", schema, table), 1);
    assertUpdate(String.format("INSERT INTO %s.%s VALUES (3, 4)", schema, table), 1);
    assertUpdate(String.format("INSERT INTO %s.%s VALUES (5, 6)", schema, table), 1);
    assertUpdate(String.format("VACUUM TABLE %s.%s FULL AND WAIT", schema, table), 3);
    TableMetadata tableMetadata = getTableMetadata("hive", schema, table);
    String tablePath = ((String) tableMetadata.getMetadata().getProperties().get("location"));
    assertFilesAfterCleanup(tablePath, 1);
    assertUpdate(String.format("DROP TABLE %s.%s", schema, table));
}

Example 7 with TableMetadata

use of io.prestosql.metadata.TableMetadata in project hetu-core by openlookeng.

the class TestHiveIntegrationSmokeTest method testVacuumOnPartitionedTable1.

@Test
public void testVacuumOnPartitionedTable1() {
    String table = "tab5";
    String schema = "default";
    assertUpdate(String.format("CREATE SCHEMA IF NOT EXISTS %s", schema));
    String partitionedColumn = "b";
    assertUpdate(String.format("CREATE TABLE %s.%s (a int, b int) with (transactional=true, format='orc', partitioned_by=Array['%s'])", schema, table, partitionedColumn));
    assertUpdate(String.format("INSERT INTO %s.%s VALUES (1, 1)", schema, table), 1);
    assertUpdate(String.format("INSERT INTO %s.%s VALUES (1, 2)", schema, table), 1);
    assertUpdate(String.format("INSERT INTO %s.%s VALUES (2, 1)", schema, table), 1);
    assertUpdate(String.format("INSERT INTO %s.%s VALUES (2, 2)", schema, table), 1);
    assertUpdate(String.format("VACUUM TABLE %s.%s AND WAIT", schema, table), 4);
    TableMetadata tableMetadata = getTableMetadata("hive", schema, table);
    String tablePath = (String) tableMetadata.getMetadata().getProperties().get("location");
    assertFilesAfterCleanupOnPartitionTable(tablePath, partitionedColumn, ImmutableList.of("1", "2"), 1);
    assertUpdate(String.format("UPDATE %s.%s SET a = -1 WHERE a = 2", schema, table), 2);
    assertUpdate(String.format("VACUUM TABLE %s.%s AND WAIT", schema, table), 8);
    assertFilesAfterCleanupOnPartitionTable(tablePath, partitionedColumn, ImmutableList.of("1", "2"), 2);
    assertUpdate(String.format("VACUUM TABLE %s.%s FULL AND WAIT", schema, table), 4);
    assertFilesAfterCleanupOnPartitionTable(tablePath, partitionedColumn, ImmutableList.of("1", "2"), 1);
    assertUpdate(String.format("DROP TABLE %s.%s", schema, table));
}

Example 8 with TableMetadata

use of io.prestosql.metadata.TableMetadata in project hetu-core by openlookeng.

the class TestHiveIntegrationSmokeTest method testInsertPartitionedTableOverwriteExistingPartition.

private void testInsertPartitionedTableOverwriteExistingPartition(Session session, HiveStorageFormat storageFormat, boolean transactional) {
    String tableName = "test_insert_partitioned_table_overwrite_existing_partition";
    @Language("SQL") String createTable = "" + "CREATE TABLE " + tableName + " " + "(" + "  order_key BIGINT," + "  comment VARCHAR," + "  order_status VARCHAR" + ") " + "WITH (" + (transactional ? "transactional=true, " : "") + "format = '" + storageFormat + "', " + "partitioned_by = ARRAY[ 'order_status' ]" + ") ";
    assertUpdate(session, createTable);
    TableMetadata tableMetadata = getTableMetadata(catalog, TPCH_SCHEMA, tableName);
    assertEquals(tableMetadata.getMetadata().getProperties().get(STORAGE_FORMAT_PROPERTY), storageFormat);
    assertEquals(tableMetadata.getMetadata().getProperties().get(PARTITIONED_BY_PROPERTY), ImmutableList.of("order_status"));
    for (int i = 0; i < 3; i++) {
        assertUpdate(session, format("INSERT OVERWRITE " + tableName + " " + "SELECT orderkey, comment, orderstatus " + "FROM tpch.tiny.orders " + "WHERE orderkey %% 3 = %d", i), format("SELECT count(*) from orders where orderkey %% 3 = %d", i));
        // verify the partitions
        List<?> partitions = getPartitions(tableName);
        assertEquals(partitions.size(), 3);
        assertQuery(session, "SELECT * from " + tableName, format("SELECT orderkey, comment, orderstatus FROM orders where orderkey %% 3 = %d", i));
        if (transactional) {
            TableMetadata metadata = getTableMetadata("hive", session.getSchema().get(), tableName);
            String tablePath = (String) tableMetadata.getMetadata().getProperties().get("location");
            File file = new File(tablePath.replace("file:", ""));
            File[] partitionsLocations = file.listFiles((a) -> a.isDirectory() && !a.getName().startsWith("."));
            int expectedBaseCount = i + 1;
            Arrays.stream(partitionsLocations).forEach((partition) -> {
                File[] baseDirectories = partition.listFiles((f) -> f.isDirectory() && f.getName().startsWith("base_"));
                // In case of transactional insert_overwrite base directory is written directly instead of delta.
                assertEquals(expectedBaseCount, baseDirectories.length);
            });
        }
    }
    assertUpdate(session, "DROP TABLE " + tableName);
    assertFalse(getQueryRunner().tableExists(session, tableName));
}

Example 9 with TableMetadata

use of io.prestosql.metadata.TableMetadata in project hetu-core by openlookeng.

the class TestHiveIntegrationSmokeTest method testCreatePartitionedTableAsWithPartitionedRedistribute.

// Presto: test case for partitioned redistribute writes type
private void testCreatePartitionedTableAsWithPartitionedRedistribute(Session session, HiveStorageFormat storageFormat) {
    @Language("SQL") String createTable = "" + "CREATE TABLE test_create_partitioned_table_as " + "WITH (" + "format = '" + storageFormat + "', " + "partitioned_by = ARRAY[ 'SHIP_PRIORITY', 'ORDER_STATUS' ]" + ") " + "AS " + "SELECT orderkey AS order_key, shippriority AS ship_priority, orderstatus AS order_status " + "FROM tpch.tiny.orders";
    Long count = (Long) computeActual("SELECT count(*) from orders").getOnlyValue();
    assertUpdate(Session.builder(getSession()).setSystemProperty("redistribute_writes_type", "PARTITIONED").build(), createTable, count, assertRemotePartitionedExchange("orderstatus"));
    TableMetadata tableMetadata = getTableMetadata(catalog, TPCH_SCHEMA, "test_create_partitioned_table_as");
    assertEquals(tableMetadata.getMetadata().getProperties().get(STORAGE_FORMAT_PROPERTY), storageFormat);
    assertEquals(tableMetadata.getMetadata().getProperties().get(PARTITIONED_BY_PROPERTY), ImmutableList.of("ship_priority", "order_status"));
    List<?> partitions = getPartitions("test_create_partitioned_table_as");
    assertEquals(partitions.size(), 3);
    assertQuery(session, "SELECT * from test_create_partitioned_table_as", "SELECT orderkey, shippriority, orderstatus FROM orders");
    assertUpdate(session, "DROP TABLE test_create_partitioned_table_as");
    assertFalse(getQueryRunner().tableExists(session, "test_create_partitioned_table_as"));
}

Example 10 with TableMetadata

use of io.prestosql.metadata.TableMetadata in project hetu-core by openlookeng.

the class TestHiveIntegrationSmokeTest method testEmptyBucketedTable.

private void testEmptyBucketedTable(Session session, HiveStorageFormat storageFormat, boolean createEmpty) {
    String tableName = "test_empty_bucketed_table";
    @Language("SQL") String createTable = "" + "CREATE TABLE " + tableName + " " + "(bucket_key VARCHAR, col_1 VARCHAR, col2 VARCHAR) " + "WITH (" + "format = '" + storageFormat + "', " + "bucketed_by = ARRAY[ 'bucket_key' ], " + "bucket_count = 11 " + ") ";
    assertUpdate(createTable);
    TableMetadata tableMetadata = getTableMetadata(catalog, TPCH_SCHEMA, tableName);
    assertEquals(tableMetadata.getMetadata().getProperties().get(STORAGE_FORMAT_PROPERTY), storageFormat);
    assertNull(tableMetadata.getMetadata().getProperties().get(PARTITIONED_BY_PROPERTY));
    assertEquals(tableMetadata.getMetadata().getProperties().get(BUCKETED_BY_PROPERTY), ImmutableList.of("bucket_key"));
    assertEquals(tableMetadata.getMetadata().getProperties().get(BUCKET_COUNT_PROPERTY), 11);
    assertEquals(computeActual("SELECT * from " + tableName).getRowCount(), 0);
    // make sure that we will get one file per bucket regardless of writer count configured
    Session parallelWriter = Session.builder(getParallelWriteSession()).setCatalogSessionProperty(catalog, "create_empty_bucket_files", String.valueOf(createEmpty)).build();
    assertUpdate(parallelWriter, "INSERT INTO " + tableName + " VALUES ('a0', 'b0', 'c0')", 1);
    assertUpdate(parallelWriter, "INSERT INTO " + tableName + " VALUES ('a1', 'b1', 'c1')", 1);
    assertQuery("SELECT * from " + tableName, "VALUES ('a0', 'b0', 'c0'), ('a1', 'b1', 'c1')");
    assertUpdate(session, "DROP TABLE " + tableName);
    assertFalse(getQueryRunner().tableExists(session, tableName));
}