Examples with ShardInfo io.trino.plugin.raptor.legacy.metadata.ShardInfo used on opensource projects

Example 6 with ShardInfo

use of io.trino.plugin.raptor.legacy.metadata.ShardInfo in project trino by trinodb.

the class TestShardOrganizerUtil method testGetOrganizationEligibleShards.

@Test
public void testGetOrganizationEligibleShards() {
    int day1 = 1111;
    int day2 = 2222;
    SchemaTableName tableName = new SchemaTableName("default", "test");
    metadata.createTable(SESSION, tableMetadataBuilder(tableName).column("orderkey", BIGINT).column("orderdate", DATE).column("orderstatus", createVarcharType(3)).property("ordering", ImmutableList.of("orderstatus", "orderkey")).property("temporal_column", "orderdate").build(), false);
    Table tableInfo = metadataDao.getTableInformation(tableName.getSchemaName(), tableName.getTableName());
    List<TableColumn> tableColumns = metadataDao.listTableColumns(tableInfo.getTableId());
    Map<String, TableColumn> tableColumnMap = Maps.uniqueIndex(tableColumns, TableColumn::getColumnName);
    long orderDate = tableColumnMap.get("orderdate").getColumnId();
    long orderKey = tableColumnMap.get("orderkey").getColumnId();
    long orderStatus = tableColumnMap.get("orderstatus").getColumnId();
    List<ShardInfo> shards = ImmutableList.<ShardInfo>builder().add(shardInfo(UUID.randomUUID(), "node1", ImmutableList.of(new ColumnStats(orderDate, day1, day1 + 10), new ColumnStats(orderKey, 13L, 14L), new ColumnStats(orderStatus, "aaa", "abc")))).add(shardInfo(UUID.randomUUID(), "node1", ImmutableList.of(new ColumnStats(orderDate, day2, day2 + 100), new ColumnStats(orderKey, 2L, 20L), new ColumnStats(orderStatus, "aaa", "abc")))).add(shardInfo(UUID.randomUUID(), "node1", ImmutableList.of(new ColumnStats(orderDate, day1, day2), new ColumnStats(orderKey, 2L, 11L), new ColumnStats(orderStatus, "aaa", "abc")))).add(shardInfo(UUID.randomUUID(), "node1", ImmutableList.of(new ColumnStats(orderDate, day1, day2), new ColumnStats(orderKey, 2L, null), new ColumnStats(orderStatus, "aaa", "abc")))).add(shardInfo(UUID.randomUUID(), "node1", ImmutableList.of(new ColumnStats(orderDate, day1, null), new ColumnStats(orderKey, 2L, 11L), new ColumnStats(orderStatus, "aaa", "abc")))).build();
    long transactionId = shardManager.beginTransaction();
    shardManager.commitShards(transactionId, tableInfo.getTableId(), COLUMNS, shards, Optional.empty(), 0);
    Set<ShardMetadata> shardMetadatas = shardManager.getNodeShards("node1");
    Long temporalColumnId = metadataDao.getTemporalColumnId(tableInfo.getTableId());
    TableColumn temporalColumn = metadataDao.getTableColumn(tableInfo.getTableId(), temporalColumnId);
    Set<ShardIndexInfo> actual = ImmutableSet.copyOf(getOrganizationEligibleShards(dbi, metadataDao, tableInfo, shardMetadatas, false));
    List<ShardIndexInfo> expected = getShardIndexInfo(tableInfo, shards, temporalColumn, Optional.empty());
    assertEquals(actual, expected);
    List<TableColumn> sortColumns = metadataDao.listSortColumns(tableInfo.getTableId());
    Set<ShardIndexInfo> actualSortRange = ImmutableSet.copyOf(getOrganizationEligibleShards(dbi, metadataDao, tableInfo, shardMetadatas, true));
    List<ShardIndexInfo> expectedSortRange = getShardIndexInfo(tableInfo, shards, temporalColumn, Optional.of(sortColumns));
    assertEquals(actualSortRange, expectedSortRange);
}

Example 7 with ShardInfo

use of io.trino.plugin.raptor.legacy.metadata.ShardInfo in project trino by trinodb.

the class TestShardOrganizerUtil method getShardIndexInfo.

private static List<ShardIndexInfo> getShardIndexInfo(Table tableInfo, List<ShardInfo> shards, TableColumn temporalColumn, Optional<List<TableColumn>> sortColumns) {
    long tableId = tableInfo.getTableId();
    Type temporalType = temporalColumn.getDataType();
    ImmutableList.Builder<ShardIndexInfo> builder = ImmutableList.builder();
    for (ShardInfo shard : shards) {
        ColumnStats temporalColumnStats = shard.getColumnStats().stream().filter(columnStats -> columnStats.getColumnId() == temporalColumn.getColumnId()).findFirst().get();
        if (temporalColumnStats.getMin() == null || temporalColumnStats.getMax() == null) {
            continue;
        }
        Optional<ShardRange> sortRange = Optional.empty();
        if (sortColumns.isPresent()) {
            Map<Long, ColumnStats> columnIdToStats = Maps.uniqueIndex(shard.getColumnStats(), ColumnStats::getColumnId);
            ImmutableList.Builder<Type> typesBuilder = ImmutableList.builder();
            ImmutableList.Builder<Object> minBuilder = ImmutableList.builder();
            ImmutableList.Builder<Object> maxBuilder = ImmutableList.builder();
            boolean isShardEligible = true;
            for (TableColumn sortColumn : sortColumns.get()) {
                ColumnStats columnStats = columnIdToStats.get(sortColumn.getColumnId());
                typesBuilder.add(sortColumn.getDataType());
                if (columnStats.getMin() == null || columnStats.getMax() == null) {
                    isShardEligible = false;
                    break;
                }
                minBuilder.add(columnStats.getMin());
                maxBuilder.add(columnStats.getMax());
            }
            if (!isShardEligible) {
                continue;
            }
            List<Type> types = typesBuilder.build();
            List<Object> minValues = minBuilder.build();
            List<Object> maxValues = maxBuilder.build();
            sortRange = Optional.of(ShardRange.of(new Tuple(types, minValues), new Tuple(types, maxValues)));
        }
        builder.add(new ShardIndexInfo(tableId, OptionalInt.empty(), shard.getShardUuid(), shard.getRowCount(), shard.getUncompressedSize(), sortRange, Optional.of(ShardRange.of(new Tuple(temporalType, temporalColumnStats.getMin()), new Tuple(temporalType, temporalColumnStats.getMax())))));
    }
    return builder.build();
}

Example 8 with ShardInfo

use of io.trino.plugin.raptor.legacy.metadata.ShardInfo in project trino by trinodb.

the class TestShardMetadataRecordCursor method testSimple.

@Test
public void testSimple() {
    ShardManager shardManager = createShardManager(dbi);
    // Add shards to the table
    long tableId = 1;
    OptionalInt bucketNumber = OptionalInt.empty();
    UUID uuid1 = UUID.randomUUID();
    UUID uuid2 = UUID.randomUUID();
    UUID uuid3 = UUID.randomUUID();
    ShardInfo shardInfo1 = new ShardInfo(uuid1, bucketNumber, ImmutableSet.of("node1"), ImmutableList.of(), 1, 10, 100, 0x1234);
    ShardInfo shardInfo2 = new ShardInfo(uuid2, bucketNumber, ImmutableSet.of("node2"), ImmutableList.of(), 2, 20, 200, 0xCAFEBABEDEADBEEFL);
    ShardInfo shardInfo3 = new ShardInfo(uuid3, bucketNumber, ImmutableSet.of("node3"), ImmutableList.of(), 3, 30, 300, 0xFEDCBA0987654321L);
    List<ShardInfo> shards = ImmutableList.of(shardInfo1, shardInfo2, shardInfo3);
    long transactionId = shardManager.beginTransaction();
    shardManager.commitShards(transactionId, tableId, ImmutableList.of(new ColumnInfo(1, BIGINT), new ColumnInfo(2, DATE)), shards, Optional.empty(), 0);
    Slice schema = utf8Slice(DEFAULT_TEST_ORDERS.getSchemaName());
    Slice table = utf8Slice(DEFAULT_TEST_ORDERS.getTableName());
    DateTime date1 = DateTime.parse("2015-01-01T00:00");
    DateTime date2 = DateTime.parse("2015-01-02T00:00");
    TupleDomain<Integer> tupleDomain = TupleDomain.withColumnDomains(ImmutableMap.<Integer, Domain>builder().put(0, Domain.singleValue(createVarcharType(10), schema)).put(1, Domain.create(ValueSet.ofRanges(lessThanOrEqual(createVarcharType(10), table)), true)).put(8, Domain.create(ValueSet.ofRanges(lessThanOrEqual(BIGINT, date1.getMillis()), greaterThan(BIGINT, date2.getMillis())), true)).put(9, Domain.create(ValueSet.ofRanges(lessThanOrEqual(BIGINT, date1.getMillis()), greaterThan(BIGINT, date2.getMillis())), true)).buildOrThrow());
    List<MaterializedRow> actual;
    try (RecordCursor cursor = new ShardMetadataSystemTable(dbi).cursor(null, SESSION, tupleDomain)) {
        actual = getMaterializedResults(cursor, SHARD_METADATA.getColumns());
    }
    assertEquals(actual.size(), 3);
    List<MaterializedRow> expected = ImmutableList.of(new MaterializedRow(DEFAULT_PRECISION, schema, table, utf8Slice(uuid1.toString()), null, 100L, 10L, 1L, utf8Slice("0000000000001234"), null, null, null, null), new MaterializedRow(DEFAULT_PRECISION, schema, table, utf8Slice(uuid2.toString()), null, 200L, 20L, 2L, utf8Slice("cafebabedeadbeef"), null, null, null, null), new MaterializedRow(DEFAULT_PRECISION, schema, table, utf8Slice(uuid3.toString()), null, 300L, 30L, 3L, utf8Slice("fedcba0987654321"), null, null, null, null));
    assertEquals(actual, expected);
}

Example 9 with ShardInfo

use of io.trino.plugin.raptor.legacy.metadata.ShardInfo in project trino by trinodb.

the class TestRaptorStorageManager method testRewriter.

@Test
public void testRewriter() throws Exception {
    RaptorStorageManager manager = createRaptorStorageManager();
    long transactionId = TRANSACTION_ID;
    List<Long> columnIds = ImmutableList.of(3L, 7L);
    List<Type> columnTypes = ImmutableList.of(BIGINT, createVarcharType(10));
    // create file with 2 rows
    StoragePageSink sink = createStoragePageSink(manager, columnIds, columnTypes);
    List<Page> pages = rowPagesBuilder(columnTypes).row(123L, "hello").row(456L, "bye").build();
    sink.appendPages(pages);
    List<ShardInfo> shards = getFutureValue(sink.commit());
    assertEquals(shardRecorder.getShards().size(), 1);
    // delete one row
    BitSet rowsToDelete = new BitSet();
    rowsToDelete.set(0);
    Collection<Slice> fragments = manager.rewriteShard(transactionId, OptionalInt.empty(), shards.get(0).getShardUuid(), rowsToDelete);
    Slice shardDelta = Iterables.getOnlyElement(fragments);
    ShardDelta shardDeltas = jsonCodec(ShardDelta.class).fromJson(shardDelta.getBytes());
    ShardInfo shardInfo = Iterables.getOnlyElement(shardDeltas.getNewShards());
    // check that output file has one row
    assertEquals(shardInfo.getRowCount(), 1);
    // check that storage file is same as backup file
    File storageFile = storageService.getStorageFile(shardInfo.getShardUuid());
    File backupFile = fileBackupStore.getBackupFile(shardInfo.getShardUuid());
    assertFileEquals(storageFile, backupFile);
    // verify recorded shard
    List<RecordedShard> recordedShards = shardRecorder.getShards();
    assertEquals(recordedShards.size(), 2);
    assertEquals(recordedShards.get(1).getTransactionId(), TRANSACTION_ID);
    assertEquals(recordedShards.get(1).getShardUuid(), shardInfo.getShardUuid());
}

Example 10 with ShardInfo

use of io.trino.plugin.raptor.legacy.metadata.ShardInfo in project trino by trinodb.

the class TestRaptorStorageManager method testReader.

@Test
public void testReader() throws Exception {
    RaptorStorageManager manager = createRaptorStorageManager();
    List<Long> columnIds = ImmutableList.of(2L, 4L, 6L, 7L, 8L, 9L);
    List<Type> columnTypes = ImmutableList.of(BIGINT, createVarcharType(10), VARBINARY, DATE, BOOLEAN, DOUBLE);
    byte[] bytes1 = octets(0x00, 0xFE, 0xFF);
    byte[] bytes3 = octets(0x01, 0x02, 0x19, 0x80);
    StoragePageSink sink = createStoragePageSink(manager, columnIds, columnTypes);
    Object[][] doubles = { { 881L, "-inf", null, null, null, Double.NEGATIVE_INFINITY }, { 882L, "+inf", null, null, null, Double.POSITIVE_INFINITY }, { 883L, "nan", null, null, null, Double.NaN }, { 884L, "min", null, null, null, Double.MIN_VALUE }, { 885L, "max", null, null, null, Double.MAX_VALUE }, { 886L, "pzero", null, null, null, 0.0 }, { 887L, "nzero", null, null, null, -0.0 } };
    List<Page> pages = rowPagesBuilder(columnTypes).row(123L, "hello", wrappedBuffer(bytes1), sqlDate(2001, 8, 22).getDays(), true, 123.45).row(null, null, null, null, null, null).row(456L, "bye", wrappedBuffer(bytes3), sqlDate(2005, 4, 22).getDays(), false, 987.65).rows(doubles).build();
    sink.appendPages(pages);
    List<ShardInfo> shards = getFutureValue(sink.commit());
    assertEquals(shards.size(), 1);
    UUID uuid = Iterables.getOnlyElement(shards).getShardUuid();
    MaterializedResult expected = resultBuilder(SESSION, columnTypes).row(123L, "hello", sqlBinary(bytes1), sqlDate(2001, 8, 22), true, 123.45).row(null, null, null, null, null, null).row(456L, "bye", sqlBinary(bytes3), sqlDate(2005, 4, 22), false, 987.65).rows(doubles).build();
    // no tuple domain (all)
    TupleDomain<RaptorColumnHandle> tupleDomain = TupleDomain.all();
    try (ConnectorPageSource pageSource = getPageSource(manager, columnIds, columnTypes, uuid, tupleDomain)) {
        MaterializedResult result = materializeSourceDataStream(SESSION, pageSource, columnTypes);
        assertEquals(result.getRowCount(), expected.getRowCount());
        assertEquals(result, expected);
    }
    // tuple domain within the column range
    tupleDomain = TupleDomain.fromFixedValues(ImmutableMap.<RaptorColumnHandle, NullableValue>builder().put(new RaptorColumnHandle("c1", 2, BIGINT), NullableValue.of(BIGINT, 124L)).buildOrThrow());
    try (ConnectorPageSource pageSource = getPageSource(manager, columnIds, columnTypes, uuid, tupleDomain)) {
        MaterializedResult result = materializeSourceDataStream(SESSION, pageSource, columnTypes);
        assertEquals(result.getRowCount(), expected.getRowCount());
    }
    // tuple domain outside the column range
    tupleDomain = TupleDomain.fromFixedValues(ImmutableMap.<RaptorColumnHandle, NullableValue>builder().put(new RaptorColumnHandle("c1", 2, BIGINT), NullableValue.of(BIGINT, 122L)).buildOrThrow());
    try (ConnectorPageSource pageSource = getPageSource(manager, columnIds, columnTypes, uuid, tupleDomain)) {
        MaterializedResult result = materializeSourceDataStream(SESSION, pageSource, columnTypes);
        assertEquals(result.getRowCount(), 0);
    }
}