Examples with Pair io.prestosql.spi.heuristicindex.Pair used on opensource projects

Example 36 with Pair

use of io.prestosql.spi.heuristicindex.Pair in project hetu-core by openlookeng.

the class TestPartitionIndexWriter method testAddValue.

@Test
public void testAddValue() throws IOException {
    List<Pair<String, Type>> columns = new ArrayList<>();
    List<String> partitions = Collections.singletonList("partition1");
    Properties properties = new Properties();
    CreateIndexMetadata createIndexMetadata = new CreateIndexMetadata("hetu_partition_idx", "testTable", "BTREE", 0L, columns, partitions, properties, "testuser", CreateIndexMetadata.Level.PARTITION);
    HetuFileSystemClient fileSystemClient = Mockito.mock(HetuFileSystemClient.class);
    Properties connectorMetadata = new Properties();
    connectorMetadata.setProperty(HetuConstant.DATASOURCE_FILE_MODIFICATION, String.valueOf(System.currentTimeMillis()));
    connectorMetadata.setProperty(HetuConstant.DATASOURCE_FILE_PATH, "hdfs://testable/testcolumn/cp=123121/file1");
    connectorMetadata.setProperty(HetuConstant.DATASOURCE_STRIPE_OFFSET, "3");
    connectorMetadata.setProperty(HetuConstant.DATASOURCE_STRIPE_LENGTH, "100");
    PartitionIndexWriter indexWriter = new PartitionIndexWriter(createIndexMetadata, fileSystemClient, Paths.get("/tmp"));
    Map<String, List<Object>> valuesMap = new HashMap<>();
    List<Object> values = new ArrayList<>();
    for (int i = 0; i < 10; i++) {
        values.add("key" + i);
    }
    valuesMap.put("testcolumne", values);
    indexWriter.addData(valuesMap, connectorMetadata);
    Map<Comparable<? extends Comparable<?>>, String> result = indexWriter.getDataMap();
    assertEquals(10, result.size());
    assertEquals(1, indexWriter.getSymbolTable().size());
}

Example 37 with Pair

use of io.prestosql.spi.heuristicindex.Pair in project hetu-core by openlookeng.

the class CreateIndexOperator method addInput.

@Override
public void addInput(Page page) {
    checkState(needsInput(), "Operator is already finishing");
    requireNonNull(page, "page is null");
    // TODO-cp-I38S9O: Operator currently not supported for Snapshot
    if (page instanceof MarkerPage) {
        throw new UnsupportedOperationException("Operator doesn't support snapshotting.");
    }
    // if operator is still receiving input, it's not finished
    finished.putIfAbsent(this, false);
    if (page.getPositionCount() == 0) {
        return;
    }
    if (createIndexMetadata.getCreateLevel() == CreateIndexMetadata.Level.UNDEFINED) {
        boolean tableIsPartitioned = getPartitionName(page.getPageMetadata().getProperty(HetuConstant.DATASOURCE_FILE_PATH), createIndexMetadata.getTableName()) != null;
        createIndexMetadata.decideIndexLevel(tableIsPartitioned);
    }
    Map<String, List<Object>> values = new HashMap<>();
    for (int blockId = 0; blockId < page.getChannelCount(); blockId++) {
        Block block = page.getBlock(blockId);
        Pair<String, Type> entry = createIndexMetadata.getIndexColumns().get(blockId);
        String indexColumn = entry.getFirst();
        Type type = entry.getSecond();
        for (int position = 0; position < block.getPositionCount(); ++position) {
            Object value = getNativeValue(type, block, position);
            value = getActualValue(type, value);
            values.computeIfAbsent(indexColumn, k -> new ArrayList<>()).add(value);
        }
    }
    Properties connectorMetadata = new Properties();
    connectorMetadata.put(HetuConstant.DATASOURCE_CATALOG, createIndexMetadata.getTableName().split("\\.")[0]);
    connectorMetadata.putAll(page.getPageMetadata());
    try {
        switch(createIndexMetadata.getCreateLevel()) {
            case STRIPE:
                {
                    String filePath = page.getPageMetadata().getProperty(HetuConstant.DATASOURCE_FILE_PATH);
                    levelWriter.computeIfAbsent(filePath, k -> heuristicIndexerManager.getIndexWriter(createIndexMetadata, connectorMetadata));
                    persistBy.putIfAbsent(levelWriter.get(filePath), this);
                    levelWriter.get(filePath).addData(values, connectorMetadata);
                    break;
                }
            case PARTITION:
                {
                    String partition = getPartitionName(page.getPageMetadata().getProperty(HetuConstant.DATASOURCE_FILE_PATH), createIndexMetadata.getTableName());
                    if (partition == null) {
                        throw new IllegalStateException("Partition level is not supported for non partitioned table.");
                    }
                    if (!createIndexMetadata.getPartitions().isEmpty()) {
                        for (String userPart : createIndexMetadata.getPartitions()) {
                            String userPartCol = userPart.split("=")[0];
                            String actualPartCol = partition.split("=")[0];
                            if (!userPartCol.equals(actualPartCol)) {
                                throw new IllegalArgumentException(String.format("Creating index on %s is not supported as it's not first-level partition", userPartCol));
                            }
                        }
                    }
                    levelWriter.putIfAbsent(partition, heuristicIndexerManager.getIndexWriter(createIndexMetadata, connectorMetadata));
                    persistBy.putIfAbsent(levelWriter.get(partition), this);
                    levelWriter.get(partition).addData(values, connectorMetadata);
                    break;
                }
            case TABLE:
                {
                    levelWriter.putIfAbsent(createIndexMetadata.getTableName(), heuristicIndexerManager.getIndexWriter(createIndexMetadata, connectorMetadata));
                    persistBy.putIfAbsent(levelWriter.get(createIndexMetadata.getTableName()), this);
                    levelWriter.get(createIndexMetadata.getTableName()).addData(values, connectorMetadata);
                    break;
                }
            default:
                throw new IllegalArgumentException("Create level not supported");
        }
    } catch (IOException e) {
        throw new UncheckedIOException(e);
    }
}

Example 38 with Pair

use of io.prestosql.spi.heuristicindex.Pair in project hetu-core by openlookeng.

the class TestMemorySelection method getSplitAndMaterializedResult.

Pair<Integer, MaterializedResult> getSplitAndMaterializedResult(String testerQuery) {
    // Select the entry with specifics
    MaterializedResult queryResult = computeActual(testerQuery);
    String doublyQuotedQuery = testerQuery.replaceAll("'", "''");
    // Get queries executed and query ID to find the task with sum of splits
    String splits = "select sum(splits) from system.runtime.tasks where query_id in " + "(select query_id from system.runtime.queries " + "where query='" + doublyQuotedQuery + "' order by created desc limit 1)";
    MaterializedResult rows = computeActual(splits);
    assertEquals(rows.getRowCount(), 1);
    MaterializedRow materializedRow = rows.getMaterializedRows().get(0);
    int fieldCount = materializedRow.getFieldCount();
    assertEquals(fieldCount, 1, "Expected only one column, but got '%d', fiedlCount: " + fieldCount);
    Object value = materializedRow.getField(0);
    return new Pair<>((int) (long) value, queryResult);
}