Examples with Pair io.prestosql.spi.heuristicindex.Pair used on opensource projects

Example 1 with Pair

use of io.prestosql.spi.heuristicindex.Pair in project hetu-core by openlookeng.

the class HeuristicIndexClient method lookUpIndexRecord.

@Override
public RecordStatus lookUpIndexRecord(CreateIndexMetadata createIndexMetadata) throws IOException {
    IndexRecord sameNameRecord = indexRecordManager.lookUpIndexRecord(createIndexMetadata.getIndexName());
    IndexRecord sameIndexRecord = indexRecordManager.lookUpIndexRecord(createIndexMetadata.getTableName(), createIndexMetadata.getIndexColumns().stream().map(Pair::getFirst).toArray(String[]::new), createIndexMetadata.getIndexType());
    if (sameNameRecord == null) {
        if (sameIndexRecord != null) {
            return sameIndexRecord.isInProgressRecord() ? RecordStatus.IN_PROGRESS_SAME_CONTENT : RecordStatus.SAME_CONTENT;
        }
    } else {
        if (sameIndexRecord != null) {
            boolean partitionMerge = createIndexMetadata.getPartitions().size() != 0;
            for (String partition : createIndexMetadata.getPartitions()) {
                if (sameIndexRecord.partitions.isEmpty() || sameIndexRecord.partitions.contains(partition)) {
                    partitionMerge = false;
                    break;
                }
            }
            if (!partitionMerge) {
                return sameIndexRecord.isInProgressRecord() ? RecordStatus.IN_PROGRESS_SAME_INDEX_PART_CONFLICT : RecordStatus.SAME_INDEX_PART_CONFLICT;
            } else {
                return sameIndexRecord.isInProgressRecord() ? RecordStatus.IN_PROGRESS_SAME_INDEX_PART_CAN_MERGE : RecordStatus.SAME_INDEX_PART_CAN_MERGE;
            }
        } else {
            return sameNameRecord.isInProgressRecord() ? RecordStatus.IN_PROGRESS_SAME_NAME : RecordStatus.SAME_NAME;
        }
    }
    return RecordStatus.NOT_FOUND;
}

Example 2 with Pair

use of io.prestosql.spi.heuristicindex.Pair in project hetu-core by openlookeng.

the class UpdateIndexOperator method addInput.

@Override
public void addInput(Page page) {
    checkState(needsInput(), "Operator is already finishing");
    requireNonNull(page, "page is null");
    // TODO-cp-I38S9O: Operator currently not supported for Snapshot
    if (page instanceof MarkerPage) {
        throw new UnsupportedOperationException("Operator doesn't support snapshotting.");
    }
    // if operator is still receiving input, it's not finished
    finished.putIfAbsent(this, false);
    if (page.getPositionCount() == 0) {
        return;
    }
    IndexRecord indexRecord;
    try {
        indexRecord = heuristicIndexerManager.getIndexClient().lookUpIndexRecord(createIndexMetadata.getIndexName());
    } catch (IOException e) {
        throw new UncheckedIOException("Error reading index records, ", e);
    }
    if (createIndexMetadata.getCreateLevel() == CreateIndexMetadata.Level.UNDEFINED) {
        boolean tableIsPartitioned = getPartitionName(page.getPageMetadata().getProperty(HetuConstant.DATASOURCE_FILE_PATH), indexRecord.qualifiedTable) != null;
        createIndexMetadata.decideIndexLevel(tableIsPartitioned);
    }
    Map<String, List<Object>> values = new HashMap<>();
    for (int blockId = 0; blockId < page.getChannelCount(); blockId++) {
        Block block = page.getBlock(blockId);
        Pair<String, Type> entry = createIndexMetadata.getIndexColumns().get(blockId);
        String indexColumn = entry.getFirst();
        Type type = entry.getSecond();
        for (int position = 0; position < block.getPositionCount(); ++position) {
            Object value = getNativeValue(type, block, position);
            value = getActualValue(type, value);
            values.computeIfAbsent(indexColumn, k -> new ArrayList<>()).add(value);
        }
    }
    Properties connectorMetadata = new Properties();
    connectorMetadata.put(HetuConstant.DATASOURCE_CATALOG, createIndexMetadata.getTableName().split("\\.")[0]);
    connectorMetadata.putAll(page.getPageMetadata());
    try {
        switch(createIndexMetadata.getCreateLevel()) {
            case STRIPE:
                {
                    String filePath = page.getPageMetadata().getProperty(HetuConstant.DATASOURCE_FILE_PATH);
                    // The orc file this page resides in wasn't modified from when the index was created/last updated
                    if (pathToModifiedTime.containsKey(filePath) && pathToModifiedTime.get(filePath).equals(page.getPageMetadata().getProperty(HetuConstant.DATASOURCE_FILE_MODIFICATION))) {
                        return;
                    }
                    levelWriter.computeIfAbsent(filePath, k -> heuristicIndexerManager.getIndexWriter(createIndexMetadata, connectorMetadata));
                    persistBy.putIfAbsent(levelWriter.get(filePath), this);
                    levelWriter.get(filePath).addData(values, connectorMetadata);
                    break;
                }
            case PARTITION:
                {
                    String partition = getPartitionName(page.getPageMetadata().getProperty(HetuConstant.DATASOURCE_FILE_PATH), createIndexMetadata.getTableName());
                    indexLevelToMaxModifiedTime.compute(partition, (k, v) -> {
                        if (v != null && v >= (Long.parseLong(page.getPageMetadata().getProperty(HetuConstant.DATASOURCE_FILE_MODIFICATION)))) {
                            return v;
                        }
                        return (Long.parseLong(page.getPageMetadata().getProperty(HetuConstant.DATASOURCE_FILE_MODIFICATION)));
                    });
                    levelWriter.putIfAbsent(partition, heuristicIndexerManager.getIndexWriter(createIndexMetadata, connectorMetadata));
                    persistBy.putIfAbsent(levelWriter.get(partition), this);
                    levelWriter.get(partition).addData(values, connectorMetadata);
                    break;
                }
            case TABLE:
                {
                    indexLevelToMaxModifiedTime.compute(createIndexMetadata.getTableName(), (k, v) -> {
                        if (v != null && v >= (Long.parseLong(page.getPageMetadata().getProperty(HetuConstant.DATASOURCE_FILE_MODIFICATION)))) {
                            return v;
                        }
                        return (Long.parseLong(page.getPageMetadata().getProperty(HetuConstant.DATASOURCE_FILE_MODIFICATION)));
                    });
                    levelWriter.putIfAbsent(createIndexMetadata.getTableName(), heuristicIndexerManager.getIndexWriter(createIndexMetadata, connectorMetadata));
                    persistBy.putIfAbsent(levelWriter.get(createIndexMetadata.getTableName()), this);
                    levelWriter.get(createIndexMetadata.getTableName()).addData(values, connectorMetadata);
                    break;
                }
            default:
                throw new IllegalArgumentException("Create level not supported");
        }
    } catch (IOException e) {
        throw new UncheckedIOException(e);
    }
}

Example 3 with Pair

use of io.prestosql.spi.heuristicindex.Pair in project hetu-core by openlookeng.

the class TestHeuristicIndexFilter method setup.

@BeforeClass
public void setup() throws IOException {
    try (TempFolder folder = new TempFolder()) {
        folder.create();
        File testFile = folder.newFile();
        bloomIndex1 = new BloomIndex();
        bloomIndex1.setExpectedNumOfEntries(2);
        bloomIndex1.addValues(Collections.singletonList(new Pair<>("testColumn", ImmutableList.of("a", "b"))));
        try (FileOutputStream fo = new FileOutputStream(testFile)) {
            bloomIndex1.serialize(fo);
        }
        try (FileInputStream fi = new FileInputStream(testFile)) {
            bloomIndex1.deserialize(fi);
        }
        bloomIndex2 = new BloomIndex();
        bloomIndex2.setExpectedNumOfEntries(2);
        bloomIndex2.addValues(Collections.singletonList(new Pair<>("testColumn", ImmutableList.of("c", "d"))));
        try (FileOutputStream fo = new FileOutputStream(testFile)) {
            bloomIndex2.serialize(fo);
        }
        try (FileInputStream fi = new FileInputStream(testFile)) {
            bloomIndex2.deserialize(fi);
        }
        minMaxIndex1 = new MinMaxIndex();
        minMaxIndex1.addValues(Collections.singletonList(new Pair<>("testColumn", ImmutableList.of(1L, 5L, 10L))));
        minMaxIndex2 = new MinMaxIndex();
        minMaxIndex2.addValues(Collections.singletonList(new Pair<>("testColumn", ImmutableList.of(50L, 80L, 100L))));
    }
}

Example 4 with Pair

use of io.prestosql.spi.heuristicindex.Pair in project hetu-core by openlookeng.

the class TestBitmapIndex method testBitmapBetweenForDataTypes.

@Test(dataProvider = "bitmapBetweenForDataTypes")
public void testBitmapBetweenForDataTypes(List<Object> columnValues, Type type, Object matchVal1, Object matchVal2, List<Object> matchedList) throws IOException {
    System.out.println("Testing for " + columnValues + " for type " + type.getDisplayName() + " from " + matchVal1.toString() + " to " + matchVal2.toString());
    try (TempFolder folder = new TempFolder();
        BitmapIndex bitmapIndexWrite = new BitmapIndex();
        BitmapIndex bitmapIndexRead = new BitmapIndex()) {
        folder.create();
        File file = folder.newFile();
        String columnName = "column";
        bitmapIndexWrite.setExpectedNumOfEntries(columnValues.size());
        bitmapIndexWrite.addValues(Collections.singletonList(new Pair<>(columnName, columnValues)));
        try (FileOutputStream os = new FileOutputStream(file);
            FileInputStream is = new FileInputStream(file)) {
            bitmapIndexWrite.serialize(os);
            bitmapIndexRead.deserialize(is);
        }
        assertEquals(iteratorToList(bitmapIndexRead.lookUp(Domain.create(ValueSet.ofRanges(Range.range(type, matchVal1, true, matchVal2, true)), false))), matchedList);
    }
}

Example 5 with Pair

use of io.prestosql.spi.heuristicindex.Pair in project hetu-core by openlookeng.

the class TestBitmapIndex method testString.

@Test
public void testString() throws IOException {
    try (TempFolder folder = new TempFolder();
        BitmapIndex bitmapIndexWrite = new BitmapIndex();
        BitmapIndex bitmapIndexRead = new BitmapIndex()) {
        folder.create();
        File file = folder.newFile();
        String columnName = "column";
        List<Object> columnValues = ImmutableList.of("foo", "bar", "baz", "foo", "foo", "baz");
        bitmapIndexWrite.setExpectedNumOfEntries(columnValues.size());
        bitmapIndexWrite.addValues(Collections.singletonList(new Pair<>(columnName, columnValues)));
        try (FileOutputStream os = new FileOutputStream(file);
            FileInputStream is = new FileInputStream(file)) {
            bitmapIndexWrite.serialize(os);
            bitmapIndexRead.deserialize(is);
        }
        assertEquals(iteratorToList(bitmapIndexRead.lookUp(Domain.create(ValueSet.ofRanges(equal(createUnboundedVarcharType(), utf8Slice("foo"))), false))), ImmutableList.of(0, 3, 4));
        assertEquals(iteratorToList(bitmapIndexRead.lookUp(Domain.create(ValueSet.ofRanges(equal(createUnboundedVarcharType(), utf8Slice("bar"))), false))), ImmutableList.of(1));
        assertEquals(iteratorToList(bitmapIndexRead.lookUp(Domain.create(ValueSet.ofRanges(equal(createUnboundedVarcharType(), utf8Slice("notfound"))), false))), ImmutableList.of());
    }
}