Examples with Index io.prestosql.spi.heuristicindex.Index used on opensource projects

Example 11 with Index

use of io.prestosql.spi.heuristicindex.Index in project hetu-core by openlookeng.

the class TestIndexCacheRemoval method testExpiredCacheIndices.

@Test
public void testExpiredCacheIndices() throws Exception {
    synchronized (this) {
        HiveSplit testHiveSplit;
        testHiveSplit = mock(HiveSplit.class);
        when(testHiveSplit.getPath()).thenReturn(testPath);
        when(testHiveSplit.getLastModifiedTime()).thenReturn(testLastModifiedTime);
        IndexMetadata indexMetadata = mock(IndexMetadata.class);
        when(indexMetadata.getLastModifiedTime()).thenReturn(testLastModifiedTime);
        Index index = mock(Index.class);
        when(indexMetadata.getIndex()).then(new Returns(index));
        when(index.getMemoryUsage()).thenReturn(new DataSize(1, KILOBYTE).toBytes());
        List<IndexMetadata> expectedIndices = new LinkedList<>();
        expectedIndices.add(indexMetadata);
        IndexCacheLoader indexCacheLoader = mock(IndexCacheLoader.class);
        when(indexCacheLoader.load(any())).then(new Returns(expectedIndices));
        IndexCache indexCache = new IndexCache(indexCacheLoader, loadDelay, new NoOpIndexClient());
        List<IndexMetadata> actualSplitIndex = indexCache.getIndices(catalog, table, testHiveSplit, effectivePredicate, testPartitions);
        assertEquals(actualSplitIndex.size(), 0);
        Thread.sleep(loadDelay + 2000);
        actualSplitIndex = indexCache.getIndices(catalog, table, testHiveSplit, effectivePredicate, testPartitions);
        assertEquals(actualSplitIndex.size(), numberOfIndexTypes);
        // now the index is in the cache, but changing the lastmodified date of the split should invalidate it
        when(testHiveSplit.getLastModifiedTime()).thenReturn(testLastModifiedTime + 1);
        actualSplitIndex = indexCache.getIndices(catalog, table, testHiveSplit, effectivePredicate, testPartitions);
        assertEquals(actualSplitIndex.size(), 0);
    }
}

Example 12 with Index

use of io.prestosql.spi.heuristicindex.Index in project boostkit-bigdata by kunpengcompute.

the class TestIndexCachePartition method testIndexCacheWithPartitions.

@Test
public void testIndexCacheWithPartitions() throws Exception {
    synchronized (this) {
        HiveSplit testHiveSplit;
        testHiveSplit = mock(HiveSplit.class);
        when(testHiveSplit.getPath()).thenReturn(testPath);
        when(testHiveSplit.getLastModifiedTime()).thenReturn(testLastModifiedTime);
        HiveColumnHandle partitionColumnHandle;
        TupleDomain<HiveColumnHandle> effectivePredicateForPartition;
        partitionColumnHandle = mock(HiveColumnHandle.class);
        // partition column should be filtered out this should never get called
        when(partitionColumnHandle.getName()).thenThrow(Exception.class);
        effectivePredicateForPartition = TupleDomain.withColumnDomains(ImmutableMap.of(testColumnHandle, domain, partitionColumnHandle, domain));
        List<HiveColumnHandle> partitionColumns = ImmutableList.of(partitionColumnHandle);
        IndexMetadata indexMetadata = mock(IndexMetadata.class);
        when(indexMetadata.getLastModifiedTime()).thenReturn(testLastModifiedTime);
        Index index = mock(Index.class);
        when(indexMetadata.getIndex()).then(new Returns(index));
        when(index.getMemoryUsage()).thenReturn(new DataSize(1, KILOBYTE).toBytes());
        List<IndexMetadata> expectedIndices = new LinkedList<>();
        expectedIndices.add(indexMetadata);
        IndexCacheLoader indexCacheLoader = mock(IndexCacheLoader.class);
        when(indexCacheLoader.load(any())).then(new Returns(expectedIndices));
        IndexCache indexCache = new IndexCache(indexCacheLoader, loadDelay, new NoOpIndexClient());
        List<IndexMetadata> actualSplitIndex = indexCache.getIndices(catalog, table, testHiveSplit, effectivePredicateForPartition, partitionColumns);
        assertEquals(actualSplitIndex.size(), 0);
        Thread.sleep(loadDelay + 2000);
        actualSplitIndex = indexCache.getIndices(catalog, table, testHiveSplit, effectivePredicateForPartition, partitionColumns);
        assertEquals(actualSplitIndex.size(), numberOfIndexTypes);
    }
}

Example 13 with Index

use of io.prestosql.spi.heuristicindex.Index in project hetu-core by openlookeng.

the class AbstractOrcRecordReader method filterStripeUsingIndex.

private boolean filterStripeUsingIndex(StripeInformation stripe, Map<Long, List<IndexMetadata>> stripeOffsetToIndex, Map<String, Domain> and, Map<String, List<Domain>> or) {
    if (stripeOffsetToIndex.isEmpty()) {
        return false;
    }
    List<IndexMetadata> stripeIndex = stripeOffsetToIndex.get(Long.valueOf(stripe.getOffset()));
    Map<Index, Domain> andDomainMap = new HashMap<>();
    Map<Index, Domain> orDomainMap = new HashMap<>();
    for (Map.Entry<String, Domain> domainEntry : and.entrySet()) {
        String columnName = domainEntry.getKey();
        Domain columnDomain = domainEntry.getValue();
        // if the index exists, there should only be one index for this column within this stripe
        List<IndexMetadata> indexMetadata = stripeIndex.stream().filter(p -> p.getColumns()[0].equalsIgnoreCase(columnName)).collect(Collectors.toList());
        if (indexMetadata.isEmpty() || indexMetadata.size() > 1) {
            continue;
        }
        Index index = indexMetadata.get(0).getIndex();
        andDomainMap.put(index, columnDomain);
    }
    for (Map.Entry<String, List<Domain>> domainEntry : or.entrySet()) {
        String columnName = domainEntry.getKey();
        List<Domain> columnDomain = domainEntry.getValue();
        // if the index exists, there should only be one index for this column within this stripe
        List<IndexMetadata> indexMetadata = stripeIndex.stream().filter(p -> p.getColumns()[0].equalsIgnoreCase(columnName)).collect(Collectors.toList());
        if (indexMetadata.isEmpty() || indexMetadata.size() > 1) {
            continue;
        }
        Index index = indexMetadata.get(0).getIndex();
        orDomainMap.put(index, columnDomain.get(0));
    }
    if (!andDomainMap.isEmpty()) {
        List<Iterator<Integer>> matchings = new ArrayList<>(andDomainMap.size());
        for (Map.Entry<Index, Domain> e : andDomainMap.entrySet()) {
            try {
                Iterator<Integer> lookUpRes = e.getKey().lookUp(e.getValue());
                if (lookUpRes != null) {
                    matchings.add(lookUpRes);
                } else if (!e.getKey().matches(e.getValue())) {
                    return true;
                }
            } catch (UnsupportedOperationException | IndexLookUpException uoe2) {
                return false;
            }
        }
        if (!matchings.isEmpty()) {
            Iterator<Integer> thisStripeMatchingRows = SequenceUtils.intersect(matchings);
            PeekingIterator<Integer> peekingIterator = Iterators.peekingIterator(thisStripeMatchingRows);
            this.stripeMatchingRows.put(stripe, peekingIterator);
        }
        return false;
    }
    if (!orDomainMap.isEmpty()) {
        for (Map.Entry<Index, Domain> e : orDomainMap.entrySet()) {
            try {
                Iterator<Integer> thisStripeMatchingRows = e.getKey().lookUp(e.getValue());
                if (thisStripeMatchingRows != null) {
                    if (thisStripeMatchingRows.hasNext()) {
                        /* any one matched; then include the stripe */
                        return false;
                    }
                } else if (e.getKey().matches(e.getValue())) {
                    return false;
                }
            } catch (UnsupportedOperationException | IndexLookUpException uoe2) {
                return false;
            }
        }
        return true;
    }
    return false;
}

Example 14 with Index

use of io.prestosql.spi.heuristicindex.Index in project hetu-core by openlookeng.

the class TestIndexCacheFetch method testIndexCacheGetIndices.

@Test
public void testIndexCacheGetIndices() throws Exception {
    synchronized (this) {
        HiveSplit testHiveSplit;
        testHiveSplit = mock(HiveSplit.class);
        when(testHiveSplit.getPath()).thenReturn(testPath);
        when(testHiveSplit.getLastModifiedTime()).thenReturn(testLastModifiedTime);
        IndexMetadata indexMetadata = mock(IndexMetadata.class);
        when(indexMetadata.getLastModifiedTime()).thenReturn(testLastModifiedTime);
        Index index = mock(Index.class);
        when(indexMetadata.getIndex()).then(new Returns(index));
        when(index.getMemoryUsage()).thenReturn(new DataSize(1, KILOBYTE).toBytes());
        List<IndexMetadata> expectedIndices = new LinkedList<>();
        expectedIndices.add(indexMetadata);
        IndexCacheLoader indexCacheLoader = mock(IndexCacheLoader.class);
        when(indexCacheLoader.load(any())).then(new Returns(expectedIndices));
        IndexCache indexCache = new IndexCache(indexCacheLoader, new NoOpIndexClient());
        List<IndexMetadata> actualSplitIndex = indexCache.getIndices(catalog, table, testHiveSplit, effectivePredicate, testPartitions);
        assertEquals(actualSplitIndex.size(), 0);
        Thread.sleep(loadDelay + 2000);
        actualSplitIndex = indexCache.getIndices(catalog, table, testHiveSplit, effectivePredicate, testPartitions);
        assertEquals(actualSplitIndex.size(), numberOfIndexTypes);
        assertEquals(actualSplitIndex.get(0), expectedIndices.get(0));
    }
}

Example 15 with Index

use of io.prestosql.spi.heuristicindex.Index in project hetu-core by openlookeng.

the class TestIndexCachePartition method testIndexCacheWithPartitions.

@Test
public void testIndexCacheWithPartitions() throws Exception {
    synchronized (this) {
        HiveSplit testHiveSplit;
        testHiveSplit = mock(HiveSplit.class);
        when(testHiveSplit.getPath()).thenReturn(testPath);
        when(testHiveSplit.getLastModifiedTime()).thenReturn(testLastModifiedTime);
        HiveColumnHandle partitionColumnHandle;
        TupleDomain<HiveColumnHandle> effectivePredicateForPartition;
        partitionColumnHandle = mock(HiveColumnHandle.class);
        // partition column should be filtered out this should never get called
        when(partitionColumnHandle.getName()).thenThrow(Exception.class);
        effectivePredicateForPartition = TupleDomain.withColumnDomains(ImmutableMap.of(testColumnHandle, domain, partitionColumnHandle, domain));
        List<HiveColumnHandle> partitionColumns = ImmutableList.of(partitionColumnHandle);
        IndexMetadata indexMetadata = mock(IndexMetadata.class);
        when(indexMetadata.getLastModifiedTime()).thenReturn(testLastModifiedTime);
        Index index = mock(Index.class);
        when(indexMetadata.getIndex()).then(new Returns(index));
        when(index.getMemoryUsage()).thenReturn(new DataSize(1, KILOBYTE).toBytes());
        List<IndexMetadata> expectedIndices = new LinkedList<>();
        expectedIndices.add(indexMetadata);
        IndexCacheLoader indexCacheLoader = mock(IndexCacheLoader.class);
        when(indexCacheLoader.load(any())).then(new Returns(expectedIndices));
        IndexCache indexCache = new IndexCache(indexCacheLoader, loadDelay, new NoOpIndexClient());
        List<IndexMetadata> actualSplitIndex = indexCache.getIndices(catalog, table, testHiveSplit, effectivePredicateForPartition, partitionColumns);
        assertEquals(actualSplitIndex.size(), 0);
        Thread.sleep(loadDelay + 2000);
        actualSplitIndex = indexCache.getIndices(catalog, table, testHiveSplit, effectivePredicateForPartition, partitionColumns);
        assertEquals(actualSplitIndex.size(), numberOfIndexTypes);
    }
}