Examples with Index io.prestosql.spi.heuristicindex.Index used on opensource projects

Example 21 with Index

use of io.prestosql.spi.heuristicindex.Index in project hetu-core by openlookeng.

the class TestBTreeIndex method testLessThan.

@Test
public void testLessThan() throws IOException, IndexLookUpException {
    BTreeIndex index = new BTreeIndex();
    for (int i = 0; i < 100; i++) {
        List<Pair> pairs = new ArrayList<>();
        Long key = Long.valueOf(100 + i);
        String value = "value" + i;
        pairs.add(new Pair(key, value));
        Pair pair = new Pair("dummyCol", pairs);
        index.addKeyValues(Collections.singletonList(pair));
    }
    File file = getFile();
    index.serialize(new FileOutputStream(file));
    BTreeIndex readIndex = new BTreeIndex();
    readIndex.deserialize(new FileInputStream(file));
    RowExpression comparisonExpression = simplePredicate(OperatorType.LESS_THAN, "dummyCol", BIGINT, 120L);
    Iterator<String> result = readIndex.lookUp(comparisonExpression);
    assertNotNull(result, "Result shouldn't be null");
    assertTrue(result.hasNext());
    Object[] arr = IntStream.iterate(0, n -> n + 1).limit(20).mapToObj(i -> "value" + i).toArray();
    Arrays.sort(arr);
    for (int i = 0; i < 20; i++) {
        assertEquals(arr[i], result.next());
    }
    assertFalse(result.hasNext());
    index.close();
}

Example 22 with Index

use of io.prestosql.spi.heuristicindex.Index in project boostkit-bigdata by kunpengcompute.

the class TestIndexCacheFetch method testIndexCacheGetIndices.

@Test
public void testIndexCacheGetIndices() throws Exception {
    synchronized (this) {
        HiveSplit testHiveSplit;
        testHiveSplit = mock(HiveSplit.class);
        when(testHiveSplit.getPath()).thenReturn(testPath);
        when(testHiveSplit.getLastModifiedTime()).thenReturn(testLastModifiedTime);
        IndexMetadata indexMetadata = mock(IndexMetadata.class);
        when(indexMetadata.getLastModifiedTime()).thenReturn(testLastModifiedTime);
        Index index = mock(Index.class);
        when(indexMetadata.getIndex()).then(new Returns(index));
        when(index.getMemoryUsage()).thenReturn(new DataSize(1, KILOBYTE).toBytes());
        List<IndexMetadata> expectedIndices = new LinkedList<>();
        expectedIndices.add(indexMetadata);
        IndexCacheLoader indexCacheLoader = mock(IndexCacheLoader.class);
        when(indexCacheLoader.load(any())).then(new Returns(expectedIndices));
        IndexCache indexCache = new IndexCache(indexCacheLoader, new NoOpIndexClient());
        List<IndexMetadata> actualSplitIndex = indexCache.getIndices(catalog, table, testHiveSplit, effectivePredicate, testPartitions);
        assertEquals(actualSplitIndex.size(), 0);
        Thread.sleep(loadDelay + 2000);
        actualSplitIndex = indexCache.getIndices(catalog, table, testHiveSplit, effectivePredicate, testPartitions);
        assertEquals(actualSplitIndex.size(), numberOfIndexTypes);
        assertEquals(actualSplitIndex.get(0), expectedIndices.get(0));
    }
}

Example 23 with Index

use of io.prestosql.spi.heuristicindex.Index in project boostkit-bigdata by kunpengcompute.

the class TestIndexCacheRemoval method testIndexCacheEviction.

@Test
public void testIndexCacheEviction() throws Exception {
    synchronized (this) {
        HiveSplit testHiveSplit;
        testHiveSplit = mock(HiveSplit.class);
        when(testHiveSplit.getPath()).thenReturn(testPath);
        when(testHiveSplit.getLastModifiedTime()).thenReturn(testLastModifiedTime);
        IndexCacheLoader indexCacheLoader = mock(IndexCacheLoader.class);
        IndexCache indexCache = new IndexCache(indexCacheLoader, loadDelay, new NoOpIndexClient());
        // get index for split1
        IndexMetadata indexMetadata1 = mock(IndexMetadata.class);
        when(indexMetadata1.getLastModifiedTime()).thenReturn(testLastModifiedTime);
        Index index1 = mock(Index.class);
        when(indexMetadata1.getIndex()).thenReturn(index1);
        when(index1.getMemoryUsage()).thenReturn(new DataSize(2, KILOBYTE).toBytes());
        List<IndexMetadata> expectedIndices1 = new LinkedList<>();
        expectedIndices1.add(indexMetadata1);
        when(indexCacheLoader.load(any())).then(new Returns(expectedIndices1));
        // each index is has memory usage of 2, and limit is 2*types of idx, so all should be loaded
        List<IndexMetadata> actualSplitIndex = indexCache.getIndices(catalog, table, testHiveSplit, effectivePredicate, testPartitions);
        assertEquals(actualSplitIndex.size(), 0);
        Thread.sleep(loadDelay + 2000);
        actualSplitIndex = indexCache.getIndices(catalog, table, testHiveSplit, effectivePredicate, testPartitions);
        assertEquals(actualSplitIndex.size(), numberOfIndexTypes);
        assertEquals(actualSplitIndex.get(0), indexMetadata1);
        assertEquals(indexCache.getCacheSize(), numberOfIndexTypes);
        // get index for split2
        when(testHiveSplit.getPath()).thenReturn(testPath2);
        IndexMetadata indexMetadata2 = mock(IndexMetadata.class);
        when(indexMetadata2.getLastModifiedTime()).thenReturn(testLastModifiedTime);
        Index index2 = mock(Index.class);
        when(indexMetadata2.getIndex()).thenReturn(index2);
        when(index2.getMemoryUsage()).thenReturn(new DataSize(2, KILOBYTE).toBytes());
        // previous indexes should be evicted bc cache was at max weight limit and new ones should be added
        List<IndexMetadata> expectedIndices2 = new LinkedList<>();
        expectedIndices2.add(indexMetadata2);
        when(indexCacheLoader.load(any())).then(new Returns(expectedIndices2));
        actualSplitIndex = indexCache.getIndices(catalog, table, testHiveSplit, effectivePredicate, testPartitions);
        assertEquals(actualSplitIndex.size(), 0);
        assertEquals(indexCache.getCacheSize(), numberOfIndexTypes);
        Thread.sleep(loadDelay + 2000);
        actualSplitIndex = indexCache.getIndices(catalog, table, testHiveSplit, effectivePredicate, testPartitions);
        assertEquals(actualSplitIndex.size(), numberOfIndexTypes);
        assertEquals(actualSplitIndex.get(0), indexMetadata2);
        assertEquals(indexCache.getCacheSize(), numberOfIndexTypes);
        // get index for split1
        when(testHiveSplit.getPath()).thenReturn(testPath);
        actualSplitIndex = indexCache.getIndices(catalog, table, testHiveSplit, effectivePredicate, testPartitions);
        assertEquals(actualSplitIndex.size(), 0);
        assertEquals(indexCache.getCacheSize(), numberOfIndexTypes);
    }
}

Example 24 with Index

use of io.prestosql.spi.heuristicindex.Index in project boostkit-bigdata by kunpengcompute.

the class TestIndexCacheRemoval method testExpiredCacheIndices.

@Test
public void testExpiredCacheIndices() throws Exception {
    synchronized (this) {
        HiveSplit testHiveSplit;
        testHiveSplit = mock(HiveSplit.class);
        when(testHiveSplit.getPath()).thenReturn(testPath);
        when(testHiveSplit.getLastModifiedTime()).thenReturn(testLastModifiedTime);
        IndexMetadata indexMetadata = mock(IndexMetadata.class);
        when(indexMetadata.getLastModifiedTime()).thenReturn(testLastModifiedTime);
        Index index = mock(Index.class);
        when(indexMetadata.getIndex()).then(new Returns(index));
        when(index.getMemoryUsage()).thenReturn(new DataSize(1, KILOBYTE).toBytes());
        List<IndexMetadata> expectedIndices = new LinkedList<>();
        expectedIndices.add(indexMetadata);
        IndexCacheLoader indexCacheLoader = mock(IndexCacheLoader.class);
        when(indexCacheLoader.load(any())).then(new Returns(expectedIndices));
        IndexCache indexCache = new IndexCache(indexCacheLoader, loadDelay, new NoOpIndexClient());
        List<IndexMetadata> actualSplitIndex = indexCache.getIndices(catalog, table, testHiveSplit, effectivePredicate, testPartitions);
        assertEquals(actualSplitIndex.size(), 0);
        Thread.sleep(loadDelay + 2000);
        actualSplitIndex = indexCache.getIndices(catalog, table, testHiveSplit, effectivePredicate, testPartitions);
        assertEquals(actualSplitIndex.size(), numberOfIndexTypes);
        // now the index is in the cache, but changing the lastmodified date of the split should invalidate it
        when(testHiveSplit.getLastModifiedTime()).thenReturn(testLastModifiedTime + 1);
        actualSplitIndex = indexCache.getIndices(catalog, table, testHiveSplit, effectivePredicate, testPartitions);
        assertEquals(actualSplitIndex.size(), 0);
    }
}