Examples with Index io.prestosql.spi.heuristicindex.Index used on opensource projects

Example 6 with Index

use of io.prestosql.spi.heuristicindex.Index in project hetu-core by openlookeng.

the class TestIndexCacheFetch method testIndexCacheGetIndices.

@Test
public void testIndexCacheGetIndices() throws Exception {
    synchronized (this) {
        IndexMetadata indexMetadata = mock(IndexMetadata.class);
        when(indexMetadata.getLastModifiedTime()).thenReturn(testLastModifiedTime);
        Index index = mock(Index.class);
        when(indexMetadata.getIndex()).then(new Returns(index));
        when(index.getMemoryUsage()).thenReturn(new DataSize(1, KILOBYTE).toBytes());
        List<IndexMetadata> expectedIndices = new LinkedList<>();
        expectedIndices.add(indexMetadata);
        IndexCacheLoader indexCacheLoader = mock(IndexCacheLoader.class);
        when(indexCacheLoader.load(any())).then(new Returns(expectedIndices));
        IndexCache indexCache = new IndexCache(indexCacheLoader, new NoOpIndexClient(), false);
        List<IndexMetadata> actualSplitIndex = indexCache.getIndices(table, column, split);
        assertEquals(actualSplitIndex.size(), 0);
        Thread.sleep(loadDelay + 2000);
        actualSplitIndex = indexCache.getIndices(table, column, split);
        assertEquals(actualSplitIndex.size(), numberOfIndexTypes);
        assertEquals(actualSplitIndex.get(0), expectedIndices.get(0));
    }
}

Example 7 with Index

use of io.prestosql.spi.heuristicindex.Index in project hetu-core by openlookeng.

the class TestIndexCacheRemoval method testExpiredCacheIndices.

@Test
public void testExpiredCacheIndices() throws Exception {
    synchronized (this) {
        IndexMetadata indexMetadata = mock(IndexMetadata.class);
        when(indexMetadata.getLastModifiedTime()).thenReturn(testLastModifiedTime);
        Index index = mock(Index.class);
        when(indexMetadata.getIndex()).then(new Returns(index));
        when(index.getMemoryUsage()).thenReturn(new DataSize(1, KILOBYTE).toBytes());
        List<IndexMetadata> expectedIndices = new LinkedList<>();
        expectedIndices.add(indexMetadata);
        IndexCacheLoader indexCacheLoader = mock(IndexCacheLoader.class);
        when(indexCacheLoader.load(any())).then(new Returns(expectedIndices));
        IndexCache indexCache = new IndexCache(indexCacheLoader, new NoOpIndexClient(), false);
        List<IndexMetadata> actualSplitIndex = indexCache.getIndices(table, column, split);
        assertEquals(actualSplitIndex.size(), 0);
        Thread.sleep(loadDelay + 2000);
        actualSplitIndex = indexCache.getIndices(table, column, split);
        assertEquals(actualSplitIndex.size(), numberOfIndexTypes);
        // now the index is in the cache, but changing the lastmodified date of the split should invalidate it
        when(indexMetadata.getLastModifiedTime()).then(new Returns(testLastModifiedTime + 1));
        actualSplitIndex = indexCache.getIndices(table, column, split);
        assertEquals(actualSplitIndex.size(), 0);
    }
}

Example 8 with Index

use of io.prestosql.spi.heuristicindex.Index in project hetu-core by openlookeng.

the class TestIndexCacheRemoval method testIndexCacheEviction.

@Test
public void testIndexCacheEviction() throws Exception {
    synchronized (this) {
        IndexCacheLoader indexCacheLoader = mock(IndexCacheLoader.class);
        IndexCache indexCache = new IndexCache(indexCacheLoader, new NoOpIndexClient(), false);
        // get index for split1
        IndexMetadata indexMetadata1 = mock(IndexMetadata.class);
        when(indexMetadata1.getLastModifiedTime()).thenReturn(testLastModifiedTime);
        Index index1 = mock(Index.class);
        when(indexMetadata1.getIndex()).thenReturn(index1);
        when(index1.getMemoryUsage()).thenReturn(new DataSize(2, KILOBYTE).toBytes());
        List<IndexMetadata> expectedIndices1 = new LinkedList<>();
        expectedIndices1.add(indexMetadata1);
        when(indexCacheLoader.load(any())).then(new Returns(expectedIndices1));
        // each index is has memory usage of 2, and limit is 2*types of idx, so all should be loaded
        List<IndexMetadata> actualSplitIndex = indexCache.getIndices(table, column, split);
        assertEquals(actualSplitIndex.size(), 0);
        Thread.sleep(loadDelay + 2000);
        actualSplitIndex = indexCache.getIndices(table, column, split);
        assertEquals(actualSplitIndex.size(), numberOfIndexTypes);
        assertEquals(actualSplitIndex.get(0), indexMetadata1);
        assertEquals(indexCache.getCacheSize(), numberOfIndexTypes);
        // get index for split2
        when(connectorSplit.getFilePath()).thenReturn(testPath2);
        IndexMetadata indexMetadata2 = mock(IndexMetadata.class);
        when(indexMetadata2.getLastModifiedTime()).thenReturn(testLastModifiedTime);
        Index index2 = mock(Index.class);
        when(indexMetadata2.getIndex()).thenReturn(index2);
        when(index2.getMemoryUsage()).thenReturn(new DataSize(2, KILOBYTE).toBytes());
        // previous indexes should be evicted bc cache was at max weight limit and new ones should be added
        List<IndexMetadata> expectedIndices2 = new LinkedList<>();
        expectedIndices2.add(indexMetadata2);
        when(indexCacheLoader.load(any())).then(new Returns(expectedIndices2));
        actualSplitIndex = indexCache.getIndices(table, column, split);
        assertEquals(actualSplitIndex.size(), 0);
        assertEquals(indexCache.getCacheSize(), numberOfIndexTypes);
        Thread.sleep(loadDelay + 2000);
        actualSplitIndex = indexCache.getIndices(table, column, split);
        assertEquals(actualSplitIndex.size(), numberOfIndexTypes);
        assertEquals(actualSplitIndex.get(0), indexMetadata2);
        assertEquals(indexCache.getCacheSize(), numberOfIndexTypes);
        // get index for split1
        when(connectorSplit.getFilePath()).thenReturn(testPath);
        actualSplitIndex = indexCache.getIndices(table, column, split);
        assertEquals(actualSplitIndex.size(), 0);
        assertEquals(indexCache.getCacheSize(), numberOfIndexTypes);
    }
}

Example 9 with Index

use of io.prestosql.spi.heuristicindex.Index in project hetu-core by openlookeng.

the class TestBTreeIndex method testLessThanEqualTo.

@Test
public void testLessThanEqualTo() throws IOException, IndexLookUpException {
    BTreeIndex index = new BTreeIndex();
    for (int i = 0; i < 100; i++) {
        List<Pair> pairs = new ArrayList<>();
        Long key = Long.valueOf(100 + i);
        String value = "value" + i;
        pairs.add(new Pair(key, value));
        Pair pair = new Pair("dummyCol", pairs);
        index.addKeyValues(Collections.singletonList(pair));
    }
    File file = getFile();
    index.serialize(new FileOutputStream(file));
    BTreeIndex readIndex = new BTreeIndex();
    readIndex.deserialize(new FileInputStream(file));
    RowExpression comparisonExpression = simplePredicate(OperatorType.LESS_THAN_OR_EQUAL, "dummyCol", BIGINT, 120L);
    Iterator<String> result = readIndex.lookUp(comparisonExpression);
    assertNotNull(result, "Result shouldn't be null");
    assertTrue(result.hasNext());
    Object[] arr = IntStream.iterate(0, n -> n + 1).limit(21).mapToObj(i -> "value" + i).toArray();
    Arrays.sort(arr);
    for (int i = 0; i <= 20; i++) {
        assertEquals(arr[i], result.next());
    }
    assertFalse(result.hasNext());
    index.close();
}

Example 10 with Index

use of io.prestosql.spi.heuristicindex.Index in project hetu-core by openlookeng.

the class TestBTreeIndex method testDeserialize.

@Test
public void testDeserialize() throws IOException, IndexLookUpException {
    BTreeIndex index = new BTreeIndex();
    String value = "foo bar";
    for (int i = 0; i < 1000; i++) {
        List<Pair> pairs = new ArrayList<>();
        Long key = Long.valueOf(100 + i);
        pairs.add(new Pair(key, value));
        Pair pair = new Pair("dummyCol", pairs);
        index.addKeyValues(Collections.singletonList(pair));
    }
    File file = File.createTempFile("test-serialize-", UUID.randomUUID().toString());
    index.serialize(new FileOutputStream(file));
    Index readindex = new BTreeIndex();
    readindex.deserialize(new FileInputStream(file));
    RowExpression comparisonExpression = simplePredicate(OperatorType.EQUAL, "column", BIGINT, 101L);
    Iterator<String> result = readindex.lookUp(comparisonExpression);
    assertNotNull(result, "Result shouldn't be null");
    assertTrue(result.hasNext());
    assertEquals(value, result.next());
    index.close();
}