Examples with IndexCacheKey io.prestosql.spi.heuristicindex.IndexCacheKey used on opensource projects

Example 1 with IndexCacheKey

use of io.prestosql.spi.heuristicindex.IndexCacheKey in project hetu-core by openlookeng.

the class IndexCache method getIndices.

public List<IndexMetadata> getIndices(String catalog, String table, HiveSplit hiveSplit, TupleDomain<HiveColumnHandle> effectivePredicate, List<HiveColumnHandle> partitions) {
    if (cache == null || catalog == null || table == null || hiveSplit == null || effectivePredicate == null) {
        return Collections.emptyList();
    }
    long lastModifiedTime = hiveSplit.getLastModifiedTime();
    Path path = new Path(hiveSplit.getPath());
    URI pathUri = URI.create(URIUtil.encodePath(path.toString()));
    String tableFqn = catalog + "." + table;
    // for each split, load indexes for each predicate (if the predicate contains an indexed column)
    List<IndexMetadata> splitIndexes = new LinkedList<>();
    effectivePredicate.getDomains().get().keySet().stream().filter(key -> partitions == null || !partitions.contains(key)).map(HiveColumnHandle::getName).map(String::toLowerCase).forEach(column -> {
        // e.g. catalog.schema.table or dc.catalog.schema.table
        if (!tableFqn.matches("([\\p{Alnum}_]+\\.){2,3}[\\p{Alnum}_]+")) {
            LOG.warn("Invalid table name " + tableFqn);
            return;
        }
        if (!column.matches("[\\p{Alnum}_]+")) {
            LOG.warn("Invalid column name " + column);
            return;
        }
        for (String indexType : INDEX_TYPES) {
            String indexCacheKeyPath = Paths.get(tableFqn, column, indexType, pathUri.getRawPath()).toString();
            IndexCacheKey indexCacheKey = new IndexCacheKey(indexCacheKeyPath, lastModifiedTime);
            // check if cache contains the key
            List<IndexMetadata> predicateIndexes = cache.getIfPresent(indexCacheKey);
            // if cache didn't contain the key, it has not been loaded, load it asynchronously
            if (predicateIndexes == null) {
                executor.schedule(() -> {
                    try {
                        cache.get(indexCacheKey);
                        LOG.debug("Loaded index for %s.", indexCacheKeyPath);
                    } catch (ExecutionException e) {
                        if (e.getCause() instanceof IndexNotCreatedException) {
                        // Do nothing. Index not registered.
                        } else if (LOG.isDebugEnabled()) {
                            LOG.debug(e, "Unable to load index for %s. ", indexCacheKeyPath);
                        }
                    }
                }, loadDelay, TimeUnit.MILLISECONDS);
            } else {
                // the index is only valid if the lastModifiedTime of the split matches the index's lastModifiedTime
                for (IndexMetadata index : predicateIndexes) {
                    if (index.getLastModifiedTime() != lastModifiedTime) {
                        cache.invalidate(indexCacheKey);
                        predicateIndexes = Collections.emptyList();
                        break;
                    }
                }
                // cache contained the key
                splitIndexes.addAll(predicateIndexes);
            }
        }
    });
    return splitIndexes;
}

Example 2 with IndexCacheKey

use of io.prestosql.spi.heuristicindex.IndexCacheKey in project hetu-core by openlookeng.

the class TestIndexCacheLoader method testNoValidIndexFilesFoundException.

@Test(expectedExceptions = Exception.class)
public void testNoValidIndexFilesFoundException() throws Exception {
    IndexClient indexclient = mock(IndexClient.class);
    IndexCacheLoader indexCacheLoader = new IndexCacheLoader(indexclient);
    long lastModifiedTime = 1L;
    IndexCacheKey indexCacheKey = new IndexCacheKey("/path/to/split", lastModifiedTime);
    when(indexclient.getLastModifiedTime((indexCacheKey.getPath()))).thenReturn(lastModifiedTime);
    when(indexclient.readSplitIndex((indexCacheKey.getPath()))).thenThrow(Exception.class);
    indexCacheLoader.load(indexCacheKey);
}

Example 3 with IndexCacheKey

use of io.prestosql.spi.heuristicindex.IndexCacheKey in project hetu-core by openlookeng.

the class TestIndexCacheLoader method testNoMatchingLastModifiedTime.

@Test(expectedExceptions = Exception.class)
public void testNoMatchingLastModifiedTime() throws Exception {
    IndexClient indexclient = mock(IndexClient.class);
    IndexCacheLoader indexCacheLoader = new IndexCacheLoader(indexclient);
    IndexCacheKey indexCacheKey = new IndexCacheKey("/path/to/split", 1L);
    // return different last modified time to simulate expired index
    when(indexclient.getLastModifiedTime((indexCacheKey.getPath()))).thenReturn(2L);
    indexCacheLoader.load(indexCacheKey);
}

Example 4 with IndexCacheKey

use of io.prestosql.spi.heuristicindex.IndexCacheKey in project hetu-core by openlookeng.

the class TestIndexCacheLoader method testIndexFound.

@Test
public void testIndexFound() throws Exception {
    IndexClient indexclient = mock(IndexClient.class);
    IndexCacheLoader indexCacheLoader = new IndexCacheLoader(indexclient);
    List<IndexMetadata> expectedSplitIndexes = new LinkedList<>();
    expectedSplitIndexes.add(mock(IndexMetadata.class));
    long lastModifiedTime = 1L;
    IndexCacheKey indexCacheKey = new IndexCacheKey("/path/to/split", lastModifiedTime);
    when(indexclient.getLastModifiedTime((indexCacheKey.getPath()))).thenReturn(lastModifiedTime);
    when(indexclient.readSplitIndex((indexCacheKey.getPath()))).thenReturn(expectedSplitIndexes);
    List<IndexMetadata> actualSplitIndexes = indexCacheLoader.load(indexCacheKey);
    assertEquals(expectedSplitIndexes.size(), actualSplitIndexes.size());
}

Example 5 with IndexCacheKey

use of io.prestosql.spi.heuristicindex.IndexCacheKey in project hetu-core by openlookeng.

the class IndexCache method getIndices.

public List<IndexMetadata> getIndices(String table, String column, Split split, Map<String, IndexRecord> indexRecordKeyToRecordMap) {
    if (cache == null) {
        return Collections.emptyList();
    }
    URI splitUri = URI.create(URIUtil.encodePath(split.getConnectorSplit().getFilePath()));
    long lastModifiedTime = split.getConnectorSplit().getLastModifiedTime();
    List<IndexMetadata> indices = new LinkedList<>();
    for (String indexType : INDEX_TYPES) {
        String filterKeyPath = table + "/" + column + "/" + indexType + splitUri.getRawPath();
        String indexRecordKey = table + "/" + column + "/" + indexType;
        IndexRecord record = indexRecordKeyToRecordMap.get(indexRecordKey);
        IndexCacheKey filterKey = new IndexCacheKey(filterKeyPath, lastModifiedTime, record);
        // it is possible to return multiple SplitIndexMetadata due to the range mismatch, especially in the case
        // where the split has a wider range than the original splits used for index creation
        // check if cache contains the key
        List<IndexMetadata> indexOfThisType;
        // if cache didn't contain the key, it has not been loaded, load it asynchronously
        indexOfThisType = cache.getIfPresent(filterKey);
        if (indexOfThisType == null) {
            executor.schedule(() -> {
                try {
                    cache.get(filterKey);
                    LOG.debug("Loaded index for %s.", filterKey);
                } catch (ExecutionException e) {
                    if (e.getCause() instanceof IndexNotCreatedException) {
                    // Do nothing. Index not registered.
                    } else if (LOG.isDebugEnabled()) {
                        LOG.debug(e, "Unable to load index for %s. ", filterKey);
                    }
                }
            }, loadDelay, TimeUnit.MILLISECONDS);
        } else {
            // the index is only valid if the lastModifiedTime of the split matches the index's lastModifiedTime
            for (IndexMetadata index : indexOfThisType) {
                if (index.getLastModifiedTime() != lastModifiedTime) {
                    cache.invalidate(filterKey);
                    indexOfThisType = Collections.emptyList();
                    break;
                }
            }
            indices.addAll(indexOfThisType);
        }
    }
    return indices;
}