Examples with IndexMetadata io.prestosql.spi.heuristicindex.IndexMetadata used on opensource projects

Example 1 with IndexMetadata

use of io.prestosql.spi.heuristicindex.IndexMetadata in project hetu-core by openlookeng.

the class HeuristicIndexClient method readSplitIndex.

@Override
public List<IndexMetadata> readSplitIndex(String path) throws IOException {
    requireNonNull(path, "no path specified");
    List<IndexMetadata> indexes = new LinkedList<>();
    Path indexKeyPath = Paths.get(path);
    IndexRecord curIndex = null;
    try {
        curIndex = indexRecordManager.lookUpIndexRecord(indexKeyPath.subpath(0, 1).toString(), new String[] { indexKeyPath.subpath(1, 2).toString() }, indexKeyPath.subpath(2, 3).toString());
        if (curIndex == null) {
            // Use index record file to pre-screen. If record does not contain the index, skip loading
            return null;
        }
    } catch (Exception e) {
        // On exception, log and continue reading from disk
        LOG.debug("Error reading index records: " + path);
    }
    for (Map.Entry<String, Index> entry : readIndexMap(path, curIndex).entrySet()) {
        String absolutePath = entry.getKey();
        Path remainder = Paths.get(absolutePath.replaceFirst(root.toString(), ""));
        Path table = remainder.subpath(0, 1);
        remainder = Paths.get(remainder.toString().replaceFirst(table.toString(), ""));
        Path column = remainder.subpath(0, 1);
        remainder = Paths.get(remainder.toString().replaceFirst(column.toString(), ""));
        Path indexType = remainder.subpath(0, 1);
        remainder = Paths.get(remainder.toString().replaceFirst(indexType.toString(), ""));
        Path filenamePath = remainder.getFileName();
        if (filenamePath == null) {
            throw new IllegalArgumentException("Split path cannot be resolved: " + path);
        }
        remainder = remainder.getParent();
        table = table.getFileName();
        column = column.getFileName();
        indexType = indexType.getFileName();
        if (remainder == null || table == null || column == null || indexType == null) {
            throw new IllegalArgumentException("Split path cannot be resolved: " + path);
        }
        String filename = filenamePath.toString();
        long splitStart = Long.parseLong(filename.substring(0, filename.lastIndexOf('.')));
        String timeDir = Paths.get(table.toString(), column.toString(), indexType.toString(), remainder.toString()).toString();
        long lastUpdated = getLastModifiedTime(timeDir);
        IndexMetadata index = new IndexMetadata(entry.getValue(), table.toString(), new String[] { column.toString() }, root.toString(), remainder.toString(), splitStart, lastUpdated);
        indexes.add(index);
    }
    return indexes;
}

Example 2 with IndexMetadata

use of io.prestosql.spi.heuristicindex.IndexMetadata in project hetu-core by openlookeng.

the class IndexCache method getIndices.

public List<IndexMetadata> getIndices(String catalog, String table, HiveSplit hiveSplit, TupleDomain<HiveColumnHandle> effectivePredicate, List<HiveColumnHandle> partitions) {
    if (cache == null || catalog == null || table == null || hiveSplit == null || effectivePredicate == null) {
        return Collections.emptyList();
    }
    long lastModifiedTime = hiveSplit.getLastModifiedTime();
    Path path = new Path(hiveSplit.getPath());
    URI pathUri = URI.create(URIUtil.encodePath(path.toString()));
    String tableFqn = catalog + "." + table;
    // for each split, load indexes for each predicate (if the predicate contains an indexed column)
    List<IndexMetadata> splitIndexes = new LinkedList<>();
    effectivePredicate.getDomains().get().keySet().stream().filter(key -> partitions == null || !partitions.contains(key)).map(HiveColumnHandle::getName).map(String::toLowerCase).forEach(column -> {
        // e.g. catalog.schema.table or dc.catalog.schema.table
        if (!tableFqn.matches("([\\p{Alnum}_]+\\.){2,3}[\\p{Alnum}_]+")) {
            LOG.warn("Invalid table name " + tableFqn);
            return;
        }
        if (!column.matches("[\\p{Alnum}_]+")) {
            LOG.warn("Invalid column name " + column);
            return;
        }
        for (String indexType : INDEX_TYPES) {
            String indexCacheKeyPath = Paths.get(tableFqn, column, indexType, pathUri.getRawPath()).toString();
            IndexCacheKey indexCacheKey = new IndexCacheKey(indexCacheKeyPath, lastModifiedTime);
            // check if cache contains the key
            List<IndexMetadata> predicateIndexes = cache.getIfPresent(indexCacheKey);
            // if cache didn't contain the key, it has not been loaded, load it asynchronously
            if (predicateIndexes == null) {
                executor.schedule(() -> {
                    try {
                        cache.get(indexCacheKey);
                        LOG.debug("Loaded index for %s.", indexCacheKeyPath);
                    } catch (ExecutionException e) {
                        if (e.getCause() instanceof IndexNotCreatedException) {
                        // Do nothing. Index not registered.
                        } else if (LOG.isDebugEnabled()) {
                            LOG.debug(e, "Unable to load index for %s. ", indexCacheKeyPath);
                        }
                    }
                }, loadDelay, TimeUnit.MILLISECONDS);
            } else {
                // the index is only valid if the lastModifiedTime of the split matches the index's lastModifiedTime
                for (IndexMetadata index : predicateIndexes) {
                    if (index.getLastModifiedTime() != lastModifiedTime) {
                        cache.invalidate(indexCacheKey);
                        predicateIndexes = Collections.emptyList();
                        break;
                    }
                }
                // cache contained the key
                splitIndexes.addAll(predicateIndexes);
            }
        }
    });
    return splitIndexes;
}

Example 3 with IndexMetadata

use of io.prestosql.spi.heuristicindex.IndexMetadata in project hetu-core by openlookeng.

the class IndexCacheLoader method load.

@Override
public List<IndexMetadata> load(IndexCacheKey key) throws Exception {
    requireNonNull(key);
    requireNonNull(indexClient);
    // only load index files if index lastModified matches key lastModified
    long lastModified;
    try {
        lastModified = indexClient.getLastModifiedTime(key.getPath());
    } catch (Exception e) {
        // no lastModified file found, i.e. index doesn't exist
        throw new IndexNotCreatedException();
    }
    if (lastModified != key.getLastModifiedTime()) {
        throw new Exception("Index files are expired for key " + key);
    }
    List<IndexMetadata> indices;
    try {
        indices = indexClient.readSplitIndex(key.getPath());
    } catch (Exception e) {
        throw new Exception("No valid index files found for key " + key, e);
    }
    // null indicates that the index is not registered in index records
    if (indices == null) {
        throw new IndexNotCreatedException();
    }
    // lastModified file was valid, but no index files for the given types
    if (indices.isEmpty()) {
        throw new Exception("No index files found for key " + key);
    }
    // Sort the indices based on split starting position
    return indices.stream().sorted(comparingLong(IndexMetadata::getSplitStart)).collect(Collectors.toList());
}

Example 4 with IndexMetadata

use of io.prestosql.spi.heuristicindex.IndexMetadata in project hetu-core by openlookeng.

the class TestIndexCacheFetch method testIndexCacheGetIndices.

@Test
public void testIndexCacheGetIndices() throws Exception {
    synchronized (this) {
        IndexMetadata indexMetadata = mock(IndexMetadata.class);
        when(indexMetadata.getLastModifiedTime()).thenReturn(testLastModifiedTime);
        Index index = mock(Index.class);
        when(indexMetadata.getIndex()).then(new Returns(index));
        when(index.getMemoryUsage()).thenReturn(new DataSize(1, KILOBYTE).toBytes());
        List<IndexMetadata> expectedIndices = new LinkedList<>();
        expectedIndices.add(indexMetadata);
        IndexCacheLoader indexCacheLoader = mock(IndexCacheLoader.class);
        when(indexCacheLoader.load(any())).then(new Returns(expectedIndices));
        IndexCache indexCache = new IndexCache(indexCacheLoader, new NoOpIndexClient(), false);
        List<IndexMetadata> actualSplitIndex = indexCache.getIndices(table, column, split);
        assertEquals(actualSplitIndex.size(), 0);
        Thread.sleep(loadDelay + 2000);
        actualSplitIndex = indexCache.getIndices(table, column, split);
        assertEquals(actualSplitIndex.size(), numberOfIndexTypes);
        assertEquals(actualSplitIndex.get(0), expectedIndices.get(0));
    }
}

Example 5 with IndexMetadata

use of io.prestosql.spi.heuristicindex.IndexMetadata in project hetu-core by openlookeng.

the class TestIndexCacheFetch method testIndexCacheThrowsExecutionException.

@Test
public void testIndexCacheThrowsExecutionException() throws Exception {
    synchronized (this) {
        IndexMetadata indexMetadata = mock(IndexMetadata.class);
        when(indexMetadata.getLastModifiedTime()).then(new Returns(testLastModifiedTime));
        List<IndexMetadata> expectedIndices = new LinkedList<>();
        expectedIndices.add(indexMetadata);
        IndexCacheLoader indexCacheLoader = mock(IndexCacheLoader.class);
        when(indexCacheLoader.load(any())).thenThrow(ExecutionException.class);
        IndexCache indexCache = new IndexCache(indexCacheLoader, new NoOpIndexClient(), false);
        List<IndexMetadata> actualSplitIndex = indexCache.getIndices(table, column, split);
        assertEquals(actualSplitIndex.size(), 0);
        Thread.sleep(loadDelay + 2000);
        actualSplitIndex = indexCache.getIndices(table, column, split);
        assertEquals(actualSplitIndex.size(), 0);
    }
}