Examples with IndexRecord io.prestosql.spi.heuristicindex.IndexRecord used on opensource projects

Example 16 with IndexRecord

use of io.prestosql.spi.heuristicindex.IndexRecord in project hetu-core by openlookeng.

the class SplitFiltering method preloadCache.

public static void preloadCache(IndexClient indexClient, List<String> preloadIndexNames) throws IOException {
    List<IndexRecord> indexToPreload = new ArrayList<>(preloadIndexNames.size());
    if (preloadIndexNames.contains(PRELOAD_ALL_KEY)) {
        indexToPreload = indexClient.getAllIndexRecords();
        LOG.info("Preloading all indices: " + indexToPreload.stream().map(r -> r.name).collect(Collectors.joining(",")));
    } else {
        for (String indexName : preloadIndexNames) {
            IndexRecord record = indexClient.lookUpIndexRecord(indexName);
            if (record != null) {
                indexToPreload.add(indexClient.lookUpIndexRecord(indexName));
            } else {
                LOG.info("Index " + indexName + " is not found. Preloading skipped.");
            }
        }
    }
    for (IndexRecord record : indexToPreload) {
        LOG.info("Preloading index %s to cache...", record.name);
        Duration timeElapsed = indexCache.loadIndexToCache(record);
        LOG.info("Index %s was loaded to cache. (Time elapsed: %s)", record.name, timeElapsed.toString());
    }
}

Example 17 with IndexRecord

use of io.prestosql.spi.heuristicindex.IndexRecord in project hetu-core by openlookeng.

the class StatementAnalyzer method validateUpdateIndex.

private void validateUpdateIndex(Table table, Optional<Scope> scope) {
    UpdateIndex updateIndex = (UpdateIndex) analysis.getOriginalStatement();
    IndexRecord indexRecord;
    try {
        indexRecord = heuristicIndexerManager.getIndexClient().lookUpIndexRecord(updateIndex.getIndexName().toString());
    } catch (IOException e) {
        throw new UncheckedIOException("Error reading index records, ", e);
    }
    QualifiedObjectName tableFullName = QualifiedObjectName.valueOf(indexRecord.qualifiedTable);
    accessControl.checkCanCreateIndex(session.getRequiredTransactionId(), session.getIdentity(), tableFullName);
    String tableName = tableFullName.toString();
    Optional<TableHandle> tableHandle = metadata.getTableHandle(session, tableFullName);
    if (!tableHandle.isPresent()) {
        throw new SemanticException(MISSING_ATTRIBUTE, table, "Unable to update index. " + "Index table '%s' may have been dropped from outside OLK. Index should also be dropped.", tableFullName);
    }
    List<Pair<String, Type>> indexColumns = new LinkedList<>();
    for (String i : indexRecord.columns) {
        indexColumns.add(new Pair<>(i, UNKNOWN));
    }
    try {
        // Use this place holder to check the existence of index and lock the place
        Properties properties = new Properties();
        properties.setProperty(INPROGRESS_PROPERTY_KEY, "TRUE");
        CreateIndexMetadata placeHolder = new CreateIndexMetadata(updateIndex.getIndexName().toString(), tableName, indexRecord.indexType, 0L, indexColumns, indexRecord.partitions, properties, session.getUser(), UNDEFINED);
        synchronized (StatementAnalyzer.class) {
            IndexClient.RecordStatus recordStatus = heuristicIndexerManager.getIndexClient().lookUpIndexRecord(placeHolder);
            switch(recordStatus) {
                case IN_PROGRESS_SAME_NAME:
                    throw new SemanticException(INDEX_ALREADY_EXISTS, updateIndex, "Index '%s' is being created by another user. Check running queries for details. If there is no running query for this index, " + "the index may be in an unexpected error state and should be dropped using 'DROP INDEX %s'", updateIndex.getIndexName().toString(), updateIndex.getIndexName().toString());
                case IN_PROGRESS_SAME_CONTENT:
                    throw new SemanticException(INDEX_ALREADY_EXISTS, updateIndex, "Index with same (table,column,indexType) is being created by another user. Check running queries for details. " + "If there is no running query for this index, the index may be in an unexpected error state and should be dropped using 'DROP INDEX'");
                case IN_PROGRESS_SAME_INDEX_PART_CONFLICT:
                    if (indexRecord.partitions.isEmpty()) {
                        throw new SemanticException(INDEX_ALREADY_EXISTS, updateIndex, "Index with same (table,column,indexType) is being created by another user. Check running queries for details. " + "If there is no running query for this index, the index may be in an unexpected error state and should be dropped using 'DROP INDEX %s'", updateIndex.getIndexName().toString());
                    }
                // allow different queries to run with explicitly same partitions
                case NOT_FOUND:
                    throw new SemanticException(MISSING_INDEX, updateIndex, "Index with name '%s' does not exist", updateIndex.getIndexName().toString());
            }
        }
    } catch (IOException e) {
        throw new UncheckedIOException(e);
    }
}

Example 18 with IndexRecord

use of io.prestosql.spi.heuristicindex.IndexRecord in project hetu-core by openlookeng.

the class HeuristicIndexClient method getLastModifiedTimes.

@Override
public Map<String, String> getLastModifiedTimes(String indexName) throws IOException {
    IndexRecord indexRecord = lookUpIndexRecord(indexName);
    CreateIndexMetadata.Level createLevel = indexRecord.getLevel();
    Path pathToIndex = Paths.get(root.toString(), indexRecord.qualifiedTable, indexRecord.columns[0], indexRecord.indexType);
    // check required for security scan since we are constructing a path using input
    checkArgument(!pathToIndex.toString().contains("../"), pathToIndex + " must be absolute and under one of the following whitelisted directories:  " + SecurePathWhiteList.getSecurePathWhiteList().toString());
    checkArgument(SecurePathWhiteList.isSecurePath(pathToIndex), pathToIndex + " must be under one of the following whitelisted directories: " + SecurePathWhiteList.getSecurePathWhiteList().toString());
    List<Path> paths = fs.walk(pathToIndex).filter(p -> !fs.isDirectory(p)).collect(Collectors.toList());
    switch(createLevel) {
        case STRIPE:
            return paths.stream().collect(Collectors.toMap(path -> "/" + path.subpath(pathToIndex.getNameCount(), path.getNameCount() - 1), path -> {
                String filename = path.getFileName().toString();
                return filename.replaceAll("\\D", "");
            }));
        case PARTITION:
            return paths.stream().collect(Collectors.toMap(path -> path.subpath(pathToIndex.getNameCount(), path.getNameCount() - 1).toString(), path -> {
                String filename = path.getFileName().toString();
                return filename.replaceAll("\\D", "");
            }));
        case TABLE:
            return paths.stream().filter(path -> path.getNameCount() - 1 == pathToIndex.getNameCount()).collect(Collectors.toMap(path -> indexRecord.qualifiedTable, path -> {
                String filename = path.getFileName().toString();
                return filename.replaceAll("\\D", "");
            }));
        default:
            return Collections.emptyMap();
    }
}

Example 19 with IndexRecord

use of io.prestosql.spi.heuristicindex.IndexRecord in project hetu-core by openlookeng.

the class IndexRecordManager method lookUpIndexRecord.

/**
 * Look up index record according to what it is for (triplet of [table, column, type]).
 */
public IndexRecord lookUpIndexRecord(String table, String[] columns, String indexType) {
    String[] tableQualified = table.split("\\.");
    if (tableQualified.length != 3) {
        throw new IllegalArgumentException(String.format("Illegal table name: %s", table));
    }
    Optional<TableEntity> tableEntity = metastore.getTable(tableQualified[0], tableQualified[1], tableQualified[2]);
    if (tableEntity.isPresent()) {
        for (Map.Entry<String, String> parameter : tableEntity.get().getParameters().entrySet()) {
            IndexRecord read = new IndexRecord(tableEntity.get(), parameter);
            if (Arrays.equals(read.columns, columns) && read.indexType.equals(indexType)) {
                return read;
            }
        }
    }
    return null;
}

Example 20 with IndexRecord

use of io.prestosql.spi.heuristicindex.IndexRecord in project hetu-core by openlookeng.

the class TestIndexRecordManager method testIndexRecordAddLookUpHelper.

private void testIndexRecordAddLookUpHelper(String name, String user, String table, String[] columns, String indexType, List<String> indexProperties, List<String> partitions) throws IOException {
    try (TempFolder folder = new TempFolder()) {
        folder.create();
        HetuMetastore testMetaStore = new HetuFsMetastore(new HetuFsMetastoreConfig().setHetuFileSystemMetastorePath(folder.getRoot().getPath()), FILE_SYSTEM_CLIENT);
        IndexRecordManager indexRecordManager = new IndexRecordManager(testMetaStore);
        IndexRecord expected = new IndexRecord(name, user, table, columns, indexType, 0L, indexProperties, partitions);
        indexRecordManager.addIndexRecord(name, user, table, columns, indexType, 0L, indexProperties, partitions);
        IndexRecord actual1 = indexRecordManager.lookUpIndexRecord(name);
        assertNotNull(actual1);
        assertEquals(actual1, expected);
        IndexRecord actual2 = indexRecordManager.lookUpIndexRecord(table, columns, indexType);
        assertNotNull(actual2);
        assertEquals(actual2, expected);
    }
}