Examples with IndexRecord io.prestosql.spi.heuristicindex.IndexRecord used on opensource projects

Example 1 with IndexRecord

use of io.prestosql.spi.heuristicindex.IndexRecord in project hetu-core by openlookeng.

the class HeuristicIndexClient method readSplitIndex.

@Override
public List<IndexMetadata> readSplitIndex(String path) throws IOException {
    requireNonNull(path, "no path specified");
    List<IndexMetadata> indexes = new LinkedList<>();
    Path indexKeyPath = Paths.get(path);
    IndexRecord curIndex = null;
    try {
        curIndex = indexRecordManager.lookUpIndexRecord(indexKeyPath.subpath(0, 1).toString(), new String[] { indexKeyPath.subpath(1, 2).toString() }, indexKeyPath.subpath(2, 3).toString());
        if (curIndex == null) {
            // Use index record file to pre-screen. If record does not contain the index, skip loading
            return null;
        }
    } catch (Exception e) {
        // On exception, log and continue reading from disk
        LOG.debug("Error reading index records: " + path);
    }
    for (Map.Entry<String, Index> entry : readIndexMap(path, curIndex).entrySet()) {
        String absolutePath = entry.getKey();
        Path remainder = Paths.get(absolutePath.replaceFirst(root.toString(), ""));
        Path table = remainder.subpath(0, 1);
        remainder = Paths.get(remainder.toString().replaceFirst(table.toString(), ""));
        Path column = remainder.subpath(0, 1);
        remainder = Paths.get(remainder.toString().replaceFirst(column.toString(), ""));
        Path indexType = remainder.subpath(0, 1);
        remainder = Paths.get(remainder.toString().replaceFirst(indexType.toString(), ""));
        Path filenamePath = remainder.getFileName();
        if (filenamePath == null) {
            throw new IllegalArgumentException("Split path cannot be resolved: " + path);
        }
        remainder = remainder.getParent();
        table = table.getFileName();
        column = column.getFileName();
        indexType = indexType.getFileName();
        if (remainder == null || table == null || column == null || indexType == null) {
            throw new IllegalArgumentException("Split path cannot be resolved: " + path);
        }
        String filename = filenamePath.toString();
        long splitStart = Long.parseLong(filename.substring(0, filename.lastIndexOf('.')));
        String timeDir = Paths.get(table.toString(), column.toString(), indexType.toString(), remainder.toString()).toString();
        long lastUpdated = getLastModifiedTime(timeDir);
        IndexMetadata index = new IndexMetadata(entry.getValue(), table.toString(), new String[] { column.toString() }, root.toString(), remainder.toString(), splitStart, lastUpdated);
        indexes.add(index);
    }
    return indexes;
}

Example 2 with IndexRecord

use of io.prestosql.spi.heuristicindex.IndexRecord in project hetu-core by openlookeng.

the class HeuristicIndexClient method deleteIndex.

@Override
public void deleteIndex(String indexName, List<String> partitionsToDelete) throws IOException {
    IndexRecord indexRecord = indexRecordManager.lookUpIndexRecord(indexName);
    // dir structure example: root/catalog.schema.table/column1,column2/BLOOM
    Path tablePath = root.resolve(indexRecord.qualifiedTable);
    Path columnPath = tablePath.resolve(String.join(",", indexRecord.columns));
    Path indexLevelPath = columnPath.resolve(indexRecord.indexType);
    if (!fs.exists(indexLevelPath)) {
        indexRecordManager.deleteIndexRecord(indexName, partitionsToDelete);
        return;
    }
    Lock lock = new FileBasedLock(fs, indexLevelPath.getParent());
    try {
        Runtime.getRuntime().addShutdownHook(new Thread(() -> {
            lock.unlock();
            try {
                fs.close();
            } catch (IOException e) {
                throw new UncheckedIOException("Error closing FileSystem Client: " + fs.getClass().getName(), e);
            }
        }));
        lock.lock();
        if (partitionsToDelete.isEmpty()) {
            fs.deleteRecursively(indexLevelPath);
        } else {
            List<Path> toDeletePartitions = fs.walk(indexLevelPath).filter(fs::isDirectory).filter(path -> partitionsToDelete.contains(path.getFileName().toString())).collect(Collectors.toList());
            for (Path path : toDeletePartitions) {
                fs.deleteRecursively(path);
            }
            // if all partitions have been deleted, remove index path
            if (fs.walk(indexLevelPath).allMatch(fs::isDirectory)) {
                fs.deleteRecursively(indexLevelPath);
            }
        }
        try {
            // clean empty directories
            if (fs.list(columnPath).allMatch(FileBasedLock::isLockUtilFile)) {
                fs.deleteRecursively(columnPath);
            }
            if (fs.list(tablePath).allMatch(FileBasedLock::isLockUtilFile)) {
                fs.deleteRecursively(tablePath);
            }
        } catch (Exception e) {
            LOG.debug("failed to clean empty index directory", e);
        }
    } finally {
        lock.unlock();
    }
    indexRecordManager.deleteIndexRecord(indexName, partitionsToDelete);
}

Example 3 with IndexRecord

use of io.prestosql.spi.heuristicindex.IndexRecord in project hetu-core by openlookeng.

the class HeuristicIndexClient method lookUpIndexRecord.

@Override
public RecordStatus lookUpIndexRecord(CreateIndexMetadata createIndexMetadata) throws IOException {
    IndexRecord sameNameRecord = indexRecordManager.lookUpIndexRecord(createIndexMetadata.getIndexName());
    IndexRecord sameIndexRecord = indexRecordManager.lookUpIndexRecord(createIndexMetadata.getTableName(), createIndexMetadata.getIndexColumns().stream().map(Pair::getFirst).toArray(String[]::new), createIndexMetadata.getIndexType());
    if (sameNameRecord == null) {
        if (sameIndexRecord != null) {
            return sameIndexRecord.isInProgressRecord() ? RecordStatus.IN_PROGRESS_SAME_CONTENT : RecordStatus.SAME_CONTENT;
        }
    } else {
        if (sameIndexRecord != null) {
            boolean partitionMerge = createIndexMetadata.getPartitions().size() != 0;
            for (String partition : createIndexMetadata.getPartitions()) {
                if (sameIndexRecord.partitions.isEmpty() || sameIndexRecord.partitions.contains(partition)) {
                    partitionMerge = false;
                    break;
                }
            }
            if (!partitionMerge) {
                return sameIndexRecord.isInProgressRecord() ? RecordStatus.IN_PROGRESS_SAME_INDEX_PART_CONFLICT : RecordStatus.SAME_INDEX_PART_CONFLICT;
            } else {
                return sameIndexRecord.isInProgressRecord() ? RecordStatus.IN_PROGRESS_SAME_INDEX_PART_CAN_MERGE : RecordStatus.SAME_INDEX_PART_CAN_MERGE;
            }
        } else {
            return sameNameRecord.isInProgressRecord() ? RecordStatus.IN_PROGRESS_SAME_NAME : RecordStatus.SAME_NAME;
        }
    }
    return RecordStatus.NOT_FOUND;
}

Example 4 with IndexRecord

use of io.prestosql.spi.heuristicindex.IndexRecord in project hetu-core by openlookeng.

the class HeuristicIndexClient method getIndexSize.

@Override
public long getIndexSize(String name) throws IOException {
    IndexRecord referenceRecord = indexRecordManager.lookUpIndexRecord(name);
    Path pathToIndex = Paths.get(root.toString(), referenceRecord.qualifiedTable, referenceRecord.columns[0], referenceRecord.indexType);
    // check required for security scan since we are constructing a path using input
    checkArgument(!pathToIndex.toString().contains("../"), pathToIndex + " must be absolute and under one of the following whitelisted directories:  " + SecurePathWhiteList.getSecurePathWhiteList().toString());
    checkArgument(SecurePathWhiteList.isSecurePath(pathToIndex), pathToIndex + " must be under one of the following whitelisted directories: " + SecurePathWhiteList.getSecurePathWhiteList().toString());
    return getDirectorySize(pathToIndex);
}

Example 5 with IndexRecord

use of io.prestosql.spi.heuristicindex.IndexRecord in project hetu-core by openlookeng.

the class IndexRecordManager method addIndexRecord.

/**
 * Add IndexRecord into record file. If the method is called with a name that already exists,
 *
 * it will OVERWRITE the existing entry but COMBINE the partition columns (if it previously was partitioned)
 */
public synchronized void addIndexRecord(String name, String user, String table, String[] columns, String indexType, long indexSize, List<String> indexProperties, List<String> partitions) {
    IndexRecord record = new IndexRecord(name, user, table, columns, indexType, indexSize, indexProperties, partitions);
    IndexRecord old = lookUpIndexRecord(name);
    if (old != null) {
        record.partitions.addAll(0, old.partitions);
    }
    Optional<CatalogEntity> oldCatalog = metastore.getCatalog(record.catalog);
    if (!oldCatalog.isPresent()) {
        CatalogEntity newCatalog = CatalogEntity.builder().setCatalogName(record.catalog).build();
        metastore.createCatalogIfNotExist(newCatalog);
    }
    Optional<DatabaseEntity> oldSchema = metastore.getDatabase(record.catalog, record.schema);
    if (!oldSchema.isPresent()) {
        DatabaseEntity newSchema = DatabaseEntity.builder().setCatalogName(record.catalog).setDatabaseName(record.schema).build();
        metastore.createDatabaseIfNotExist(newSchema);
    }
    Optional<TableEntity> oldTable = metastore.getTable(record.catalog, record.schema, record.table);
    if (!oldTable.isPresent()) {
        TableEntity newTable = TableEntity.builder().setCatalogName(record.catalog).setDatabaseName(record.schema).setTableName(record.table).setTableType(TableEntityType.TABLE.toString()).build();
        metastore.createTableIfNotExist(newTable);
    }
    metastore.alterTableParameter(record.catalog, record.schema, record.table, record.serializeKey(), record.serializeValue());
}