use of io.prestosql.spi.connector.CreateIndexMetadata in project hetu-core by openlookeng.
the class QueryPlanner method createIndex.
/**
* CREATE INDEX statements are rewritten as SELECT statements,
* if the original statement was CREATE INDEX, create the necessary plan nodes
* to create the index
*/
private PlanBuilder createIndex(PlanBuilder subPlan, Statement originalStatement) {
if (!(originalStatement instanceof CreateIndex)) {
return subPlan;
}
// rewrite sub queries
CreateIndex createIndex = (CreateIndex) originalStatement;
String tableName = MetadataUtil.createQualifiedObjectName(session, originalStatement, createIndex.getTableName()).toString();
List<String> partitions = new ArrayList<>();
if (createIndex.getExpression().isPresent()) {
partitions = HeuristicIndexUtils.extractPartitions(createIndex.getExpression().get());
}
Map<String, Type> columnTypes = new HashMap<>();
for (Field field : analysis.getRootScope().getRelationType().getAllFields()) {
if (INDEX_SUPPORTED_TYPES.get(createIndex.getIndexType().toLowerCase(Locale.ENGLISH)).stream().noneMatch(supportType -> field.getType().getDisplayName().contains(supportType))) {
throw new UnsupportedOperationException("Index creation on " + field.getType().getDisplayName() + " column is not supported");
}
columnTypes.put(field.getOriginColumnName().get(), field.getType());
}
Properties indexProperties = new Properties();
CreateIndexMetadata.Level indexCreationLevel = CreateIndexMetadata.Level.UNDEFINED;
indexProperties.setProperty(LEVEL_PROP_KEY, indexCreationLevel.toString());
boolean autoLoadFound = false;
for (Property property : createIndex.getProperties()) {
String key = extractPropertyValue(property.getName());
String val = extractPropertyValue(property.getValue()).toUpperCase(Locale.ENGLISH);
if (key.equals(LEVEL_PROP_KEY)) {
indexCreationLevel = CreateIndexMetadata.Level.valueOf(val);
continue;
}
if (key.equals(AUTOLOAD_PROP_KEY)) {
autoLoadFound = true;
String valInLowerCase = val.toLowerCase(Locale.ROOT);
if (valInLowerCase.equals("true") || valInLowerCase.equals("false")) {
indexProperties.setProperty(key, valInLowerCase);
} else {
throw new IllegalArgumentException("Unrecognized value for key '" + AUTOLOAD_PROP_KEY + "', only 'true' or 'false' are allowed");
}
continue;
}
indexProperties.setProperty(key, val);
}
if (!autoLoadFound) {
boolean defaultAutoloadProp = PropertyService.getBooleanProperty(HetuConstant.FILTER_CACHE_AUTOLOAD_DEFAULT);
indexProperties.setProperty(AUTOLOAD_PROP_KEY, String.valueOf(defaultAutoloadProp));
}
return subPlan.withNewRoot(new CreateIndexNode(idAllocator.getNextId(), ExchangeNode.gatheringExchange(idAllocator.getNextId(), ExchangeNode.Scope.REMOTE, subPlan.getRoot()), new CreateIndexMetadata(createIndex.getIndexName().toString(), tableName, createIndex.getIndexType(), 0L, createIndex.getColumnAliases().stream().map(identifier -> new Pair<>(identifier.toString(), columnTypes.get(identifier.toString().toLowerCase(Locale.ROOT)))).collect(Collectors.toList()), partitions, indexProperties, session.getUser(), indexCreationLevel)));
}
use of io.prestosql.spi.connector.CreateIndexMetadata in project hetu-core by openlookeng.
the class UpdateIndexOperator method addInput.
@Override
public void addInput(Page page) {
checkState(needsInput(), "Operator is already finishing");
requireNonNull(page, "page is null");
// TODO-cp-I38S9O: Operator currently not supported for Snapshot
if (page instanceof MarkerPage) {
throw new UnsupportedOperationException("Operator doesn't support snapshotting.");
}
// if operator is still receiving input, it's not finished
finished.putIfAbsent(this, false);
if (page.getPositionCount() == 0) {
return;
}
IndexRecord indexRecord;
try {
indexRecord = heuristicIndexerManager.getIndexClient().lookUpIndexRecord(createIndexMetadata.getIndexName());
} catch (IOException e) {
throw new UncheckedIOException("Error reading index records, ", e);
}
if (createIndexMetadata.getCreateLevel() == CreateIndexMetadata.Level.UNDEFINED) {
boolean tableIsPartitioned = getPartitionName(page.getPageMetadata().getProperty(HetuConstant.DATASOURCE_FILE_PATH), indexRecord.qualifiedTable) != null;
createIndexMetadata.decideIndexLevel(tableIsPartitioned);
}
Map<String, List<Object>> values = new HashMap<>();
for (int blockId = 0; blockId < page.getChannelCount(); blockId++) {
Block block = page.getBlock(blockId);
Pair<String, Type> entry = createIndexMetadata.getIndexColumns().get(blockId);
String indexColumn = entry.getFirst();
Type type = entry.getSecond();
for (int position = 0; position < block.getPositionCount(); ++position) {
Object value = getNativeValue(type, block, position);
value = getActualValue(type, value);
values.computeIfAbsent(indexColumn, k -> new ArrayList<>()).add(value);
}
}
Properties connectorMetadata = new Properties();
connectorMetadata.put(HetuConstant.DATASOURCE_CATALOG, createIndexMetadata.getTableName().split("\\.")[0]);
connectorMetadata.putAll(page.getPageMetadata());
try {
switch(createIndexMetadata.getCreateLevel()) {
case STRIPE:
{
String filePath = page.getPageMetadata().getProperty(HetuConstant.DATASOURCE_FILE_PATH);
// The orc file this page resides in wasn't modified from when the index was created/last updated
if (pathToModifiedTime.containsKey(filePath) && pathToModifiedTime.get(filePath).equals(page.getPageMetadata().getProperty(HetuConstant.DATASOURCE_FILE_MODIFICATION))) {
return;
}
levelWriter.computeIfAbsent(filePath, k -> heuristicIndexerManager.getIndexWriter(createIndexMetadata, connectorMetadata));
persistBy.putIfAbsent(levelWriter.get(filePath), this);
levelWriter.get(filePath).addData(values, connectorMetadata);
break;
}
case PARTITION:
{
String partition = getPartitionName(page.getPageMetadata().getProperty(HetuConstant.DATASOURCE_FILE_PATH), createIndexMetadata.getTableName());
indexLevelToMaxModifiedTime.compute(partition, (k, v) -> {
if (v != null && v >= (Long.parseLong(page.getPageMetadata().getProperty(HetuConstant.DATASOURCE_FILE_MODIFICATION)))) {
return v;
}
return (Long.parseLong(page.getPageMetadata().getProperty(HetuConstant.DATASOURCE_FILE_MODIFICATION)));
});
levelWriter.putIfAbsent(partition, heuristicIndexerManager.getIndexWriter(createIndexMetadata, connectorMetadata));
persistBy.putIfAbsent(levelWriter.get(partition), this);
levelWriter.get(partition).addData(values, connectorMetadata);
break;
}
case TABLE:
{
indexLevelToMaxModifiedTime.compute(createIndexMetadata.getTableName(), (k, v) -> {
if (v != null && v >= (Long.parseLong(page.getPageMetadata().getProperty(HetuConstant.DATASOURCE_FILE_MODIFICATION)))) {
return v;
}
return (Long.parseLong(page.getPageMetadata().getProperty(HetuConstant.DATASOURCE_FILE_MODIFICATION)));
});
levelWriter.putIfAbsent(createIndexMetadata.getTableName(), heuristicIndexerManager.getIndexWriter(createIndexMetadata, connectorMetadata));
persistBy.putIfAbsent(levelWriter.get(createIndexMetadata.getTableName()), this);
levelWriter.get(createIndexMetadata.getTableName()).addData(values, connectorMetadata);
break;
}
default:
throw new IllegalArgumentException("Create level not supported");
}
} catch (IOException e) {
throw new UncheckedIOException(e);
}
}
use of io.prestosql.spi.connector.CreateIndexMetadata in project hetu-core by openlookeng.
the class TestHeuristicIndexClient method testDeleteSelectedColumnsHelper.
@Test
public void testDeleteSelectedColumnsHelper() throws IOException {
String tableName = "catalog.schema.UT_test";
try (TempFolder folder = new TempFolder()) {
folder.create();
File tableFolder = new File(folder.getRoot().getPath(), tableName);
assertTrue(tableFolder.mkdir());
File columnFolder = new File(tableFolder, "test_column");
assertTrue(columnFolder.mkdirs());
File indexTypeFolder = new File(columnFolder, "BLOOM");
assertTrue(indexTypeFolder.mkdirs());
assertTrue(new File(indexTypeFolder, "testIndex.index").createNewFile());
HetuFileSystemClient fs = new HetuLocalFileSystemClient(new LocalConfig(new Properties()), folder.getRoot().toPath());
HetuMetastore testMetaStore = new HetuFsMetastore(new HetuFsMetastoreConfig().setHetuFileSystemMetastorePath(folder.getRoot().getPath()), fs);
HeuristicIndexClient client = new HeuristicIndexClient(fs, testMetaStore, folder.getRoot().toPath());
client.addIndexRecord(new CreateIndexMetadata("idx1", tableName, "BLOOM", 0L, Collections.singletonList(new Pair<>("test_column", VARCHAR)), Collections.emptyList(), new Properties(), "user", CreateIndexMetadata.Level.UNDEFINED));
client.deleteIndex("idx1", Collections.emptyList());
assertFalse(indexTypeFolder.exists());
}
}
use of io.prestosql.spi.connector.CreateIndexMetadata in project hetu-core by openlookeng.
the class TestPartitionIndexWriter method testAddValueMultThread.
@Test
public void testAddValueMultThread() throws InterruptedException {
List<Pair<String, Type>> columns = new ArrayList<>();
List<String> partitions = Collections.singletonList("partition1");
Properties properties = new Properties();
CreateIndexMetadata createIndexMetadata = new CreateIndexMetadata("hetu_partition_idx", "testTable", "BTREE", 0L, columns, partitions, properties, "testuser", CreateIndexMetadata.Level.PARTITION);
HetuFileSystemClient fileSystemClient = Mockito.mock(HetuFileSystemClient.class);
Properties connectorMetadata1 = new Properties();
connectorMetadata1.setProperty(HetuConstant.DATASOURCE_FILE_MODIFICATION, String.valueOf(System.currentTimeMillis()));
connectorMetadata1.setProperty(HetuConstant.DATASOURCE_FILE_PATH, "hdfs://testable/testcolumn/cp=123121/file1");
connectorMetadata1.setProperty(HetuConstant.DATASOURCE_STRIPE_OFFSET, "3");
connectorMetadata1.setProperty(HetuConstant.DATASOURCE_STRIPE_LENGTH, "100");
Properties connectorMetadata2 = new Properties();
connectorMetadata2.setProperty(HetuConstant.DATASOURCE_FILE_MODIFICATION, String.valueOf(System.currentTimeMillis()));
connectorMetadata2.setProperty(HetuConstant.DATASOURCE_FILE_PATH, "hdfs://testable/testcolumn/cp=123121/file2");
connectorMetadata2.setProperty(HetuConstant.DATASOURCE_STRIPE_OFFSET, "3");
connectorMetadata2.setProperty(HetuConstant.DATASOURCE_STRIPE_LENGTH, "100");
PartitionIndexWriter indexWriter = new PartitionIndexWriter(createIndexMetadata, fileSystemClient, Paths.get("/tmp"));
ExecutorService executorService = Executors.newFixedThreadPool(2);
CountDownLatch latch = new CountDownLatch(2);
executorService.submit(new TestDriver(indexWriter, connectorMetadata1, latch));
executorService.submit(new TestDriver(indexWriter, connectorMetadata2, latch));
latch.await(5, TimeUnit.SECONDS);
Map<Comparable<? extends Comparable<?>>, String> result = indexWriter.getDataMap();
assertEquals(10, result.size());
assertEquals(2, indexWriter.getSymbolTable().size());
}
use of io.prestosql.spi.connector.CreateIndexMetadata in project hetu-core by openlookeng.
the class StatementAnalyzer method validateCreateIndex.
private void validateCreateIndex(Table table, Optional<Scope> scope) {
CreateIndex createIndex = (CreateIndex) analysis.getOriginalStatement();
QualifiedObjectName tableFullName = createQualifiedObjectName(session, createIndex, createIndex.getTableName());
accessControl.checkCanCreateIndex(session.getRequiredTransactionId(), session.getIdentity(), tableFullName);
String tableName = tableFullName.toString();
// check whether catalog support create index
if (!metadata.isHeuristicIndexSupported(session, tableFullName)) {
throw new SemanticException(NOT_SUPPORTED, createIndex, "CREATE INDEX is not supported in catalog '%s'", tableFullName.getCatalogName());
}
List<String> partitions = new ArrayList<>();
String partitionColumn = null;
if (createIndex.getExpression().isPresent()) {
partitions = HeuristicIndexUtils.extractPartitions(createIndex.getExpression().get());
// check partition name validate, create index …… where pt_d = xxx;
// pt_d must be partition column
Set<String> partitionColumns = partitions.stream().map(k -> k.substring(0, k.indexOf("="))).collect(Collectors.toSet());
if (partitionColumns.size() > 1) {
// currently only support one partition column
throw new IllegalArgumentException("Heuristic index only supports predicates on one column");
}
// The only entry in set should be the only partition column name
partitionColumn = partitionColumns.iterator().next();
}
Optional<TableHandle> tableHandle = metadata.getTableHandle(session, tableFullName);
if (tableHandle.isPresent()) {
if (!tableHandle.get().getConnectorHandle().isHeuristicIndexSupported()) {
throw new SemanticException(NOT_SUPPORTED, table, "Catalog supported, but table storage format is not supported by heuristic index");
}
TableMetadata tableMetadata = metadata.getTableMetadata(session, tableHandle.get());
List<String> availableColumns = tableMetadata.getColumns().stream().map(ColumnMetadata::getName).collect(Collectors.toList());
for (Identifier column : createIndex.getColumnAliases()) {
if (!availableColumns.contains(column.getValue().toLowerCase(Locale.ROOT))) {
throw new SemanticException(MISSING_ATTRIBUTE, table, "Column '%s' cannot be resolved", column.getValue());
}
}
if (partitionColumn != null && !tableHandle.get().getConnectorHandle().isPartitionColumn(partitionColumn)) {
throw new SemanticException(NOT_SUPPORTED, table, "Heuristic index creation is only supported for predicates on partition columns");
}
} else {
throw new SemanticException(MISSING_ATTRIBUTE, table, "Table '%s' is invalid", tableFullName);
}
List<Pair<String, Type>> indexColumns = new LinkedList<>();
for (Identifier i : createIndex.getColumnAliases()) {
indexColumns.add(new Pair<>(i.toString(), UNKNOWN));
}
// For now, creating index for multiple columns is not supported
if (indexColumns.size() > 1) {
throw new SemanticException(NOT_SUPPORTED, table, "Multi-column indexes are currently not supported");
}
try {
// Use this place holder to check the existence of index and lock the place
Properties properties = new Properties();
properties.setProperty(INPROGRESS_PROPERTY_KEY, "TRUE");
CreateIndexMetadata placeHolder = new CreateIndexMetadata(createIndex.getIndexName().toString(), tableName, createIndex.getIndexType(), 0L, indexColumns, partitions, properties, session.getUser(), UNDEFINED);
synchronized (StatementAnalyzer.class) {
IndexClient.RecordStatus recordStatus = heuristicIndexerManager.getIndexClient().lookUpIndexRecord(placeHolder);
switch(recordStatus) {
case SAME_NAME:
throw new SemanticException(INDEX_ALREADY_EXISTS, createIndex, "Index '%s' already exists", createIndex.getIndexName().toString());
case SAME_CONTENT:
throw new SemanticException(INDEX_ALREADY_EXISTS, createIndex, "Index with same (table,column,indexType) already exists");
case SAME_INDEX_PART_CONFLICT:
throw new SemanticException(INDEX_ALREADY_EXISTS, createIndex, "Index with same (table,column,indexType) already exists and partition(s) contain conflicts");
case IN_PROGRESS_SAME_NAME:
throw new SemanticException(INDEX_ALREADY_EXISTS, createIndex, "Index '%s' is being created by another user. Check running queries for details. If there is no running query for this index, " + "the index may be in an unexpected error state and should be dropped using 'DROP INDEX %s'", createIndex.getIndexName().toString(), createIndex.getIndexName().toString());
case IN_PROGRESS_SAME_CONTENT:
throw new SemanticException(INDEX_ALREADY_EXISTS, createIndex, "Index with same (table,column,indexType) is being created by another user. Check running queries for details. " + "If there is no running query for this index, the index may be in an unexpected error state and should be dropped using 'DROP INDEX'");
case IN_PROGRESS_SAME_INDEX_PART_CONFLICT:
if (partitions.isEmpty()) {
throw new SemanticException(INDEX_ALREADY_EXISTS, createIndex, "Index with same (table,column,indexType) is being created by another user. Check running queries for details. " + "If there is no running query for this index, the index may be in an unexpected error state and should be dropped using 'DROP INDEX %s'", createIndex.getIndexName().toString());
}
// allow different queries to run with explicitly same partitions
case SAME_INDEX_PART_CAN_MERGE:
case IN_PROGRESS_SAME_INDEX_PART_CAN_MERGE:
break;
case NOT_FOUND:
heuristicIndexerManager.getIndexClient().addIndexRecord(placeHolder);
}
}
} catch (IOException e) {
throw new UncheckedIOException(e);
}
}
Aggregations