Examples with ConnectorTableHandle com.facebook.presto.spi.ConnectorTableHandle used on opensource projects

Example 71 with ConnectorTableHandle

use of com.facebook.presto.spi.ConnectorTableHandle in project presto by prestodb.

the class HiveMetadata method beginInsertInternal.

private HiveInsertTableHandle beginInsertInternal(ConnectorSession session, ConnectorTableHandle tableHandle) {
    verifyJvmTimeZone();
    MetastoreContext metastoreContext = getMetastoreContext(session);
    SchemaTableName tableName = ((HiveTableHandle) tableHandle).getSchemaTableName();
    Table table = metastore.getTable(metastoreContext, tableName.getSchemaName(), tableName.getTableName()).orElseThrow(() -> new TableNotFoundException(tableName));
    checkTableIsWritable(table, writesToNonManagedTablesEnabled);
    for (Column column : table.getDataColumns()) {
        if (!isWritableType(column.getType())) {
            throw new PrestoException(NOT_SUPPORTED, format("Inserting into Hive table %s with column type %s not supported", tableName, column.getType()));
        }
    }
    List<HiveColumnHandle> handles = hiveColumnHandles(table).stream().filter(columnHandle -> !columnHandle.isHidden()).collect(toList());
    HiveStorageFormat tableStorageFormat = extractHiveStorageFormat(table);
    LocationHandle locationHandle;
    boolean isTemporaryTable = table.getTableType().equals(TEMPORARY_TABLE);
    boolean tempPathRequired = isTempPathRequired(session, table.getStorage().getBucketProperty(), decodePreferredOrderingColumnsFromStorage(table.getStorage()));
    if (isTemporaryTable) {
        locationHandle = locationService.forTemporaryTable(metastore, session, table, tempPathRequired);
    } else {
        locationHandle = locationService.forExistingTable(metastore, session, table, tempPathRequired);
    }
    Optional<? extends TableEncryptionProperties> tableEncryptionProperties = getTableEncryptionPropertiesFromHiveProperties(table.getParameters(), tableStorageFormat);
    HiveStorageFormat partitionStorageFormat = isRespectTableFormat(session) ? tableStorageFormat : getHiveStorageFormat(session);
    HiveStorageFormat actualStorageFormat = table.getPartitionColumns().isEmpty() ? tableStorageFormat : partitionStorageFormat;
    if (tableEncryptionProperties.isPresent() && actualStorageFormat != tableStorageFormat) {
        throw new PrestoException(INVALID_TABLE_PROPERTY, format("For encrypted tables, partition format (%s) should match table format (%s). Using the session property %s or appropriately setting %s can help with ensuring this", actualStorageFormat.name(), tableStorageFormat.name(), RESPECT_TABLE_FORMAT, HIVE_STORAGE_FORMAT));
    }
    HiveInsertTableHandle result = new HiveInsertTableHandle(tableName.getSchemaName(), tableName.getTableName(), handles, metastore.generatePageSinkMetadata(metastoreContext, tableName), locationHandle, table.getStorage().getBucketProperty(), decodePreferredOrderingColumnsFromStorage(table.getStorage()), tableStorageFormat, partitionStorageFormat, actualStorageFormat, getHiveCompressionCodec(session, isTemporaryTable, actualStorageFormat), encryptionInformationProvider.getWriteEncryptionInformation(session, tableEncryptionProperties.map(identity()), tableName.getSchemaName(), tableName.getTableName()));
    WriteInfo writeInfo = locationService.getQueryWriteInfo(locationHandle);
    metastore.declareIntentionToWrite(new HdfsContext(session, tableName.getSchemaName(), tableName.getTableName(), table.getStorage().getLocation(), false), metastoreContext, writeInfo.getWriteMode(), writeInfo.getWritePath(), writeInfo.getTempPath(), tableName, isTemporaryTable);
    return result;
}

Example 72 with ConnectorTableHandle

use of com.facebook.presto.spi.ConnectorTableHandle in project presto by prestodb.

the class TestMemoryMetadata method tableAlreadyExists.

@Test
public void tableAlreadyExists() {
    assertNoTables();
    SchemaTableName test1Table = new SchemaTableName("default", "test1");
    SchemaTableName test2Table = new SchemaTableName("default", "test2");
    metadata.createTable(SESSION, new ConnectorTableMetadata(test1Table, ImmutableList.of()), false);
    try {
        metadata.createTable(SESSION, new ConnectorTableMetadata(test1Table, ImmutableList.of()), false);
        fail("Should fail because table already exists");
    } catch (PrestoException ex) {
        assertEquals(ex.getErrorCode(), ALREADY_EXISTS.toErrorCode());
        assertEquals(ex.getMessage(), "Table [default.test1] already exists");
    }
    ConnectorTableHandle test1TableHandle = metadata.getTableHandle(SESSION, test1Table);
    metadata.createTable(SESSION, new ConnectorTableMetadata(test2Table, ImmutableList.of()), false);
    try {
        metadata.renameTable(SESSION, test1TableHandle, test2Table);
        fail("Should fail because table already exists");
    } catch (PrestoException ex) {
        assertEquals(ex.getErrorCode(), ALREADY_EXISTS.toErrorCode());
        assertEquals(ex.getMessage(), "Table [default.test2] already exists");
    }
}

Example 73 with ConnectorTableHandle

use of com.facebook.presto.spi.ConnectorTableHandle in project presto by prestodb.

the class TestMemoryMetadata method testRenameTable.

@Test
public void testRenameTable() {
    SchemaTableName tableName = new SchemaTableName("test_schema", "test_table_to_be_renamed");
    metadata.createSchema(SESSION, "test_schema", ImmutableMap.of());
    ConnectorOutputTableHandle table = metadata.beginCreateTable(SESSION, new ConnectorTableMetadata(tableName, ImmutableList.of(), ImmutableMap.of()), Optional.empty());
    metadata.finishCreateTable(SESSION, table, ImmutableList.of(), ImmutableList.of());
    // rename table to schema which does not exist
    SchemaTableName invalidSchemaTableName = new SchemaTableName("test_schema_not_exist", "test_table_renamed");
    ConnectorTableHandle tableHandle = metadata.getTableHandle(SESSION, tableName);
    Throwable throwable = expectThrows(SchemaNotFoundException.class, () -> metadata.renameTable(SESSION, tableHandle, invalidSchemaTableName));
    assertTrue(throwable.getMessage().equals("Schema test_schema_not_exist not found"));
    // rename table to same schema
    SchemaTableName sameSchemaTableName = new SchemaTableName("test_schema", "test_renamed");
    metadata.renameTable(SESSION, metadata.getTableHandle(SESSION, tableName), sameSchemaTableName);
    assertEquals(metadata.listTables(SESSION, "test_schema"), ImmutableList.of(sameSchemaTableName));
    // rename table to different schema
    metadata.createSchema(SESSION, "test_different_schema", ImmutableMap.of());
    SchemaTableName differentSchemaTableName = new SchemaTableName("test_different_schema", "test_renamed");
    metadata.renameTable(SESSION, metadata.getTableHandle(SESSION, sameSchemaTableName), differentSchemaTableName);
    assertEquals(metadata.listTables(SESSION, "test_schema"), ImmutableList.of());
    assertEquals(metadata.listTables(SESSION, "test_different_schema"), ImmutableList.of(differentSchemaTableName));
}

Example 74 with ConnectorTableHandle

use of com.facebook.presto.spi.ConnectorTableHandle in project presto by prestodb.

the class HiveFilterPushdown method pushdownFilter.

@VisibleForTesting
public static ConnectorPushdownFilterResult pushdownFilter(ConnectorSession session, ConnectorMetadata metadata, SemiTransactionalHiveMetastore metastore, RowExpressionService rowExpressionService, StandardFunctionResolution functionResolution, HivePartitionManager partitionManager, FunctionMetadataManager functionMetadataManager, ConnectorTableHandle tableHandle, RowExpression filter, Optional<ConnectorTableLayoutHandle> currentLayoutHandle) {
    checkArgument(!FALSE_CONSTANT.equals(filter), "Cannot pushdown filter that is always false");
    if (TRUE_CONSTANT.equals(filter) && currentLayoutHandle.isPresent()) {
        return new ConnectorPushdownFilterResult(metadata.getTableLayout(session, currentLayoutHandle.get()), TRUE_CONSTANT);
    }
    // Split the filter into 3 groups of conjuncts:
    // - range filters that apply to entire columns,
    // - range filters that apply to subfields,
    // - the rest. Intersect these with possibly pre-existing filters.
    DomainTranslator.ExtractionResult<Subfield> decomposedFilter = rowExpressionService.getDomainTranslator().fromPredicate(session, filter, new SubfieldExtractor(functionResolution, rowExpressionService.getExpressionOptimizer(), session).toColumnExtractor());
    if (currentLayoutHandle.isPresent()) {
        HiveTableLayoutHandle currentHiveLayout = (HiveTableLayoutHandle) currentLayoutHandle.get();
        decomposedFilter = intersectExtractionResult(new DomainTranslator.ExtractionResult(currentHiveLayout.getDomainPredicate(), currentHiveLayout.getRemainingPredicate()), decomposedFilter);
    }
    if (decomposedFilter.getTupleDomain().isNone()) {
        return new ConnectorPushdownFilterResult(EMPTY_TABLE_LAYOUT, FALSE_CONSTANT);
    }
    RowExpression optimizedRemainingExpression = rowExpressionService.getExpressionOptimizer().optimize(decomposedFilter.getRemainingExpression(), OPTIMIZED, session);
    if (optimizedRemainingExpression instanceof ConstantExpression) {
        ConstantExpression constantExpression = (ConstantExpression) optimizedRemainingExpression;
        if (FALSE_CONSTANT.equals(constantExpression) || constantExpression.getValue() == null) {
            return new ConnectorPushdownFilterResult(EMPTY_TABLE_LAYOUT, FALSE_CONSTANT);
        }
    }
    Map<String, ColumnHandle> columnHandles = metadata.getColumnHandles(session, tableHandle);
    TupleDomain<ColumnHandle> entireColumnDomain = decomposedFilter.getTupleDomain().transform(subfield -> isEntireColumn(subfield) ? subfield.getRootName() : null).transform(columnHandles::get);
    if (currentLayoutHandle.isPresent()) {
        entireColumnDomain = entireColumnDomain.intersect(((HiveTableLayoutHandle) (currentLayoutHandle.get())).getPartitionColumnPredicate());
    }
    Constraint<ColumnHandle> constraint = new Constraint<>(entireColumnDomain);
    // Extract deterministic conjuncts that apply to partition columns and specify these as Constraint#predicate
    if (!TRUE_CONSTANT.equals(decomposedFilter.getRemainingExpression())) {
        LogicalRowExpressions logicalRowExpressions = new LogicalRowExpressions(rowExpressionService.getDeterminismEvaluator(), functionResolution, functionMetadataManager);
        RowExpression deterministicPredicate = logicalRowExpressions.filterDeterministicConjuncts(decomposedFilter.getRemainingExpression());
        if (!TRUE_CONSTANT.equals(deterministicPredicate)) {
            ConstraintEvaluator evaluator = new ConstraintEvaluator(rowExpressionService, session, columnHandles, deterministicPredicate);
            constraint = new Constraint<>(entireColumnDomain, evaluator::isCandidate);
        }
    }
    HivePartitionResult hivePartitionResult = partitionManager.getPartitions(metastore, tableHandle, constraint, session);
    TupleDomain<Subfield> domainPredicate = withColumnDomains(ImmutableMap.<Subfield, Domain>builder().putAll(hivePartitionResult.getUnenforcedConstraint().transform(HiveFilterPushdown::toSubfield).getDomains().orElse(ImmutableMap.of())).putAll(decomposedFilter.getTupleDomain().transform(subfield -> !isEntireColumn(subfield) ? subfield : null).getDomains().orElse(ImmutableMap.of())).build());
    Set<String> predicateColumnNames = new HashSet<>();
    domainPredicate.getDomains().get().keySet().stream().map(Subfield::getRootName).forEach(predicateColumnNames::add);
    // Include only columns referenced in the optimized expression. Although the expression is sent to the worker node
    // unoptimized, the worker is expected to optimize the expression before executing.
    extractAll(optimizedRemainingExpression).stream().map(VariableReferenceExpression::getName).forEach(predicateColumnNames::add);
    Map<String, HiveColumnHandle> predicateColumns = predicateColumnNames.stream().map(columnHandles::get).map(HiveColumnHandle.class::cast).collect(toImmutableMap(HiveColumnHandle::getName, Functions.identity()));
    SchemaTableName tableName = ((HiveTableHandle) tableHandle).getSchemaTableName();
    LogicalRowExpressions logicalRowExpressions = new LogicalRowExpressions(rowExpressionService.getDeterminismEvaluator(), functionResolution, functionMetadataManager);
    List<RowExpression> conjuncts = extractConjuncts(decomposedFilter.getRemainingExpression());
    RowExpression dynamicFilterExpression = extractDynamicConjuncts(conjuncts, logicalRowExpressions);
    RowExpression remainingExpression = extractStaticConjuncts(conjuncts, logicalRowExpressions);
    remainingExpression = removeNestedDynamicFilters(remainingExpression);
    Table table = metastore.getTable(new MetastoreContext(session.getIdentity(), session.getQueryId(), session.getClientInfo(), session.getSource(), getMetastoreHeaders(session), isUserDefinedTypeEncodingEnabled(session), metastore.getColumnConverterProvider()), tableName.getSchemaName(), tableName.getTableName()).orElseThrow(() -> new TableNotFoundException(tableName));
    return new ConnectorPushdownFilterResult(metadata.getTableLayout(session, new HiveTableLayoutHandle(tableName, table.getStorage().getLocation(), hivePartitionResult.getPartitionColumns(), // remove comments to optimize serialization costs
    pruneColumnComments(hivePartitionResult.getDataColumns()), hivePartitionResult.getTableParameters(), hivePartitionResult.getPartitions(), domainPredicate, remainingExpression, predicateColumns, hivePartitionResult.getEnforcedConstraint(), hivePartitionResult.getBucketHandle(), hivePartitionResult.getBucketFilter(), true, createTableLayoutString(session, rowExpressionService, tableName, hivePartitionResult.getBucketHandle(), hivePartitionResult.getBucketFilter(), remainingExpression, domainPredicate), currentLayoutHandle.map(layout -> ((HiveTableLayoutHandle) layout).getRequestedColumns()).orElse(Optional.empty()), false)), dynamicFilterExpression);
}

Example 75 with ConnectorTableHandle

use of com.facebook.presto.spi.ConnectorTableHandle in project presto by prestodb.

the class CassandraPartitionManager method getPartitions.

public CassandraPartitionResult getPartitions(ConnectorTableHandle tableHandle, TupleDomain<ColumnHandle> tupleDomain) {
    CassandraTableHandle cassandraTableHandle = (CassandraTableHandle) tableHandle;
    CassandraTable table = cassandraSession.getTable(cassandraTableHandle.getSchemaTableName());
    List<CassandraColumnHandle> partitionKeys = table.getPartitionKeyColumns();
    // fetch the partitions
    List<CassandraPartition> allPartitions = getCassandraPartitions(table, tupleDomain);
    log.debug("%s.%s #partitions: %d", cassandraTableHandle.getSchemaName(), cassandraTableHandle.getTableName(), allPartitions.size());
    // do a final pass to filter based on fields that could not be used to build the prefix
    List<CassandraPartition> partitions = allPartitions.stream().filter(partition -> tupleDomain.overlaps(partition.getTupleDomain())).collect(toList());
    // All partition key domains will be fully evaluated, so we don't need to include those
    TupleDomain<ColumnHandle> remainingTupleDomain = TupleDomain.none();
    if (!tupleDomain.isNone()) {
        if (partitions.size() == 1 && partitions.get(0).isUnpartitioned()) {
            remainingTupleDomain = tupleDomain;
        } else {
            @SuppressWarnings({ "rawtypes", "unchecked" }) List<ColumnHandle> partitionColumns = (List) partitionKeys;
            remainingTupleDomain = TupleDomain.withColumnDomains(Maps.filterKeys(tupleDomain.getDomains().get(), not(in(partitionColumns))));
        }
    }
    // push down indexed column fixed value predicates only for unpartitioned partition which uses token range query
    if ((partitions.size() == 1) && partitions.get(0).isUnpartitioned()) {
        Map<ColumnHandle, Domain> domains = tupleDomain.getDomains().get();
        List<ColumnHandle> indexedColumns = new ArrayList<>();
        // compose partitionId by using indexed column
        StringBuilder sb = new StringBuilder();
        for (Map.Entry<ColumnHandle, Domain> entry : domains.entrySet()) {
            CassandraColumnHandle column = (CassandraColumnHandle) entry.getKey();
            Domain domain = entry.getValue();
            if (column.isIndexed() && domain.isSingleValue()) {
                sb.append(CassandraCqlUtils.validColumnName(column.getName())).append(" = ").append(CassandraCqlUtils.cqlValue(toCQLCompatibleString(entry.getValue().getSingleValue()), column.getCassandraType()));
                indexedColumns.add(column);
                // Only one indexed column predicate can be pushed down.
                break;
            }
        }
        if (sb.length() > 0) {
            CassandraPartition partition = partitions.get(0);
            TupleDomain<ColumnHandle> filterIndexedColumn = TupleDomain.withColumnDomains(Maps.filterKeys(remainingTupleDomain.getDomains().get(), not(in(indexedColumns))));
            partitions = new ArrayList<>();
            partitions.add(new CassandraPartition(partition.getKey(), sb.toString(), filterIndexedColumn, true));
            return new CassandraPartitionResult(partitions, filterIndexedColumn);
        }
    }
    return new CassandraPartitionResult(partitions, remainingTupleDomain);
}