Examples with NOT_SUPPORTED com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED used on opensource projects

Example 1 with NOT_SUPPORTED

use of com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED in project presto by prestodb.

the class HiveMetadata method beginInsert.

@Override
public HiveInsertTableHandle beginInsert(ConnectorSession session, ConnectorTableHandle tableHandle) {
    verifyJvmTimeZone();
    SchemaTableName tableName = schemaTableName(tableHandle);
    Optional<Table> table = metastore.getTable(tableName.getSchemaName(), tableName.getTableName());
    if (!table.isPresent()) {
        throw new TableNotFoundException(tableName);
    }
    checkTableIsWritable(table.get());
    for (Column column : table.get().getDataColumns()) {
        if (!isWritableType(column.getType())) {
            throw new PrestoException(NOT_SUPPORTED, format("Inserting into Hive table %s.%s with column type %s not supported", table.get().getDatabaseName(), table.get().getTableName(), column.getType()));
        }
    }
    List<HiveColumnHandle> handles = hiveColumnHandles(connectorId, table.get()).stream().filter(columnHandle -> !columnHandle.isHidden()).collect(toList());
    HiveStorageFormat tableStorageFormat = extractHiveStorageFormat(table.get());
    LocationHandle locationHandle = locationService.forExistingTable(metastore, session.getUser(), session.getQueryId(), table.get());
    HiveInsertTableHandle result = new HiveInsertTableHandle(connectorId, tableName.getSchemaName(), tableName.getTableName(), handles, session.getQueryId(), metastore.generatePageSinkMetadata(tableName), locationHandle, table.get().getStorage().getBucketProperty(), tableStorageFormat, respectTableFormat ? tableStorageFormat : defaultStorageFormat);
    Optional<Path> writePathRoot = locationService.writePathRoot(locationHandle);
    Path targetPathRoot = locationService.targetPathRoot(locationHandle);
    if (writePathRoot.isPresent()) {
        WriteMode mode = writePathRoot.get().equals(targetPathRoot) ? DIRECT_TO_TARGET_NEW_DIRECTORY : STAGE_AND_MOVE_TO_TARGET_DIRECTORY;
        metastore.declareIntentionToWrite(session, mode, writePathRoot.get(), result.getFilePrefix(), tableName);
    } else {
        metastore.declareIntentionToWrite(session, DIRECT_TO_TARGET_EXISTING_DIRECTORY, targetPathRoot, result.getFilePrefix(), tableName);
    }
    return result;
}

Example 2 with NOT_SUPPORTED

use of com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED in project presto by prestodb.

the class StoragePartitionLoader method getBucketedSplits.

private List<InternalHiveSplit> getBucketedSplits(Path path, ExtendedFileSystem fileSystem, InternalHiveSplitFactory splitFactory, BucketSplitInfo bucketSplitInfo, Optional<HiveSplit.BucketConversion> bucketConversion, String partitionName, boolean splittable, PathFilter pathFilter) {
    int readBucketCount = bucketSplitInfo.getReadBucketCount();
    int tableBucketCount = bucketSplitInfo.getTableBucketCount();
    int partitionBucketCount = bucketConversion.map(HiveSplit.BucketConversion::getPartitionBucketCount).orElse(tableBucketCount);
    int bucketCount = max(readBucketCount, partitionBucketCount);
    checkState(readBucketCount <= tableBucketCount, "readBucketCount(%s) should be less than or equal to tableBucketCount(%s)", readBucketCount, tableBucketCount);
    // list all files in the partition
    List<HiveFileInfo> fileInfos = new ArrayList<>(partitionBucketCount);
    try {
        Iterators.addAll(fileInfos, directoryLister.list(fileSystem, table, path, namenodeStats, pathFilter, new HiveDirectoryContext(FAIL, isUseListDirectoryCache(session))));
    } catch (HiveFileIterator.NestedDirectoryNotAllowedException e) {
        // Fail here to be on the safe side. This seems to be the same as what Hive does
        throw new PrestoException(HIVE_INVALID_BUCKET_FILES, format("Hive table '%s' is corrupt. Found sub-directory in bucket directory for partition: %s", table.getSchemaTableName(), partitionName));
    }
    ListMultimap<Integer, HiveFileInfo> bucketToFileInfo = ArrayListMultimap.create();
    if (!shouldCreateFilesForMissingBuckets(table, session)) {
        fileInfos.stream().forEach(fileInfo -> {
            String fileName = fileInfo.getPath().getName();
            OptionalInt bucket = getBucketNumber(fileName);
            if (bucket.isPresent()) {
                bucketToFileInfo.put(bucket.getAsInt(), fileInfo);
            } else {
                throw new PrestoException(HIVE_INVALID_BUCKET_FILES, format("invalid hive bucket file name: %s", fileName));
            }
        });
    } else {
        // build mapping of file name to bucket
        for (HiveFileInfo file : fileInfos) {
            String fileName = file.getPath().getName();
            OptionalInt bucket = getBucketNumber(fileName);
            if (bucket.isPresent()) {
                bucketToFileInfo.put(bucket.getAsInt(), file);
                continue;
            }
            // legacy mode requires exactly one file per bucket
            if (fileInfos.size() != partitionBucketCount) {
                throw new PrestoException(HIVE_INVALID_BUCKET_FILES, format("Hive table '%s' is corrupt. File '%s' does not match the standard naming pattern, and the number " + "of files in the directory (%s) does not match the declared bucket count (%s) for partition: %s", table.getSchemaTableName(), fileName, fileInfos.size(), partitionBucketCount, partitionName));
            }
            if (fileInfos.get(0).getPath().getName().matches("\\d+")) {
                try {
                    // File names are integer if they are created when file_renaming_enabled is set to true
                    fileInfos.sort(Comparator.comparingInt(fileInfo -> Integer.parseInt(fileInfo.getPath().getName())));
                } catch (NumberFormatException e) {
                    throw new PrestoException(HIVE_INVALID_FILE_NAMES, format("Hive table '%s' is corrupt. Some of the filenames in the partition: %s are not integers", new SchemaTableName(table.getDatabaseName(), table.getTableName()), partitionName));
                }
            } else {
                // Sort FileStatus objects (instead of, e.g., fileStatus.getPath().toString). This matches org.apache.hadoop.hive.ql.metadata.Table.getSortedPaths
                fileInfos.sort(null);
            }
            // Use position in sorted list as the bucket number
            bucketToFileInfo.clear();
            for (int i = 0; i < fileInfos.size(); i++) {
                bucketToFileInfo.put(i, fileInfos.get(i));
            }
            break;
        }
    }
    // convert files internal splits
    List<InternalHiveSplit> splitList = new ArrayList<>();
    for (int bucketNumber = 0; bucketNumber < bucketCount; bucketNumber++) {
        // Physical bucket #. This determine file name. It also determines the order of splits in the result.
        int partitionBucketNumber = bucketNumber % partitionBucketCount;
        if (!bucketToFileInfo.containsKey(partitionBucketNumber)) {
            continue;
        }
        // Logical bucket #. Each logical bucket corresponds to a "bucket" from engine's perspective.
        int readBucketNumber = bucketNumber % readBucketCount;
        boolean containsIneligibleTableBucket = false;
        List<Integer> eligibleTableBucketNumbers = new ArrayList<>();
        for (int tableBucketNumber = bucketNumber % tableBucketCount; tableBucketNumber < tableBucketCount; tableBucketNumber += bucketCount) {
            // table bucket number: this is used for evaluating "$bucket" filters.
            if (bucketSplitInfo.isTableBucketEnabled(tableBucketNumber)) {
                eligibleTableBucketNumbers.add(tableBucketNumber);
            } else {
                containsIneligibleTableBucket = true;
            }
        }
        if (!eligibleTableBucketNumbers.isEmpty() && containsIneligibleTableBucket) {
            throw new PrestoException(NOT_SUPPORTED, "The bucket filter cannot be satisfied. There are restrictions on the bucket filter when all the following is true: " + "1. a table has a different buckets count as at least one of its partitions that is read in this query; " + "2. the table has a different but compatible bucket number with another table in the query; " + "3. some buckets of the table is filtered out from the query, most likely using a filter on \"$bucket\". " + "(table name: " + table.getTableName() + ", table bucket count: " + tableBucketCount + ", " + "partition bucket count: " + partitionBucketCount + ", effective reading bucket count: " + readBucketCount + ")");
        }
        if (!eligibleTableBucketNumbers.isEmpty()) {
            for (HiveFileInfo fileInfo : bucketToFileInfo.get(partitionBucketNumber)) {
                eligibleTableBucketNumbers.stream().map(tableBucketNumber -> splitFactory.createInternalHiveSplit(fileInfo, readBucketNumber, tableBucketNumber, splittable)).forEach(optionalSplit -> optionalSplit.ifPresent(splitList::add));
            }
        }
    }
    return splitList;
}

Example 3 with NOT_SUPPORTED

use of com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED in project presto by prestodb.

the class ConnectorMetadata method getInsertLayout.

/**
 * Get the physical layout for a inserting into an existing table.
 */
default Optional<ConnectorNewTableLayout> getInsertLayout(ConnectorSession session, ConnectorTableHandle tableHandle) {
    List<ConnectorTableLayout> layouts = getTableLayouts(session, tableHandle, new Constraint<>(TupleDomain.all(), map -> true), Optional.empty()).stream().map(ConnectorTableLayoutResult::getTableLayout).filter(layout -> layout.getTablePartitioning().isPresent()).collect(toList());
    if (layouts.isEmpty()) {
        return Optional.empty();
    }
    if (layouts.size() > 1) {
        throw new PrestoException(NOT_SUPPORTED, "Tables with multiple layouts can not be written");
    }
    ConnectorTableLayout layout = layouts.get(0);
    ConnectorPartitioningHandle partitioningHandle = layout.getTablePartitioning().get().getPartitioningHandle();
    Map<ColumnHandle, String> columnNamesByHandle = getColumnHandles(session, tableHandle).entrySet().stream().collect(Collectors.toMap(Map.Entry::getValue, Map.Entry::getKey));
    List<String> partitionColumns = layout.getTablePartitioning().get().getPartitioningColumns().stream().map(columnNamesByHandle::get).collect(toList());
    return Optional.of(new ConnectorNewTableLayout(partitioningHandle, partitionColumns));
}

Example 4 with NOT_SUPPORTED

use of com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED in project presto by prestodb.

the class HiveMetadata method prepareTable.

private Table prepareTable(ConnectorSession session, ConnectorTableMetadata tableMetadata, PrestoTableType tableType) {
    SchemaTableName schemaTableName = tableMetadata.getTable();
    String schemaName = schemaTableName.getSchemaName();
    String tableName = schemaTableName.getTableName();
    List<String> partitionedBy = getPartitionedBy(tableMetadata.getProperties());
    Optional<HiveBucketProperty> bucketProperty = getBucketProperty(tableMetadata.getProperties());
    if ((bucketProperty.isPresent() || !partitionedBy.isEmpty()) && getAvroSchemaUrl(tableMetadata.getProperties()) != null) {
        throw new PrestoException(NOT_SUPPORTED, "Bucketing/Partitioning columns not supported when Avro schema url is set");
    }
    List<HiveColumnHandle> columnHandles = getColumnHandles(tableMetadata, ImmutableSet.copyOf(partitionedBy), typeTranslator);
    HiveStorageFormat hiveStorageFormat = getHiveStorageFormat(tableMetadata.getProperties());
    List<SortingColumn> preferredOrderingColumns = getPreferredOrderingColumns(tableMetadata.getProperties());
    Optional<TableEncryptionProperties> tableEncryptionProperties = getTableEncryptionPropertiesFromTableProperties(tableMetadata, hiveStorageFormat, partitionedBy);
    if (tableEncryptionProperties.isPresent() && partitionedBy.isEmpty()) {
        throw new PrestoException(HIVE_UNSUPPORTED_ENCRYPTION_OPERATION, "Creating an encrypted table without partitions is not supported. Use CREATE TABLE AS SELECT to " + "create an encrypted table without partitions");
    }
    validateColumns(hiveStorageFormat, columnHandles);
    MetastoreContext metastoreContext = getMetastoreContext(session);
    Map<String, HiveColumnHandle> columnHandlesByName = Maps.uniqueIndex(columnHandles, HiveColumnHandle::getName);
    List<Column> partitionColumns = partitionedBy.stream().map(columnHandlesByName::get).map(columnHandle -> columnHandleToColumn(metastoreContext, columnHandle)).collect(toList());
    checkPartitionTypesSupported(partitionColumns);
    Path targetPath;
    if (tableType.equals(EXTERNAL_TABLE)) {
        if (!createsOfNonManagedTablesEnabled) {
            throw new PrestoException(NOT_SUPPORTED, "Cannot create non-managed Hive table");
        }
        String externalLocation = getExternalLocation(tableMetadata.getProperties());
        targetPath = getExternalPath(new HdfsContext(session, schemaName, tableName, externalLocation, true), externalLocation);
    } else if (tableType.equals(MANAGED_TABLE) || tableType.equals(MATERIALIZED_VIEW)) {
        LocationHandle locationHandle = locationService.forNewTable(metastore, session, schemaName, tableName, isTempPathRequired(session, bucketProperty, preferredOrderingColumns));
        targetPath = locationService.getQueryWriteInfo(locationHandle).getTargetPath();
    } else {
        throw new IllegalStateException(format("%s is not a valid table type to be created.", tableType));
    }
    Map<String, String> tableProperties = getEmptyTableProperties(tableMetadata, new HdfsContext(session, schemaName, tableName, targetPath.toString(), true), hiveStorageFormat, tableEncryptionProperties);
    return buildTableObject(session.getQueryId(), schemaName, tableName, session.getUser(), columnHandles, hiveStorageFormat, partitionedBy, bucketProperty, preferredOrderingColumns, tableProperties, targetPath, tableType, prestoVersion, metastoreContext);
}

Example 5 with NOT_SUPPORTED

use of com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED in project presto by prestodb.

the class HiveMaterializedViewUtils method validateMaterializedViewPartitionColumns.

/**
 * Validate the partition columns of a materialized view to ensure 1) a materialized view is partitioned; 2) it has at least one partition
 * directly mapped to all base tables and 3) Outer join conditions have common partitions that are partitions in the view as well
 * <p>
 * A column is directly mapped to a base table column if it is derived directly or transitively from the base table column,
 * by only selecting a column or an aliased column without any function or operator applied.
 * For example, with SELECT column_b AS column_a, column_a is directly mapped to column_b.
 * With SELECT column_b + column_c AS column_a, column_a is not directly mapped to any column.
 * <p>
 * {@code viewToBaseColumnMap} only contains direct column mappings.
 */
public static void validateMaterializedViewPartitionColumns(SemiTransactionalHiveMetastore metastore, MetastoreContext metastoreContext, Table viewTable, ConnectorMaterializedViewDefinition viewDefinition) {
    SchemaTableName viewName = new SchemaTableName(viewTable.getDatabaseName(), viewTable.getTableName());
    Map<String, Map<SchemaTableName, String>> viewToBaseDirectColumnMap = viewDefinition.getDirectColumnMappingsAsMap();
    if (viewToBaseDirectColumnMap.isEmpty()) {
        throw new PrestoException(NOT_SUPPORTED, format("Materialized view %s must have at least one column directly defined by a base table column.", viewName));
    }
    List<Column> viewPartitions = viewTable.getPartitionColumns();
    if (viewPartitions.isEmpty()) {
        throw new PrestoException(NOT_SUPPORTED, "Unpartitioned materialized view is not supported.");
    }
    List<Table> baseTables = viewDefinition.getBaseTables().stream().map(baseTableName -> metastore.getTable(metastoreContext, baseTableName.getSchemaName(), baseTableName.getTableName()).orElseThrow(() -> new TableNotFoundException(baseTableName))).collect(toImmutableList());
    Map<Table, List<Column>> baseTablePartitions = baseTables.stream().collect(toImmutableMap(table -> table, Table::getPartitionColumns));
    for (Table baseTable : baseTablePartitions.keySet()) {
        SchemaTableName schemaBaseTable = new SchemaTableName(baseTable.getDatabaseName(), baseTable.getTableName());
        if (!isCommonPartitionFound(schemaBaseTable, baseTablePartitions.get(baseTable), viewPartitions, viewToBaseDirectColumnMap)) {
            throw new PrestoException(NOT_SUPPORTED, format("Materialized view %s must have at least one partition column that exists in %s as well", viewName, baseTable.getTableName()));
        }
        if (viewDefinition.getBaseTablesOnOuterJoinSide().contains(schemaBaseTable) && viewToBaseTableOnOuterJoinSideIndirectMappedPartitions(viewDefinition, baseTable).get().isEmpty()) {
            throw new PrestoException(NOT_SUPPORTED, format("Outer join conditions in Materialized view %s must have at least one common partition equality constraint", viewName));
        }
    }
}