Examples with WriteInfo com.facebook.presto.hive.LocationService.WriteInfo used on opensource projects

Example 1 with WriteInfo

use of com.facebook.presto.hive.LocationService.WriteInfo in project presto by prestodb.

the class HiveWriterFactory method getWriterParametersForOverwritePartition.

private WriterParameters getWriterParametersForOverwritePartition(String partitionName) {
    // Overwrite an existing partition
    // 
    // The behavior of overwrite considered as if first dropping the partition and inserting a new partition, thus:
    // * No partition writable check is required.
    // * Table schema and storage format is used for the new partition (instead of existing partition schema and storage format).
    WriteInfo writeInfo = locationService.getPartitionWriteInfo(locationHandle, Optional.empty(), partitionName);
    checkState(writeInfo.getWriteMode() != DIRECT_TO_TARGET_EXISTING_DIRECTORY, "Overwriting existing partition doesn't support DIRECT_TO_TARGET_EXISTING_DIRECTORY write mode");
    return new WriterParameters(UpdateMode.OVERWRITE, getHiveSchema(table), writeInfo, fromHiveStorageFormat(partitionStorageFormat));
}

Example 2 with WriteInfo

use of com.facebook.presto.hive.LocationService.WriteInfo in project presto by prestodb.

the class AbstractTestHiveClient method getStagingPathRoot.

// These are protected so extensions to the hive connector can replace the handle classes
protected Path getStagingPathRoot(ConnectorInsertTableHandle insertTableHandle) {
    HiveInsertTableHandle handle = (HiveInsertTableHandle) insertTableHandle;
    WriteInfo writeInfo = getLocationService().getQueryWriteInfo(handle.getLocationHandle());
    if (writeInfo.getWriteMode() != STAGE_AND_MOVE_TO_TARGET_DIRECTORY) {
        throw new AssertionError("writeMode is not STAGE_AND_MOVE_TO_TARGET_DIRECTORY");
    }
    return writeInfo.getWritePath();
}

Example 3 with WriteInfo

use of com.facebook.presto.hive.LocationService.WriteInfo in project presto by prestodb.

the class CreateEmptyPartitionProcedure method doCreateEmptyPartition.

private void doCreateEmptyPartition(ConnectorSession session, String schema, String table, List<Object> partitionColumnNames, List<Object> partitionValues) {
    TransactionalMetadata hiveMetadata = hiveMetadataFactory.get();
    HiveInsertTableHandle hiveInsertTableHandle = (HiveInsertTableHandle) hiveMetadata.beginInsert(session, new HiveTableHandle(schema, table));
    List<String> actualPartitionColumnNames = hiveInsertTableHandle.getInputColumns().stream().filter(HiveColumnHandle::isPartitionKey).map(HiveColumnHandle::getName).collect(toImmutableList());
    if (!Objects.equals(partitionColumnNames, actualPartitionColumnNames)) {
        throw new PrestoException(INVALID_PROCEDURE_ARGUMENT, "input partition column names doesn't match actual partition column names");
    }
    List<String> partitionStringValues = partitionValues.stream().map(String.class::cast).collect(toImmutableList());
    if (metastore.getPartition(new MetastoreContext(session.getIdentity(), session.getQueryId(), session.getClientInfo(), session.getSource(), getMetastoreHeaders(session), false, HiveColumnConverterProvider.DEFAULT_COLUMN_CONVERTER_PROVIDER), schema, table, partitionStringValues).isPresent()) {
        throw new PrestoException(ALREADY_EXISTS, "Partition already exists");
    }
    String partitionName = FileUtils.makePartName(actualPartitionColumnNames, partitionStringValues);
    WriteInfo writeInfo = locationService.getPartitionWriteInfo(hiveInsertTableHandle.getLocationHandle(), Optional.empty(), partitionName);
    PartitionUpdate partitionUpdate = new PartitionUpdate(partitionName, UpdateMode.NEW, writeInfo.getWritePath(), writeInfo.getTargetPath(), ImmutableList.of(), 0, 0, 0, writeInfo.getWritePath().getName().matches("\\d+"));
    byte[] serializedPartitionUpdate;
    if (isOptimizedPartitionUpdateSerializationEnabled(session)) {
        serializedPartitionUpdate = serializeZstdCompressed(partitionUpdateSmileCodec, partitionUpdate);
    } else {
        serializedPartitionUpdate = partitionUpdateJsonCodec.toJsonBytes(partitionUpdate);
    }
    hiveMetadata.finishInsert(session, hiveInsertTableHandle, ImmutableList.of(Slices.wrappedBuffer(serializedPartitionUpdate)), ImmutableList.of());
    hiveMetadata.commit();
}

Example 4 with WriteInfo

use of com.facebook.presto.hive.LocationService.WriteInfo in project presto by prestodb.

the class HiveMetadata method beginInsertInternal.

private HiveInsertTableHandle beginInsertInternal(ConnectorSession session, ConnectorTableHandle tableHandle) {
    verifyJvmTimeZone();
    MetastoreContext metastoreContext = getMetastoreContext(session);
    SchemaTableName tableName = ((HiveTableHandle) tableHandle).getSchemaTableName();
    Table table = metastore.getTable(metastoreContext, tableName.getSchemaName(), tableName.getTableName()).orElseThrow(() -> new TableNotFoundException(tableName));
    checkTableIsWritable(table, writesToNonManagedTablesEnabled);
    for (Column column : table.getDataColumns()) {
        if (!isWritableType(column.getType())) {
            throw new PrestoException(NOT_SUPPORTED, format("Inserting into Hive table %s with column type %s not supported", tableName, column.getType()));
        }
    }
    List<HiveColumnHandle> handles = hiveColumnHandles(table).stream().filter(columnHandle -> !columnHandle.isHidden()).collect(toList());
    HiveStorageFormat tableStorageFormat = extractHiveStorageFormat(table);
    LocationHandle locationHandle;
    boolean isTemporaryTable = table.getTableType().equals(TEMPORARY_TABLE);
    boolean tempPathRequired = isTempPathRequired(session, table.getStorage().getBucketProperty(), decodePreferredOrderingColumnsFromStorage(table.getStorage()));
    if (isTemporaryTable) {
        locationHandle = locationService.forTemporaryTable(metastore, session, table, tempPathRequired);
    } else {
        locationHandle = locationService.forExistingTable(metastore, session, table, tempPathRequired);
    }
    Optional<? extends TableEncryptionProperties> tableEncryptionProperties = getTableEncryptionPropertiesFromHiveProperties(table.getParameters(), tableStorageFormat);
    HiveStorageFormat partitionStorageFormat = isRespectTableFormat(session) ? tableStorageFormat : getHiveStorageFormat(session);
    HiveStorageFormat actualStorageFormat = table.getPartitionColumns().isEmpty() ? tableStorageFormat : partitionStorageFormat;
    if (tableEncryptionProperties.isPresent() && actualStorageFormat != tableStorageFormat) {
        throw new PrestoException(INVALID_TABLE_PROPERTY, format("For encrypted tables, partition format (%s) should match table format (%s). Using the session property %s or appropriately setting %s can help with ensuring this", actualStorageFormat.name(), tableStorageFormat.name(), RESPECT_TABLE_FORMAT, HIVE_STORAGE_FORMAT));
    }
    HiveInsertTableHandle result = new HiveInsertTableHandle(tableName.getSchemaName(), tableName.getTableName(), handles, metastore.generatePageSinkMetadata(metastoreContext, tableName), locationHandle, table.getStorage().getBucketProperty(), decodePreferredOrderingColumnsFromStorage(table.getStorage()), tableStorageFormat, partitionStorageFormat, actualStorageFormat, getHiveCompressionCodec(session, isTemporaryTable, actualStorageFormat), encryptionInformationProvider.getWriteEncryptionInformation(session, tableEncryptionProperties.map(identity()), tableName.getSchemaName(), tableName.getTableName()));
    WriteInfo writeInfo = locationService.getQueryWriteInfo(locationHandle);
    metastore.declareIntentionToWrite(new HdfsContext(session, tableName.getSchemaName(), tableName.getTableName(), table.getStorage().getLocation(), false), metastoreContext, writeInfo.getWriteMode(), writeInfo.getWritePath(), writeInfo.getTempPath(), tableName, isTemporaryTable);
    return result;
}

Example 5 with WriteInfo

use of com.facebook.presto.hive.LocationService.WriteInfo in project presto by prestodb.

the class HiveMetadata method finishCreateTable.

@Override
public Optional<ConnectorOutputMetadata> finishCreateTable(ConnectorSession session, ConnectorOutputTableHandle tableHandle, Collection<Slice> fragments, Collection<ComputedStatistics> computedStatistics) {
    HiveOutputTableHandle handle = (HiveOutputTableHandle) tableHandle;
    List<PartitionUpdate> partitionUpdates = getPartitionUpdates(session, fragments);
    Map<String, String> tableEncryptionParameters = ImmutableMap.of();
    Map<String, String> partitionEncryptionParameters = ImmutableMap.of();
    if (handle.getEncryptionInformation().isPresent()) {
        EncryptionInformation encryptionInformation = handle.getEncryptionInformation().get();
        if (encryptionInformation.getDwrfEncryptionMetadata().isPresent()) {
            if (handle.getPartitionedBy().isEmpty()) {
                tableEncryptionParameters = ImmutableMap.copyOf(encryptionInformation.getDwrfEncryptionMetadata().get().getExtraMetadata());
            } else {
                partitionEncryptionParameters = ImmutableMap.copyOf(encryptionInformation.getDwrfEncryptionMetadata().get().getExtraMetadata());
            }
        }
    }
    WriteInfo writeInfo = locationService.getQueryWriteInfo(handle.getLocationHandle());
    MetastoreContext metastoreContext = getMetastoreContext(session);
    Table table = buildTableObject(session.getQueryId(), handle.getSchemaName(), handle.getTableName(), handle.getTableOwner(), handle.getInputColumns(), handle.getTableStorageFormat(), handle.getPartitionedBy(), handle.getBucketProperty(), handle.getPreferredOrderingColumns(), ImmutableMap.<String, String>builder().putAll(handle.getAdditionalTableParameters()).putAll(tableEncryptionParameters).build(), writeInfo.getTargetPath(), MANAGED_TABLE, prestoVersion, metastoreContext);
    PrincipalPrivileges principalPrivileges = buildInitialPrivilegeSet(handle.getTableOwner());
    partitionUpdates = PartitionUpdate.mergePartitionUpdates(partitionUpdates);
    if (handle.getBucketProperty().isPresent() && isCreateEmptyBucketFiles(session)) {
        List<PartitionUpdate> partitionUpdatesForMissingBuckets = computePartitionUpdatesForMissingBuckets(session, handle, table, partitionUpdates);
        // replace partitionUpdates before creating the zero-row files so that those files will be cleaned up if we end up rollback
        partitionUpdates = PartitionUpdate.mergePartitionUpdates(concat(partitionUpdates, partitionUpdatesForMissingBuckets));
        HdfsContext hdfsContext = new HdfsContext(session, table.getDatabaseName(), table.getTableName(), table.getStorage().getLocation(), true);
        for (PartitionUpdate partitionUpdate : partitionUpdatesForMissingBuckets) {
            Optional<Partition> partition = table.getPartitionColumns().isEmpty() ? Optional.empty() : Optional.of(partitionObjectBuilder.buildPartitionObject(session, table, partitionUpdate, prestoVersion, partitionEncryptionParameters));
            zeroRowFileCreator.createFiles(session, hdfsContext, partitionUpdate.getWritePath(), getTargetFileNames(partitionUpdate.getFileWriteInfos()), getStorageFormat(partition, table), handle.getCompressionCodec(), getSchema(partition, table));
        }
    }
    Map<String, Type> columnTypes = handle.getInputColumns().stream().collect(toImmutableMap(HiveColumnHandle::getName, column -> column.getHiveType().getType(typeManager)));
    Map<List<String>, ComputedStatistics> partitionComputedStatistics = createComputedStatisticsToPartitionMap(computedStatistics, handle.getPartitionedBy(), columnTypes);
    PartitionStatistics tableStatistics;
    if (table.getPartitionColumns().isEmpty()) {
        HiveBasicStatistics basicStatistics = partitionUpdates.stream().map(PartitionUpdate::getStatistics).reduce((first, second) -> reduce(first, second, ADD)).orElse(createZeroStatistics());
        tableStatistics = createPartitionStatistics(session, basicStatistics, columnTypes, getColumnStatistics(partitionComputedStatistics, ImmutableList.of()));
    } else {
        tableStatistics = new PartitionStatistics(createEmptyStatistics(), ImmutableMap.of());
    }
    metastore.createTable(session, table, principalPrivileges, Optional.of(writeInfo.getWritePath()), false, tableStatistics);
    if (handle.getPartitionedBy().isEmpty()) {
        return Optional.of(new HiveWrittenPartitions(ImmutableList.of(UNPARTITIONED_ID)));
    }
    if (isRespectTableFormat(session)) {
        verify(handle.getPartitionStorageFormat() == handle.getTableStorageFormat());
    }
    for (PartitionUpdate update : partitionUpdates) {
        Map<String, String> partitionParameters = partitionEncryptionParameters;
        if (isPreferManifestsToListFiles(session) && isFileRenamingEnabled(session)) {
            // Store list of file names and sizes in partition metadata when prefer_manifests_to_list_files and file_renaming_enabled are set to true
            partitionParameters = updatePartitionMetadataWithFileNamesAndSizes(update, partitionParameters);
        }
        Partition partition = partitionObjectBuilder.buildPartitionObject(session, table, update, prestoVersion, partitionParameters);
        PartitionStatistics partitionStatistics = createPartitionStatistics(session, update.getStatistics(), columnTypes, getColumnStatistics(partitionComputedStatistics, partition.getValues()));
        metastore.addPartition(session, handle.getSchemaName(), handle.getTableName(), table.getStorage().getLocation(), true, partitionObjectBuilder.buildPartitionObject(session, table, update, prestoVersion, partitionParameters), update.getWritePath(), partitionStatistics);
    }
    return Optional.of(new HiveWrittenPartitions(partitionUpdates.stream().map(PartitionUpdate::getName).collect(toList())));
}