Examples with HdfsContext io.prestosql.plugin.hive.HdfsEnvironment.HdfsContext used on opensource projects

Example 11 with HdfsContext

use of io.prestosql.plugin.hive.HdfsEnvironment.HdfsContext in project hetu-core by openlookeng.

the class SemiTransactionalHiveMetastore method finishInsertIntoExistingPartition.

public synchronized void finishInsertIntoExistingPartition(ConnectorSession session, String databaseName, String tableName, List<String> partitionValues, Path currentLocation, List<String> fileNames, PartitionStatistics statisticsUpdate, HiveACIDWriteType acidWriteType) {
    setShared();
    isVacuumIncluded |= HiveACIDWriteType.isVacuum(acidWriteType);
    HiveIdentity identity = new HiveIdentity(session);
    SchemaTableName schemaTableName = new SchemaTableName(databaseName, tableName);
    Map<List<String>, Action<PartitionAndMore>> partitionActionsOfTable = partitionActions.computeIfAbsent(schemaTableName, k -> new LinkedHashMap<>());
    Action<PartitionAndMore> oldPartitionAction = partitionActionsOfTable.get(partitionValues);
    if (oldPartitionAction == null) {
        Partition partition = delegate.getPartition(identity, databaseName, tableName, partitionValues).orElseThrow(() -> new PartitionNotFoundException(schemaTableName, partitionValues));
        String partitionName = getPartitionName(identity, databaseName, tableName, partitionValues);
        PartitionStatistics mergedStatistics = statisticsUpdate;
        boolean updateStats = canUpdateStats(session, acidWriteType);
        if (updateStats) {
            PartitionStatistics currentStatistics = closure.getPartitionStatistics(identity, databaseName, tableName, ImmutableSet.of(partitionName)).get(partitionName);
            if (currentStatistics == null) {
                throw new PrestoException(HiveErrorCode.HIVE_METASTORE_ERROR, "currentStatistics is null");
            }
            mergedStatistics = merge(currentStatistics, statisticsUpdate);
        }
        HdfsContext context = new HdfsContext(session, databaseName, tableName);
        partitionActionsOfTable.put(partitionValues, new Action<>(ActionType.INSERT_EXISTING, new PartitionAndMore(identity, partition, currentLocation, Optional.of(fileNames), mergedStatistics, statisticsUpdate, updateStats), context, identity));
        return;
    }
    switch(oldPartitionAction.getType()) {
        case DROP:
            throw new PartitionNotFoundException(schemaTableName, partitionValues);
        case ADD:
        case ALTER:
        case INSERT_EXISTING:
            throw new UnsupportedOperationException("Inserting into a partition that were added, altered, or inserted into in the same transaction is not supported");
        default:
            throw new IllegalStateException("Unknown action type");
    }
}

Example 12 with HdfsContext

use of io.prestosql.plugin.hive.HdfsEnvironment.HdfsContext in project hetu-core by openlookeng.

the class SemiTransactionalHiveMetastore method addPartition.

public synchronized void addPartition(ConnectorSession session, String databaseName, String tableName, Partition partition, Path currentLocation, PartitionStatistics statistics, HiveACIDWriteType acidWriteType) {
    setShared();
    checkArgument(getPrestoQueryId(partition).isPresent());
    Map<List<String>, Action<PartitionAndMore>> partitionActionsOfTable = partitionActions.computeIfAbsent(new SchemaTableName(databaseName, tableName), k -> new ConcurrentHashMap<>());
    Action<PartitionAndMore> oldPartitionAction = partitionActionsOfTable.get(partition.getValues());
    HdfsContext hdfsContext = new HdfsContext(session, databaseName, tableName);
    HiveIdentity identity = new HiveIdentity(session);
    boolean canUpdateStats = canUpdateStats(session, acidWriteType);
    this.partitionCommitBatchSize = HiveSessionProperties.getMetastoreWriteBatchSize(session);
    if (oldPartitionAction == null) {
        partitionActionsOfTable.put(partition.getValues(), new Action<>(ActionType.ADD, new PartitionAndMore(identity, partition, currentLocation, Optional.empty(), statistics, statistics, canUpdateStats), hdfsContext, identity));
        return;
    }
    switch(oldPartitionAction.getType()) {
        case DROP:
            {
                if (!oldPartitionAction.getHdfsContext().getIdentity().getUser().equals(session.getUser())) {
                    throw new PrestoException(TRANSACTION_CONFLICT, "Operation on the same partition with different user in the same transaction is not supported");
                }
                partitionActionsOfTable.put(partition.getValues(), new Action<>(ActionType.ALTER, new PartitionAndMore(identity, partition, currentLocation, Optional.empty(), statistics, statistics, canUpdateStats), hdfsContext, identity));
                break;
            }
        case ADD:
        case ALTER:
        case INSERT_EXISTING:
            throw new PrestoException(ALREADY_EXISTS, format("Partition already exists for table '%s.%s': %s", databaseName, tableName, partition.getValues()));
        default:
            throw new IllegalStateException("Unknown action type");
    }
}

Example 13 with HdfsContext

use of io.prestosql.plugin.hive.HdfsEnvironment.HdfsContext in project hetu-core by openlookeng.

the class HiveMetadata method createEmptyFiles.

private void createEmptyFiles(ConnectorSession session, Path path, Table table, Optional<Partition> partition, List<String> fileNames) {
    JobConf conf = ConfigurationUtils.toJobConf(hdfsEnvironment.getConfiguration(new HdfsContext(session, table.getDatabaseName(), table.getTableName()), path));
    Properties schema;
    StorageFormat format;
    if (partition.isPresent()) {
        schema = MetastoreUtil.getHiveSchema(partition.get(), table);
        format = partition.get().getStorage().getStorageFormat();
    } else {
        schema = MetastoreUtil.getHiveSchema(table);
        format = table.getStorage().getStorageFormat();
    }
    hdfsEnvironment.doAs(session.getUser(), () -> {
        for (String fileName : fileNames) {
            writeEmptyFile(session, new Path(path, fileName), conf, schema, format.getSerDe(), format.getOutputFormat());
        }
    });
}

Example 14 with HdfsContext

use of io.prestosql.plugin.hive.HdfsEnvironment.HdfsContext in project hetu-core by openlookeng.

the class HiveMetadata method getTableModificationTime.

@Override
public long getTableModificationTime(ConnectorSession session, ConnectorTableHandle tableHandle) {
    SchemaTableName tableName = ((HiveTableHandle) tableHandle).getSchemaTableName();
    Table table = metastore.getTable(new HiveIdentity(session), tableName.getSchemaName(), tableName.getTableName()).orElseThrow(() -> new TableNotFoundException(tableName));
    String tableLocation = table.getStorage().getLocation();
    Path tablePath = new Path(tableLocation);
    try {
        FileSystem fileSystem = this.hdfsEnvironment.getFileSystem(new HdfsContext(session, tableName.getSchemaName()), tablePath);
        // since HDFS is append-only and any table modification will trigger directory update.
        return fileSystem.getFileStatus(tablePath).getModificationTime();
    }// We want to make sure the query doesn't fail because of star-tree not being able to get last modified time
     catch (Exception e) {
        log.error("Exception thrown while trying to get modified time", e);
        return -1L;
    }
}

Example 15 with HdfsContext

use of io.prestosql.plugin.hive.HdfsEnvironment.HdfsContext in project hetu-core by openlookeng.

the class HiveMetadata method computeFileNamesForMissingBuckets.

private List<String> computeFileNamesForMissingBuckets(ConnectorSession session, Table table, HiveStorageFormat storageFormat, Path targetPath, int bucketCount, PartitionUpdate partitionUpdate) {
    if (partitionUpdate.getFileNames().size() == bucketCount) {
        // fast path for common case
        return ImmutableList.of();
    }
    HdfsContext hdfsContext = new HdfsContext(session, table.getDatabaseName(), table.getTableName());
    JobConf conf = ConfigurationUtils.toJobConf(hdfsEnvironment.getConfiguration(hdfsContext, targetPath));
    String fileExtension = HiveWriterFactory.getFileExtension(conf, StorageFormat.fromHiveStorageFormat(storageFormat));
    Set<String> fileNames = ImmutableSet.copyOf(partitionUpdate.getFileNames());
    ImmutableList.Builder<String> missingFileNamesBuilder = ImmutableList.builder();
    for (int i = 0; i < bucketCount; i++) {
        String fileName = HiveWriterFactory.computeBucketedFileName(session.getQueryId(), i) + fileExtension;
        if (!fileNames.contains(fileName)) {
            missingFileNamesBuilder.add(fileName);
        }
    }
    List<String> missingFileNames = missingFileNamesBuilder.build();
    verify(fileNames.size() + missingFileNames.size() == bucketCount);
    return missingFileNames;
}