Search in sources :

Example 1 with PartitionsSaveRequest

use of com.netflix.metacat.common.server.connectors.model.PartitionsSaveRequest in project metacat by Netflix.

the class HiveConnectorPartitionService method savePartitions.

/**
     * {@inheritDoc}.
     */
@Override
public PartitionsSaveResponse savePartitions(@Nonnull @NonNull final ConnectorContext requestContext, @Nonnull @NonNull final QualifiedName tableName, @Nonnull @NonNull final PartitionsSaveRequest partitionsSaveRequest) {
    final String databasename = tableName.getDatabaseName();
    final String tablename = tableName.getTableName();
    // New partitions
    final List<Partition> hivePartitions = Lists.newArrayList();
    try {
        final Table table = metacatHiveClient.getTableByName(databasename, tablename);
        final List<PartitionInfo> partitionInfos = partitionsSaveRequest.getPartitions();
        // New partition ids
        final List<String> addedPartitionIds = Lists.newArrayList();
        // Updated partition ids
        final List<String> existingPartitionIds = Lists.newArrayList();
        // Existing partitions
        final List<Partition> existingHivePartitions = Lists.newArrayList();
        // Existing partition map
        Map<String, Partition> existingPartitionMap = Collections.emptyMap();
        if (partitionsSaveRequest.getCheckIfExists()) {
            final List<String> partitionNames = partitionInfos.stream().map(partition -> {
                final String partitionName = partition.getName().getPartitionName();
                PartitionUtil.validatePartitionName(partitionName, getPartitionKeys(table.getPartitionKeys()));
                return partitionName;
            }).collect(Collectors.toList());
            existingPartitionMap = getPartitionsByNames(table, partitionNames);
        }
        final TableInfo tableInfo = hiveMetacatConverters.toTableInfo(tableName, table);
        for (PartitionInfo partitionInfo : partitionInfos) {
            final String partitionName = partitionInfo.getName().getPartitionName();
            final Partition hivePartition = existingPartitionMap.get(partitionName);
            if (hivePartition == null) {
                addedPartitionIds.add(partitionName);
                hivePartitions.add(hiveMetacatConverters.fromPartitionInfo(tableInfo, partitionInfo));
            } else {
                //unless we alterifExists
                if (partitionsSaveRequest.getAlterIfExists()) {
                    final Partition existingPartition = hiveMetacatConverters.fromPartitionInfo(tableInfo, partitionInfo);
                    existingPartitionIds.add(partitionName);
                    existingPartition.setParameters(hivePartition.getParameters());
                    existingPartition.setCreateTime(hivePartition.getCreateTime());
                    existingPartition.setLastAccessTime(hivePartition.getLastAccessTime());
                    existingHivePartitions.add(existingPartition);
                }
            }
        }
        final Set<String> deletePartitionIds = Sets.newHashSet();
        if (!partitionsSaveRequest.getAlterIfExists()) {
            deletePartitionIds.addAll(existingPartitionIds);
        }
        if (partitionsSaveRequest.getPartitionIdsForDeletes() != null) {
            deletePartitionIds.addAll(partitionsSaveRequest.getPartitionIdsForDeletes());
        }
        if (partitionsSaveRequest.getAlterIfExists() && !existingHivePartitions.isEmpty()) {
            copyTableSdToPartitionSd(existingHivePartitions, table);
            metacatHiveClient.alterPartitions(databasename, tablename, existingHivePartitions);
        }
        copyTableSdToPartitionSd(hivePartitions, table);
        metacatHiveClient.addDropPartitions(databasename, tablename, hivePartitions, Lists.newArrayList(deletePartitionIds));
        final PartitionsSaveResponse result = new PartitionsSaveResponse();
        result.setAdded(addedPartitionIds);
        result.setUpdated(existingPartitionIds);
        return result;
    } catch (NoSuchObjectException exception) {
        if (exception.getMessage() != null && exception.getMessage().startsWith("Partition doesn't exist")) {
            throw new PartitionNotFoundException(tableName, "", exception);
        } else {
            throw new TableNotFoundException(tableName, exception);
        }
    } catch (MetaException | InvalidObjectException exception) {
        throw new InvalidMetaException("One or more partitions are invalid.", exception);
    } catch (AlreadyExistsException e) {
        final List<String> ids = getFakePartitionName(hivePartitions);
        throw new PartitionAlreadyExistsException(tableName, ids, e);
    } catch (TException exception) {
        throw new ConnectorException(String.format("Failed savePartitions hive table %s", tableName), exception);
    }
}
Also used : MetaException(org.apache.hadoop.hive.metastore.api.MetaException) SortOrder(com.netflix.metacat.common.dto.SortOrder) HashMap(java.util.HashMap) SerDeInfo(org.apache.hadoop.hive.metastore.api.SerDeInfo) Partition(org.apache.hadoop.hive.metastore.api.Partition) Function(java.util.function.Function) Warehouse(org.apache.hadoop.hive.metastore.Warehouse) ArrayList(java.util.ArrayList) AlreadyExistsException(org.apache.hadoop.hive.metastore.api.AlreadyExistsException) Inject(javax.inject.Inject) LinkedHashMap(java.util.LinkedHashMap) Strings(com.google.common.base.Strings) ConnectorPartitionService(com.netflix.metacat.common.server.connectors.ConnectorPartitionService) InvalidMetaException(com.netflix.metacat.common.server.connectors.exception.InvalidMetaException) Lists(com.google.common.collect.Lists) ConnectorException(com.netflix.metacat.common.server.connectors.exception.ConnectorException) PartitionInfo(com.netflix.metacat.common.server.connectors.model.PartitionInfo) Map(java.util.Map) ConnectorContext(com.netflix.metacat.common.server.connectors.ConnectorContext) Named(javax.inject.Named) HiveConnectorInfoConverter(com.netflix.metacat.connector.hive.converters.HiveConnectorInfoConverter) PartitionUtil(com.netflix.metacat.common.server.partition.util.PartitionUtil) StorageDescriptor(org.apache.hadoop.hive.metastore.api.StorageDescriptor) Nonnull(javax.annotation.Nonnull) Nullable(javax.annotation.Nullable) NonNull(lombok.NonNull) Pageable(com.netflix.metacat.common.dto.Pageable) TException(org.apache.thrift.TException) Set(java.util.Set) QualifiedName(com.netflix.metacat.common.QualifiedName) InvalidObjectException(org.apache.hadoop.hive.metastore.api.InvalidObjectException) TableNotFoundException(com.netflix.metacat.common.server.connectors.exception.TableNotFoundException) Collectors(java.util.stream.Collectors) Sets(com.google.common.collect.Sets) Table(org.apache.hadoop.hive.metastore.api.Table) PartitionsSaveResponse(com.netflix.metacat.common.server.connectors.model.PartitionsSaveResponse) FieldSchema(org.apache.hadoop.hive.metastore.api.FieldSchema) List(java.util.List) TableInfo(com.netflix.metacat.common.server.connectors.model.TableInfo) PartitionAlreadyExistsException(com.netflix.metacat.common.server.connectors.exception.PartitionAlreadyExistsException) PartitionsSaveRequest(com.netflix.metacat.common.server.connectors.model.PartitionsSaveRequest) PartitionListRequest(com.netflix.metacat.common.server.connectors.model.PartitionListRequest) ConnectorUtils(com.netflix.metacat.common.server.connectors.ConnectorUtils) PartitionNotFoundException(com.netflix.metacat.common.server.connectors.exception.PartitionNotFoundException) Collections(java.util.Collections) NoSuchObjectException(org.apache.hadoop.hive.metastore.api.NoSuchObjectException) Sort(com.netflix.metacat.common.dto.Sort) TException(org.apache.thrift.TException) Partition(org.apache.hadoop.hive.metastore.api.Partition) Table(org.apache.hadoop.hive.metastore.api.Table) AlreadyExistsException(org.apache.hadoop.hive.metastore.api.AlreadyExistsException) PartitionAlreadyExistsException(com.netflix.metacat.common.server.connectors.exception.PartitionAlreadyExistsException) InvalidMetaException(com.netflix.metacat.common.server.connectors.exception.InvalidMetaException) TableNotFoundException(com.netflix.metacat.common.server.connectors.exception.TableNotFoundException) PartitionNotFoundException(com.netflix.metacat.common.server.connectors.exception.PartitionNotFoundException) ConnectorException(com.netflix.metacat.common.server.connectors.exception.ConnectorException) PartitionsSaveResponse(com.netflix.metacat.common.server.connectors.model.PartitionsSaveResponse) TableInfo(com.netflix.metacat.common.server.connectors.model.TableInfo) NoSuchObjectException(org.apache.hadoop.hive.metastore.api.NoSuchObjectException) InvalidObjectException(org.apache.hadoop.hive.metastore.api.InvalidObjectException) ArrayList(java.util.ArrayList) List(java.util.List) PartitionInfo(com.netflix.metacat.common.server.connectors.model.PartitionInfo) PartitionAlreadyExistsException(com.netflix.metacat.common.server.connectors.exception.PartitionAlreadyExistsException) MetaException(org.apache.hadoop.hive.metastore.api.MetaException) InvalidMetaException(com.netflix.metacat.common.server.connectors.exception.InvalidMetaException)

Example 2 with PartitionsSaveRequest

use of com.netflix.metacat.common.server.connectors.model.PartitionsSaveRequest in project metacat by Netflix.

the class S3ConnectorPartitionService method savePartitions.

@Override
public PartitionsSaveResponse savePartitions(@Nonnull final ConnectorContext context, @Nonnull final QualifiedName tableName, @Nonnull final PartitionsSaveRequest partitionsSaveRequest) {
    log.debug("Start: Save partitions for table {}", tableName);
    // Table
    final Table table = getTable(tableName);
    // New partition ids
    final List<String> addedPartitionIds = Lists.newArrayList();
    // Updated partition ids
    final List<String> existingPartitionIds = Lists.newArrayList();
    //
    Map<String, Partition> existingPartitionMap = Maps.newHashMap();
    if (partitionsSaveRequest.getCheckIfExists()) {
        final List<String> partitionNames = partitionsSaveRequest.getPartitions().stream().map(partition -> {
            final String partitionName = partition.getName().getPartitionName();
            PartitionUtil.validatePartitionName(partitionName, infoConverter.partitionKeys(table));
            return partitionName;
        }).collect(Collectors.toList());
        existingPartitionMap = getPartitionsByNames(table.getId(), partitionNames);
    }
    // New partitions
    final List<Partition> s3Partitions = Lists.newArrayList();
    for (PartitionInfo partition : partitionsSaveRequest.getPartitions()) {
        final String partitionName = partition.getName().getPartitionName();
        final Partition s3Partition = existingPartitionMap.get(partitionName);
        if (s3Partition == null) {
            addedPartitionIds.add(partitionName);
            s3Partitions.add(infoConverter.toPartition(table, partition));
        } else {
            final String partitionUri = infoConverter.getUri(partition);
            final String s3PartitionUri = s3Partition.getUri();
            if (partitionUri != null && !partitionUri.equals(s3PartitionUri)) {
                s3Partition.setUri(partitionUri);
                existingPartitionIds.add(partitionName);
                s3Partitions.add(s3Partition);
            }
        }
    }
    final List<String> partitionIdsForDeletes = partitionsSaveRequest.getPartitionIdsForDeletes();
    if (partitionIdsForDeletes != null && !partitionIdsForDeletes.isEmpty()) {
        partitionDao.deleteByNames(catalogName, tableName.getDatabaseName(), tableName.getTableName(), partitionIdsForDeletes);
    }
    partitionDao.save(s3Partitions);
    log.debug("End: Save partitions for table {}", tableName);
    return PartitionsSaveResponse.builder().added(addedPartitionIds).updated(existingPartitionIds).build();
}
Also used : TableDao(com.netflix.metacat.connector.s3.dao.TableDao) PartitionKeyParserEval(com.netflix.metacat.common.server.partition.visitor.PartitionKeyParserEval) PartitionParamParserEval(com.netflix.metacat.common.server.partition.visitor.PartitionParamParserEval) Transactional(com.google.inject.persist.Transactional) Partition(com.netflix.metacat.connector.s3.model.Partition) Inject(javax.inject.Inject) Strings(com.google.common.base.Strings) ConnectorPartitionService(com.netflix.metacat.common.server.connectors.ConnectorPartitionService) Lists(com.google.common.collect.Lists) PartitionInfo(com.netflix.metacat.common.server.connectors.model.PartitionInfo) Map(java.util.Map) PartitionParser(com.netflix.metacat.common.server.partition.parser.PartitionParser) ConnectorContext(com.netflix.metacat.common.server.connectors.ConnectorContext) Named(javax.inject.Named) PartitionUtil(com.netflix.metacat.common.server.partition.util.PartitionUtil) Nonnull(javax.annotation.Nonnull) PartitionDao(com.netflix.metacat.connector.s3.dao.PartitionDao) Nullable(javax.annotation.Nullable) Collection(java.util.Collection) Pageable(com.netflix.metacat.common.dto.Pageable) QualifiedName(com.netflix.metacat.common.QualifiedName) BaseInfo(com.netflix.metacat.common.server.connectors.model.BaseInfo) TableNotFoundException(com.netflix.metacat.common.server.connectors.exception.TableNotFoundException) Maps(com.google.common.collect.Maps) Collectors(java.util.stream.Collectors) FilterPartition(com.netflix.metacat.common.server.partition.util.FilterPartition) PartitionsSaveResponse(com.netflix.metacat.common.server.connectors.model.PartitionsSaveResponse) Slf4j(lombok.extern.slf4j.Slf4j) List(java.util.List) PartitionAlreadyExistsException(com.netflix.metacat.common.server.connectors.exception.PartitionAlreadyExistsException) StringReader(java.io.StringReader) PartitionsSaveRequest(com.netflix.metacat.common.server.connectors.model.PartitionsSaveRequest) Table(com.netflix.metacat.connector.s3.model.Table) PartitionListRequest(com.netflix.metacat.common.server.connectors.model.PartitionListRequest) PartitionNotFoundException(com.netflix.metacat.common.server.connectors.exception.PartitionNotFoundException) Sort(com.netflix.metacat.common.dto.Sort) Partition(com.netflix.metacat.connector.s3.model.Partition) FilterPartition(com.netflix.metacat.common.server.partition.util.FilterPartition) Table(com.netflix.metacat.connector.s3.model.Table) PartitionInfo(com.netflix.metacat.common.server.connectors.model.PartitionInfo)

Example 3 with PartitionsSaveRequest

use of com.netflix.metacat.common.server.connectors.model.PartitionsSaveRequest in project metacat by Netflix.

the class S3ConnectorPartitionService method savePartitions.

@Override
public PartitionsSaveResponse savePartitions(@Nonnull final ConnectorRequestContext context, @Nonnull final QualifiedName tableName, @Nonnull final PartitionsSaveRequest partitionsSaveRequest) {
    log.debug("Start: Save partitions for table {}", tableName);
    // Table
    final Table table = getTable(tableName);
    // New partition ids
    final List<String> addedPartitionIds = Lists.newArrayList();
    // Updated partition ids
    final List<String> existingPartitionIds = Lists.newArrayList();
    // 
    Map<String, Partition> existingPartitionMap = Maps.newHashMap();
    if (partitionsSaveRequest.getCheckIfExists()) {
        final List<String> partitionNames = partitionsSaveRequest.getPartitions().stream().map(partition -> {
            final String partitionName = partition.getName().getPartitionName();
            PartitionUtil.validatePartitionName(partitionName, infoConverter.partitionKeys(table));
            return partitionName;
        }).collect(Collectors.toList());
        existingPartitionMap = getPartitionsByNames(table.getId(), partitionNames);
    }
    // New partitions
    final List<Partition> s3Partitions = Lists.newArrayList();
    for (PartitionInfo partition : partitionsSaveRequest.getPartitions()) {
        final String partitionName = partition.getName().getPartitionName();
        final Partition s3Partition = existingPartitionMap.get(partitionName);
        if (s3Partition == null) {
            addedPartitionIds.add(partitionName);
            s3Partitions.add(infoConverter.toPartition(table, partition));
        } else {
            final String partitionUri = infoConverter.getUri(partition);
            final String s3PartitionUri = s3Partition.getUri();
            if (partitionUri != null && !partitionUri.equals(s3PartitionUri)) {
                s3Partition.setUri(partitionUri);
                existingPartitionIds.add(partitionName);
                s3Partitions.add(s3Partition);
            }
        }
    }
    final List<String> partitionIdsForDeletes = partitionsSaveRequest.getPartitionIdsForDeletes();
    if (partitionIdsForDeletes != null && !partitionIdsForDeletes.isEmpty()) {
        partitionDao.deleteByNames(catalogName, tableName.getDatabaseName(), tableName.getTableName(), partitionIdsForDeletes);
    }
    partitionDao.save(s3Partitions);
    log.debug("End: Save partitions for table {}", tableName);
    return PartitionsSaveResponse.builder().added(addedPartitionIds).updated(existingPartitionIds).build();
}
Also used : TableDao(com.netflix.metacat.connector.s3.dao.TableDao) PartitionKeyParserEval(com.netflix.metacat.common.server.partition.visitor.PartitionKeyParserEval) PartitionParamParserEval(com.netflix.metacat.common.server.partition.visitor.PartitionParamParserEval) Transactional(com.google.inject.persist.Transactional) Partition(com.netflix.metacat.connector.s3.model.Partition) Inject(javax.inject.Inject) Strings(com.google.common.base.Strings) ConnectorPartitionService(com.netflix.metacat.common.server.connectors.ConnectorPartitionService) Lists(com.google.common.collect.Lists) PartitionInfo(com.netflix.metacat.common.server.connectors.model.PartitionInfo) Map(java.util.Map) PartitionParser(com.netflix.metacat.common.server.partition.parser.PartitionParser) Named(javax.inject.Named) PartitionUtil(com.netflix.metacat.common.server.partition.util.PartitionUtil) Nonnull(javax.annotation.Nonnull) ConnectorRequestContext(com.netflix.metacat.common.server.connectors.ConnectorRequestContext) PartitionDao(com.netflix.metacat.connector.s3.dao.PartitionDao) Nullable(javax.annotation.Nullable) Collection(java.util.Collection) Pageable(com.netflix.metacat.common.dto.Pageable) QualifiedName(com.netflix.metacat.common.QualifiedName) BaseInfo(com.netflix.metacat.common.server.connectors.model.BaseInfo) TableNotFoundException(com.netflix.metacat.common.server.connectors.exception.TableNotFoundException) Maps(com.google.common.collect.Maps) Collectors(java.util.stream.Collectors) FilterPartition(com.netflix.metacat.common.server.partition.util.FilterPartition) PartitionsSaveResponse(com.netflix.metacat.common.server.connectors.model.PartitionsSaveResponse) Slf4j(lombok.extern.slf4j.Slf4j) List(java.util.List) Stream(java.util.stream.Stream) TableInfo(com.netflix.metacat.common.server.connectors.model.TableInfo) PartitionAlreadyExistsException(com.netflix.metacat.common.server.connectors.exception.PartitionAlreadyExistsException) StringReader(java.io.StringReader) PartitionsSaveRequest(com.netflix.metacat.common.server.connectors.model.PartitionsSaveRequest) Table(com.netflix.metacat.connector.s3.model.Table) PartitionListRequest(com.netflix.metacat.common.server.connectors.model.PartitionListRequest) PartitionNotFoundException(com.netflix.metacat.common.server.connectors.exception.PartitionNotFoundException) Sort(com.netflix.metacat.common.dto.Sort) Partition(com.netflix.metacat.connector.s3.model.Partition) FilterPartition(com.netflix.metacat.common.server.partition.util.FilterPartition) Table(com.netflix.metacat.connector.s3.model.Table) PartitionInfo(com.netflix.metacat.common.server.connectors.model.PartitionInfo)

Example 4 with PartitionsSaveRequest

use of com.netflix.metacat.common.server.connectors.model.PartitionsSaveRequest in project metacat by Netflix.

the class HiveConnectorPartitionService method savePartitions.

/**
 * By default(checkIfExists=true and aletrIfExists=false), this method adds the provided list of partitions.
 * If a partition already exists, it is dropped first before adding it.
 * If checkIfExists=false, the method adds the partitions to the table. If a partition already exists,
 * an AlreadyExistsException error is thrown.
 * If alterIfExists=true, the method updates existing partitions and adds non-existant partitions.
 * If a partition in the provided partition list has all the details, then it is used. If the details are missing,
 * then the table details are inherited. This is mostly for the storage information.
 */
@Override
public PartitionsSaveResponse savePartitions(final ConnectorRequestContext requestContext, final QualifiedName tableQName, final PartitionsSaveRequest partitionsSaveRequest) {
    final String databaseName = tableQName.getDatabaseName();
    final String tableName = tableQName.getTableName();
    final Table table;
    try {
        table = metacatHiveClient.getTableByName(databaseName, tableName);
    } catch (NoSuchObjectException exception) {
        throw new TableNotFoundException(tableQName, exception);
    } catch (TException e) {
        throw new ConnectorException(String.format("Failed getting hive table %s", tableQName), e);
    }
    // New partitions
    final List<PartitionInfo> addedPartitionInfos = Lists.newArrayList();
    final List<PartitionInfo> partitionInfos = partitionsSaveRequest.getPartitions();
    final List<String> partitionNames = partitionInfos.stream().map(part -> {
        final String partitionName = part.getName().getPartitionName();
        PartitionUtil.validatePartitionName(partitionName, getPartitionKeys(table.getPartitionKeys()));
        return partitionName;
    }).collect(Collectors.toList());
    // New partition names
    final List<String> addedPartitionNames = Lists.newArrayList();
    // Updated partition names
    final List<String> existingPartitionNames = Lists.newArrayList();
    // Existing partitions
    final List<PartitionHolder> existingPartitionHolders = Lists.newArrayList();
    // Existing partition map
    Map<String, PartitionHolder> existingPartitionMap = Collections.emptyMap();
    // 
    if (partitionsSaveRequest.getCheckIfExists() || partitionsSaveRequest.getAlterIfExists()) {
        existingPartitionMap = getPartitionsByNames(table, partitionNames);
    }
    for (PartitionInfo partitionInfo : partitionInfos) {
        final String partitionName = partitionInfo.getName().getPartitionName();
        final PartitionHolder existingPartitionHolder = existingPartitionMap.get(partitionName);
        if (existingPartitionHolder == null) {
            addedPartitionNames.add(partitionName);
            addedPartitionInfos.add(partitionInfo);
        } else {
            final String partitionUri = partitionInfo.getSerde() != null ? partitionInfo.getSerde().getUri() : null;
            final String existingPartitionUri = getPartitionUri(existingPartitionHolder);
            if (partitionUri == null || !partitionUri.equals(existingPartitionUri)) {
                existingPartitionNames.add(partitionName);
                // We need to copy the existing partition info and
                if (partitionInfo.getSerde() == null) {
                    partitionInfo.setSerde(new StorageInfo());
                }
                if (partitionInfo.getAudit() == null) {
                    partitionInfo.setAudit(new AuditInfo());
                }
                if (StringUtils.isBlank(partitionUri)) {
                    partitionInfo.getSerde().setUri(existingPartitionUri);
                }
                // unless we alterifExists
                if (partitionsSaveRequest.getAlterIfExists()) {
                    if (existingPartitionHolder.getPartition() != null) {
                        final Partition existingPartition = existingPartitionHolder.getPartition();
                        partitionInfo.getSerde().setParameters(existingPartition.getParameters());
                        partitionInfo.getAudit().setCreatedDate(HiveConnectorInfoConverter.epochSecondsToDate(existingPartition.getCreateTime()));
                        partitionInfo.getAudit().setLastModifiedDate(HiveConnectorInfoConverter.epochSecondsToDate(existingPartition.getLastAccessTime()));
                    } else {
                        final PartitionInfo existingPartitionInfo = existingPartitionHolder.getPartitionInfo();
                        if (existingPartitionInfo.getSerde() != null) {
                            partitionInfo.getSerde().setParameters(existingPartitionInfo.getSerde().getParameters());
                        }
                        if (existingPartitionInfo.getAudit() != null) {
                            partitionInfo.getAudit().setCreatedDate(existingPartitionInfo.getAudit().getCreatedDate());
                            partitionInfo.getAudit().setLastModifiedDate(existingPartitionInfo.getAudit().getLastModifiedDate());
                        }
                    }
                    existingPartitionHolder.setPartitionInfo(partitionInfo);
                    existingPartitionHolders.add(existingPartitionHolder);
                } else {
                    addedPartitionInfos.add(partitionInfo);
                }
            }
        }
    }
    final Set<String> deletePartitionNames = Sets.newHashSet();
    if (!partitionsSaveRequest.getAlterIfExists()) {
        deletePartitionNames.addAll(existingPartitionNames);
    }
    if (partitionsSaveRequest.getPartitionIdsForDeletes() != null) {
        deletePartitionNames.addAll(partitionsSaveRequest.getPartitionIdsForDeletes());
    }
    addUpdateDropPartitions(tableQName, table, partitionNames, addedPartitionInfos, existingPartitionHolders, deletePartitionNames);
    final PartitionsSaveResponse result = new PartitionsSaveResponse();
    result.setAdded(addedPartitionNames);
    result.setUpdated(existingPartitionNames);
    return result;
}
Also used : TException(org.apache.thrift.TException) StringUtils(org.apache.commons.lang.StringUtils) Getter(lombok.Getter) MetaException(org.apache.hadoop.hive.metastore.api.MetaException) SortOrder(com.netflix.metacat.common.dto.SortOrder) AuditInfo(com.netflix.metacat.common.server.connectors.model.AuditInfo) HashMap(java.util.HashMap) SerDeInfo(org.apache.hadoop.hive.metastore.api.SerDeInfo) Partition(org.apache.hadoop.hive.metastore.api.Partition) Function(java.util.function.Function) Warehouse(org.apache.hadoop.hive.metastore.Warehouse) ArrayList(java.util.ArrayList) AlreadyExistsException(org.apache.hadoop.hive.metastore.api.AlreadyExistsException) Strings(com.google.common.base.Strings) ConnectorPartitionService(com.netflix.metacat.common.server.connectors.ConnectorPartitionService) InvalidMetaException(com.netflix.metacat.common.server.connectors.exception.InvalidMetaException) Lists(com.google.common.collect.Lists) ConnectorException(com.netflix.metacat.common.server.connectors.exception.ConnectorException) PartitionInfo(com.netflix.metacat.common.server.connectors.model.PartitionInfo) Map(java.util.Map) ConnectorContext(com.netflix.metacat.common.server.connectors.ConnectorContext) StorageInfo(com.netflix.metacat.common.server.connectors.model.StorageInfo) HiveConnectorInfoConverter(com.netflix.metacat.connector.hive.converters.HiveConnectorInfoConverter) PartitionUtil(com.netflix.metacat.common.server.partition.util.PartitionUtil) StorageDescriptor(org.apache.hadoop.hive.metastore.api.StorageDescriptor) ConnectorRequestContext(com.netflix.metacat.common.server.connectors.ConnectorRequestContext) Nullable(javax.annotation.Nullable) PartitionHolder(com.netflix.metacat.connector.hive.sql.PartitionHolder) Pageable(com.netflix.metacat.common.dto.Pageable) TException(org.apache.thrift.TException) Set(java.util.Set) QualifiedName(com.netflix.metacat.common.QualifiedName) InvalidObjectException(org.apache.hadoop.hive.metastore.api.InvalidObjectException) TableNotFoundException(com.netflix.metacat.common.server.connectors.exception.TableNotFoundException) Collectors(java.util.stream.Collectors) Sets(com.google.common.collect.Sets) Table(org.apache.hadoop.hive.metastore.api.Table) PartitionsSaveResponse(com.netflix.metacat.common.server.connectors.model.PartitionsSaveResponse) FieldSchema(org.apache.hadoop.hive.metastore.api.FieldSchema) List(java.util.List) PartitionAlreadyExistsException(com.netflix.metacat.common.server.connectors.exception.PartitionAlreadyExistsException) TableInfo(com.netflix.metacat.common.server.connectors.model.TableInfo) PartitionsSaveRequest(com.netflix.metacat.common.server.connectors.model.PartitionsSaveRequest) PartitionListRequest(com.netflix.metacat.common.server.connectors.model.PartitionListRequest) ConnectorUtils(com.netflix.metacat.common.server.connectors.ConnectorUtils) PartitionNotFoundException(com.netflix.metacat.common.server.connectors.exception.PartitionNotFoundException) Collections(java.util.Collections) NoSuchObjectException(org.apache.hadoop.hive.metastore.api.NoSuchObjectException) Sort(com.netflix.metacat.common.dto.Sort) Partition(org.apache.hadoop.hive.metastore.api.Partition) PartitionHolder(com.netflix.metacat.connector.hive.sql.PartitionHolder) AuditInfo(com.netflix.metacat.common.server.connectors.model.AuditInfo) Table(org.apache.hadoop.hive.metastore.api.Table) TableNotFoundException(com.netflix.metacat.common.server.connectors.exception.TableNotFoundException) ConnectorException(com.netflix.metacat.common.server.connectors.exception.ConnectorException) StorageInfo(com.netflix.metacat.common.server.connectors.model.StorageInfo) PartitionsSaveResponse(com.netflix.metacat.common.server.connectors.model.PartitionsSaveResponse) NoSuchObjectException(org.apache.hadoop.hive.metastore.api.NoSuchObjectException) PartitionInfo(com.netflix.metacat.common.server.connectors.model.PartitionInfo)

Aggregations

Strings (com.google.common.base.Strings)4 Lists (com.google.common.collect.Lists)4 QualifiedName (com.netflix.metacat.common.QualifiedName)4 Pageable (com.netflix.metacat.common.dto.Pageable)4 Sort (com.netflix.metacat.common.dto.Sort)4 ConnectorPartitionService (com.netflix.metacat.common.server.connectors.ConnectorPartitionService)4 PartitionAlreadyExistsException (com.netflix.metacat.common.server.connectors.exception.PartitionAlreadyExistsException)4 PartitionNotFoundException (com.netflix.metacat.common.server.connectors.exception.PartitionNotFoundException)4 TableNotFoundException (com.netflix.metacat.common.server.connectors.exception.TableNotFoundException)4 PartitionInfo (com.netflix.metacat.common.server.connectors.model.PartitionInfo)4 PartitionListRequest (com.netflix.metacat.common.server.connectors.model.PartitionListRequest)4 PartitionsSaveRequest (com.netflix.metacat.common.server.connectors.model.PartitionsSaveRequest)4 PartitionsSaveResponse (com.netflix.metacat.common.server.connectors.model.PartitionsSaveResponse)4 PartitionUtil (com.netflix.metacat.common.server.partition.util.PartitionUtil)4 List (java.util.List)4 Map (java.util.Map)4 Collectors (java.util.stream.Collectors)4 Nullable (javax.annotation.Nullable)4 ConnectorContext (com.netflix.metacat.common.server.connectors.ConnectorContext)3 TableInfo (com.netflix.metacat.common.server.connectors.model.TableInfo)3