Search in sources :

Example 71 with QualifiedName

use of com.netflix.metacat.common.QualifiedName in project metacat by Netflix.

the class S3ConnectorPartitionService method savePartitions.

@Override
public PartitionsSaveResponse savePartitions(@Nonnull final ConnectorContext context, @Nonnull final QualifiedName tableName, @Nonnull final PartitionsSaveRequest partitionsSaveRequest) {
    log.debug("Start: Save partitions for table {}", tableName);
    // Table
    final Table table = getTable(tableName);
    // New partition ids
    final List<String> addedPartitionIds = Lists.newArrayList();
    // Updated partition ids
    final List<String> existingPartitionIds = Lists.newArrayList();
    //
    Map<String, Partition> existingPartitionMap = Maps.newHashMap();
    if (partitionsSaveRequest.getCheckIfExists()) {
        final List<String> partitionNames = partitionsSaveRequest.getPartitions().stream().map(partition -> {
            final String partitionName = partition.getName().getPartitionName();
            PartitionUtil.validatePartitionName(partitionName, infoConverter.partitionKeys(table));
            return partitionName;
        }).collect(Collectors.toList());
        existingPartitionMap = getPartitionsByNames(table.getId(), partitionNames);
    }
    // New partitions
    final List<Partition> s3Partitions = Lists.newArrayList();
    for (PartitionInfo partition : partitionsSaveRequest.getPartitions()) {
        final String partitionName = partition.getName().getPartitionName();
        final Partition s3Partition = existingPartitionMap.get(partitionName);
        if (s3Partition == null) {
            addedPartitionIds.add(partitionName);
            s3Partitions.add(infoConverter.toPartition(table, partition));
        } else {
            final String partitionUri = infoConverter.getUri(partition);
            final String s3PartitionUri = s3Partition.getUri();
            if (partitionUri != null && !partitionUri.equals(s3PartitionUri)) {
                s3Partition.setUri(partitionUri);
                existingPartitionIds.add(partitionName);
                s3Partitions.add(s3Partition);
            }
        }
    }
    final List<String> partitionIdsForDeletes = partitionsSaveRequest.getPartitionIdsForDeletes();
    if (partitionIdsForDeletes != null && !partitionIdsForDeletes.isEmpty()) {
        partitionDao.deleteByNames(catalogName, tableName.getDatabaseName(), tableName.getTableName(), partitionIdsForDeletes);
    }
    partitionDao.save(s3Partitions);
    log.debug("End: Save partitions for table {}", tableName);
    return PartitionsSaveResponse.builder().added(addedPartitionIds).updated(existingPartitionIds).build();
}
Also used : TableDao(com.netflix.metacat.connector.s3.dao.TableDao) PartitionKeyParserEval(com.netflix.metacat.common.server.partition.visitor.PartitionKeyParserEval) PartitionParamParserEval(com.netflix.metacat.common.server.partition.visitor.PartitionParamParserEval) Transactional(com.google.inject.persist.Transactional) Partition(com.netflix.metacat.connector.s3.model.Partition) Inject(javax.inject.Inject) Strings(com.google.common.base.Strings) ConnectorPartitionService(com.netflix.metacat.common.server.connectors.ConnectorPartitionService) Lists(com.google.common.collect.Lists) PartitionInfo(com.netflix.metacat.common.server.connectors.model.PartitionInfo) Map(java.util.Map) PartitionParser(com.netflix.metacat.common.server.partition.parser.PartitionParser) ConnectorContext(com.netflix.metacat.common.server.connectors.ConnectorContext) Named(javax.inject.Named) PartitionUtil(com.netflix.metacat.common.server.partition.util.PartitionUtil) Nonnull(javax.annotation.Nonnull) PartitionDao(com.netflix.metacat.connector.s3.dao.PartitionDao) Nullable(javax.annotation.Nullable) Collection(java.util.Collection) Pageable(com.netflix.metacat.common.dto.Pageable) QualifiedName(com.netflix.metacat.common.QualifiedName) BaseInfo(com.netflix.metacat.common.server.connectors.model.BaseInfo) TableNotFoundException(com.netflix.metacat.common.server.connectors.exception.TableNotFoundException) Maps(com.google.common.collect.Maps) Collectors(java.util.stream.Collectors) FilterPartition(com.netflix.metacat.common.server.partition.util.FilterPartition) PartitionsSaveResponse(com.netflix.metacat.common.server.connectors.model.PartitionsSaveResponse) Slf4j(lombok.extern.slf4j.Slf4j) List(java.util.List) PartitionAlreadyExistsException(com.netflix.metacat.common.server.connectors.exception.PartitionAlreadyExistsException) StringReader(java.io.StringReader) PartitionsSaveRequest(com.netflix.metacat.common.server.connectors.model.PartitionsSaveRequest) Table(com.netflix.metacat.connector.s3.model.Table) PartitionListRequest(com.netflix.metacat.common.server.connectors.model.PartitionListRequest) PartitionNotFoundException(com.netflix.metacat.common.server.connectors.exception.PartitionNotFoundException) Sort(com.netflix.metacat.common.dto.Sort) Partition(com.netflix.metacat.connector.s3.model.Partition) FilterPartition(com.netflix.metacat.common.server.partition.util.FilterPartition) Table(com.netflix.metacat.connector.s3.model.Table) PartitionInfo(com.netflix.metacat.common.server.connectors.model.PartitionInfo)

Example 72 with QualifiedName

use of com.netflix.metacat.common.QualifiedName in project metacat by Netflix.

the class S3ConnectorPartitionService method get.

@Override
public PartitionInfo get(@Nonnull final ConnectorContext context, @Nonnull final QualifiedName name) {
    final QualifiedName tableName = QualifiedName.ofTable(catalogName, name.getDatabaseName(), name.getTableName());
    final Table table = getTable(tableName);
    final List<Partition> partitions = partitionDao.getPartitions(table.getId(), Lists.newArrayList(name.getPartitionName()), null, null, null, null);
    if (partitions.isEmpty()) {
        throw new PartitionNotFoundException(tableName, name.getPartitionName());
    }
    log.debug("Get partition for table {}", tableName);
    return infoConverter.toPartitionInfo(tableName, table, partitions.get(0));
}
Also used : Partition(com.netflix.metacat.connector.s3.model.Partition) FilterPartition(com.netflix.metacat.common.server.partition.util.FilterPartition) PartitionNotFoundException(com.netflix.metacat.common.server.connectors.exception.PartitionNotFoundException) Table(com.netflix.metacat.connector.s3.model.Table) QualifiedName(com.netflix.metacat.common.QualifiedName)

Example 73 with QualifiedName

use of com.netflix.metacat.common.QualifiedName in project metacat by Netflix.

the class S3ConnectorTableService method update.

@Override
public void update(@Nonnull final ConnectorContext context, @Nonnull final TableInfo tableInfo) {
    log.debug("Start: Update table {}", tableInfo.getName());
    final QualifiedName tableName = tableInfo.getName();
    final Table table = tableDao.getBySourceDatabaseTableName(catalogName, tableName.getDatabaseName(), tableName.getTableName());
    if (table == null) {
        throw new TableNotFoundException(tableName);
    }
    //we can update the fields, the uri, or the full serde
    final Location newLocation = infoConverter.toLocation(tableInfo);
    Location location = table.getLocation();
    if (location == null) {
        location = new Location();
        location.setTable(table);
        table.setLocation(location);
    }
    if (newLocation.getUri() != null) {
        location.setUri(newLocation.getUri());
    }
    final Info newInfo = newLocation.getInfo();
    if (newInfo != null) {
        final Info info = location.getInfo();
        if (info == null) {
            location.setInfo(newInfo);
            newInfo.setLocation(location);
        } else {
            if (newInfo.getInputFormat() != null) {
                info.setInputFormat(newInfo.getInputFormat());
            }
            if (newInfo.getOutputFormat() != null) {
                info.setOutputFormat(newInfo.getOutputFormat());
            }
            if (newInfo.getOwner() != null) {
                info.setOwner(newInfo.getOwner());
            }
            if (newInfo.getSerializationLib() != null) {
                info.setSerializationLib(newInfo.getSerializationLib());
            }
            if (newInfo.getParameters() != null && !newInfo.getParameters().isEmpty()) {
                info.setParameters(newInfo.getParameters());
            }
        }
    }
    final Schema newSchema = newLocation.getSchema();
    if (newSchema != null) {
        final List<Field> newFields = newSchema.getFields();
        if (newFields != null && !newFields.isEmpty()) {
            final Schema schema = location.getSchema();
            if (schema == null) {
                location.setSchema(newSchema);
                newSchema.setLocation(location);
            } else {
                final List<Field> fields = schema.getFields();
                if (fields.isEmpty()) {
                    newFields.forEach(field -> {
                        field.setSchema(schema);
                        fields.add(field);
                    });
                } else {
                    for (int i = 0; i < newFields.size(); i++) {
                        final Field newField = newFields.get(i);
                        newField.setPos(i);
                        newField.setSchema(schema);
                        if (newField.getType() == null) {
                            newField.setType(newField.getSourceType());
                        }
                    }
                    schema.setFields(null);
                    fieldDao.delete(fields);
                    tableDao.save(table, true);
                    schema.setFields(newFields);
                }
            }
        }
    }
    log.debug("End: Update table {}", tableInfo.getName());
}
Also used : TableNotFoundException(com.netflix.metacat.common.server.connectors.exception.TableNotFoundException) Field(com.netflix.metacat.connector.s3.model.Field) Table(com.netflix.metacat.connector.s3.model.Table) QualifiedName(com.netflix.metacat.common.QualifiedName) Schema(com.netflix.metacat.connector.s3.model.Schema) Info(com.netflix.metacat.connector.s3.model.Info) TableInfo(com.netflix.metacat.common.server.connectors.model.TableInfo) Location(com.netflix.metacat.connector.s3.model.Location)

Example 74 with QualifiedName

use of com.netflix.metacat.common.QualifiedName in project metacat by Netflix.

the class PartitionV1Resource method deletePartitions.

/**
     * {@inheritDoc}
     */
@Override
public void deletePartitions(final String catalogName, final String databaseName, final String tableName, final String viewName, final List<String> partitionIds) {
    final QualifiedName name = requestWrapper.qualifyName(() -> QualifiedName.ofView(catalogName, databaseName, tableName, viewName));
    requestWrapper.processRequest(name, "deleteMViewPartition", () -> {
        if (partitionIds == null || partitionIds.isEmpty()) {
            throw new IllegalArgumentException("partitionIds are required");
        }
        mViewService.deletePartitions(name, partitionIds);
        return null;
    });
}
Also used : QualifiedName(com.netflix.metacat.common.QualifiedName)

Example 75 with QualifiedName

use of com.netflix.metacat.common.QualifiedName in project metacat by Netflix.

the class S3ConnectorPartitionService method create.

@Override
public void create(@Nonnull final ConnectorContext context, @Nonnull final PartitionInfo partitionInfo) {
    final QualifiedName name = partitionInfo.getName();
    log.debug("Start: Create partition {}", name);
    final QualifiedName tableName = QualifiedName.ofTable(catalogName, name.getDatabaseName(), name.getTableName());
    // Table
    final Table table = getTable(tableName);
    final List<Partition> partitions = partitionDao.getPartitions(table.getId(), Lists.newArrayList(name.getPartitionName()), null, null, null, null);
    if (!partitions.isEmpty()) {
        throw new PartitionAlreadyExistsException(tableName, name.getPartitionName());
    }
    partitionDao.save(infoConverter.toPartition(table, partitionInfo));
    log.debug("End: Create partition {}", name);
}
Also used : Partition(com.netflix.metacat.connector.s3.model.Partition) FilterPartition(com.netflix.metacat.common.server.partition.util.FilterPartition) Table(com.netflix.metacat.connector.s3.model.Table) QualifiedName(com.netflix.metacat.common.QualifiedName) PartitionAlreadyExistsException(com.netflix.metacat.common.server.connectors.exception.PartitionAlreadyExistsException)

Aggregations

QualifiedName (com.netflix.metacat.common.QualifiedName)144 List (java.util.List)52 Lists (com.google.common.collect.Lists)44 Collectors (java.util.stream.Collectors)41 Map (java.util.Map)38 Slf4j (lombok.extern.slf4j.Slf4j)36 Strings (com.google.common.base.Strings)35 Nullable (javax.annotation.Nullable)33 Pageable (com.netflix.metacat.common.dto.Pageable)29 Sort (com.netflix.metacat.common.dto.Sort)29 Nonnull (javax.annotation.Nonnull)29 MetacatRequestContext (com.netflix.metacat.common.MetacatRequestContext)28 TableDto (com.netflix.metacat.common.dto.TableDto)27 Maps (com.google.common.collect.Maps)25 ConnectorException (com.netflix.metacat.common.server.connectors.exception.ConnectorException)25 DatabaseNotFoundException (com.netflix.metacat.common.server.connectors.exception.DatabaseNotFoundException)24 TableNotFoundException (com.netflix.metacat.common.server.connectors.exception.TableNotFoundException)24 Optional (java.util.Optional)22 Registry (com.netflix.spectator.api.Registry)21 Set (java.util.Set)21