Search in sources :

Example 6 with AuditInfo

use of com.netflix.metacat.common.server.connectors.model.AuditInfo in project metacat by Netflix.

the class HiveConnectorInfoConverter method toPartitionInfo.

/**
     * Converts to PartitionDto.
     *
     * @param partition connector partition
     * @return Metacat partition Info
     */
@Override
public PartitionInfo toPartitionInfo(@Nonnull @NonNull final TableInfo tableInfo, @Nonnull @NonNull final Partition partition) {
    final QualifiedName tableName = tableInfo.getName();
    final QualifiedName partitionName = QualifiedName.ofPartition(tableName.getCatalogName(), tableName.getDatabaseName(), tableName.getTableName(), getNameFromPartVals(tableInfo, partition.getValues()));
    final String owner = notNull(tableInfo.getSerde()) ? tableInfo.getSerde().getOwner() : "";
    final AuditInfo auditInfo = AuditInfo.builder().createdDate(epochSecondsToDate(partition.getCreateTime())).lastModifiedDate(epochSecondsToDate(partition.getLastAccessTime())).build();
    return PartitionInfo.builder().serde(toStorageInfo(partition.getSd(), owner)).name(partitionName).auditInfo(auditInfo).metadata(partition.getParameters()).build();
}
Also used : AuditInfo(com.netflix.metacat.common.server.connectors.model.AuditInfo) QualifiedName(com.netflix.metacat.common.QualifiedName)

Example 7 with AuditInfo

use of com.netflix.metacat.common.server.connectors.model.AuditInfo in project metacat by Netflix.

the class S3ConnectorInfoConverter method fromPartitionInfo.

/**
     * Converts from partition info to s3 partition object.
     * @param partitionInfo partition info
     * @return s3 partition
     */
Partition fromPartitionInfo(final PartitionInfo partitionInfo) {
    final Partition result = new Partition();
    result.setName(partitionInfo.getName().getPartitionName());
    result.setUri(partitionInfo.getSerde().getUri());
    final AuditInfo auditInfo = partitionInfo.getAudit();
    if (auditInfo != null) {
        result.setCreatedDate(auditInfo.getCreatedDate());
        result.setLastUpdatedDate(auditInfo.getLastModifiedDate());
    }
    return result;
}
Also used : Partition(com.netflix.metacat.connector.s3.model.Partition) AuditInfo(com.netflix.metacat.common.server.connectors.model.AuditInfo)

Example 8 with AuditInfo

use of com.netflix.metacat.common.server.connectors.model.AuditInfo in project metacat by Netflix.

the class S3ConnectorInfoConverter method toPartitionInfo.

@Override
public PartitionInfo toPartitionInfo(final TableInfo tableInfo, final Partition partition) {
    final QualifiedName tableName = tableInfo.getName();
    final StorageInfo storageInfo = tableInfo.getSerde();
    storageInfo.setUri(partition.getUri());
    final AuditInfo auditInfo = AuditInfo.builder().createdDate(partition.getCreatedDate()).lastModifiedDate(partition.getLastUpdatedDate()).build();
    final AuditInfo tableAuditInfo = tableInfo.getAudit();
    if (tableAuditInfo != null) {
        auditInfo.setCreatedBy(tableAuditInfo.getCreatedBy());
        auditInfo.setLastModifiedBy(tableAuditInfo.getLastModifiedBy());
    }
    return PartitionInfo.builder().name(QualifiedName.ofPartition(tableName.getCatalogName(), tableName.getDatabaseName(), tableName.getTableName(), partition.getName())).serde(storageInfo).auditInfo(auditInfo).build();
}
Also used : AuditInfo(com.netflix.metacat.common.server.connectors.model.AuditInfo) QualifiedName(com.netflix.metacat.common.QualifiedName) StorageInfo(com.netflix.metacat.common.server.connectors.model.StorageInfo)

Example 9 with AuditInfo

use of com.netflix.metacat.common.server.connectors.model.AuditInfo in project metacat by Netflix.

the class HiveConnectorInfoConverter method fromTableInfo.

/**
     * Converts from TableDto to the connector table.
     *
     * @param tableInfo Metacat table Info
     * @return connector table
     */
@Override
public Table fromTableInfo(@Nonnull @NonNull final TableInfo tableInfo) {
    final QualifiedName name = tableInfo.getName();
    final String tableName = (name != null) ? name.getTableName() : "";
    final String databaseName = (name != null) ? name.getDatabaseName() : "";
    final StorageInfo storageInfo = tableInfo.getSerde();
    final String owner = (storageInfo != null && storageInfo.getOwner() != null) ? storageInfo.getOwner() : "";
    final AuditInfo auditInfo = tableInfo.getAudit();
    final int createTime = (auditInfo != null && auditInfo.getCreatedDate() != null) ? dateToEpochSeconds(auditInfo.getCreatedDate()) : 0;
    final Map<String, String> params = (tableInfo.getMetadata() != null) ? tableInfo.getMetadata() : new HashMap<>();
    final List<FieldInfo> fields = tableInfo.getFields();
    List<FieldSchema> partitionFields = Collections.emptyList();
    List<FieldSchema> nonPartitionFields = Collections.emptyList();
    if (fields != null) {
        nonPartitionFields = Lists.newArrayListWithCapacity(fields.size());
        partitionFields = Lists.newArrayListWithCapacity(fields.size());
        for (FieldInfo fieldInfo : fields) {
            if (fieldInfo.isPartitionKey()) {
                partitionFields.add(metacatToHiveField(fieldInfo));
            } else {
                nonPartitionFields.add(metacatToHiveField(fieldInfo));
            }
        }
    }
    final StorageDescriptor sd = fromStorageInfo(storageInfo, nonPartitionFields);
    return new Table(tableName, databaseName, owner, createTime, 0, 0, sd, partitionFields, params, null, null, "EXTERNAL_TABLE");
}
Also used : AuditInfo(com.netflix.metacat.common.server.connectors.model.AuditInfo) Table(org.apache.hadoop.hive.metastore.api.Table) QualifiedName(com.netflix.metacat.common.QualifiedName) FieldSchema(org.apache.hadoop.hive.metastore.api.FieldSchema) StorageDescriptor(org.apache.hadoop.hive.metastore.api.StorageDescriptor) StorageInfo(com.netflix.metacat.common.server.connectors.model.StorageInfo) FieldInfo(com.netflix.metacat.common.server.connectors.model.FieldInfo)

Aggregations

AuditInfo (com.netflix.metacat.common.server.connectors.model.AuditInfo)9 StorageInfo (com.netflix.metacat.common.server.connectors.model.StorageInfo)7 QualifiedName (com.netflix.metacat.common.QualifiedName)6 FieldInfo (com.netflix.metacat.common.server.connectors.model.FieldInfo)4 PartitionInfo (com.netflix.metacat.common.server.connectors.model.PartitionInfo)4 FieldSchema (org.apache.hadoop.hive.metastore.api.FieldSchema)4 Strings (com.google.common.base.Strings)3 Lists (com.google.common.collect.Lists)3 DatabaseInfo (com.netflix.metacat.common.server.connectors.model.DatabaseInfo)3 TableInfo (com.netflix.metacat.common.server.connectors.model.TableInfo)3 Instant (java.time.Instant)3 Date (java.util.Date)3 HashMap (java.util.HashMap)3 List (java.util.List)3 Map (java.util.Map)3 Collectors (java.util.stream.Collectors)3 Nonnull (javax.annotation.Nonnull)3 NonNull (lombok.NonNull)3 Slf4j (lombok.extern.slf4j.Slf4j)3 StorageDescriptor (org.apache.hadoop.hive.metastore.api.StorageDescriptor)3