Examples with QualifiedName com.netflix.metacat.common.QualifiedName used on opensource projects

Example 31 with QualifiedName

use of com.netflix.metacat.common.QualifiedName in project metacat by Netflix.

the class HiveConnectorInfoConverter method toTableInfo.

/**
 * Converts to TableDto.
 *
 * @param table connector table
 * @return Metacat table Info
 */
@Override
public TableInfo toTableInfo(final QualifiedName name, final Table table) {
    final List<FieldSchema> nonPartitionColumns = (table.getSd() != null) ? table.getSd().getCols() : Collections.emptyList();
    // ignore all exceptions
    try {
        if (nonPartitionColumns.isEmpty()) {
            for (StructField field : HiveTableUtil.getTableStructFields(table)) {
                final FieldSchema fieldSchema = new FieldSchema(field.getFieldName(), field.getFieldObjectInspector().getTypeName(), field.getFieldComment());
                nonPartitionColumns.add(fieldSchema);
            }
        }
    } catch (final Exception e) {
        log.error(e.getMessage(), e);
    }
    final List<FieldSchema> partitionColumns = table.getPartitionKeys();
    final Date creationDate = table.isSetCreateTime() ? epochSecondsToDate(table.getCreateTime()) : null;
    final List<FieldInfo> allFields = Lists.newArrayListWithCapacity(nonPartitionColumns.size() + partitionColumns.size());
    nonPartitionColumns.stream().map(field -> hiveToMetacatField(field, false)).forEachOrdered(allFields::add);
    partitionColumns.stream().map(field -> hiveToMetacatField(field, true)).forEachOrdered(allFields::add);
    final AuditInfo auditInfo = AuditInfo.builder().createdDate(creationDate).build();
    if (null != table.getTableType() && table.getTableType().equals(TableType.VIRTUAL_VIEW.name())) {
        return TableInfo.builder().serde(toStorageInfo(table.getSd(), table.getOwner())).fields(allFields).metadata(table.getParameters()).name(name).auditInfo(auditInfo).view(ViewInfo.builder().viewOriginalText(table.getViewOriginalText()).viewExpandedText(table.getViewExpandedText()).build()).build();
    } else {
        return TableInfo.builder().serde(toStorageInfo(table.getSd(), table.getOwner())).fields(allFields).metadata(table.getParameters()).name(name).auditInfo(auditInfo).build();
    }
}

Example 32 with QualifiedName

use of com.netflix.metacat.common.QualifiedName in project metacat by Netflix.

the class HiveConnectorInfoConverter method fromPartitionInfo.

/**
 * Converts from PartitionDto to the connector partition.
 *
 * @param partition Metacat partition Info
 * @return connector partition
 */
@Override
public Partition fromPartitionInfo(final TableInfo tableInfo, final PartitionInfo partition) {
    final QualifiedName name = partition.getName();
    final List<String> values = Lists.newArrayListWithCapacity(16);
    Map<String, String> metadata = partition.getMetadata();
    if (metadata == null) {
        metadata = new HashMap<>();
    // can't use Collections.emptyMap()
    // which is immutable and can't be
    // modifed by add parts in the embedded
    }
    final List<FieldInfo> fields = tableInfo.getFields();
    List<FieldSchema> fieldSchemas = Collections.emptyList();
    if (notNull(fields)) {
        fieldSchemas = fields.stream().filter(field -> !field.isPartitionKey()).map(this::metacatToHiveField).collect(Collectors.toList());
    }
    final StorageDescriptor sd = fromStorageInfo(partition.getSerde(), fieldSchemas);
    // using the table level serialization lib
    if (notNull(sd.getSerdeInfo()) && notNull(tableInfo.getSerde()) && Strings.isNullOrEmpty(sd.getSerdeInfo().getSerializationLib())) {
        sd.getSerdeInfo().setSerializationLib(tableInfo.getSerde().getSerializationLib());
    }
    final AuditInfo auditInfo = partition.getAudit();
    final int createTime = (notNull(auditInfo) && notNull(auditInfo.getCreatedDate())) ? dateToEpochSeconds(auditInfo.getCreatedDate()) : 0;
    final int lastAccessTime = (notNull(auditInfo) && notNull(auditInfo.getLastModifiedDate())) ? dateToEpochSeconds(auditInfo.getLastModifiedDate()) : 0;
    if (null == name) {
        return new Partition(values, "", "", createTime, lastAccessTime, sd, metadata);
    }
    if (notNull(name.getPartitionName())) {
        for (String partialPartName : SLASH_SPLITTER.split(partition.getName().getPartitionName())) {
            final List<String> nameValues = ImmutableList.copyOf(EQUAL_SPLITTER.split(partialPartName));
            Preconditions.checkState(nameValues.size() == 2, "Unrecognized partition name: " + partition.getName());
            values.add(nameValues.get(1));
        }
    }
    final String databaseName = notNull(name.getDatabaseName()) ? name.getDatabaseName() : "";
    final String tableName = notNull(name.getTableName()) ? name.getTableName() : "";
    return new Partition(values, databaseName, tableName, createTime, lastAccessTime, sd, metadata);
}

Example 33 with QualifiedName

use of com.netflix.metacat.common.QualifiedName in project metacat by Netflix.

the class HiveConnectorInfoConverter method toPartitionInfo.

/**
 * Converts to PartitionDto.
 *
 * @param partition connector partition
 * @return Metacat partition Info
 */
@Override
public PartitionInfo toPartitionInfo(final TableInfo tableInfo, final Partition partition) {
    final QualifiedName tableName = tableInfo.getName();
    final QualifiedName partitionName = QualifiedName.ofPartition(tableName.getCatalogName(), tableName.getDatabaseName(), tableName.getTableName(), getNameFromPartVals(tableInfo, partition.getValues()));
    final String owner = notNull(tableInfo.getSerde()) ? tableInfo.getSerde().getOwner() : "";
    final AuditInfo auditInfo = AuditInfo.builder().createdDate(epochSecondsToDate(partition.getCreateTime())).lastModifiedDate(epochSecondsToDate(partition.getLastAccessTime())).build();
    return PartitionInfo.builder().serde(toStorageInfo(partition.getSd(), owner)).name(partitionName).auditInfo(auditInfo).metadata(partition.getParameters()).build();
}

Example 34 with QualifiedName

use of com.netflix.metacat.common.QualifiedName in project metacat by Netflix.

the class DirectSqlGetPartition method getPartitionKeys.

/**
 * Gets the partition names/keys based on a filter expression for the specified table.
 *
 * @param requestContext    The Metacat request context
 * @param tableName         table handle to get partition for
 * @param partitionsRequest The metadata for what kind of partitions to get from the table
 * @return filtered list of partition names
 */
@Transactional(readOnly = true)
public List<String> getPartitionKeys(final ConnectorRequestContext requestContext, final QualifiedName tableName, final PartitionListRequest partitionsRequest) {
    final long start = registry.clock().wallTime();
    final List<String> result;
    final List<String> partitionNames = partitionsRequest.getPartitionNames();
    final Sort sort = partitionsRequest.getSort();
    final Pageable pageable = partitionsRequest.getPageable();
    final String filterExpression = partitionsRequest.getFilter();
    if (filterExpression != null) {
        return filterPartitionsColumn(tableName.getDatabaseName(), tableName.getTableName(), partitionNames, PARTITION_NAME, filterExpression, sort, pageable, partitionsRequest.getIncludeAuditOnly());
    } else {
        final ResultSetExtractor<List<String>> handler = rs -> {
            final List<String> names = Lists.newArrayList();
            while (rs.next()) {
                names.add(rs.getString("name"));
            }
            return names;
        };
        result = getHandlerResults(tableName.getDatabaseName(), tableName.getTableName(), null, partitionNames, SQL.SQL_GET_PARTITIONS_WITH_KEY, handler, sort, pageable, partitionsRequest.getIncludeAuditOnly());
    }
    this.fastServiceMetric.recordTimer(HiveMetrics.TagGetPartitionKeys.getMetricName(), registry.clock().wallTime() - start);
    return result;
}

Example 35 with QualifiedName

use of com.netflix.metacat.common.QualifiedName in project metacat by Netflix.

the class DirectSqlGetPartition method getSourceTableName.

/**
 * Check if an audit table, i.e. the database is audit and the table name matches WAP table pattern
 *
 * @param databaseName database
 * @param tableName    table name
 * @return true or false
 */
private Optional<QualifiedName> getSourceTableName(final String databaseName, final String tableName, final boolean forceDisableAudit) {
    Optional<QualifiedName> sourceTable = Optional.empty();
    if (!forceDisableAudit && isAuditProcessingEnabled && databaseName.equals(AUDIT_DB)) {
        final Matcher matcher = AUDIT_TABLENAME_PATTERN.matcher(tableName);
        if (matcher.matches()) {
            final String sourceDatabaseName = matcher.group("db");
            final String sourceTableName = matcher.group("table");
            sourceTable = Optional.of(QualifiedName.ofTable(this.catalogName, sourceDatabaseName, sourceTableName));
        }
    }
    return sourceTable;
}