Examples with StorageInfo com.netflix.metacat.common.server.connectors.model.StorageInfo used on opensource projects

Example 11 with StorageInfo

use of com.netflix.metacat.common.server.connectors.model.StorageInfo in project metacat by Netflix.

the class DruidConnectorInfoConverter method getPartitionInfoFromSegment.

/**
 * Convert from data source to partitionInfo.
 *
 * @param segment segment object
 * @return partition info object
 */
public PartitionInfo getPartitionInfoFromSegment(final Segment segment) {
    final Map<String, String> metadata = new HashMap<>();
    metadata.put(DruidConfigConstants.LOADSPEC_KEY, segment.getLoadSpec().getKeys().toString());
    metadata.put(DruidConfigConstants.LOADSPEC_BUCKET, segment.getLoadSpec().getBucket());
    metadata.put(DruidConfigConstants.LOADSPEC_TYPE, segment.getLoadSpec().getType());
    metadata.put(DruidConfigConstants.DIMENSIONS, segment.getDimensions());
    metadata.put(DruidConfigConstants.METRICS, segment.getMetric());
    final StorageInfo storageInfo = StorageInfo.builder().uri(segment.getLoadSpec().getUri()).build();
    return PartitionInfo.builder().metadata(metadata).serde(storageInfo).build();
}

Example 12 with StorageInfo

use of com.netflix.metacat.common.server.connectors.model.StorageInfo in project metacat by Netflix.

the class HiveConnectorFastPartitionService method copyTableSdToPartitionInfoSd.

private void copyTableSdToPartitionInfoSd(final PartitionInfo partitionInfo, final Table table) {
    StorageInfo sd = partitionInfo.getSerde();
    // 
    if (sd == null) {
        sd = new StorageInfo();
        partitionInfo.setSerde(sd);
    }
    final StorageDescriptor tableSd = table.getSd();
    if (StringUtils.isBlank(sd.getInputFormat())) {
        sd.setInputFormat(tableSd.getInputFormat());
    }
    if (StringUtils.isBlank(sd.getOutputFormat())) {
        sd.setOutputFormat(tableSd.getOutputFormat());
    }
    if (sd.getParameters() == null || sd.getParameters().isEmpty()) {
        sd.setParameters(tableSd.getParameters());
    }
    final SerDeInfo tableSerde = tableSd.getSerdeInfo();
    if (tableSerde != null) {
        if (StringUtils.isBlank(sd.getSerializationLib())) {
            sd.setSerializationLib(tableSerde.getSerializationLib());
        }
        if (sd.getSerdeInfoParameters() == null || sd.getSerdeInfoParameters().isEmpty()) {
            sd.setSerdeInfoParameters(tableSerde.getParameters());
        }
    }
}

Example 13 with StorageInfo

use of com.netflix.metacat.common.server.connectors.model.StorageInfo in project metacat by Netflix.

the class S3ConnectorInfoConverter method fromStorageInfo.

/**
 * Converts from storage info to s3 location.
 * @param storageInfo storage info
 * @return location
 */
Location fromStorageInfo(final StorageInfo storageInfo) {
    final Location result = new Location();
    if (storageInfo != null) {
        result.setUri(storageInfo.getUri());
        final Info info = new Info();
        info.setLocation(result);
        info.setOwner(storageInfo.getOwner());
        info.setInputFormat(storageInfo.getInputFormat());
        info.setOutputFormat(storageInfo.getOutputFormat());
        info.setSerializationLib(storageInfo.getSerializationLib());
        final Map<String, String> parameters = Maps.newHashMap();
        if (storageInfo.getParameters() != null) {
            parameters.putAll(storageInfo.getParameters());
        }
        if (storageInfo.getSerdeInfoParameters() != null) {
            parameters.putAll(storageInfo.getSerdeInfoParameters());
        }
        info.setParameters(parameters);
        result.setInfo(info);
    }
    return result;
}

Example 14 with StorageInfo

use of com.netflix.metacat.common.server.connectors.model.StorageInfo in project metacat by Netflix.

the class S3ConnectorInfoConverter method toStorageInfo.

/**
 * Converts from s3 table info to storage info.
 * @param table table info
 * @return table info
 */
StorageInfo toStorageInfo(final Table table) {
    StorageInfo result = null;
    final Location location = table.getLocation();
    if (location != null) {
        final Map<String, String> infoParameters = Maps.newHashMap();
        result = new StorageInfo();
        result.setUri(location.getUri());
        final Info info = location.getInfo();
        if (info != null) {
            result.setOwner(info.getOwner());
            result.setInputFormat(info.getInputFormat());
            result.setOutputFormat(info.getOutputFormat());
            result.setSerializationLib(info.getSerializationLib());
            if (info.getParameters() != null) {
                infoParameters.putAll(info.getParameters());
            }
        }
        result.setSerdeInfoParameters(infoParameters);
        result.setParameters(Maps.newHashMap());
    }
    return result;
}