Examples with HdfsDataSegmentPusher io.druid.storage.hdfs.HdfsDataSegmentPusher used on opensource projects

Example 1 with HdfsDataSegmentPusher

use of io.druid.storage.hdfs.HdfsDataSegmentPusher in project hive by apache.

the class DruidStorageHandler method loadDruidSegments.

protected void loadDruidSegments(Table table, boolean overwrite) throws MetaException {
    // at this point we have Druid segments from reducers but we need to atomically
    // rename and commit to metadata
    final String dataSourceName = table.getParameters().get(Constants.DRUID_DATA_SOURCE);
    final List<DataSegment> segmentList = Lists.newArrayList();
    final Path tableDir = getSegmentDescriptorDir();
    // Read the created segments metadata from the table staging directory
    try {
        segmentList.addAll(DruidStorageHandlerUtils.getCreatedSegments(tableDir, getConf()));
    } catch (IOException e) {
        LOG.error("Failed to load segments descriptor from directory {}", tableDir.toString());
        Throwables.propagate(e);
        cleanWorkingDir();
    }
    // Moving Druid segments and committing to druid metadata as one transaction.
    final HdfsDataSegmentPusherConfig hdfsSegmentPusherConfig = new HdfsDataSegmentPusherConfig();
    List<DataSegment> publishedDataSegmentList = Lists.newArrayList();
    final String segmentDirectory = table.getParameters().get(Constants.DRUID_SEGMENT_DIRECTORY) != null ? table.getParameters().get(Constants.DRUID_SEGMENT_DIRECTORY) : HiveConf.getVar(getConf(), HiveConf.ConfVars.DRUID_SEGMENT_DIRECTORY);
    LOG.info(String.format("Moving [%s] Druid segments from staging directory [%s] to Deep storage [%s]", segmentList.size(), getStagingWorkingDir(), segmentDirectory));
    hdfsSegmentPusherConfig.setStorageDirectory(segmentDirectory);
    try {
        DataSegmentPusher dataSegmentPusher = new HdfsDataSegmentPusher(hdfsSegmentPusherConfig, getConf(), DruidStorageHandlerUtils.JSON_MAPPER);
        publishedDataSegmentList = DruidStorageHandlerUtils.publishSegmentsAndCommit(getConnector(), getDruidMetadataStorageTablesConfig(), dataSourceName, segmentList, overwrite, getConf(), dataSegmentPusher);
    } catch (CallbackFailedException | IOException e) {
        LOG.error("Failed to move segments from staging directory");
        if (e instanceof CallbackFailedException) {
            Throwables.propagate(e.getCause());
        }
        Throwables.propagate(e);
    } finally {
        cleanWorkingDir();
    }
    checkLoadStatus(publishedDataSegmentList);
}