Examples with ISE io.druid.java.util.common.ISE used on opensource projects

Example 86 with ISE

use of io.druid.java.util.common.ISE in project druid by druid-io.

the class MoveTask method run.

@Override
public TaskStatus run(TaskToolbox toolbox) throws Exception {
    // Confirm we have a lock (will throw if there isn't exactly one element)
    final TaskLock myLock = Iterables.getOnlyElement(getTaskLocks(toolbox));
    if (!myLock.getDataSource().equals(getDataSource())) {
        throw new ISE("WTF?! Lock dataSource[%s] != task dataSource[%s]", myLock.getDataSource(), getDataSource());
    }
    if (!myLock.getInterval().equals(getInterval())) {
        throw new ISE("WTF?! Lock interval[%s] != task interval[%s]", myLock.getInterval(), getInterval());
    }
    // List unused segments
    final List<DataSegment> unusedSegments = toolbox.getTaskActionClient().submit(new SegmentListUnusedAction(myLock.getDataSource(), myLock.getInterval()));
    // Verify none of these segments have versions > lock version
    for (final DataSegment unusedSegment : unusedSegments) {
        if (unusedSegment.getVersion().compareTo(myLock.getVersion()) > 0) {
            throw new ISE("WTF?! Unused segment[%s] has version[%s] > task version[%s]", unusedSegment.getIdentifier(), unusedSegment.getVersion(), myLock.getVersion());
        }
        log.info("OK to move segment: %s", unusedSegment.getIdentifier());
    }
    // Move segments
    for (DataSegment segment : unusedSegments) {
        final DataSegment movedSegment = toolbox.getDataSegmentMover().move(segment, targetLoadSpec);
        toolbox.getTaskActionClient().submit(new SegmentMetadataUpdateAction(ImmutableSet.of(movedSegment)));
    }
    return TaskStatus.success(getId());
}

Example 87 with ISE

use of io.druid.java.util.common.ISE in project druid by druid-io.

the class RestoreTask method run.

@Override
public TaskStatus run(TaskToolbox toolbox) throws Exception {
    // Confirm we have a lock (will throw if there isn't exactly one element)
    final TaskLock myLock = Iterables.getOnlyElement(getTaskLocks(toolbox));
    if (!myLock.getDataSource().equals(getDataSource())) {
        throw new ISE("WTF?! Lock dataSource[%s] != task dataSource[%s]", myLock.getDataSource(), getDataSource());
    }
    if (!myLock.getInterval().equals(getInterval())) {
        throw new ISE("WTF?! Lock interval[%s] != task interval[%s]", myLock.getInterval(), getInterval());
    }
    // List unused segments
    final List<DataSegment> unusedSegments = toolbox.getTaskActionClient().submit(new SegmentListUnusedAction(myLock.getDataSource(), myLock.getInterval()));
    // Verify none of these segments have versions > lock version
    for (final DataSegment unusedSegment : unusedSegments) {
        if (unusedSegment.getVersion().compareTo(myLock.getVersion()) > 0) {
            throw new ISE("WTF?! Unused segment[%s] has version[%s] > task version[%s]", unusedSegment.getIdentifier(), unusedSegment.getVersion(), myLock.getVersion());
        }
        log.info("OK to restore segment: %s", unusedSegment.getIdentifier());
    }
    final List<DataSegment> restoredSegments = new ArrayList<>();
    // Move segments
    for (DataSegment segment : unusedSegments) {
        final DataSegment restored = toolbox.getDataSegmentArchiver().restore(segment);
        if (restored != null) {
            restoredSegments.add(restored);
        } else {
            log.info("Segment [%s] did not move, not updating metadata", segment);
        }
    }
    if (restoredSegments.isEmpty()) {
        log.info("No segments restored");
    } else {
        // Update metadata for moved segments
        toolbox.getTaskActionClient().submit(new SegmentMetadataUpdateAction(ImmutableSet.copyOf(restoredSegments)));
    }
    return TaskStatus.success(getId());
}

Example 88 with ISE

use of io.druid.java.util.common.ISE in project druid by druid-io.

the class DatasourcePathSpec method addInputPaths.

@Override
public Job addInputPaths(HadoopDruidIndexerConfig config, Job job) throws IOException {
    if (segments == null || segments.isEmpty()) {
        if (ingestionSpec.isIgnoreWhenNoSegments()) {
            logger.warn("No segments found for ingestionSpec [%s]", ingestionSpec);
            return job;
        } else {
            throw new ISE("No segments found for ingestion spec [%s]", ingestionSpec);
        }
    }
    logger.info("Found total [%d] segments for [%s]  in interval [%s]", segments.size(), ingestionSpec.getDataSource(), ingestionSpec.getIntervals());
    DatasourceIngestionSpec updatedIngestionSpec = ingestionSpec;
    if (updatedIngestionSpec.getDimensions() == null) {
        List<String> dims;
        if (config.getParser().getParseSpec().getDimensionsSpec().hasCustomDimensions()) {
            dims = config.getParser().getParseSpec().getDimensionsSpec().getDimensionNames();
        } else {
            Set<String> dimSet = Sets.newHashSet(Iterables.concat(Iterables.transform(segments, new Function<WindowedDataSegment, Iterable<String>>() {

                @Override
                public Iterable<String> apply(WindowedDataSegment dataSegment) {
                    return dataSegment.getSegment().getDimensions();
                }
            })));
            dims = Lists.newArrayList(Sets.difference(dimSet, config.getParser().getParseSpec().getDimensionsSpec().getDimensionExclusions()));
        }
        updatedIngestionSpec = updatedIngestionSpec.withDimensions(dims);
    }
    if (updatedIngestionSpec.getMetrics() == null) {
        Set<String> metrics = Sets.newHashSet();
        final AggregatorFactory[] cols = config.getSchema().getDataSchema().getAggregators();
        if (cols != null) {
            for (AggregatorFactory col : cols) {
                metrics.add(col.getName());
            }
        }
        updatedIngestionSpec = updatedIngestionSpec.withMetrics(Lists.newArrayList(metrics));
    }
    updatedIngestionSpec = updatedIngestionSpec.withQueryGranularity(config.getGranularitySpec().getQueryGranularity());
    job.getConfiguration().set(DatasourceInputFormat.CONF_DRUID_SCHEMA, mapper.writeValueAsString(updatedIngestionSpec));
    job.getConfiguration().set(DatasourceInputFormat.CONF_INPUT_SEGMENTS, mapper.writeValueAsString(segments));
    job.getConfiguration().set(DatasourceInputFormat.CONF_MAX_SPLIT_SIZE, String.valueOf(maxSplitSize));
    MultipleInputs.addInputPath(job, new Path("/dummy/tobe/ignored"), DatasourceInputFormat.class);
    return job;
}

Example 89 with ISE

use of io.druid.java.util.common.ISE in project druid by druid-io.

the class DruidKerberosUtil method authenticateIfRequired.

public static void authenticateIfRequired(AuthenticationKerberosConfig config) throws IOException {
    String principal = config.getPrincipal();
    String keytab = config.getKeytab();
    if (!Strings.isNullOrEmpty(principal) && !Strings.isNullOrEmpty(keytab)) {
        Configuration conf = new Configuration();
        conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos");
        UserGroupInformation.setConfiguration(conf);
        try {
            if (UserGroupInformation.getCurrentUser().hasKerberosCredentials() == false || !UserGroupInformation.getCurrentUser().getUserName().equals(principal)) {
                log.info("trying to authenticate user [%s] with keytab [%s]", principal, keytab);
                UserGroupInformation.loginUserFromKeytab(principal, keytab);
            }
        } catch (IOException e) {
            throw new ISE(e, "Failed to authenticate user principal [%s] with keytab [%s]", principal, keytab);
        }
    }
}

Example 90 with ISE

use of io.druid.java.util.common.ISE in project druid by druid-io.

the class IndexMergerV9 method setupMetricsWriters.

private ArrayList<GenericColumnSerializer> setupMetricsWriters(final IOPeon ioPeon, final List<String> mergedMetrics, final Map<String, ValueType> metricsValueTypes, final Map<String, String> metricTypeNames, final IndexSpec indexSpec) throws IOException {
    ArrayList<GenericColumnSerializer> metWriters = Lists.newArrayListWithCapacity(mergedMetrics.size());
    final CompressedObjectStrategy.CompressionStrategy metCompression = indexSpec.getMetricCompression();
    final CompressionFactory.LongEncodingStrategy longEncoding = indexSpec.getLongEncoding();
    for (String metric : mergedMetrics) {
        ValueType type = metricsValueTypes.get(metric);
        GenericColumnSerializer writer;
        switch(type) {
            case LONG:
                writer = LongColumnSerializer.create(ioPeon, metric, metCompression, longEncoding);
                break;
            case FLOAT:
                writer = FloatColumnSerializer.create(ioPeon, metric, metCompression);
                break;
            case COMPLEX:
                final String typeName = metricTypeNames.get(metric);
                ComplexMetricSerde serde = ComplexMetrics.getSerdeForType(typeName);
                if (serde == null) {
                    throw new ISE("Unknown type[%s]", typeName);
                }
                writer = serde.getSerializer(ioPeon, metric);
                break;
            default:
                throw new ISE("Unknown type[%s]", type);
        }
        writer.open();
        // we will close these writers in another method after we added all the metrics
        metWriters.add(writer);
    }
    return metWriters;
}