Examples with IAE org.apache.druid.java.util.common.IAE used on opensource projects

Example 36 with IAE

use of org.apache.druid.java.util.common.IAE in project druid by druid-io.

the class LocalIntermediaryDataManager method addSegment.

/**
 * Write a segment into one of configured locations. The location to write is chosen in a round-robin manner per
 * supervisorTaskId.
 */
@Override
public DataSegment addSegment(String supervisorTaskId, String subTaskId, DataSegment segment, File segmentDir) throws IOException {
    // Get or create the location iterator for supervisorTask.
    final Iterator<StorageLocation> iterator = locationIterators.computeIfAbsent(supervisorTaskId, k -> {
        final Iterator<StorageLocation> cyclicIterator = Iterators.cycle(shuffleDataLocations);
        // Random start of the iterator
        final int random = ThreadLocalRandom.current().nextInt(shuffleDataLocations.size());
        IntStream.range(0, random).forEach(i -> cyclicIterator.next());
        return cyclicIterator;
    });
    // Create a zipped segment in a temp directory.
    final File taskTempDir = taskConfig.getTaskTempDir(subTaskId);
    final Closer closer = Closer.create();
    closer.register(() -> {
        try {
            org.apache.commons.io.FileUtils.forceDelete(taskTempDir);
        } catch (IOException e) {
            LOG.warn(e, "Failed to delete directory[%s]", taskTempDir.getAbsolutePath());
        }
    });
    if (!(segment.getShardSpec() instanceof BucketNumberedShardSpec)) {
        throw new IAE("Invalid shardSpec type. Expected [%s] but got [%s]", BucketNumberedShardSpec.class.getName(), segment.getShardSpec().getClass().getName());
    }
    final BucketNumberedShardSpec<?> bucketNumberedShardSpec = (BucketNumberedShardSpec<?>) segment.getShardSpec();
    // noinspection unused
    try (final Closer resourceCloser = closer) {
        FileUtils.mkdirp(taskTempDir);
        // Tempary compressed file. Will be removed when taskTempDir is deleted.
        final File tempZippedFile = new File(taskTempDir, segment.getId().toString());
        final long unzippedSizeBytes = CompressionUtils.zip(segmentDir, tempZippedFile);
        if (unzippedSizeBytes == 0) {
            throw new IOE("Read 0 bytes from segmentDir[%s]", segmentDir.getAbsolutePath());
        }
        // Try copying the zipped segment to one of storage locations
        for (int i = 0; i < shuffleDataLocations.size(); i++) {
            final StorageLocation location = iterator.next();
            final String partitionFilePath = getPartitionFilePath(supervisorTaskId, subTaskId, segment.getInterval(), // we must use the bucket ID instead of partition ID
            bucketNumberedShardSpec.getBucketId());
            final File destFile = location.reserve(partitionFilePath, segment.getId().toString(), tempZippedFile.length());
            if (destFile != null) {
                try {
                    FileUtils.mkdirp(destFile.getParentFile());
                    FileUtils.writeAtomically(destFile, out -> Files.asByteSource(tempZippedFile).copyTo(out));
                    LOG.info("Wrote intermediary segment[%s] for subtask[%s] at [%s]", segment.getId(), subTaskId, destFile);
                    return segment.withSize(unzippedSizeBytes).withBinaryVersion(SegmentUtils.getVersionFromDir(segmentDir));
                } catch (Exception e) {
                    location.release(partitionFilePath, tempZippedFile.length());
                    org.apache.commons.io.FileUtils.deleteQuietly(destFile);
                    LOG.warn(e, "Failed to write segment[%s] at [%s]. Trying again with the next location", segment.getId(), destFile);
                }
            }
        }
        throw new ISE("Can't find location to handle segment[%s]", segment);
    }
}

Example 37 with IAE

use of org.apache.druid.java.util.common.IAE in project druid by druid-io.

the class InlineDataSource method fromJson.

/**
 * Factory method for Jackson. Used for inline datasources that were originally encoded as JSON. Private because
 * non-Jackson callers should use {@link #fromIterable}.
 */
@JsonCreator
private static InlineDataSource fromJson(@JsonProperty("columnNames") List<String> columnNames, @JsonProperty("columnTypes") List<ColumnType> columnTypes, @JsonProperty("rows") List<Object[]> rows) {
    Preconditions.checkNotNull(columnNames, "'columnNames' must be nonnull");
    if (columnTypes != null && columnNames.size() != columnTypes.size()) {
        throw new IAE("columnNames and columnTypes must be the same length");
    }
    final RowSignature.Builder builder = RowSignature.builder();
    for (int i = 0; i < columnNames.size(); i++) {
        final String name = columnNames.get(i);
        final ColumnType type = columnTypes != null ? columnTypes.get(i) : null;
        builder.add(name, type);
    }
    return new InlineDataSource(rows, builder.build());
}

Example 38 with IAE

use of org.apache.druid.java.util.common.IAE in project druid by druid-io.

the class IPv4AddressMatchExprMacro method apply.

@Override
public Expr apply(final List<Expr> args) {
    if (args.size() != 2) {
        throw new IAE(ExprUtils.createErrMsg(name(), "must have 2 arguments"));
    }
    SubnetUtils.SubnetInfo subnetInfo = getSubnetInfo(args);
    Expr arg = args.get(0);
    class IPv4AddressMatchExpr extends ExprMacroTable.BaseScalarUnivariateMacroFunctionExpr {

        private final SubnetUtils.SubnetInfo subnetInfo;

        private IPv4AddressMatchExpr(Expr arg, SubnetUtils.SubnetInfo subnetInfo) {
            super(FN_NAME, arg);
            this.subnetInfo = subnetInfo;
        }

        @Nonnull
        @Override
        public ExprEval eval(final ObjectBinding bindings) {
            ExprEval eval = arg.eval(bindings);
            boolean match;
            switch(eval.type().getType()) {
                case STRING:
                    match = isStringMatch(eval.asString());
                    break;
                case LONG:
                    match = !eval.isNumericNull() && isLongMatch(eval.asLong());
                    break;
                default:
                    match = false;
            }
            return ExprEval.ofLongBoolean(match);
        }

        private boolean isStringMatch(String stringValue) {
            return IPv4AddressExprUtils.isValidAddress(stringValue) && subnetInfo.isInRange(stringValue);
        }

        private boolean isLongMatch(long longValue) {
            return !IPv4AddressExprUtils.overflowsUnsignedInt(longValue) && subnetInfo.isInRange((int) longValue);
        }

        @Override
        public Expr visit(Shuttle shuttle) {
            return shuttle.visit(apply(shuttle.visitAll(args)));
        }

        @Nullable
        @Override
        public ExpressionType getOutputType(InputBindingInspector inspector) {
            return ExpressionType.LONG;
        }

        @Override
        public String stringify() {
            return StringUtils.format("%s(%s, %s)", FN_NAME, arg.stringify(), args.get(ARG_SUBNET).stringify());
        }
    }
    return new IPv4AddressMatchExpr(arg, subnetInfo);
}

Example 39 with IAE

use of org.apache.druid.java.util.common.IAE in project druid by druid-io.

the class IPv4AddressMatchExprMacro method getSubnetInfo.

private SubnetUtils.SubnetInfo getSubnetInfo(List<Expr> args) {
    String subnetArgName = "subnet";
    Expr arg = args.get(ARG_SUBNET);
    ExprUtils.checkLiteralArgument(name(), arg, subnetArgName);
    String subnet = (String) arg.getLiteralValue();
    SubnetUtils subnetUtils;
    try {
        subnetUtils = new SubnetUtils(subnet);
    } catch (IllegalArgumentException e) {
        throw new IAE(e, ExprUtils.createErrMsg(name(), subnetArgName + " arg has an invalid format: " + subnet));
    }
    // make network and broadcast addresses match
    subnetUtils.setInclusiveHostCount(true);
    return subnetUtils.getInfo();
}

Example 40 with IAE

use of org.apache.druid.java.util.common.IAE in project druid by druid-io.

the class TimestampExtractExprMacro method apply.

@Override
public Expr apply(final List<Expr> args) {
    if (args.size() < 2 || args.size() > 3) {
        throw new IAE("Function[%s] must have 2 to 3 arguments", name());
    }
    if (!args.get(1).isLiteral() || args.get(1).getLiteralValue() == null) {
        throw new IAE("Function[%s] unit arg must be literal", name());
    }
    if (args.size() > 2 && !args.get(2).isLiteral()) {
        throw new IAE("Function[%s] timezone arg must be literal", name());
    }
    final Expr arg = args.get(0);
    final Unit unit = Unit.valueOf(StringUtils.toUpperCase((String) args.get(1).getLiteralValue()));
    final DateTimeZone timeZone;
    if (args.size() > 2) {
        timeZone = ExprUtils.toTimeZone(args.get(2));
    } else {
        timeZone = DateTimeZone.UTC;
    }
    final ISOChronology chronology = ISOChronology.getInstance(timeZone);
    class TimestampExtractExpr extends ExprMacroTable.BaseScalarUnivariateMacroFunctionExpr {

        private TimestampExtractExpr(Expr arg) {
            super(FN_NAME, arg);
        }

        @Nonnull
        @Override
        public ExprEval eval(final ObjectBinding bindings) {
            Object val = arg.eval(bindings).value();
            if (val == null) {
                // Return null if the argument if null.
                return ExprEval.of(null);
            }
            final DateTime dateTime = new DateTime(val, chronology);
            long epoch = dateTime.getMillis() / 1000;
            switch(unit) {
                case EPOCH:
                    return ExprEval.of(epoch);
                case MICROSECOND:
                    return ExprEval.of(epoch / 1000);
                case MILLISECOND:
                    return ExprEval.of(dateTime.millisOfSecond().get());
                case SECOND:
                    return ExprEval.of(dateTime.secondOfMinute().get());
                case MINUTE:
                    return ExprEval.of(dateTime.minuteOfHour().get());
                case HOUR:
                    return ExprEval.of(dateTime.hourOfDay().get());
                case DAY:
                    return ExprEval.of(dateTime.dayOfMonth().get());
                case DOW:
                    return ExprEval.of(dateTime.dayOfWeek().get());
                case ISODOW:
                    return ExprEval.of(dateTime.dayOfWeek().get());
                case DOY:
                    return ExprEval.of(dateTime.dayOfYear().get());
                case WEEK:
                    return ExprEval.of(dateTime.weekOfWeekyear().get());
                case MONTH:
                    return ExprEval.of(dateTime.monthOfYear().get());
                case QUARTER:
                    return ExprEval.of((dateTime.monthOfYear().get() - 1) / 3 + 1);
                case YEAR:
                    return ExprEval.of(dateTime.year().get());
                case ISOYEAR:
                    return ExprEval.of(dateTime.year().get());
                case DECADE:
                    // The year field divided by 10, See https://www.postgresql.org/docs/10/functions-datetime.html
                    return ExprEval.of(dateTime.year().get() / 10);
                case CENTURY:
                    return ExprEval.of(Math.ceil((double) dateTime.year().get() / 100));
                case MILLENNIUM:
                    // See https://www.postgresql.org/docs/10/functions-datetime.html
                    return ExprEval.of(Math.ceil((double) dateTime.year().get() / 1000));
                default:
                    throw new ISE("Unhandled unit[%s]", unit);
            }
        }

        @Override
        public Expr visit(Shuttle shuttle) {
            return shuttle.visit(apply(shuttle.visitAll(args)));
        }

        @Nullable
        @Override
        public ExpressionType getOutputType(InputBindingInspector inspector) {
            switch(unit) {
                case CENTURY:
                case MILLENNIUM:
                    return ExpressionType.DOUBLE;
                default:
                    return ExpressionType.LONG;
            }
        }

        @Override
        public String stringify() {
            if (args.size() > 2) {
                return StringUtils.format("%s(%s, %s, %s)", FN_NAME, arg.stringify(), args.get(1).stringify(), args.get(2).stringify());
            }
            return StringUtils.format("%s(%s, %s)", FN_NAME, arg.stringify(), args.get(1).stringify());
        }
    }
    return new TimestampExtractExpr(arg);
}