Examples with ISE org.apache.druid.java.util.common.ISE used on opensource projects

Example 16 with ISE

use of org.apache.druid.java.util.common.ISE in project druid by druid-io.

the class JoinableClauses method createClauses.

/**
 * Builds a list of {@link JoinableClause} corresponding to a list of {@link PreJoinableClause}. This will call
 * {@link JoinableFactory#build} on each one and therefore may be an expensive operation.
 */
public static JoinableClauses createClauses(final List<PreJoinableClause> preClauses, final JoinableFactory joinableFactory) {
    // Since building a JoinableClause can be expensive, check for prefix conflicts before building
    checkPreJoinableClausesForDuplicatesAndShadowing(preClauses);
    List<JoinableClause> joinableClauses = preClauses.stream().map(preJoinableClause -> {
        final Optional<Joinable> joinable = joinableFactory.build(preJoinableClause.getDataSource(), preJoinableClause.getCondition());
        return new JoinableClause(preJoinableClause.getPrefix(), joinable.orElseThrow(() -> new ISE("dataSource is not joinable: %s", preJoinableClause.getDataSource())), preJoinableClause.getJoinType(), preJoinableClause.getCondition());
    }).collect(Collectors.toList());
    return new JoinableClauses(joinableClauses);
}

Example 17 with ISE

use of org.apache.druid.java.util.common.ISE in project druid by druid-io.

the class ScanQueryRunnerTest method verify.

public static void verify(Iterable<ScanResultValue> expectedResults, Iterable<ScanResultValue> actualResults) {
    Iterator<ScanResultValue> expectedIter = expectedResults.iterator();
    Iterator<ScanResultValue> actualIter = actualResults.iterator();
    while (expectedIter.hasNext()) {
        ScanResultValue expected = expectedIter.next();
        ScanResultValue actual = actualIter.next();
        Assert.assertEquals(expected.getSegmentId(), actual.getSegmentId());
        Set exColumns = Sets.newTreeSet(expected.getColumns());
        Set acColumns = Sets.newTreeSet(actual.getColumns());
        Assert.assertEquals(exColumns, acColumns);
        Iterator<Map<String, Object>> expectedEvts = ((List<Map<String, Object>>) expected.getEvents()).iterator();
        Iterator<Map<String, Object>> actualEvts = ((List<Map<String, Object>>) actual.getEvents()).iterator();
        while (expectedEvts.hasNext()) {
            Map<String, Object> exHolder = expectedEvts.next();
            Map<String, Object> acHolder = actualEvts.next();
            for (Map.Entry<String, Object> ex : exHolder.entrySet()) {
                Object actVal = acHolder.get(ex.getKey());
                if (actVal instanceof String[]) {
                    actVal = Arrays.asList((String[]) actVal);
                }
                Object exValue = ex.getValue();
                if (exValue instanceof Double || exValue instanceof Float) {
                    final double expectedDoubleValue = ((Number) exValue).doubleValue();
                    Assert.assertEquals("invalid value for " + ex.getKey(), expectedDoubleValue, ((Number) actVal).doubleValue(), expectedDoubleValue * 1e-6);
                } else {
                    Assert.assertEquals("invalid value for " + ex.getKey(), ex.getValue(), actVal);
                }
            }
            for (Map.Entry<String, Object> ac : acHolder.entrySet()) {
                Object exVal = exHolder.get(ac.getKey());
                Object actVal = ac.getValue();
                if (actVal instanceof String[]) {
                    actVal = Arrays.asList((String[]) actVal);
                }
                if (exVal instanceof Double || exVal instanceof Float) {
                    final double exDoubleValue = ((Number) exVal).doubleValue();
                    Assert.assertEquals("invalid value for " + ac.getKey(), exDoubleValue, ((Number) actVal).doubleValue(), exDoubleValue * 1e-6);
                } else {
                    Assert.assertEquals("invalid value for " + ac.getKey(), exVal, actVal);
                }
            }
        }
        if (actualEvts.hasNext()) {
            throw new ISE("This event iterator should be exhausted!");
        }
    }
    if (actualIter.hasNext()) {
        throw new ISE("This iterator should be exhausted!");
    }
}

Example 18 with ISE

use of org.apache.druid.java.util.common.ISE in project druid by druid-io.

the class BaseAppenderatorDriver method append.

/**
 * Add a row. Must not be called concurrently from multiple threads.
 *
 * @param row                      the row to add
 * @param sequenceName             sequenceName for this row's segment
 * @param committerSupplier        supplier of a committer associated with all data that has been added, including this row
 *                                 if {@param allowIncrementalPersists} is set to false then this will not be used
 * @param skipSegmentLineageCheck  if false, perform lineage validation using previousSegmentId for this sequence.
 *                                 Should be set to false if replica tasks would index events in same order
 * @param allowIncrementalPersists whether to allow persist to happen when maxRowsInMemory or intermediate persist period
 *                                 threshold is hit
 *
 * @return {@link AppenderatorDriverAddResult}
 *
 * @throws IOException if there is an I/O error while allocating or writing to a segment
 */
protected AppenderatorDriverAddResult append(final InputRow row, final String sequenceName, @Nullable final Supplier<Committer> committerSupplier, final boolean skipSegmentLineageCheck, final boolean allowIncrementalPersists) throws IOException {
    Preconditions.checkNotNull(row, "row");
    Preconditions.checkNotNull(sequenceName, "sequenceName");
    final SegmentIdWithShardSpec identifier = getSegment(row, sequenceName, skipSegmentLineageCheck);
    if (identifier != null) {
        try {
            final Appenderator.AppenderatorAddResult result = appenderator.add(identifier, row, committerSupplier == null ? null : wrapCommitterSupplier(committerSupplier), allowIncrementalPersists);
            return AppenderatorDriverAddResult.ok(identifier, result.getNumRowsInSegment(), appenderator.getTotalRowCount(), result.isPersistRequired());
        } catch (SegmentNotWritableException e) {
            throw new ISE(e, "Segment[%s] not writable when it should have been.", identifier);
        }
    } else {
        return AppenderatorDriverAddResult.fail();
    }
}

Example 19 with ISE

use of org.apache.druid.java.util.common.ISE in project druid by druid-io.

the class BatchAppenderator method persistHydrant.

/**
 * Persists the given hydrant and returns the number of rows persisted.
 *
 * @param indexToPersist hydrant to persist
 * @param identifier     the segment this hydrant is going to be part of
 * @return the number of rows persisted
 */
private int persistHydrant(FireHydrant indexToPersist, SegmentIdWithShardSpec identifier) {
    if (indexToPersist.hasSwapped()) {
        throw new ISE("Segment[%s] hydrant[%s] already swapped. This cannot happen.", identifier, indexToPersist);
    }
    log.debug("Segment[%s], persisting Hydrant[%s]", identifier, indexToPersist);
    try {
        final long startTime = System.nanoTime();
        int numRows = indexToPersist.getIndex().size();
        // since the sink may have been persisted before it may have lost its
        // hydrant count, we remember that value in the sinks' metadata, so we have
        // to pull it from there....
        SinkMetadata sm = sinksMetadata.get(identifier);
        if (sm == null) {
            throw new ISE("Sink must not be null for identifier when persisting hydrant[%s]", identifier);
        }
        final File persistDir = createPersistDirIfNeeded(identifier);
        indexMerger.persist(indexToPersist.getIndex(), identifier.getInterval(), new File(persistDir, String.valueOf(sm.getNumHydrants())), tuningConfig.getIndexSpecForIntermediatePersists(), tuningConfig.getSegmentWriteOutMediumFactory());
        sm.setPersistedFileDir(persistDir);
        log.info("Persisted in-memory data for segment[%s] spill[%s] to disk in [%,d] ms (%,d rows).", indexToPersist.getSegmentId(), sm.getNumHydrants(), (System.nanoTime() - startTime) / 1000000, numRows);
        indexToPersist.swapSegment(null);
        // remember hydrant count:
        sm.addHydrants(1);
        return numRows;
    } catch (IOException e) {
        log.makeAlert("Incremental persist failed").addData("segment", identifier.toString()).addData("dataSource", schema.getDataSource()).addData("count", indexToPersist.getCount()).emit();
        throw new RuntimeException(e);
    }
}

Example 20 with ISE

use of org.apache.druid.java.util.common.ISE in project druid by druid-io.

the class BatchAppenderator method persistAll.

@Override
public ListenableFuture<Object> persistAll(@Nullable final Committer committer) {
    throwPersistErrorIfExists();
    if (committer != null) {
        throw new ISE("committer must be null for BatchAppenderator");
    }
    // Get ready to persist all sinks:
    final Map<SegmentIdWithShardSpec, Sink> sinksToPersist = swapSinks();
    final Stopwatch runExecStopwatch = Stopwatch.createStarted();
    ListenableFuture<Object> future = persistExecutor.submit(() -> {
        log.info("Spawning intermediate persist");
        // figure out hydrants (indices) to persist:
        final List<Pair<FireHydrant, SegmentIdWithShardSpec>> indexesToPersist = new ArrayList<>();
        int numPersistedRows = 0;
        long bytesPersisted = 0;
        int totalHydrantsCount = 0;
        final long totalSinks = sinksToPersist.size();
        for (Map.Entry<SegmentIdWithShardSpec, Sink> entry : sinksToPersist.entrySet()) {
            final SegmentIdWithShardSpec identifier = entry.getKey();
            final Sink sink = entry.getValue();
            if (sink == null) {
                throw new ISE("No sink for identifier: %s", identifier);
            }
            final List<FireHydrant> hydrants = Lists.newArrayList(sink);
            // Since everytime we persist we also get rid of the in-memory references to sink & hydrants
            // the invariant of exactly one, always swappable, sink with exactly one unpersisted hydrant must hold
            int totalHydrantsForSink = hydrants.size();
            if (totalHydrantsForSink != 1) {
                throw new ISE("There should be only one hydrant for identifier[%s] but there are[%s]", identifier, totalHydrantsForSink);
            }
            totalHydrantsCount++;
            numPersistedRows += sink.getNumRowsInMemory();
            bytesPersisted += sink.getBytesInMemory();
            if (!sink.swappable()) {
                throw new ISE("Sink is not swappable![%s]", identifier);
            }
            indexesToPersist.add(Pair.of(sink.swap(), identifier));
        }
        if (indexesToPersist.isEmpty()) {
            log.info("No indexes will be persisted");
        }
        final Stopwatch persistStopwatch = Stopwatch.createStarted();
        try {
            for (Pair<FireHydrant, SegmentIdWithShardSpec> pair : indexesToPersist) {
                metrics.incrementRowOutputCount(persistHydrant(pair.lhs, pair.rhs));
            }
            log.info("Persisted in-memory data for segments: %s", indexesToPersist.stream().filter(itp -> itp.rhs != null).map(itp -> itp.rhs.asSegmentId().toString()).distinct().collect(Collectors.joining(", ")));
            log.info("Persisted stats: processed rows: [%d], persisted rows[%d], persisted sinks: [%d], persisted fireHydrants (across sinks): [%d]", rowIngestionMeters.getProcessed(), numPersistedRows, totalSinks, totalHydrantsCount);
        // note that we do not need to reset sinks metadata since we did it at the start...
        } catch (Exception e) {
            metrics.incrementFailedPersists();
            throw e;
        } finally {
            metrics.incrementNumPersists();
            long persistMillis = persistStopwatch.elapsed(TimeUnit.MILLISECONDS);
            metrics.incrementPersistTimeMillis(persistMillis);
            persistStopwatch.stop();
            // make sure no push can start while persisting:
            log.info("Persisted rows[%,d] and bytes[%,d] and removed all sinks & hydrants from memory in[%d] millis", numPersistedRows, bytesPersisted, persistMillis);
            log.info("Persist is done.");
        }
        return null;
    });
    final long startDelay = runExecStopwatch.elapsed(TimeUnit.MILLISECONDS);
    metrics.incrementPersistBackPressureMillis(startDelay);
    if (startDelay > PERSIST_WARN_DELAY) {
        log.warn("Ingestion was throttled for [%,d] millis because persists were pending.", startDelay);
    }
    runExecStopwatch.stop();
    return future;
}