Examples with ISE io.druid.java.util.common.ISE used on opensource projects

Example 11 with ISE

use of io.druid.java.util.common.ISE in project druid by druid-io.

the class Plumbers method addNextRow.

public static void addNextRow(final Supplier<Committer> committerSupplier, final Firehose firehose, final Plumber plumber, final boolean reportParseExceptions, final FireDepartmentMetrics metrics) {
    final InputRow inputRow;
    try {
        inputRow = firehose.nextRow();
    } catch (ParseException e) {
        if (reportParseExceptions) {
            throw e;
        } else {
            log.debug(e, "Discarded row due to exception, considering unparseable.");
            metrics.incrementUnparseable();
            return;
        }
    }
    if (inputRow == null) {
        if (reportParseExceptions) {
            throw new ParseException("null input row");
        } else {
            log.debug("Discarded null input row, considering unparseable.");
            metrics.incrementUnparseable();
            return;
        }
    }
    final int numRows;
    try {
        numRows = plumber.add(inputRow, committerSupplier);
    } catch (IndexSizeExceededException e) {
        // plumber.add should be swapping out indexes before they fill up.
        throw new ISE(e, "WTF?! Index size exceeded, this shouldn't happen. Bad Plumber!");
    }
    if (numRows == -1) {
        metrics.incrementThrownAway();
        log.debug("Discarded row[%s], considering thrownAway.", inputRow);
        return;
    }
    metrics.incrementProcessed();
}

Example 12 with ISE

use of io.druid.java.util.common.ISE in project druid by druid-io.

the class Initialization method getExtensionFilesToLoad.

/**
   * Find all the extension files that should be loaded by druid.
   * <p/>
   * If user explicitly specifies druid.extensions.loadList, then it will look for those extensions under root
   * extensions directory. If one of them is not found, druid will fail loudly.
   * <p/>
   * If user doesn't specify druid.extension.toLoad (or its value is empty), druid will load all the extensions
   * under the root extensions directory.
   *
   * @param config ExtensionsConfig configured by druid.extensions.xxx
   *
   * @return an array of druid extension files that will be loaded by druid process
   */
public static File[] getExtensionFilesToLoad(ExtensionsConfig config) {
    final File rootExtensionsDir = new File(config.getDirectory());
    if (rootExtensionsDir.exists() && !rootExtensionsDir.isDirectory()) {
        throw new ISE("Root extensions directory [%s] is not a directory!?", rootExtensionsDir);
    }
    File[] extensionsToLoad;
    final List<String> toLoad = config.getLoadList();
    if (toLoad == null) {
        extensionsToLoad = rootExtensionsDir.listFiles();
    } else {
        int i = 0;
        extensionsToLoad = new File[toLoad.size()];
        for (final String extensionName : toLoad) {
            File extensionDir = new File(extensionName);
            if (!extensionDir.isAbsolute()) {
                extensionDir = new File(rootExtensionsDir, extensionName);
            }
            if (!extensionDir.isDirectory()) {
                throw new ISE(String.format("Extension [%s] specified in \"druid.extensions.loadList\" didn't exist!?", extensionDir.getAbsolutePath()));
            }
            extensionsToLoad[i++] = extensionDir;
        }
    }
    return extensionsToLoad == null ? new File[] {} : extensionsToLoad;
}

Example 13 with ISE

use of io.druid.java.util.common.ISE in project druid by druid-io.

the class BatchDataSegmentAnnouncer method announceSegments.

@Override
public void announceSegments(Iterable<DataSegment> segments) throws IOException {
    Iterable<DataSegment> toAnnounce = Iterables.transform(segments, segmentTransformer);
    SegmentZNode segmentZNode = new SegmentZNode(makeServedSegmentPath());
    Set<DataSegment> batch = Sets.newHashSet();
    int byteSize = 0;
    int count = 0;
    for (DataSegment segment : toAnnounce) {
        int newBytesLen = jsonMapper.writeValueAsBytes(segment).length;
        if (newBytesLen > config.getMaxBytesPerNode()) {
            throw new ISE("byte size %,d exceeds %,d", newBytesLen, config.getMaxBytesPerNode());
        }
        if (count >= config.getSegmentsPerNode() || byteSize + newBytesLen > config.getMaxBytesPerNode()) {
            segmentZNode.addSegments(batch);
            announcer.announce(segmentZNode.getPath(), segmentZNode.getBytes());
            segmentZNode = new SegmentZNode(makeServedSegmentPath());
            batch = Sets.newHashSet();
            count = 0;
            byteSize = 0;
        }
        log.info("Announcing segment[%s] at path[%s]", segment.getIdentifier(), segmentZNode.getPath());
        segmentLookup.put(segment, segmentZNode);
        batch.add(segment);
        count++;
        byteSize += newBytesLen;
    }
    segmentZNode.addSegments(batch);
    announcer.announce(segmentZNode.getPath(), segmentZNode.getBytes());
}

Example 14 with ISE

use of io.druid.java.util.common.ISE in project druid by druid-io.

the class InventoryViewUtils method getSecuredDataSources.

public static Set<DruidDataSource> getSecuredDataSources(InventoryView inventoryView, final AuthorizationInfo authorizationInfo) {
    if (authorizationInfo == null) {
        throw new ISE("Invalid to call a secured method with null AuthorizationInfo!!");
    } else {
        final Map<Pair<Resource, Action>, Access> resourceAccessMap = new HashMap<>();
        return ImmutableSet.copyOf(Iterables.filter(getDataSources(inventoryView), new Predicate<DruidDataSource>() {

            @Override
            public boolean apply(DruidDataSource input) {
                Resource resource = new Resource(input.getName(), ResourceType.DATASOURCE);
                Action action = Action.READ;
                Pair<Resource, Action> key = new Pair<>(resource, action);
                if (resourceAccessMap.containsKey(key)) {
                    return resourceAccessMap.get(key).isAllowed();
                } else {
                    Access access = authorizationInfo.isAuthorized(key.lhs, key.rhs);
                    resourceAccessMap.put(key, access);
                    return access.isAllowed();
                }
            }
        }));
    }
}

Example 15 with ISE

use of io.druid.java.util.common.ISE in project druid by druid-io.

the class LookupCoordinatorManager method start.

@LifecycleStart
public void start() {
    synchronized (startStopSync) {
        if (started) {
            return;
        }
        if (executorService.isShutdown()) {
            throw new ISE("Cannot restart after stop!");
        }
        lookupMapConfigRef = configManager.watch(LOOKUP_CONFIG_KEY, new TypeReference<Map<String, Map<String, Map<String, Object>>>>() {
        }, null);
        final ListenableScheduledFuture backgroundManagerFuture = this.backgroundManagerFuture = executorService.scheduleWithFixedDelay(new Runnable() {

            @Override
            public void run() {
                final Map<String, Map<String, Map<String, Object>>> allLookupTiers = lookupMapConfigRef.get();
                // Sanity check for if we are shutting down
                if (Thread.currentThread().isInterrupted()) {
                    LOG.info("Not updating lookups because process was interrupted");
                    return;
                }
                if (!started) {
                    LOG.info("Not started. Returning");
                    return;
                }
                if (allLookupTiers == null) {
                    LOG.info("Not updating lookups because no data exists");
                    return;
                }
                for (final String tier : allLookupTiers.keySet()) {
                    try {
                        final Map<String, Map<String, Object>> allLookups = allLookupTiers.get(tier);
                        final Map<String, Map<String, Object>> oldLookups = prior_update.get(tier);
                        final Collection<String> drops;
                        if (oldLookups == null) {
                            drops = ImmutableList.of();
                        } else {
                            drops = Sets.difference(oldLookups.keySet(), allLookups.keySet());
                        }
                        if (allLookupTiers == prior_update) {
                            LOG.debug("No updates");
                            updateAllNewOnTier(tier, allLookups);
                        } else {
                            updateAllOnTier(tier, allLookups);
                            deleteAllOnTier(tier, drops);
                        }
                    } catch (InterruptedException e) {
                        Thread.currentThread().interrupt();
                        throw Throwables.propagate(e);
                    } catch (Exception e) {
                        LOG.error(e, "Error updating lookups for tier [%s]. Will try again soon", tier);
                    }
                }
                prior_update = allLookupTiers;
            }
        }, 0, lookupCoordinatorManagerConfig.getPeriod(), TimeUnit.MILLISECONDS);
        Futures.addCallback(backgroundManagerFuture, new FutureCallback<Object>() {

            @Override
            public void onSuccess(@Nullable Object result) {
                backgroundManagerExitedLatch.countDown();
                LOG.debug("Exited background lookup manager");
            }

            @Override
            public void onFailure(Throwable t) {
                backgroundManagerExitedLatch.countDown();
                if (backgroundManagerFuture.isCancelled()) {
                    LOG.info("Background lookup manager exited");
                    LOG.trace(t, "Background lookup manager exited with throwable");
                } else {
                    LOG.makeAlert(t, "Background lookup manager exited with error!").emit();
                }
            }
        });
        started = true;
        LOG.debug("Started");
    }
}