Examples with IAE io.druid.java.util.common.IAE used on opensource projects

Example 46 with IAE

use of io.druid.java.util.common.IAE in project druid by druid-io.

the class GroupByQueryEngine method process.

public Sequence<Row> process(final GroupByQuery query, final StorageAdapter storageAdapter) {
    if (storageAdapter == null) {
        throw new ISE("Null storage adapter found. Probably trying to issue a query against a segment being memory unmapped.");
    }
    final List<Interval> intervals = query.getQuerySegmentSpec().getIntervals();
    if (intervals.size() != 1) {
        throw new IAE("Should only have one interval, got[%s]", intervals);
    }
    Filter filter = Filters.convertToCNFFromQueryContext(query, Filters.toFilter(query.getDimFilter()));
    final Sequence<Cursor> cursors = storageAdapter.makeCursors(filter, intervals.get(0), query.getVirtualColumns(), query.getGranularity(), false);
    final ResourceHolder<ByteBuffer> bufferHolder = intermediateResultsBufferPool.take();
    return Sequences.concat(Sequences.withBaggage(Sequences.map(cursors, new Function<Cursor, Sequence<Row>>() {

        @Override
        public Sequence<Row> apply(final Cursor cursor) {
            return new BaseSequence<>(new BaseSequence.IteratorMaker<Row, RowIterator>() {

                @Override
                public RowIterator make() {
                    return new RowIterator(query, cursor, bufferHolder.get(), config.get());
                }

                @Override
                public void cleanup(RowIterator iterFromMake) {
                    CloseQuietly.close(iterFromMake);
                }
            });
        }
    }), new Closeable() {

        @Override
        public void close() throws IOException {
            CloseQuietly.close(bufferHolder);
        }
    }));
}

Example 47 with IAE

use of io.druid.java.util.common.IAE in project druid by druid-io.

the class RowBasedGrouperHelper method getValueSuppliersForDimensions.

@SuppressWarnings("unchecked")
private static Supplier<Comparable>[] getValueSuppliersForDimensions(final ColumnSelectorFactory columnSelectorFactory, final List<DimensionSpec> dimensions, final Map<String, ValueType> rawInputRowSignature) {
    final Supplier[] inputRawSuppliers = new Supplier[dimensions.size()];
    for (int i = 0; i < dimensions.size(); i++) {
        final ColumnValueSelector selector = DimensionHandlerUtils.getColumnValueSelectorFromDimensionSpec(dimensions.get(i), columnSelectorFactory);
        ValueType type = rawInputRowSignature.get(dimensions.get(i).getDimension());
        if (type == null) {
            // Subquery post-aggs aren't added to the rowSignature (see rowSignatureFor() in GroupByQueryHelper) because
            // their types aren't known, so default to String handling.
            type = ValueType.STRING;
        }
        switch(type) {
            case STRING:
                inputRawSuppliers[i] = new Supplier<Comparable>() {

                    @Override
                    public Comparable get() {
                        final String value;
                        IndexedInts index = ((DimensionSelector) selector).getRow();
                        value = index.size() == 0 ? "" : ((DimensionSelector) selector).lookupName(index.get(0));
                        return Strings.nullToEmpty(value);
                    }
                };
                break;
            case LONG:
                inputRawSuppliers[i] = new Supplier<Comparable>() {

                    @Override
                    public Comparable get() {
                        return ((LongColumnSelector) selector).get();
                    }
                };
                break;
            case FLOAT:
                inputRawSuppliers[i] = new Supplier<Comparable>() {

                    @Override
                    public Comparable get() {
                        return ((FloatColumnSelector) selector).get();
                    }
                };
                break;
            default:
                throw new IAE("invalid type: [%s]", type);
        }
    }
    return inputRawSuppliers;
}

Example 48 with IAE

use of io.druid.java.util.common.IAE in project druid by druid-io.

the class Announcer method announce.

/**
   * Announces the provided bytes at the given path.  Announcement means that it will create an ephemeral node
   * and monitor it to make sure that it always exists until it is unannounced or this object is closed.
   *
   * @param path                  The path to announce at
   * @param bytes                 The payload to announce
   * @param removeParentIfCreated remove parent of "path" if we had created that parent
   */
public void announce(String path, byte[] bytes, boolean removeParentIfCreated) {
    synchronized (toAnnounce) {
        if (!started) {
            toAnnounce.add(new Announceable(path, bytes, removeParentIfCreated));
            return;
        }
    }
    final ZKPaths.PathAndNode pathAndNode = ZKPaths.getPathAndNode(path);
    final String parentPath = pathAndNode.getPath();
    boolean buildParentPath = false;
    ConcurrentMap<String, byte[]> subPaths = announcements.get(parentPath);
    if (subPaths == null) {
        try {
            if (curator.checkExists().forPath(parentPath) == null) {
                buildParentPath = true;
            }
        } catch (Exception e) {
            log.debug(e, "Problem checking if the parent existed, ignoring.");
        }
        // I don't have a watcher on this path yet, create a Map and start watching.
        announcements.putIfAbsent(parentPath, new MapMaker().<String, byte[]>makeMap());
        // Guaranteed to be non-null, but might be a map put in there by another thread.
        final ConcurrentMap<String, byte[]> finalSubPaths = announcements.get(parentPath);
        // Synchronize to make sure that I only create a listener once.
        synchronized (finalSubPaths) {
            if (!listeners.containsKey(parentPath)) {
                final PathChildrenCache cache = factory.make(curator, parentPath);
                cache.getListenable().addListener(new PathChildrenCacheListener() {

                    private final AtomicReference<Set<String>> pathsLost = new AtomicReference<Set<String>>(null);

                    @Override
                    public void childEvent(CuratorFramework client, PathChildrenCacheEvent event) throws Exception {
                        log.debug("Path[%s] got event[%s]", parentPath, event);
                        switch(event.getType()) {
                            case CHILD_REMOVED:
                                final ChildData child = event.getData();
                                final ZKPaths.PathAndNode childPath = ZKPaths.getPathAndNode(child.getPath());
                                final byte[] value = finalSubPaths.get(childPath.getNode());
                                if (value != null) {
                                    log.info("Node[%s] dropped, reinstating.", child.getPath());
                                    createAnnouncement(child.getPath(), value);
                                }
                                break;
                            case CONNECTION_LOST:
                                // Lost connection, which means session is broken, take inventory of what has been seen.
                                // This is to protect from a race condition in which the ephemeral node could have been
                                // created but not actually seen by the PathChildrenCache, which means that it won't know
                                // that it disappeared and thus will not generate a CHILD_REMOVED event for us.  Under normal
                                // circumstances, this can only happen upon connection loss; but technically if you have
                                // an adversary in the system, they could also delete the ephemeral node before the cache sees
                                // it.  This does not protect from that case, so don't have adversaries.
                                Set<String> pathsToReinstate = Sets.newHashSet();
                                for (String node : finalSubPaths.keySet()) {
                                    String path = ZKPaths.makePath(parentPath, node);
                                    log.info("Node[%s] is added to reinstate.", path);
                                    pathsToReinstate.add(path);
                                }
                                if (!pathsToReinstate.isEmpty() && !pathsLost.compareAndSet(null, pathsToReinstate)) {
                                    log.info("Already had a pathsLost set!?[%s]", parentPath);
                                }
                                break;
                            case CONNECTION_RECONNECTED:
                                final Set<String> thePathsLost = pathsLost.getAndSet(null);
                                if (thePathsLost != null) {
                                    for (String path : thePathsLost) {
                                        log.info("Reinstating [%s]", path);
                                        final ZKPaths.PathAndNode split = ZKPaths.getPathAndNode(path);
                                        createAnnouncement(path, announcements.get(split.getPath()).get(split.getNode()));
                                    }
                                }
                                break;
                        }
                    }
                });
                synchronized (toAnnounce) {
                    if (started) {
                        if (buildParentPath) {
                            createPath(parentPath, removeParentIfCreated);
                        }
                        startCache(cache);
                        listeners.put(parentPath, cache);
                    }
                }
            }
        }
        subPaths = finalSubPaths;
    }
    boolean created = false;
    synchronized (toAnnounce) {
        if (started) {
            byte[] oldBytes = subPaths.putIfAbsent(pathAndNode.getNode(), bytes);
            if (oldBytes == null) {
                created = true;
            } else if (!Arrays.equals(oldBytes, bytes)) {
                throw new IAE("Cannot reannounce different values under the same path");
            }
        }
    }
    if (created) {
        try {
            createAnnouncement(path, bytes);
        } catch (Exception e) {
            throw Throwables.propagate(e);
        }
    }
}

Example 49 with IAE

use of io.druid.java.util.common.IAE in project druid by druid-io.

the class LocalFirehoseFactory method connect.

@Override
public Firehose connect(StringInputRowParser firehoseParser) throws IOException {
    if (baseDir == null) {
        throw new IAE("baseDir is null");
    }
    log.info("Searching for all [%s] in and beneath [%s]", filter, baseDir.getAbsoluteFile());
    Collection<File> foundFiles = FileUtils.listFiles(baseDir.getAbsoluteFile(), new WildcardFileFilter(filter), TrueFileFilter.INSTANCE);
    if (foundFiles == null || foundFiles.isEmpty()) {
        throw new ISE("Found no files to ingest! Check your schema.");
    }
    log.info("Found files: " + foundFiles);
    final LinkedList<File> files = Lists.newLinkedList(foundFiles);
    return new FileIteratingFirehose(new Iterator<LineIterator>() {

        @Override
        public boolean hasNext() {
            return !files.isEmpty();
        }

        @Override
        public LineIterator next() {
            try {
                return FileUtils.lineIterator(files.poll());
            } catch (Exception e) {
                throw Throwables.propagate(e);
            }
        }

        @Override
        public void remove() {
            throw new UnsupportedOperationException();
        }
    }, firehoseParser);
}

Example 50 with IAE

use of io.druid.java.util.common.IAE in project druid by druid-io.

the class IndexerSQLMetadataStorageCoordinator method getTimelineForIntervalsWithHandle.

private VersionedIntervalTimeline<String, DataSegment> getTimelineForIntervalsWithHandle(final Handle handle, final String dataSource, final List<Interval> intervals) throws IOException {
    if (intervals == null || intervals.isEmpty()) {
        throw new IAE("null/empty intervals");
    }
    final StringBuilder sb = new StringBuilder();
    sb.append("SELECT payload FROM %s WHERE used = true AND dataSource = ? AND (");
    for (int i = 0; i < intervals.size(); i++) {
        sb.append(String.format("(start <= ? AND %1$send%1$s >= ?)", connector.getQuoteString()));
        if (i == intervals.size() - 1) {
            sb.append(")");
        } else {
            sb.append(" OR ");
        }
    }
    Query<Map<String, Object>> sql = handle.createQuery(String.format(sb.toString(), dbTables.getSegmentsTable())).bind(0, dataSource);
    for (int i = 0; i < intervals.size(); i++) {
        Interval interval = intervals.get(i);
        sql = sql.bind(2 * i + 1, interval.getEnd().toString()).bind(2 * i + 2, interval.getStart().toString());
    }
    final ResultIterator<byte[]> dbSegments = sql.map(ByteArrayMapper.FIRST).iterator();
    final VersionedIntervalTimeline<String, DataSegment> timeline = new VersionedIntervalTimeline<>(Ordering.natural());
    while (dbSegments.hasNext()) {
        final byte[] payload = dbSegments.next();
        DataSegment segment = jsonMapper.readValue(payload, DataSegment.class);
        timeline.add(segment.getInterval(), segment.getVersion(), segment.getShardSpec().createChunk(segment));
    }
    dbSegments.close();
    return timeline;
}