use of org.apache.druid.java.util.common.IAE in project druid by druid-io.
the class SearchQueryQueryToolChest method getCacheStrategy.
@Override
public CacheStrategy<Result<SearchResultValue>, Object, SearchQuery> getCacheStrategy(final SearchQuery query) {
return new CacheStrategy<Result<SearchResultValue>, Object, SearchQuery>() {
private final List<DimensionSpec> dimensionSpecs = query.getDimensions() != null ? query.getDimensions() : Collections.emptyList();
private final List<String> dimOutputNames = dimensionSpecs.size() > 0 ? Lists.transform(dimensionSpecs, DimensionSpec::getOutputName) : Collections.emptyList();
@Override
public boolean isCacheable(SearchQuery query, boolean willMergeRunners) {
return true;
}
@Override
public byte[] computeCacheKey(SearchQuery query) {
final DimFilter dimFilter = query.getDimensionsFilter();
final byte[] filterBytes = dimFilter == null ? new byte[] {} : dimFilter.getCacheKey();
final byte[] querySpecBytes = query.getQuery().getCacheKey();
final byte[] granularityBytes = query.getGranularity().getCacheKey();
final List<DimensionSpec> dimensionSpecs = query.getDimensions() != null ? query.getDimensions() : Collections.emptyList();
final byte[][] dimensionsBytes = new byte[dimensionSpecs.size()][];
int dimensionsBytesSize = 0;
int index = 0;
for (DimensionSpec dimensionSpec : dimensionSpecs) {
dimensionsBytes[index] = dimensionSpec.getCacheKey();
dimensionsBytesSize += dimensionsBytes[index].length;
++index;
}
final byte[] sortSpecBytes = query.getSort().getCacheKey();
final ByteBuffer queryCacheKey = ByteBuffer.allocate(1 + 4 + granularityBytes.length + filterBytes.length + querySpecBytes.length + dimensionsBytesSize + sortSpecBytes.length).put(SEARCH_QUERY).put(Ints.toByteArray(query.getLimit())).put(granularityBytes).put(filterBytes).put(querySpecBytes).put(sortSpecBytes);
for (byte[] bytes : dimensionsBytes) {
queryCacheKey.put(bytes);
}
return queryCacheKey.array();
}
@Override
public byte[] computeResultLevelCacheKey(SearchQuery query) {
return computeCacheKey(query);
}
@Override
public TypeReference<Object> getCacheObjectClazz() {
return OBJECT_TYPE_REFERENCE;
}
@Override
public Function<Result<SearchResultValue>, Object> prepareForCache(boolean isResultLevelCache) {
return new Function<Result<SearchResultValue>, Object>() {
@Override
public Object apply(Result<SearchResultValue> input) {
return dimensionSpecs.size() > 0 ? Lists.newArrayList(input.getTimestamp().getMillis(), input.getValue(), dimOutputNames) : Lists.newArrayList(input.getTimestamp().getMillis(), input.getValue());
}
};
}
@Override
public Function<Object, Result<SearchResultValue>> pullFromCache(boolean isResultLevelCache) {
return new Function<Object, Result<SearchResultValue>>() {
@Override
@SuppressWarnings("unchecked")
public Result<SearchResultValue> apply(Object input) {
List<Object> result = (List<Object>) input;
boolean needsRename = false;
final Map<String, String> outputNameMap = new HashMap<>();
if (hasOutputName(result)) {
List<String> cachedOutputNames = (List) result.get(2);
Preconditions.checkArgument(cachedOutputNames.size() == dimOutputNames.size(), "cache hit, but number of dimensions mismatch");
needsRename = false;
for (int idx = 0; idx < cachedOutputNames.size(); idx++) {
String cachedOutputName = cachedOutputNames.get(idx);
String outputName = dimOutputNames.get(idx);
if (!cachedOutputName.equals(outputName)) {
needsRename = true;
}
outputNameMap.put(cachedOutputName, outputName);
}
}
return !needsRename ? new Result<>(DateTimes.utc(((Number) result.get(0)).longValue()), new SearchResultValue(Lists.transform((List) result.get(1), new Function<Object, SearchHit>() {
@Override
public SearchHit apply(@Nullable Object input) {
if (input instanceof Map) {
return new SearchHit((String) ((Map) input).get("dimension"), (String) ((Map) input).get("value"), (Integer) ((Map) input).get("count"));
} else if (input instanceof SearchHit) {
return (SearchHit) input;
} else {
throw new IAE("Unknown format [%s]", input.getClass());
}
}
}))) : new Result<>(DateTimes.utc(((Number) result.get(0)).longValue()), new SearchResultValue(Lists.transform((List) result.get(1), new Function<Object, SearchHit>() {
@Override
public SearchHit apply(@Nullable Object input) {
String dim;
String val;
Integer count;
if (input instanceof Map) {
dim = outputNameMap.get((String) ((Map) input).get("dimension"));
val = (String) ((Map) input).get("value");
count = (Integer) ((Map) input).get("count");
} else if (input instanceof SearchHit) {
SearchHit cached = (SearchHit) input;
dim = outputNameMap.get(cached.getDimension());
val = cached.getValue();
count = cached.getCount();
} else {
throw new IAE("Unknown format [%s]", input.getClass());
}
return new SearchHit(dim, val, count);
}
})));
}
};
}
private boolean hasOutputName(List<Object> cachedEntry) {
/*
* cached entry is list of two or three objects
* 1. timestamp
* 2. SearchResultValue
* 3. outputName of each dimension (optional)
*
* if a cached entry has three objects, dimension name of SearchResultValue should be check if rename is needed
*/
return cachedEntry.size() == 3;
}
};
}
use of org.apache.druid.java.util.common.IAE in project druid by druid-io.
the class ByteBufferWriteOutBytes method readFully.
@Override
public void readFully(long pos, ByteBuffer buffer) {
checkOpen();
if (pos < 0 || pos > size) {
throw new IAE("pos %d out of range [%d, %d]", pos, 0, size);
}
int ourBufferIndex = Ints.checkedCast(pos / BUFFER_SIZE);
int ourBufferOffset = Ints.checkedCast(pos % BUFFER_SIZE);
for (int bytesLeft = buffer.remaining(); bytesLeft > 0; ) {
int bytesToWrite = Math.min(BUFFER_SIZE - ourBufferOffset, bytesLeft);
ByteBuffer ourBuffer = buffers.get(ourBufferIndex);
int ourBufferPosition = ourBuffer.position();
if (bytesToWrite > ourBufferPosition - ourBufferOffset) {
throw new BufferUnderflowException();
}
try {
ourBuffer.position(ourBufferOffset);
ourBuffer.limit(ourBufferOffset + bytesToWrite);
buffer.put(ourBuffer);
} finally {
// switch back to the initial state
ourBuffer.limit(ourBuffer.capacity());
ourBuffer.position(ourBufferPosition);
}
ourBufferIndex++;
ourBufferOffset = 0;
bytesLeft -= bytesToWrite;
}
}
use of org.apache.druid.java.util.common.IAE in project druid by druid-io.
the class LookupCoordinatorResource method createOrUpdateLookup.
@POST
@Produces({ MediaType.APPLICATION_JSON, SmileMediaTypes.APPLICATION_JACKSON_SMILE })
@Path("/config/{tier}/{lookup}")
public Response createOrUpdateLookup(@PathParam("tier") String tier, @PathParam("lookup") String lookup, @HeaderParam(AuditManager.X_DRUID_AUTHOR) @DefaultValue("") final String author, @HeaderParam(AuditManager.X_DRUID_COMMENT) @DefaultValue("") final String comment, InputStream in, @Context HttpServletRequest req) {
try {
if (Strings.isNullOrEmpty(tier)) {
return Response.status(Response.Status.BAD_REQUEST).entity(ServletResourceUtils.sanitizeException(new NullPointerException("`tier` required"))).build();
}
if (Strings.isNullOrEmpty(lookup)) {
return Response.status(Response.Status.BAD_REQUEST).entity(ServletResourceUtils.sanitizeException(new IAE("`lookup` required"))).build();
}
final boolean isSmile = SmileMediaTypes.APPLICATION_JACKSON_SMILE.equals(req.getContentType());
final ObjectMapper mapper = isSmile ? smileMapper : jsonMapper;
final LookupExtractorFactoryMapContainer lookupSpec;
try {
lookupSpec = mapper.readValue(in, LookupExtractorFactoryMapContainer.class);
} catch (IOException e) {
return Response.status(Response.Status.BAD_REQUEST).entity(ServletResourceUtils.sanitizeException(e)).build();
}
if (lookupCoordinatorManager.updateLookup(tier, lookup, lookupSpec, new AuditInfo(author, comment, req.getRemoteAddr()))) {
return Response.status(Response.Status.ACCEPTED).build();
} else {
throw new RuntimeException("Unknown error updating configuration");
}
} catch (Exception e) {
LOG.error(e, "Error updating tier [%s] lookup [%s]", tier, lookup);
return Response.serverError().entity(ServletResourceUtils.sanitizeException(e)).build();
}
}
use of org.apache.druid.java.util.common.IAE in project druid by druid-io.
the class AggregationTestHelper method readQueryResultArrayFromString.
private List readQueryResultArrayFromString(String str) throws Exception {
List result = new ArrayList();
JsonParser jp = mapper.getFactory().createParser(str);
if (jp.nextToken() != JsonToken.START_ARRAY) {
throw new IAE("not an array [%s]", str);
}
ObjectCodec objectCodec = jp.getCodec();
while (jp.nextToken() != JsonToken.END_ARRAY) {
result.add(objectCodec.readValue(jp, toolChest.getBaseResultType()));
}
return result;
}
use of org.apache.druid.java.util.common.IAE in project druid by druid-io.
the class Announcer method announce.
/**
* Announces the provided bytes at the given path. Announcement means that it will create an ephemeral node
* and monitor it to make sure that it always exists until it is unannounced or this object is closed.
*
* @param path The path to announce at
* @param bytes The payload to announce
* @param removeParentIfCreated remove parent of "path" if we had created that parent
*/
public void announce(String path, byte[] bytes, boolean removeParentIfCreated) {
synchronized (toAnnounce) {
if (!started) {
toAnnounce.add(new Announceable(path, bytes, removeParentIfCreated));
return;
}
}
final ZKPaths.PathAndNode pathAndNode = ZKPaths.getPathAndNode(path);
final String parentPath = pathAndNode.getPath();
boolean buildParentPath = false;
ConcurrentMap<String, byte[]> subPaths = announcements.get(parentPath);
if (subPaths == null) {
try {
if (curator.checkExists().forPath(parentPath) == null) {
buildParentPath = true;
}
} catch (Exception e) {
log.debug(e, "Problem checking if the parent existed, ignoring.");
}
// I don't have a watcher on this path yet, create a Map and start watching.
announcements.putIfAbsent(parentPath, new ConcurrentHashMap<>());
// Guaranteed to be non-null, but might be a map put in there by another thread.
final ConcurrentMap<String, byte[]> finalSubPaths = announcements.get(parentPath);
// Synchronize to make sure that I only create a listener once.
synchronized (finalSubPaths) {
if (!listeners.containsKey(parentPath)) {
final PathChildrenCache cache = factory.make(curator, parentPath);
cache.getListenable().addListener(new PathChildrenCacheListener() {
private final AtomicReference<Set<String>> pathsLost = new AtomicReference<Set<String>>(null);
@Override
public void childEvent(CuratorFramework client, PathChildrenCacheEvent event) throws Exception {
// NOTE: ZooKeeper does not guarantee that we will get every event, and thus PathChildrenCache doesn't
// as well. If one of the below events are missed, Announcer might not work properly.
log.debug("Path[%s] got event[%s]", parentPath, event);
switch(event.getType()) {
case CHILD_REMOVED:
final ChildData child = event.getData();
final ZKPaths.PathAndNode childPath = ZKPaths.getPathAndNode(child.getPath());
final byte[] value = finalSubPaths.get(childPath.getNode());
if (value != null) {
log.info("Node[%s] dropped, reinstating.", child.getPath());
createAnnouncement(child.getPath(), value);
}
break;
case CONNECTION_LOST:
// Lost connection, which means session is broken, take inventory of what has been seen.
// This is to protect from a race condition in which the ephemeral node could have been
// created but not actually seen by the PathChildrenCache, which means that it won't know
// that it disappeared and thus will not generate a CHILD_REMOVED event for us. Under normal
// circumstances, this can only happen upon connection loss; but technically if you have
// an adversary in the system, they could also delete the ephemeral node before the cache sees
// it. This does not protect from that case, so don't have adversaries.
Set<String> pathsToReinstate = new HashSet<>();
for (String node : finalSubPaths.keySet()) {
String path = ZKPaths.makePath(parentPath, node);
log.info("Node[%s] is added to reinstate.", path);
pathsToReinstate.add(path);
}
if (!pathsToReinstate.isEmpty() && !pathsLost.compareAndSet(null, pathsToReinstate)) {
log.info("Already had a pathsLost set!?[%s]", parentPath);
}
break;
case CONNECTION_RECONNECTED:
final Set<String> thePathsLost = pathsLost.getAndSet(null);
if (thePathsLost != null) {
for (String path : thePathsLost) {
log.info("Reinstating [%s]", path);
final ZKPaths.PathAndNode split = ZKPaths.getPathAndNode(path);
createAnnouncement(path, announcements.get(split.getPath()).get(split.getNode()));
}
}
break;
case CHILD_ADDED:
if (addedChildren != null) {
addedChildren.add(event.getData().getPath());
}
// fall through
case INITIALIZED:
case CHILD_UPDATED:
case CONNECTION_SUSPENDED:
}
}
});
synchronized (toAnnounce) {
if (started) {
if (buildParentPath) {
createPath(parentPath, removeParentIfCreated);
}
startCache(cache);
listeners.put(parentPath, cache);
}
}
}
}
subPaths = finalSubPaths;
}
boolean created = false;
synchronized (toAnnounce) {
if (started) {
byte[] oldBytes = subPaths.putIfAbsent(pathAndNode.getNode(), bytes);
if (oldBytes == null) {
created = true;
} else if (!Arrays.equals(oldBytes, bytes)) {
throw new IAE("Cannot reannounce different values under the same path");
}
}
}
if (created) {
try {
createAnnouncement(path, bytes);
} catch (Exception e) {
throw new RuntimeException(e);
}
}
}
Aggregations