use of com.google.common.base.Predicate in project hive by apache.
the class DruidStorageHandler method commitCreateTable.
@Override
public void commitCreateTable(Table table) throws MetaException {
if (MetaStoreUtils.isExternalTable(table)) {
return;
}
Lifecycle lifecycle = new Lifecycle();
LOG.info(String.format("Committing table [%s] to the druid metastore", table.getDbName()));
final Path tableDir = getSegmentDescriptorDir();
try {
List<DataSegment> segmentList = DruidStorageHandlerUtils.getPublishedSegments(tableDir, getConf());
LOG.info(String.format("Found [%d] segments under path [%s]", segmentList.size(), tableDir));
druidSqlMetadataStorageUpdaterJobHandler.publishSegments(druidMetadataStorageTablesConfig.getSegmentsTable(), segmentList, DruidStorageHandlerUtils.JSON_MAPPER);
final String coordinatorAddress = HiveConf.getVar(getConf(), HiveConf.ConfVars.HIVE_DRUID_COORDINATOR_DEFAULT_ADDRESS);
int maxTries = HiveConf.getIntVar(getConf(), HiveConf.ConfVars.HIVE_DRUID_MAX_TRIES);
final String dataSourceName = table.getParameters().get(Constants.DRUID_DATA_SOURCE);
LOG.info(String.format("checking load status from coordinator [%s]", coordinatorAddress));
// check if the coordinator is up
httpClient = makeHttpClient(lifecycle);
try {
lifecycle.start();
} catch (Exception e) {
Throwables.propagate(e);
}
String coordinatorResponse = null;
try {
coordinatorResponse = RetryUtils.retry(new Callable<String>() {
@Override
public String call() throws Exception {
return DruidStorageHandlerUtils.getURL(httpClient, new URL(String.format("http://%s/status", coordinatorAddress)));
}
}, new Predicate<Throwable>() {
@Override
public boolean apply(@Nullable Throwable input) {
return input instanceof IOException;
}
}, maxTries);
} catch (Exception e) {
console.printInfo("Will skip waiting for data loading");
return;
}
if (Strings.isNullOrEmpty(coordinatorResponse)) {
console.printInfo("Will skip waiting for data loading");
return;
}
console.printInfo(String.format("Waiting for the loading of [%s] segments", segmentList.size()));
long passiveWaitTimeMs = HiveConf.getLongVar(getConf(), HiveConf.ConfVars.HIVE_DRUID_PASSIVE_WAIT_TIME);
ImmutableSet<URL> setOfUrls = FluentIterable.from(segmentList).transform(new Function<DataSegment, URL>() {
@Override
public URL apply(DataSegment dataSegment) {
try {
//Need to make sure that we are using UTC since most of the druid cluster use UTC by default
return new URL(String.format("http://%s/druid/coordinator/v1/datasources/%s/segments/%s", coordinatorAddress, dataSourceName, DataSegment.makeDataSegmentIdentifier(dataSegment.getDataSource(), new DateTime(dataSegment.getInterval().getStartMillis(), DateTimeZone.UTC), new DateTime(dataSegment.getInterval().getEndMillis(), DateTimeZone.UTC), dataSegment.getVersion(), dataSegment.getShardSpec())));
} catch (MalformedURLException e) {
Throwables.propagate(e);
}
return null;
}
}).toSet();
int numRetries = 0;
while (numRetries++ < maxTries && !setOfUrls.isEmpty()) {
setOfUrls = ImmutableSet.copyOf(Sets.filter(setOfUrls, new Predicate<URL>() {
@Override
public boolean apply(URL input) {
try {
String result = DruidStorageHandlerUtils.getURL(httpClient, input);
LOG.debug(String.format("Checking segment [%s] response is [%s]", input, result));
return Strings.isNullOrEmpty(result);
} catch (IOException e) {
LOG.error(String.format("Error while checking URL [%s]", input), e);
return true;
}
}
}));
try {
if (!setOfUrls.isEmpty()) {
Thread.sleep(passiveWaitTimeMs);
}
} catch (InterruptedException e) {
Thread.interrupted();
Throwables.propagate(e);
}
}
if (!setOfUrls.isEmpty()) {
// We are not Throwing an exception since it might be a transient issue that is blocking loading
console.printError(String.format("Wait time exhausted and we have [%s] out of [%s] segments not loaded yet", setOfUrls.size(), segmentList.size()));
}
} catch (IOException e) {
LOG.error("Exception while commit", e);
Throwables.propagate(e);
} finally {
cleanWorkingDir();
lifecycle.stop();
}
}
use of com.google.common.base.Predicate in project zeppelin by apache.
the class ConfigurationsRestApiTest method testGetAll.
@Test
public void testGetAll() throws IOException {
GetMethod get = httpGet("/configurations/all");
Map<String, Object> resp = gson.fromJson(get.getResponseBodyAsString(), new TypeToken<Map<String, Object>>() {
}.getType());
Map<String, String> body = (Map<String, String>) resp.get("body");
assertTrue(body.size() > 0);
// it shouldn't have key/value pair which key contains "password"
assertTrue(Iterators.all(body.keySet().iterator(), new Predicate<String>() {
@Override
public boolean apply(String key) {
return !key.contains("password");
}
}));
}
use of com.google.common.base.Predicate in project zeppelin by apache.
the class ConfigurationsRestApiTest method testGetViaPrefix.
@Test
public void testGetViaPrefix() throws IOException {
final String prefix = "zeppelin.server";
GetMethod get = httpGet("/configurations/prefix/" + prefix);
Map<String, Object> resp = gson.fromJson(get.getResponseBodyAsString(), new TypeToken<Map<String, Object>>() {
}.getType());
Map<String, String> body = (Map<String, String>) resp.get("body");
assertTrue(body.size() > 0);
assertTrue(Iterators.all(body.keySet().iterator(), new Predicate<String>() {
@Override
public boolean apply(String key) {
return !key.contains("password") && key.startsWith(prefix);
}
}));
}
use of com.google.common.base.Predicate in project druid by druid-io.
the class InventoryViewUtils method getSecuredDataSources.
public static Set<DruidDataSource> getSecuredDataSources(InventoryView inventoryView, final AuthorizationInfo authorizationInfo) {
if (authorizationInfo == null) {
throw new ISE("Invalid to call a secured method with null AuthorizationInfo!!");
} else {
final Map<Pair<Resource, Action>, Access> resourceAccessMap = new HashMap<>();
return ImmutableSet.copyOf(Iterables.filter(getDataSources(inventoryView), new Predicate<DruidDataSource>() {
@Override
public boolean apply(DruidDataSource input) {
Resource resource = new Resource(input.getName(), ResourceType.DATASOURCE);
Action action = Action.READ;
Pair<Resource, Action> key = new Pair<>(resource, action);
if (resourceAccessMap.containsKey(key)) {
return resourceAccessMap.get(key).isAllowed();
} else {
Access access = authorizationInfo.isAuthorized(key.lhs, key.rhs);
resourceAccessMap.put(key, access);
return access.isAllowed();
}
}
}));
}
}
use of com.google.common.base.Predicate in project druid by druid-io.
the class RulesResourceFilter method filter.
@Override
public ContainerRequest filter(ContainerRequest request) {
if (getAuthConfig().isEnabled()) {
// This is an experimental feature, see - https://github.com/druid-io/druid/pull/2424
final String dataSourceName = request.getPathSegments().get(Iterables.indexOf(request.getPathSegments(), new Predicate<PathSegment>() {
@Override
public boolean apply(PathSegment input) {
return input.getPath().equals("rules");
}
}) + 1).getPath();
Preconditions.checkNotNull(dataSourceName);
final AuthorizationInfo authorizationInfo = (AuthorizationInfo) getReq().getAttribute(AuthConfig.DRUID_AUTH_TOKEN);
Preconditions.checkNotNull(authorizationInfo, "Security is enabled but no authorization info found in the request");
final Access authResult = authorizationInfo.isAuthorized(new Resource(dataSourceName, ResourceType.DATASOURCE), getAction(request));
if (!authResult.isAllowed()) {
throw new WebApplicationException(Response.status(Response.Status.FORBIDDEN).entity(String.format("Access-Check-Result: %s", authResult.toString())).build());
}
}
return request;
}
Aggregations