Search in sources :

Example 1 with Predicate

use of com.google.common.base.Predicate in project hive by apache.

the class DruidStorageHandler method commitCreateTable.

@Override
public void commitCreateTable(Table table) throws MetaException {
    if (MetaStoreUtils.isExternalTable(table)) {
        return;
    }
    Lifecycle lifecycle = new Lifecycle();
    LOG.info(String.format("Committing table [%s] to the druid metastore", table.getDbName()));
    final Path tableDir = getSegmentDescriptorDir();
    try {
        List<DataSegment> segmentList = DruidStorageHandlerUtils.getPublishedSegments(tableDir, getConf());
        LOG.info(String.format("Found [%d] segments under path [%s]", segmentList.size(), tableDir));
        druidSqlMetadataStorageUpdaterJobHandler.publishSegments(druidMetadataStorageTablesConfig.getSegmentsTable(), segmentList, DruidStorageHandlerUtils.JSON_MAPPER);
        final String coordinatorAddress = HiveConf.getVar(getConf(), HiveConf.ConfVars.HIVE_DRUID_COORDINATOR_DEFAULT_ADDRESS);
        int maxTries = HiveConf.getIntVar(getConf(), HiveConf.ConfVars.HIVE_DRUID_MAX_TRIES);
        final String dataSourceName = table.getParameters().get(Constants.DRUID_DATA_SOURCE);
        LOG.info(String.format("checking load status from coordinator [%s]", coordinatorAddress));
        // check if the coordinator is up
        httpClient = makeHttpClient(lifecycle);
        try {
            lifecycle.start();
        } catch (Exception e) {
            Throwables.propagate(e);
        }
        String coordinatorResponse = null;
        try {
            coordinatorResponse = RetryUtils.retry(new Callable<String>() {

                @Override
                public String call() throws Exception {
                    return DruidStorageHandlerUtils.getURL(httpClient, new URL(String.format("http://%s/status", coordinatorAddress)));
                }
            }, new Predicate<Throwable>() {

                @Override
                public boolean apply(@Nullable Throwable input) {
                    return input instanceof IOException;
                }
            }, maxTries);
        } catch (Exception e) {
            console.printInfo("Will skip waiting for data loading");
            return;
        }
        if (Strings.isNullOrEmpty(coordinatorResponse)) {
            console.printInfo("Will skip waiting for data loading");
            return;
        }
        console.printInfo(String.format("Waiting for the loading of [%s] segments", segmentList.size()));
        long passiveWaitTimeMs = HiveConf.getLongVar(getConf(), HiveConf.ConfVars.HIVE_DRUID_PASSIVE_WAIT_TIME);
        ImmutableSet<URL> setOfUrls = FluentIterable.from(segmentList).transform(new Function<DataSegment, URL>() {

            @Override
            public URL apply(DataSegment dataSegment) {
                try {
                    //Need to make sure that we are using UTC since most of the druid cluster use UTC by default
                    return new URL(String.format("http://%s/druid/coordinator/v1/datasources/%s/segments/%s", coordinatorAddress, dataSourceName, DataSegment.makeDataSegmentIdentifier(dataSegment.getDataSource(), new DateTime(dataSegment.getInterval().getStartMillis(), DateTimeZone.UTC), new DateTime(dataSegment.getInterval().getEndMillis(), DateTimeZone.UTC), dataSegment.getVersion(), dataSegment.getShardSpec())));
                } catch (MalformedURLException e) {
                    Throwables.propagate(e);
                }
                return null;
            }
        }).toSet();
        int numRetries = 0;
        while (numRetries++ < maxTries && !setOfUrls.isEmpty()) {
            setOfUrls = ImmutableSet.copyOf(Sets.filter(setOfUrls, new Predicate<URL>() {

                @Override
                public boolean apply(URL input) {
                    try {
                        String result = DruidStorageHandlerUtils.getURL(httpClient, input);
                        LOG.debug(String.format("Checking segment [%s] response is [%s]", input, result));
                        return Strings.isNullOrEmpty(result);
                    } catch (IOException e) {
                        LOG.error(String.format("Error while checking URL [%s]", input), e);
                        return true;
                    }
                }
            }));
            try {
                if (!setOfUrls.isEmpty()) {
                    Thread.sleep(passiveWaitTimeMs);
                }
            } catch (InterruptedException e) {
                Thread.interrupted();
                Throwables.propagate(e);
            }
        }
        if (!setOfUrls.isEmpty()) {
            // We are not Throwing an exception since it might be a transient issue that is blocking loading
            console.printError(String.format("Wait time exhausted and we have [%s] out of [%s] segments not loaded yet", setOfUrls.size(), segmentList.size()));
        }
    } catch (IOException e) {
        LOG.error("Exception while commit", e);
        Throwables.propagate(e);
    } finally {
        cleanWorkingDir();
        lifecycle.stop();
    }
}
Also used : Path(org.apache.hadoop.fs.Path) MalformedURLException(java.net.MalformedURLException) Lifecycle(com.metamx.common.lifecycle.Lifecycle) IOException(java.io.IOException) DataSegment(io.druid.timeline.DataSegment) MetaException(org.apache.hadoop.hive.metastore.api.MetaException) SegmentLoadingException(io.druid.segment.loading.SegmentLoadingException) HiveException(org.apache.hadoop.hive.ql.metadata.HiveException) MalformedURLException(java.net.MalformedURLException) IOException(java.io.IOException) Callable(java.util.concurrent.Callable) URL(java.net.URL) DateTime(org.joda.time.DateTime) Predicate(com.google.common.base.Predicate) Function(com.google.common.base.Function) Nullable(javax.annotation.Nullable)

Example 2 with Predicate

use of com.google.common.base.Predicate in project zeppelin by apache.

the class ConfigurationsRestApiTest method testGetAll.

@Test
public void testGetAll() throws IOException {
    GetMethod get = httpGet("/configurations/all");
    Map<String, Object> resp = gson.fromJson(get.getResponseBodyAsString(), new TypeToken<Map<String, Object>>() {
    }.getType());
    Map<String, String> body = (Map<String, String>) resp.get("body");
    assertTrue(body.size() > 0);
    // it shouldn't have key/value pair which key contains "password"
    assertTrue(Iterators.all(body.keySet().iterator(), new Predicate<String>() {

        @Override
        public boolean apply(String key) {
            return !key.contains("password");
        }
    }));
}
Also used : TypeToken(com.google.gson.reflect.TypeToken) GetMethod(org.apache.commons.httpclient.methods.GetMethod) Map(java.util.Map) Predicate(com.google.common.base.Predicate) Test(org.junit.Test)

Example 3 with Predicate

use of com.google.common.base.Predicate in project zeppelin by apache.

the class ConfigurationsRestApiTest method testGetViaPrefix.

@Test
public void testGetViaPrefix() throws IOException {
    final String prefix = "zeppelin.server";
    GetMethod get = httpGet("/configurations/prefix/" + prefix);
    Map<String, Object> resp = gson.fromJson(get.getResponseBodyAsString(), new TypeToken<Map<String, Object>>() {
    }.getType());
    Map<String, String> body = (Map<String, String>) resp.get("body");
    assertTrue(body.size() > 0);
    assertTrue(Iterators.all(body.keySet().iterator(), new Predicate<String>() {

        @Override
        public boolean apply(String key) {
            return !key.contains("password") && key.startsWith(prefix);
        }
    }));
}
Also used : TypeToken(com.google.gson.reflect.TypeToken) GetMethod(org.apache.commons.httpclient.methods.GetMethod) Map(java.util.Map) Predicate(com.google.common.base.Predicate) Test(org.junit.Test)

Example 4 with Predicate

use of com.google.common.base.Predicate in project druid by druid-io.

the class InventoryViewUtils method getSecuredDataSources.

public static Set<DruidDataSource> getSecuredDataSources(InventoryView inventoryView, final AuthorizationInfo authorizationInfo) {
    if (authorizationInfo == null) {
        throw new ISE("Invalid to call a secured method with null AuthorizationInfo!!");
    } else {
        final Map<Pair<Resource, Action>, Access> resourceAccessMap = new HashMap<>();
        return ImmutableSet.copyOf(Iterables.filter(getDataSources(inventoryView), new Predicate<DruidDataSource>() {

            @Override
            public boolean apply(DruidDataSource input) {
                Resource resource = new Resource(input.getName(), ResourceType.DATASOURCE);
                Action action = Action.READ;
                Pair<Resource, Action> key = new Pair<>(resource, action);
                if (resourceAccessMap.containsKey(key)) {
                    return resourceAccessMap.get(key).isAllowed();
                } else {
                    Access access = authorizationInfo.isAuthorized(key.lhs, key.rhs);
                    resourceAccessMap.put(key, access);
                    return access.isAllowed();
                }
            }
        }));
    }
}
Also used : Action(io.druid.server.security.Action) HashMap(java.util.HashMap) Access(io.druid.server.security.Access) Resource(io.druid.server.security.Resource) ISE(io.druid.java.util.common.ISE) DruidDataSource(io.druid.client.DruidDataSource) Pair(io.druid.java.util.common.Pair) Predicate(com.google.common.base.Predicate)

Example 5 with Predicate

use of com.google.common.base.Predicate in project druid by druid-io.

the class RulesResourceFilter method filter.

@Override
public ContainerRequest filter(ContainerRequest request) {
    if (getAuthConfig().isEnabled()) {
        // This is an experimental feature, see - https://github.com/druid-io/druid/pull/2424
        final String dataSourceName = request.getPathSegments().get(Iterables.indexOf(request.getPathSegments(), new Predicate<PathSegment>() {

            @Override
            public boolean apply(PathSegment input) {
                return input.getPath().equals("rules");
            }
        }) + 1).getPath();
        Preconditions.checkNotNull(dataSourceName);
        final AuthorizationInfo authorizationInfo = (AuthorizationInfo) getReq().getAttribute(AuthConfig.DRUID_AUTH_TOKEN);
        Preconditions.checkNotNull(authorizationInfo, "Security is enabled but no authorization info found in the request");
        final Access authResult = authorizationInfo.isAuthorized(new Resource(dataSourceName, ResourceType.DATASOURCE), getAction(request));
        if (!authResult.isAllowed()) {
            throw new WebApplicationException(Response.status(Response.Status.FORBIDDEN).entity(String.format("Access-Check-Result: %s", authResult.toString())).build());
        }
    }
    return request;
}
Also used : WebApplicationException(javax.ws.rs.WebApplicationException) Access(io.druid.server.security.Access) Resource(io.druid.server.security.Resource) PathSegment(javax.ws.rs.core.PathSegment) AuthorizationInfo(io.druid.server.security.AuthorizationInfo) Predicate(com.google.common.base.Predicate)

Aggregations

Predicate (com.google.common.base.Predicate)180 ArrayList (java.util.ArrayList)29 Nullable (javax.annotation.Nullable)29 Test (org.junit.Test)29 List (java.util.List)28 UUID (java.util.UUID)21 IOException (java.io.IOException)20 Map (java.util.Map)20 File (java.io.File)15 Test (org.testng.annotations.Test)14 ImmutableList (com.google.common.collect.ImmutableList)12 HashMap (java.util.HashMap)12 Set (java.util.Set)9 DateTime (org.joda.time.DateTime)9 BigDecimal (java.math.BigDecimal)8 Path (javax.ws.rs.Path)8 PaymentTransactionModelDao (org.killbill.billing.payment.dao.PaymentTransactionModelDao)8 Account (org.killbill.billing.account.api.Account)7 PlanPhasePriceOverride (org.killbill.billing.catalog.api.PlanPhasePriceOverride)7 PaymentModelDao (org.killbill.billing.payment.dao.PaymentModelDao)7