Search in sources :

Example 1 with Cluster

use of com.google.bigtable.admin.v2.Cluster in project grpc-java by grpc.

the class ClientXdsClient method handleCdsResponse.

@Override
public void handleCdsResponse(ServerInfo serverInfo, String versionInfo, List<Any> resources, String nonce) {
    syncContext.throwIfNotInThisSynchronizationContext();
    Map<String, ParsedResource> parsedResources = new HashMap<>(resources.size());
    Set<String> unpackedResources = new HashSet<>(resources.size());
    Set<String> invalidResources = new HashSet<>();
    List<String> errors = new ArrayList<>();
    Set<String> retainedEdsResources = new HashSet<>();
    for (int i = 0; i < resources.size(); i++) {
        Any resource = resources.get(i);
        // Unpack the Cluster.
        Cluster cluster;
        try {
            cluster = unpackCompatibleType(resource, Cluster.class, ResourceType.CDS.typeUrl(), ResourceType.CDS.typeUrlV2());
        } catch (InvalidProtocolBufferException e) {
            errors.add("CDS response Resource index " + i + " - can't decode Cluster: " + e);
            continue;
        }
        if (!isResourceNameValid(cluster.getName(), resource.getTypeUrl())) {
            errors.add("Unsupported resource name: " + cluster.getName() + " for type: " + ResourceType.CDS);
            continue;
        }
        String clusterName = canonifyResourceName(cluster.getName());
        // unrequested resources.
        if (!cdsResourceSubscribers.containsKey(clusterName)) {
            continue;
        }
        unpackedResources.add(clusterName);
        // Process Cluster into CdsUpdate.
        CdsUpdate cdsUpdate;
        try {
            Set<String> certProviderInstances = null;
            if (getBootstrapInfo() != null && getBootstrapInfo().certProviders() != null) {
                certProviderInstances = getBootstrapInfo().certProviders().keySet();
            }
            cdsUpdate = processCluster(cluster, retainedEdsResources, certProviderInstances, serverInfo);
        } catch (ResourceInvalidException e) {
            errors.add("CDS response Cluster '" + clusterName + "' validation error: " + e.getMessage());
            invalidResources.add(clusterName);
            continue;
        }
        parsedResources.put(clusterName, new ParsedResource(cdsUpdate, resource));
    }
    logger.log(XdsLogLevel.INFO, "Received CDS Response version {0} nonce {1}. Parsed resources: {2}", versionInfo, nonce, unpackedResources);
    handleResourceUpdate(serverInfo, ResourceType.CDS, parsedResources, invalidResources, retainedEdsResources, versionInfo, nonce, errors);
}
Also used : LinkedHashMap(java.util.LinkedHashMap) HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) InvalidProtocolBufferException(com.google.protobuf.InvalidProtocolBufferException) Cluster(io.envoyproxy.envoy.config.cluster.v3.Cluster) Any(com.google.protobuf.Any) LbEndpoint(io.grpc.xds.Endpoints.LbEndpoint) HashSet(java.util.HashSet)

Example 2 with Cluster

use of com.google.bigtable.admin.v2.Cluster in project grpc-java by grpc.

the class ClientXdsClientDataTest method parseCluster_validateEdsSourceConfig.

@Test
public void parseCluster_validateEdsSourceConfig() throws ResourceInvalidException {
    Set<String> retainedEdsResources = new HashSet<>();
    Cluster cluster1 = Cluster.newBuilder().setName("cluster-foo.googleapis.com").setType(DiscoveryType.EDS).setEdsClusterConfig(EdsClusterConfig.newBuilder().setEdsConfig(ConfigSource.newBuilder().setAds(AggregatedConfigSource.getDefaultInstance())).setServiceName("service-foo.googleapis.com")).setLbPolicy(LbPolicy.ROUND_ROBIN).build();
    ClientXdsClient.processCluster(cluster1, retainedEdsResources, null, LRS_SERVER_INFO);
    Cluster cluster2 = Cluster.newBuilder().setName("cluster-foo.googleapis.com").setType(DiscoveryType.EDS).setEdsClusterConfig(EdsClusterConfig.newBuilder().setEdsConfig(ConfigSource.newBuilder().setSelf(SelfConfigSource.getDefaultInstance())).setServiceName("service-foo.googleapis.com")).setLbPolicy(LbPolicy.ROUND_ROBIN).build();
    ClientXdsClient.processCluster(cluster2, retainedEdsResources, null, LRS_SERVER_INFO);
    Cluster cluster3 = Cluster.newBuilder().setName("cluster-foo.googleapis.com").setType(DiscoveryType.EDS).setEdsClusterConfig(EdsClusterConfig.newBuilder().setEdsConfig(ConfigSource.newBuilder().setPath("foo-path")).setServiceName("service-foo.googleapis.com")).setLbPolicy(LbPolicy.ROUND_ROBIN).build();
    thrown.expect(ResourceInvalidException.class);
    thrown.expectMessage("Cluster cluster-foo.googleapis.com: field eds_cluster_config must be set to indicate to" + " use EDS over ADS or self ConfigSource");
    ClientXdsClient.processCluster(cluster3, retainedEdsResources, null, LRS_SERVER_INFO);
}
Also used : WeightedCluster(io.envoyproxy.envoy.config.route.v3.WeightedCluster) Cluster(io.envoyproxy.envoy.config.cluster.v3.Cluster) HashSet(java.util.HashSet) Test(org.junit.Test)

Example 3 with Cluster

use of com.google.bigtable.admin.v2.Cluster in project grpc-java by grpc.

the class ClientXdsClientDataTest method parseCluster_leastRequestLbPolicy_defaultLbConfig.

@Test
public void parseCluster_leastRequestLbPolicy_defaultLbConfig() throws ResourceInvalidException {
    ClientXdsClient.enableLeastRequest = true;
    Cluster cluster = Cluster.newBuilder().setName("cluster-foo.googleapis.com").setType(DiscoveryType.EDS).setEdsClusterConfig(EdsClusterConfig.newBuilder().setEdsConfig(ConfigSource.newBuilder().setAds(AggregatedConfigSource.getDefaultInstance())).setServiceName("service-foo.googleapis.com")).setLbPolicy(LbPolicy.LEAST_REQUEST).build();
    CdsUpdate update = ClientXdsClient.processCluster(cluster, new HashSet<String>(), null, LRS_SERVER_INFO);
    assertThat(update.lbPolicy()).isEqualTo(CdsUpdate.LbPolicy.LEAST_REQUEST);
    assertThat(update.choiceCount()).isEqualTo(ClientXdsClient.DEFAULT_LEAST_REQUEST_CHOICE_COUNT);
}
Also used : WeightedCluster(io.envoyproxy.envoy.config.route.v3.WeightedCluster) Cluster(io.envoyproxy.envoy.config.cluster.v3.Cluster) CdsUpdate(io.grpc.xds.XdsClient.CdsUpdate) Test(org.junit.Test)

Example 4 with Cluster

use of com.google.bigtable.admin.v2.Cluster in project java-docs-samples by GoogleCloudPlatform.

the class BulkWrite method getClusterNodeCount.

// Get the number of nodes for the Bigtable instance. This only works for single cluster instances
// so it will treat multi-cluster instances as single node clusters.
private static int getClusterNodeCount(String projectId, String instanceId) throws IOException, GeneralSecurityException {
    try {
        BigtableClusterUtilities clusterUtility = BigtableClusterUtilities.forInstance(projectId, instanceId);
        Cluster cluster = clusterUtility.getSingleCluster();
        String clusterId = new BigtableClusterName(cluster.getName()).getClusterId();
        String zoneId = BigtableClusterUtilities.getZoneId(cluster);
        int clusterNodeCount = clusterUtility.getClusterNodeCount(clusterId, zoneId);
        System.out.println("Cluster size " + clusterNodeCount);
        return clusterNodeCount;
    } catch (IllegalStateException e) {
        System.out.println("Unable to get cluster size. Treating as single-node cluster.");
        return 1;
    }
}
Also used : BigtableClusterName(com.google.cloud.bigtable.grpc.BigtableClusterName) Cluster(com.google.bigtable.admin.v2.Cluster) BigtableClusterUtilities(com.google.cloud.bigtable.grpc.BigtableClusterUtilities)

Example 5 with Cluster

use of com.google.bigtable.admin.v2.Cluster in project cdap by caskdata.

the class DataprocClient method getClusterStatus.

/**
 * Get the status of the specified cluster.
 *
 * @param name the cluster name
 * @return the cluster status
 * @throws RetryableProvisionException if there was a non 4xx error code returned
 */
io.cdap.cdap.runtime.spi.provisioner.ClusterStatus getClusterStatus(String name) throws RetryableProvisionException {
    io.cdap.cdap.runtime.spi.provisioner.ClusterStatus status = getDataprocCluster(name).map(cluster -> convertStatus(cluster.getStatus())).orElse(io.cdap.cdap.runtime.spi.provisioner.ClusterStatus.NOT_EXISTS);
    // if it failed, try to get the create operation and log the error message
    try {
        if (status == io.cdap.cdap.runtime.spi.provisioner.ClusterStatus.FAILED) {
            String resourceName = String.format("projects/%s/regions/%s/operations", conf.getProjectId(), conf.getRegion());
            String filter = String.format("clusterName=%s AND operationType=CREATE", name);
            OperationsClient.ListOperationsPagedResponse operationsResponse = client.getOperationsClient().listOperations(resourceName, filter);
            OperationsClient.ListOperationsPage page = operationsResponse.getPage();
            if (page == null) {
                LOG.warn("Unable to get the cause of the cluster creation failure.");
                return status;
            }
            if (page.getPageElementCount() > 1) {
                // shouldn't be possible
                LOG.warn("Multiple create operations found for cluster {}, may not be able to find the failure message.", name);
            }
            if (page.getPageElementCount() > 0) {
                Operation operation = page.getValues().iterator().next();
                Status operationError = operation.getError();
                if (operationError != null) {
                    LOG.warn("Failed to create cluster {}: {}", name, operationError.getMessage());
                }
            }
        }
    } catch (Exception e) {
        // if we failed to get the operations list, log an error and proceed with normal execution
        LOG.warn("Unable to get the cause of the cluster creation failure.", e);
    }
    return status;
}
Also used : HttpURLConnection(java.net.HttpURLConnection) NetworkPeering(com.google.api.services.compute.model.NetworkPeering) Arrays(java.util.Arrays) OperationFuture(com.google.api.gax.longrunning.OperationFuture) NotFoundException(com.google.api.gax.rpc.NotFoundException) LoggerFactory(org.slf4j.LoggerFactory) HttpStatus(org.apache.http.HttpStatus) FixedCredentialsProvider(com.google.api.gax.core.FixedCredentialsProvider) Network(com.google.api.services.compute.model.Network) DeleteClusterRequest(com.google.cloud.dataproc.v1.DeleteClusterRequest) GoogleJsonResponseException(com.google.api.client.googleapis.json.GoogleJsonResponseException) GeneralSecurityException(java.security.GeneralSecurityException) GetClusterRequest(com.google.cloud.dataproc.v1.GetClusterRequest) Cluster(com.google.cloud.dataproc.v1.Cluster) NodeInitializationAction(com.google.cloud.dataproc.v1.NodeInitializationAction) Map(java.util.Map) CredentialsProvider(com.google.api.gax.core.CredentialsProvider) ParseException(java.text.ParseException) EnumSet(java.util.EnumSet) AutoscalingConfig(com.google.cloud.dataproc.v1.AutoscalingConfig) ImmutableSet(com.google.common.collect.ImmutableSet) ClusterStatus(com.google.cloud.dataproc.v1.ClusterStatus) JacksonFactory(com.google.api.client.json.jackson2.JacksonFactory) Predicate(java.util.function.Predicate) Collection(java.util.Collection) HttpTransport(com.google.api.client.http.HttpTransport) Status(com.google.rpc.Status) Set(java.util.Set) GoogleNetHttpTransport(com.google.api.client.googleapis.javanet.GoogleNetHttpTransport) HttpResponseException(com.google.api.client.http.HttpResponseException) Collectors(java.util.stream.Collectors) AlreadyExistsException(com.google.api.gax.rpc.AlreadyExistsException) Node(io.cdap.cdap.runtime.spi.provisioner.Node) InstanceGroupConfig(com.google.cloud.dataproc.v1.InstanceGroupConfig) Objects(java.util.Objects) List(java.util.List) HttpStatusCodes(com.google.api.client.http.HttpStatusCodes) Stream(java.util.stream.Stream) OperationsClient(com.google.longrunning.OperationsClient) HttpCredentialsAdapter(com.google.auth.http.HttpCredentialsAdapter) FirewallList(com.google.api.services.compute.model.FirewallList) FieldMask(com.google.protobuf.FieldMask) IPRange(io.cdap.cdap.runtime.spi.common.IPRange) SSHPublicKey(io.cdap.cdap.runtime.spi.ssh.SSHPublicKey) Optional(java.util.Optional) Compute(com.google.api.services.compute.Compute) SoftwareConfig(com.google.cloud.dataproc.v1.SoftwareConfig) DataprocUtils(io.cdap.cdap.runtime.spi.common.DataprocUtils) Instance(com.google.api.services.compute.model.Instance) ClusterConfig(com.google.cloud.dataproc.v1.ClusterConfig) SimpleDateFormat(java.text.SimpleDateFormat) HashMap(java.util.HashMap) HttpRequest(com.google.api.client.http.HttpRequest) UpdateClusterRequest(com.google.cloud.dataproc.v1.UpdateClusterRequest) Operation(com.google.longrunning.Operation) GceClusterConfig(com.google.cloud.dataproc.v1.GceClusterConfig) ArrayList(java.util.ArrayList) ClusterOperationMetadata(com.google.cloud.dataproc.v1.ClusterOperationMetadata) HashSet(java.util.HashSet) Strings(com.google.common.base.Strings) NetworkList(com.google.api.services.compute.model.NetworkList) DiskConfig(com.google.cloud.dataproc.v1.DiskConfig) HttpRequestInitializer(com.google.api.client.http.HttpRequestInitializer) AccessConfig(com.google.api.services.compute.model.AccessConfig) Firewall(com.google.api.services.compute.model.Firewall) SocketTimeoutException(java.net.SocketTimeoutException) ShieldedInstanceConfig(com.google.cloud.dataproc.v1.ShieldedInstanceConfig) StreamSupport(java.util.stream.StreamSupport) Nullable(javax.annotation.Nullable) ClusterControllerClient(com.google.cloud.dataproc.v1.ClusterControllerClient) RetryableProvisionException(io.cdap.cdap.runtime.spi.provisioner.RetryableProvisionException) Logger(org.slf4j.Logger) EncryptionConfig(com.google.cloud.dataproc.v1.EncryptionConfig) IOException(java.io.IOException) ApiException(com.google.api.gax.rpc.ApiException) ClusterControllerSettings(com.google.cloud.dataproc.v1.ClusterControllerSettings) ExecutionException(java.util.concurrent.ExecutionException) TimeUnit(java.util.concurrent.TimeUnit) Duration(com.google.protobuf.Duration) EndpointConfig(com.google.cloud.dataproc.v1.EndpointConfig) LifecycleConfig(com.google.cloud.dataproc.v1.LifecycleConfig) Collections(java.util.Collections) HttpStatus(org.apache.http.HttpStatus) ClusterStatus(com.google.cloud.dataproc.v1.ClusterStatus) Status(com.google.rpc.Status) OperationsClient(com.google.longrunning.OperationsClient) Operation(com.google.longrunning.Operation) NotFoundException(com.google.api.gax.rpc.NotFoundException) GoogleJsonResponseException(com.google.api.client.googleapis.json.GoogleJsonResponseException) GeneralSecurityException(java.security.GeneralSecurityException) ParseException(java.text.ParseException) HttpResponseException(com.google.api.client.http.HttpResponseException) AlreadyExistsException(com.google.api.gax.rpc.AlreadyExistsException) SocketTimeoutException(java.net.SocketTimeoutException) RetryableProvisionException(io.cdap.cdap.runtime.spi.provisioner.RetryableProvisionException) IOException(java.io.IOException) ApiException(com.google.api.gax.rpc.ApiException) ExecutionException(java.util.concurrent.ExecutionException)

Aggregations

Test (org.junit.Test)59 AbstractMessage (com.google.protobuf.AbstractMessage)28 ByteString (com.google.protobuf.ByteString)27 InvalidArgumentException (com.google.api.gax.rpc.InvalidArgumentException)20 StatusRuntimeException (io.grpc.StatusRuntimeException)20 Cluster (com.google.bigtable.admin.v2.Cluster)16 Operation (com.google.longrunning.Operation)16 ExecutionException (java.util.concurrent.ExecutionException)15 ClusterName (com.google.bigtable.admin.v2.ClusterName)14 Cluster (io.envoyproxy.envoy.config.cluster.v3.Cluster)10 HashMap (java.util.HashMap)10 SnapshotName (com.google.bigtable.admin.v2.SnapshotName)8 Backup (com.google.bigtable.admin.v2.Backup)7 Snapshot (com.google.bigtable.admin.v2.Snapshot)7 WeightedCluster (io.envoyproxy.envoy.config.route.v3.WeightedCluster)7 PartialUpdateClusterRequest (com.google.bigtable.admin.v2.PartialUpdateClusterRequest)5 FieldMask (com.google.protobuf.FieldMask)5 ArrayList (java.util.ArrayList)5 BackupName (com.google.bigtable.admin.v2.BackupName)4 InstanceName (com.google.bigtable.admin.v2.InstanceName)4