Search in sources :

Example 6 with Table

use of com.google.common.collect.Table in project cdap by caskdata.

the class HBaseQueueDebugger method scanQueues.

private void scanQueues(List<NamespaceMeta> namespaceMetas) throws Exception {
    final QueueStatistics totalStats = new QueueStatistics();
    for (NamespaceMeta namespaceMeta : namespaceMetas) {
        final NamespaceId namespaceId = new NamespaceId(namespaceMeta.getName());
        final Collection<ApplicationSpecification> apps = store.getAllApplications(namespaceId);
        for (final ApplicationSpecification app : apps) {
            ApplicationId appId = new ApplicationId(namespaceMeta.getName(), app.getName(), app.getAppVersion());
            Collection<FlowSpecification> flows = app.getFlows().values();
            for (final FlowSpecification flow : flows) {
                final ProgramId flowId = appId.program(ProgramType.FLOW, flow.getName());
                impersonator.doAs(flowId, new Callable<Void>() {

                    @Override
                    public Void call() throws Exception {
                        SimpleQueueSpecificationGenerator queueSpecGenerator = new SimpleQueueSpecificationGenerator(flowId.getParent());
                        Table<QueueSpecificationGenerator.Node, String, Set<QueueSpecification>> table = queueSpecGenerator.create(flow);
                        for (Table.Cell<QueueSpecificationGenerator.Node, String, Set<QueueSpecification>> cell : table.cellSet()) {
                            if (cell.getRowKey().getType() == FlowletConnection.Type.FLOWLET) {
                                for (QueueSpecification queue : cell.getValue()) {
                                    QueueStatistics queueStats = scanQueue(queue.getQueueName(), null);
                                    totalStats.add(queueStats);
                                }
                            }
                        }
                        return null;
                    }
                });
            }
        }
    }
    System.out.printf("Total results for all queues: %s\n", totalStats.getReport(showTxTimestampOnly()));
}
Also used : ApplicationSpecification(co.cask.cdap.api.app.ApplicationSpecification) HTable(org.apache.hadoop.hbase.client.HTable) Table(com.google.common.collect.Table) ProgramId(co.cask.cdap.proto.id.ProgramId) TransactionNotInProgressException(org.apache.tephra.TransactionNotInProgressException) TransactionFailureException(org.apache.tephra.TransactionFailureException) NotFoundException(co.cask.cdap.common.NotFoundException) SimpleQueueSpecificationGenerator(co.cask.cdap.internal.app.queue.SimpleQueueSpecificationGenerator) SimpleQueueSpecificationGenerator(co.cask.cdap.internal.app.queue.SimpleQueueSpecificationGenerator) QueueSpecificationGenerator(co.cask.cdap.app.queue.QueueSpecificationGenerator) FlowSpecification(co.cask.cdap.api.flow.FlowSpecification) NamespaceMeta(co.cask.cdap.proto.NamespaceMeta) NamespaceId(co.cask.cdap.proto.id.NamespaceId) QueueSpecification(co.cask.cdap.app.queue.QueueSpecification) ApplicationId(co.cask.cdap.proto.id.ApplicationId)

Example 7 with Table

use of com.google.common.collect.Table in project mule by mulesoft.

the class MetadataComponentModelValidator method validateResolversName.

private void validateResolversName(ComponentModel model, MetadataResolverFactory resolverFactory, Table<String, String, Class<?>> names, ProblemsReporter problemsReporter) {
    List<NamedTypeResolver> resolvers = new LinkedList<>();
    resolvers.addAll(getAllInputResolvers(model, resolverFactory));
    resolvers.add(resolverFactory.getOutputResolver());
    resolvers.stream().filter(r -> !r.getClass().equals(NullMetadataResolver.class)).forEach(r -> {
        if (isBlank(r.getResolverName())) {
            problemsReporter.addError(new Problem(model, format(EMPTY_RESOLVER_NAME, getComponentModelTypeName(model), model.getName(), r.getClass().getSimpleName(), "resolver")));
        } else {
            if (names.get(r.getCategoryName(), r.getResolverName()) != null && names.get(r.getCategoryName(), r.getResolverName()) != r.getClass()) {
                problemsReporter.addError(new Problem(model, format("%s [%s] specifies metadata resolvers with repeated name [%s] for the same category [%s]. Resolver names should be unique for a given category. Affected resolvers are '%s' and '%s'", getComponentModelTypeName(model), model.getName(), r.getResolverName(), r.getCategoryName(), names.get(r.getCategoryName(), r.getResolverName()).getSimpleName(), r.getClass().getSimpleName())));
            }
            names.put(r.getCategoryName(), r.getResolverName(), r.getClass());
        }
    });
}
Also used : OperationModel(org.mule.runtime.api.meta.model.operation.OperationModel) Message(org.mule.runtime.api.message.Message) NamedObject(org.mule.runtime.api.meta.NamedObject) HashBasedTable(com.google.common.collect.HashBasedTable) MetadataKeyIdModelProperty(org.mule.runtime.extension.api.property.MetadataKeyIdModelProperty) SourceModel(org.mule.runtime.api.meta.model.source.SourceModel) ModelLoaderUtils.isScope(org.mule.runtime.module.extension.internal.loader.utils.ModelLoaderUtils.isScope) ArrayType(org.mule.metadata.api.model.ArrayType) MetadataKeyPartModelProperty(org.mule.runtime.extension.api.property.MetadataKeyPartModelProperty) Map(java.util.Map) StringUtils(org.mule.runtime.core.api.util.StringUtils) Collectors.toSet(java.util.stream.Collectors.toSet) ConnectableComponentModel(org.mule.runtime.api.meta.model.ConnectableComponentModel) ExtensionMetadataTypeUtils.isMap(org.mule.runtime.extension.api.util.ExtensionMetadataTypeUtils.isMap) ObjectType(org.mule.metadata.api.model.ObjectType) OutputTypeResolver(org.mule.runtime.api.metadata.resolving.OutputTypeResolver) ProblemsReporter(org.mule.runtime.extension.api.loader.ProblemsReporter) InputTypeResolver(org.mule.runtime.api.metadata.resolving.InputTypeResolver) Set(java.util.Set) String.format(java.lang.String.format) MetadataTypeVisitor(org.mule.metadata.api.visitor.MetadataTypeVisitor) MuleExtensionUtils(org.mule.runtime.module.extension.internal.util.MuleExtensionUtils) Serializable(java.io.Serializable) List(java.util.List) HasOperationModels(org.mule.runtime.api.meta.model.operation.HasOperationModels) NameUtils.getComponentModelTypeName(org.mule.runtime.extension.api.util.NameUtils.getComponentModelTypeName) MetadataType(org.mule.metadata.api.model.MetadataType) Optional(java.util.Optional) ExtensionOperationDescriptorModelProperty(org.mule.runtime.module.extension.internal.loader.java.type.property.ExtensionOperationDescriptorModelProperty) ParameterModel(org.mule.runtime.api.meta.model.parameter.ParameterModel) ComponentModel(org.mule.runtime.api.meta.model.ComponentModel) ExtensionTypeDescriptorModelProperty(org.mule.runtime.module.extension.internal.loader.java.type.property.ExtensionTypeDescriptorModelProperty) CustomDefinedStaticTypeAnnotation(org.mule.runtime.module.extension.internal.loader.annotations.CustomDefinedStaticTypeAnnotation) StringUtils.join(org.apache.commons.lang3.StringUtils.join) NullMetadataResolver(org.mule.runtime.extension.api.metadata.NullMetadataResolver) Problem(org.mule.runtime.extension.api.loader.Problem) LinkedList(java.util.LinkedList) MetadataResolverUtils.isNullResolver(org.mule.runtime.extension.api.metadata.MetadataResolverUtils.isNullResolver) ExtensionModelValidator(org.mule.runtime.extension.api.loader.ExtensionModelValidator) MetadataTypeUtils.isVoid(org.mule.metadata.api.utils.MetadataTypeUtils.isVoid) MetadataResolverUtils.getAllResolvers(org.mule.runtime.extension.api.metadata.MetadataResolverUtils.getAllResolvers) MetadataTypeUtils.isCollection(org.mule.metadata.api.utils.MetadataTypeUtils.isCollection) ExtensionModel(org.mule.runtime.api.meta.model.ExtensionModel) ModelLoaderUtils.isRouter(org.mule.runtime.module.extension.internal.loader.utils.ModelLoaderUtils.isRouter) Collectors.toList(java.util.stream.Collectors.toList) ExtensionMetadataTypeUtils.getType(org.mule.runtime.extension.api.util.ExtensionMetadataTypeUtils.getType) ExtensionMetadataTypeUtils(org.mule.runtime.extension.api.util.ExtensionMetadataTypeUtils) StringUtils.isBlank(org.apache.commons.lang3.StringUtils.isBlank) ExtensionWalker(org.mule.runtime.api.meta.model.util.ExtensionWalker) MetadataResolverFactory(org.mule.runtime.extension.api.metadata.MetadataResolverFactory) NamedTypeResolver(org.mule.runtime.api.metadata.resolving.NamedTypeResolver) Table(com.google.common.collect.Table) HasSourceModels(org.mule.runtime.api.meta.model.source.HasSourceModels) NullMetadataResolver(org.mule.runtime.extension.api.metadata.NullMetadataResolver) NamedTypeResolver(org.mule.runtime.api.metadata.resolving.NamedTypeResolver) Problem(org.mule.runtime.extension.api.loader.Problem) LinkedList(java.util.LinkedList)

Example 8 with Table

use of com.google.common.collect.Table in project cdap by caskdata.

the class DistributedProgramRuntimeService method list.

@Override
public synchronized Map<RunId, RuntimeInfo> list(ProgramType type) {
    Map<RunId, RuntimeInfo> result = Maps.newHashMap();
    result.putAll(super.list(type));
    // Table holds the Twill RunId and TwillController associated with the program matching the input type
    Table<ProgramId, RunId, TwillController> twillProgramInfo = HashBasedTable.create();
    // Goes through all live application and fill the twillProgramInfo table
    for (TwillRunner.LiveInfo liveInfo : twillRunner.lookupLive()) {
        String appName = liveInfo.getApplicationName();
        ProgramId programId = TwillAppNames.fromTwillAppName(appName, false);
        if (programId == null) {
            continue;
        }
        if (!type.equals(programId.getType())) {
            continue;
        }
        for (TwillController controller : liveInfo.getControllers()) {
            RunId twillRunId = controller.getRunId();
            if (isTwillRunIdCached(twillRunId)) {
                continue;
            }
            twillProgramInfo.put(programId, twillRunId, controller);
        }
    }
    if (twillProgramInfo.isEmpty()) {
        return ImmutableMap.copyOf(result);
    }
    final Set<RunId> twillRunIds = twillProgramInfo.columnKeySet();
    Collection<RunRecordMeta> activeRunRecords = store.getRuns(ProgramRunStatus.RUNNING, record -> record.getTwillRunId() != null && twillRunIds.contains(org.apache.twill.internal.RunIds.fromString(record.getTwillRunId()))).values();
    for (RunRecordMeta record : activeRunRecords) {
        String twillRunId = record.getTwillRunId();
        if (twillRunId == null) {
            // This is unexpected. Just log and ignore the run record
            LOG.warn("No twill runId for in run record {}.", record);
            continue;
        }
        RunId twillRunIdFromRecord = org.apache.twill.internal.RunIds.fromString(twillRunId);
        // Get the CDAP RunId from RunRecord
        RunId runId = RunIds.fromString(record.getPid());
        // Get the Program and TwillController for the current twillRunId
        Map<ProgramId, TwillController> mapForTwillId = twillProgramInfo.columnMap().get(twillRunIdFromRecord);
        Map.Entry<ProgramId, TwillController> entry = mapForTwillId.entrySet().iterator().next();
        // Create RuntimeInfo for the current Twill RunId
        RuntimeInfo runtimeInfo = createRuntimeInfo(entry.getKey(), entry.getValue(), runId);
        if (runtimeInfo != null) {
            result.put(runId, runtimeInfo);
            updateRuntimeInfo(type, runId, runtimeInfo);
        } else {
            LOG.warn("Unable to find program {} {}", type, entry.getKey());
        }
    }
    return ImmutableMap.copyOf(result);
}
Also used : CConfiguration(co.cask.cdap.common.conf.CConfiguration) ContainerInfo(co.cask.cdap.proto.Containers.ContainerInfo) RunRecordMeta(co.cask.cdap.internal.app.store.RunRecordMeta) Inject(com.google.inject.Inject) LoggerFactory(org.slf4j.LoggerFactory) HashBasedTable(com.google.common.collect.HashBasedTable) MetricsCollectionService(co.cask.cdap.api.metrics.MetricsCollectionService) RunIds(co.cask.cdap.common.app.RunIds) ArtifactRepository(co.cask.cdap.internal.app.runtime.artifact.ArtifactRepository) FLOWLET(co.cask.cdap.proto.Containers.ContainerType.FLOWLET) ResourceReport(org.apache.twill.api.ResourceReport) ProgramType(co.cask.cdap.proto.ProgramType) Configuration(org.apache.hadoop.conf.Configuration) Map(java.util.Map) Locations(co.cask.cdap.common.io.Locations) DistributedProgramLiveInfo(co.cask.cdap.proto.DistributedProgramLiveInfo) Resource(org.apache.hadoop.yarn.api.records.Resource) NodeReport(org.apache.hadoop.yarn.api.records.NodeReport) RunId(org.apache.twill.api.RunId) ProgramId(co.cask.cdap.proto.id.ProgramId) ProgramRunnerFactory(co.cask.cdap.app.runtime.ProgramRunnerFactory) ArtifactId(co.cask.cdap.proto.id.ArtifactId) ProgramTypeMetricTag(co.cask.cdap.internal.app.program.ProgramTypeMetricTag) AbstractResourceReporter(co.cask.cdap.internal.app.runtime.AbstractResourceReporter) TwillController(org.apache.twill.api.TwillController) ProgramResourceReporter(co.cask.cdap.app.runtime.ProgramResourceReporter) ImmutableMap(com.google.common.collect.ImmutableMap) Collection(java.util.Collection) ProgramRunner(co.cask.cdap.app.runtime.ProgramRunner) Set(java.util.Set) Impersonator(co.cask.cdap.security.impersonation.Impersonator) ProgramStateWriter(co.cask.cdap.app.runtime.ProgramStateWriter) Containers(co.cask.cdap.proto.Containers) NotRunningProgramLiveInfo(co.cask.cdap.proto.NotRunningProgramLiveInfo) ProgramDescriptor(co.cask.cdap.app.program.ProgramDescriptor) MoreExecutors(com.google.common.util.concurrent.MoreExecutors) ProgramController(co.cask.cdap.app.runtime.ProgramController) ProgramLiveInfo(co.cask.cdap.proto.ProgramLiveInfo) YarnClient(org.apache.hadoop.yarn.client.api.YarnClient) NodeState(org.apache.hadoop.yarn.api.records.NodeState) AppFabricServiceRuntimeModule(co.cask.cdap.app.guice.AppFabricServiceRuntimeModule) TwillRunResources(org.apache.twill.api.TwillRunResources) YarnUtils(org.apache.twill.internal.yarn.YarnUtils) Callable(java.util.concurrent.Callable) AbstractProgramRuntimeService(co.cask.cdap.app.runtime.AbstractProgramRuntimeService) ProgramRunStatus(co.cask.cdap.proto.ProgramRunStatus) TwillRunner(org.apache.twill.api.TwillRunner) Store(co.cask.cdap.app.store.Store) ArtifactDetail(co.cask.cdap.internal.app.runtime.artifact.ArtifactDetail) Constants(co.cask.cdap.common.conf.Constants) Nullable(javax.annotation.Nullable) AbstractListener(co.cask.cdap.internal.app.runtime.AbstractListener) Logger(org.slf4j.Logger) Iterator(java.util.Iterator) MetricsContext(co.cask.cdap.api.metrics.MetricsContext) Throwables(com.google.common.base.Throwables) IOException(java.io.IOException) Maps(com.google.common.collect.Maps) File(java.io.File) Named(com.google.inject.name.Named) Table(com.google.common.collect.Table) SimpleRuntimeInfo(co.cask.cdap.internal.app.runtime.service.SimpleRuntimeInfo) SimpleRuntimeInfo(co.cask.cdap.internal.app.runtime.service.SimpleRuntimeInfo) TwillRunner(org.apache.twill.api.TwillRunner) RunRecordMeta(co.cask.cdap.internal.app.store.RunRecordMeta) ProgramId(co.cask.cdap.proto.id.ProgramId) TwillController(org.apache.twill.api.TwillController) RunId(org.apache.twill.api.RunId) Map(java.util.Map) ImmutableMap(com.google.common.collect.ImmutableMap)

Example 9 with Table

use of com.google.common.collect.Table in project atlas by alibaba.

the class AtlasBuilder method removeSymbol.

private void removeSymbol(SymbolTable.Builder builder, Symbol symbol) {
    Table table = (Table) ReflectUtils.getField(builder, "symbols");
    table.remove(symbol.getResourceType(), symbol.getName());
}
Also used : Table(com.google.common.collect.Table)

Example 10 with Table

use of com.google.common.collect.Table in project coprhd-controller by CoprHD.

the class RPBlockServiceApiImpl method validateSourceVolumesInRGForMigrationRequest.

/**
 * Given a list of RP Source volumes, check to see if all the volumes passed in are a part of the Replication
 * Group. If we are not passed in all the volumes from the RG, throw an exception. We need to migrate
 * these volumes together.
 *
 * @param volumes List of Source volumes to check
 */
private void validateSourceVolumesInRGForMigrationRequest(List<Volume> volumes) {
    // Group all volumes in the request by RG. If there are no volumes in the request
    // that are in an RG then the table will be empty.
    Table<URI, String, List<Volume>> groupVolumes = VPlexUtil.groupVPlexVolumesByRG(volumes, null, null, _dbClient);
    for (Table.Cell<URI, String, List<Volume>> cell : groupVolumes.cellSet()) {
        // Get all the volumes in the request that have been grouped by RG
        List<Volume> volumesInRGRequest = cell.getValue();
        // Grab the first volume
        Volume firstVolume = volumesInRGRequest.get(0);
        // Get all the volumes from the RG
        List<Volume> rgVolumes = VPlexUtil.getVolumesInSameReplicationGroup(cell.getColumnKey(), cell.getRowKey(), firstVolume.getPersonality(), _dbClient);
        // We need to migrate all the volumes from the RG together.
        if (volumesInRGRequest.size() != rgVolumes.size()) {
            throw APIException.badRequests.cantMigrateNotAllRPSourceVolumesInRequest();
        }
    }
}
Also used : Table(com.google.common.collect.Table) Volume(com.emc.storageos.db.client.model.Volume) ApplicationAddVolumeList(com.emc.storageos.volumecontroller.ApplicationAddVolumeList) ArrayList(java.util.ArrayList) TaskList(com.emc.storageos.model.TaskList) VolumeGroupVolumeList(com.emc.storageos.model.application.VolumeGroupUpdateParam.VolumeGroupVolumeList) URIQueryResultList(com.emc.storageos.db.client.constraint.URIQueryResultList) StorageSystemConnectivityList(com.emc.storageos.model.systems.StorageSystemConnectivityList) List(java.util.List) NamedURI(com.emc.storageos.db.client.model.NamedURI) URI(java.net.URI)

Aggregations

Table (com.google.common.collect.Table)15 HashBasedTable (com.google.common.collect.HashBasedTable)7 List (java.util.List)6 Set (java.util.Set)6 Map (java.util.Map)5 ArrayList (java.util.ArrayList)4 ImmutableTable (com.google.common.collect.ImmutableTable)3 QueueSpecification (co.cask.cdap.app.queue.QueueSpecification)2 SimpleQueueSpecificationGenerator (co.cask.cdap.internal.app.queue.SimpleQueueSpecificationGenerator)2 ProgramId (co.cask.cdap.proto.id.ProgramId)2 Maps (com.google.common.collect.Maps)2 File (java.io.File)2 IOException (java.io.IOException)2 Collectors (java.util.stream.Collectors)2 ApplicationSpecification (co.cask.cdap.api.app.ApplicationSpecification)1 FlowSpecification (co.cask.cdap.api.flow.FlowSpecification)1 MetricsCollectionService (co.cask.cdap.api.metrics.MetricsCollectionService)1 MetricsContext (co.cask.cdap.api.metrics.MetricsContext)1 AppFabricServiceRuntimeModule (co.cask.cdap.app.guice.AppFabricServiceRuntimeModule)1 ProgramDescriptor (co.cask.cdap.app.program.ProgramDescriptor)1