Examples with FlowSpecification co.cask.cdap.api.flow.FlowSpecification used on opensource projects

Example 1 with FlowSpecification

use of co.cask.cdap.api.flow.FlowSpecification in project cdap by caskdata.

the class FlowProgramRunner method run.

@Override
public ProgramController run(Program program, ProgramOptions options) {
    // Extract and verify parameters
    ApplicationSpecification appSpec = program.getApplicationSpecification();
    Preconditions.checkNotNull(appSpec, "Missing application specification.");
    ProgramType processorType = program.getType();
    Preconditions.checkNotNull(processorType, "Missing processor type.");
    Preconditions.checkArgument(processorType == ProgramType.FLOW, "Only FLOW process type is supported.");
    FlowSpecification flowSpec = appSpec.getFlows().get(program.getName());
    Preconditions.checkNotNull(flowSpec, "Missing FlowSpecification for %s", program.getName());
    try {
        // Launch flowlet program runners
        RunId runId = ProgramRunners.getRunId(options);
        Multimap<String, QueueName> consumerQueues = FlowUtils.configureQueue(program, flowSpec, streamAdmin, queueAdmin, txExecutorFactory);
        final Table<String, Integer, ProgramController> flowlets = createFlowlets(program, options, flowSpec);
        return new FlowProgramController(flowlets, program, options, flowSpec, consumerQueues);
    } catch (Exception e) {
        throw Throwables.propagate(e);
    }
}

Example 2 with FlowSpecification

use of co.cask.cdap.api.flow.FlowSpecification in project cdap by caskdata.

the class HBaseQueueDebugger method scanQueues.

private void scanQueues(List<NamespaceMeta> namespaceMetas) throws Exception {
    final QueueStatistics totalStats = new QueueStatistics();
    for (NamespaceMeta namespaceMeta : namespaceMetas) {
        final NamespaceId namespaceId = new NamespaceId(namespaceMeta.getName());
        final Collection<ApplicationSpecification> apps = store.getAllApplications(namespaceId);
        for (final ApplicationSpecification app : apps) {
            ApplicationId appId = new ApplicationId(namespaceMeta.getName(), app.getName(), app.getAppVersion());
            Collection<FlowSpecification> flows = app.getFlows().values();
            for (final FlowSpecification flow : flows) {
                final ProgramId flowId = appId.program(ProgramType.FLOW, flow.getName());
                impersonator.doAs(flowId, new Callable<Void>() {

                    @Override
                    public Void call() throws Exception {
                        SimpleQueueSpecificationGenerator queueSpecGenerator = new SimpleQueueSpecificationGenerator(flowId.getParent());
                        Table<QueueSpecificationGenerator.Node, String, Set<QueueSpecification>> table = queueSpecGenerator.create(flow);
                        for (Table.Cell<QueueSpecificationGenerator.Node, String, Set<QueueSpecification>> cell : table.cellSet()) {
                            if (cell.getRowKey().getType() == FlowletConnection.Type.FLOWLET) {
                                for (QueueSpecification queue : cell.getValue()) {
                                    QueueStatistics queueStats = scanQueue(queue.getQueueName(), null);
                                    totalStats.add(queueStats);
                                }
                            }
                        }
                        return null;
                    }
                });
            }
        }
    }
    System.out.printf("Total results for all queues: %s\n", totalStats.getReport(showTxTimestampOnly()));
}

Example 3 with FlowSpecification

use of co.cask.cdap.api.flow.FlowSpecification in project cdap by caskdata.

the class FlowQueuePendingCorrector method run.

/**
 * Corrects queue.pending metric for all flowlets in an application.
 */
public void run(ApplicationId appId, ApplicationSpecification appSpec) throws Exception {
    System.out.println("Running queue.pending correction on app " + appId);
    Preconditions.checkArgument(appSpec.getName().equals(appId.getApplication()), String.format("Expected appSpec name '%s' to be equal to appId name '%s'", appSpec.getName(), appId.getApplication()));
    for (FlowSpecification flow : appSpec.getFlows().values()) {
        run(appId.flow(flow.getName()));
    }
}

Example 4 with FlowSpecification

use of co.cask.cdap.api.flow.FlowSpecification in project cdap by caskdata.

the class FlowQueuePendingCorrector method run.

/**
 * Corrects queue.pending metric for a flowlet.
 */
public void run(FlowId flowId, String producerFlowlet, String consumerFlowlet, String flowletQueue) throws Exception {
    ApplicationSpecification app = store.getApplication(flowId.getParent());
    Preconditions.checkArgument(app != null, flowId.getApplication() + " not found");
    Preconditions.checkArgument(app.getFlows().containsKey(flowId.getProgram()), flowId + " not found");
    FlowSpecification flow = app.getFlows().get(flowId.getProgram());
    run(flowId, producerFlowlet, consumerFlowlet, flowletQueue, flow);
}

Example 5 with FlowSpecification

use of co.cask.cdap.api.flow.FlowSpecification in project cdap by caskdata.

the class FlowVerificationTest method testFlowMissingConnection.

/**
 * This test that verification of flow connections
 */
@Test
public void testFlowMissingConnection() throws Exception {
    ApplicationSpecification appSpec = Specifications.from(new NoConsumerApp());
    ApplicationSpecificationAdapter adapter = ApplicationSpecificationAdapter.create(new ReflectionSchemaGenerator());
    ApplicationSpecification newSpec = adapter.fromJson(adapter.toJson(appSpec));
    FlowVerification flowVerifier = new FlowVerification();
    for (FlowSpecification flowSpec : appSpec.getFlows().values()) {
        VerifyResult result = flowVerifier.verify(new ApplicationId("test", newSpec.getName()), flowSpec);
        Assert.assertTrue(result.getStatus() == VerifyResult.Status.FAILED);
    }
}