Examples with FlowletId co.cask.cdap.proto.id.FlowletId used on opensource projects

Example 1 with FlowletId

use of co.cask.cdap.proto.id.FlowletId in project cdap by caskdata.

the class LineageDatasetTest method testMultipleRelations.

@Test
public void testMultipleRelations() throws Exception {
    final LineageDataset lineageDataset = getLineageDataset("testMultipleRelations");
    Assert.assertNotNull(lineageDataset);
    TransactionExecutor txnl = dsFrameworkUtil.newInMemoryTransactionExecutor((TransactionAware) lineageDataset);
    final RunId runId1 = RunIds.generate(10000);
    final RunId runId2 = RunIds.generate(20000);
    final RunId runId3 = RunIds.generate(30000);
    final RunId runId4 = RunIds.generate(40000);
    final DatasetId datasetInstance1 = NamespaceId.DEFAULT.dataset("dataset1");
    final DatasetId datasetInstance2 = NamespaceId.DEFAULT.dataset("dataset2");
    final StreamId stream1 = NamespaceId.DEFAULT.stream("stream1");
    final StreamId stream2 = NamespaceId.DEFAULT.stream("stream2");
    final ProgramId program1 = NamespaceId.DEFAULT.app("app1").flow("flow1");
    final FlowletId flowlet1 = program1.flowlet("flowlet1");
    final ProgramId program2 = NamespaceId.DEFAULT.app("app2").worker("worker2");
    final ProgramId program3 = NamespaceId.DEFAULT.app("app3").service("service3");
    final ProgramRunId run11 = program1.run(runId1.getId());
    final ProgramRunId run22 = program2.run(runId2.getId());
    final ProgramRunId run23 = program2.run(runId3.getId());
    final ProgramRunId run34 = program3.run(runId4.getId());
    final long now = System.currentTimeMillis();
    final long run11Data1AccessTime = now;
    final long run22Data2AccessTime = now + 1;
    final long run22Stream1AccessTime = now + 2;
    final long run23Stream2AccessTime = now + 1;
    final long run23Data2AccessTime = now + 3;
    // noinspection UnnecessaryLocalVariable
    txnl.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            lineageDataset.addAccess(run11, datasetInstance1, AccessType.READ, run11Data1AccessTime, flowlet1);
            lineageDataset.addAccess(run22, datasetInstance2, AccessType.WRITE, run22Data2AccessTime);
            lineageDataset.addAccess(run22, stream1, AccessType.READ, run22Stream1AccessTime);
            lineageDataset.addAccess(run23, stream2, AccessType.READ, run23Stream2AccessTime);
            lineageDataset.addAccess(run23, datasetInstance2, AccessType.WRITE, run23Data2AccessTime);
            lineageDataset.addAccess(run34, datasetInstance2, AccessType.READ_WRITE, System.currentTimeMillis());
            lineageDataset.addAccess(run34, stream2, AccessType.UNKNOWN, System.currentTimeMillis());
        }
    });
    txnl.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            Assert.assertEquals(ImmutableSet.of(new Relation(datasetInstance1, program1, AccessType.READ, runId1, ImmutableSet.of(flowlet1))), lineageDataset.getRelations(datasetInstance1, 0, 100000, Predicates.<Relation>alwaysTrue()));
            Assert.assertEquals(ImmutableSet.of(new Relation(datasetInstance2, program2, AccessType.WRITE, runId2), new Relation(datasetInstance2, program2, AccessType.WRITE, runId3), new Relation(datasetInstance2, program3, AccessType.READ_WRITE, runId4)), lineageDataset.getRelations(datasetInstance2, 0, 100000, Predicates.<Relation>alwaysTrue()));
            Assert.assertEquals(ImmutableSet.of(new Relation(stream1, program2, AccessType.READ, runId2)), lineageDataset.getRelations(stream1, 0, 100000, Predicates.<Relation>alwaysTrue()));
            Assert.assertEquals(ImmutableSet.of(new Relation(stream2, program2, AccessType.READ, runId3), new Relation(stream2, program3, AccessType.UNKNOWN, runId4)), lineageDataset.getRelations(stream2, 0, 100000, Predicates.<Relation>alwaysTrue()));
            Assert.assertEquals(ImmutableSet.of(new Relation(datasetInstance2, program2, AccessType.WRITE, runId2), new Relation(stream1, program2, AccessType.READ, runId2), new Relation(datasetInstance2, program2, AccessType.WRITE, runId3), new Relation(stream2, program2, AccessType.READ, runId3)), lineageDataset.getRelations(program2, 0, 100000, Predicates.<Relation>alwaysTrue()));
            // Reduced time range
            Assert.assertEquals(ImmutableSet.of(new Relation(datasetInstance2, program2, AccessType.WRITE, runId2), new Relation(datasetInstance2, program2, AccessType.WRITE, runId3)), lineageDataset.getRelations(datasetInstance2, 0, 35000, Predicates.<Relation>alwaysTrue()));
            Assert.assertEquals(toSet(program1, datasetInstance1), lineageDataset.getEntitiesForRun(run11));
            Assert.assertEquals(ImmutableList.of(run11Data1AccessTime), lineageDataset.getAccessTimesForRun(run11));
            Assert.assertEquals(toSet(program2, datasetInstance2, stream1), lineageDataset.getEntitiesForRun(run22));
            Assert.assertEquals(ImmutableList.of(run22Data2AccessTime, run22Stream1AccessTime), lineageDataset.getAccessTimesForRun(run22));
            Assert.assertEquals(toSet(program2, datasetInstance2, stream2), lineageDataset.getEntitiesForRun(run23));
            Assert.assertEquals(ImmutableList.of(run23Data2AccessTime, run23Stream2AccessTime), lineageDataset.getAccessTimesForRun(run23));
            Assert.assertEquals(toSet(program3, datasetInstance2, stream2), lineageDataset.getEntitiesForRun(run34));
        }
    });
}

Example 2 with FlowletId

use of co.cask.cdap.proto.id.FlowletId in project cdap by caskdata.

the class MetricsClientTestRun method testAll.

@Test
public void testAll() throws Exception {
    appClient.deploy(NamespaceId.DEFAULT, createAppJarFile(FakeApp.class));
    ApplicationId app = NamespaceId.DEFAULT.app(FakeApp.NAME);
    ProgramId flow = app.flow(FakeFlow.NAME);
    StreamId stream = NamespaceId.DEFAULT.stream(FakeApp.STREAM_NAME);
    try {
        programClient.start(flow);
        streamClient.sendEvent(stream, "hello world");
        // TODO: remove arbitrary sleep
        TimeUnit.SECONDS.sleep(5);
        FlowletId flowletId = flow.flowlet(FakeFlow.FLOWLET_NAME);
        MetricQueryResult result = metricsClient.query(MetricsTags.flowlet(flowletId), Constants.Metrics.Name.Flow.FLOWLET_INPUT);
        Assert.assertEquals(1, result.getSeries()[0].getData()[0].getValue());
        result = metricsClient.query(MetricsTags.flowlet(flowletId), ImmutableList.of(Constants.Metrics.Name.Flow.FLOWLET_INPUT), ImmutableList.<String>of(), ImmutableMap.of("aggregate", "true"));
        Assert.assertEquals(1, result.getSeries()[0].getData()[0].getValue());
        result = metricsClient.query(MetricsTags.flowlet(flowletId), ImmutableList.of(Constants.Metrics.Name.Flow.FLOWLET_INPUT), ImmutableList.<String>of(), ImmutableMap.of("start", "now-20s", "end", "now"));
        Assert.assertEquals(1, result.getSeries()[0].getData()[0].getValue());
        List<MetricTagValue> tags = metricsClient.searchTags(MetricsTags.flowlet(flowletId));
        Assert.assertEquals(1, tags.size());
        Assert.assertEquals("run", tags.get(0).getName());
        List<String> metrics = metricsClient.searchMetrics(MetricsTags.flowlet(flowletId));
        Assert.assertTrue(metrics.contains(Constants.Metrics.Name.Flow.FLOWLET_INPUT));
    } finally {
        programClient.stop(flow);
        assertProgramRuns(programClient, flow, ProgramRunStatus.KILLED, 1, 10);
        appClient.delete(app);
    }
}

Example 3 with FlowletId

use of co.cask.cdap.proto.id.FlowletId in project cdap by caskdata.

the class SetProgramInstancesCommand method perform.

@Override
public void perform(Arguments arguments, PrintStream output) throws Exception {
    String[] programIdParts = arguments.get(elementType.getArgumentName().toString()).split("\\.");
    ApplicationId appId = cliConfig.getCurrentNamespace().app(programIdParts[0]);
    int numInstances = arguments.getInt(ArgumentName.NUM_INSTANCES.toString());
    switch(elementType) {
        case FLOWLET:
            if (programIdParts.length < 3) {
                throw new CommandInputError(this);
            }
            String flowId = programIdParts[1];
            String flowletName = programIdParts[2];
            FlowletId flowletId = appId.flow(flowId).flowlet(flowletName);
            programClient.setFlowletInstances(flowletId, numInstances);
            output.printf("Successfully set flowlet '%s' of flow '%s' of app '%s' to %d instances\n", flowId, flowletId, appId.getEntityName(), numInstances);
            break;
        case WORKER:
            if (programIdParts.length < 2) {
                throw new CommandInputError(this);
            }
            String workerName = programIdParts[1];
            ProgramId workerId = appId.worker(workerName);
            programClient.setWorkerInstances(workerId, numInstances);
            output.printf("Successfully set worker '%s' of app '%s' to %d instances\n", workerName, appId.getEntityName(), numInstances);
            break;
        case SERVICE:
            if (programIdParts.length < 2) {
                throw new CommandInputError(this);
            }
            String serviceName = programIdParts[1];
            ServiceId service = appId.service(serviceName);
            programClient.setServiceInstances(service, numInstances);
            output.printf("Successfully set service '%s' of app '%s' to %d instances\n", serviceName, appId.getEntityName(), numInstances);
            break;
        default:
            // TODO: remove this
            throw new IllegalArgumentException("Unrecognized program element type for scaling: " + elementType);
    }
}

Example 4 with FlowletId

use of co.cask.cdap.proto.id.FlowletId in project cdap by caskdata.

the class GetProgramInstancesCommand method perform.

@Override
@SuppressWarnings("deprecation")
public void perform(Arguments arguments, PrintStream output) throws Exception {
    String[] programIdParts = arguments.get(elementType.getArgumentName().toString()).split("\\.");
    ApplicationId appId = cliConfig.getCurrentNamespace().app(programIdParts[0]);
    int instances;
    switch(elementType) {
        case FLOWLET:
            if (programIdParts.length < 3) {
                throw new CommandInputError(this);
            }
            String flowId = programIdParts[1];
            String flowletName = programIdParts[2];
            FlowletId flowlet = appId.flow(flowId).flowlet(flowletName);
            instances = programClient.getFlowletInstances(flowlet);
            break;
        case WORKER:
            if (programIdParts.length < 2) {
                throw new CommandInputError(this);
            }
            String workerId = programIdParts[1];
            ProgramId worker = appId.worker(workerId);
            instances = programClient.getWorkerInstances(worker);
            break;
        case SERVICE:
            if (programIdParts.length < 2) {
                throw new CommandInputError(this);
            }
            String serviceName = programIdParts[1];
            instances = programClient.getServiceInstances(appId.service(serviceName));
            break;
        default:
            // TODO: remove this
            throw new IllegalArgumentException("Unrecognized program element type for scaling: " + elementType);
    }
    output.println(instances);
}

Example 5 with FlowletId

use of co.cask.cdap.proto.id.FlowletId in project cdap by caskdata.

the class LineageDatasetTest method testOneRelation.

@Test
public void testOneRelation() throws Exception {
    final LineageDataset lineageDataset = getLineageDataset("testOneRelation");
    Assert.assertNotNull(lineageDataset);
    TransactionExecutor txnl = dsFrameworkUtil.newInMemoryTransactionExecutor((TransactionAware) lineageDataset);
    final RunId runId = RunIds.generate(10000);
    final DatasetId datasetInstance = new DatasetId("default", "dataset1");
    final ProgramId program = new ProgramId("default", "app1", ProgramType.FLOW, "flow1");
    final FlowletId flowlet = program.flowlet("flowlet1");
    final ProgramRunId run = program.run(runId.getId());
    final long accessTimeMillis = System.currentTimeMillis();
    txnl.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            lineageDataset.addAccess(run, datasetInstance, AccessType.READ, accessTimeMillis, flowlet);
        }
    });
    txnl.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            Relation expected = new Relation(datasetInstance, program, AccessType.READ, runId, ImmutableSet.of(flowlet));
            Set<Relation> relations = lineageDataset.getRelations(datasetInstance, 0, 100000, Predicates.<Relation>alwaysTrue());
            Assert.assertEquals(1, relations.size());
            Assert.assertEquals(expected, relations.iterator().next());
            Assert.assertEquals(toSet(program, datasetInstance), lineageDataset.getEntitiesForRun(run));
            Assert.assertEquals(ImmutableList.of(accessTimeMillis), lineageDataset.getAccessTimesForRun(run));
        }
    });
}