Search in sources :

Example 16 with ApplicationId

use of io.cdap.cdap.proto.id.ApplicationId in project cdap by caskdata.

the class DefaultStoreTest method testRemoveApplication.

@Test
public void testRemoveApplication() {
    ApplicationSpecification spec = Specifications.from(new AllProgramsApp());
    NamespaceId namespaceId = new NamespaceId("account1");
    ApplicationId appId = namespaceId.app(spec.getName());
    store.addApplication(appId, spec);
    Assert.assertNotNull(store.getApplication(appId));
    // removing application
    store.removeApplication(appId);
    Assert.assertNull(store.getApplication(appId));
}
Also used : ApplicationSpecification(io.cdap.cdap.api.app.ApplicationSpecification) AllProgramsApp(io.cdap.cdap.AllProgramsApp) NamespaceId(io.cdap.cdap.proto.id.NamespaceId) ApplicationId(io.cdap.cdap.proto.id.ApplicationId) Test(org.junit.Test)

Example 17 with ApplicationId

use of io.cdap.cdap.proto.id.ApplicationId in project cdap by caskdata.

the class AppMetadataStoreTest method testGetActiveRuns.

@Test
public void testGetActiveRuns() throws Exception {
    // write a run record for each state for two programs in two apps in two namespaces
    String app1 = "app1";
    String app2 = "app2";
    String program1 = "prog1";
    String program2 = "prog2";
    Collection<NamespaceId> namespaces = Arrays.asList(new NamespaceId("ns1"), new NamespaceId("ns2"));
    Collection<ApplicationId> apps = namespaces.stream().flatMap(ns -> Stream.of(ns.app(app1), ns.app(app2))).collect(Collectors.toList());
    Collection<ProgramId> programs = apps.stream().flatMap(app -> Stream.of(app.mr(program1), app.mr(program2))).collect(Collectors.toList());
    for (ProgramId programId : programs) {
        TransactionRunners.run(transactionRunner, context -> {
            AppMetadataStore store = AppMetadataStore.create(context);
            // one run in pending state
            ProgramRunId runId = programId.run(RunIds.generate());
            store.recordProgramProvisioning(runId, Collections.emptyMap(), SINGLETON_PROFILE_MAP, AppFabricTestHelper.createSourceId(sourceId.incrementAndGet()), ARTIFACT_ID);
            // one run in starting state
            runId = programId.run(RunIds.generate());
            store.recordProgramProvisioning(runId, Collections.emptyMap(), SINGLETON_PROFILE_MAP, AppFabricTestHelper.createSourceId(sourceId.incrementAndGet()), ARTIFACT_ID);
            store.recordProgramProvisioned(runId, 3, AppFabricTestHelper.createSourceId(sourceId.incrementAndGet()));
            store.recordProgramStart(runId, UUID.randomUUID().toString(), Collections.emptyMap(), AppFabricTestHelper.createSourceId(sourceId.incrementAndGet()));
            // one run in running state
            runId = programId.run(RunIds.generate());
            store.recordProgramProvisioning(runId, Collections.emptyMap(), SINGLETON_PROFILE_MAP, AppFabricTestHelper.createSourceId(sourceId.incrementAndGet()), ARTIFACT_ID);
            store.recordProgramProvisioned(runId, 3, AppFabricTestHelper.createSourceId(sourceId.incrementAndGet()));
            String twillRunId = UUID.randomUUID().toString();
            store.recordProgramStart(runId, twillRunId, Collections.emptyMap(), AppFabricTestHelper.createSourceId(sourceId.incrementAndGet()));
            store.recordProgramRunning(runId, System.currentTimeMillis(), twillRunId, AppFabricTestHelper.createSourceId(sourceId.incrementAndGet()));
            // one in suspended state
            runId = programId.run(RunIds.generate());
            store.recordProgramProvisioning(runId, Collections.emptyMap(), SINGLETON_PROFILE_MAP, AppFabricTestHelper.createSourceId(sourceId.incrementAndGet()), ARTIFACT_ID);
            store.recordProgramProvisioned(runId, 3, AppFabricTestHelper.createSourceId(sourceId.incrementAndGet()));
            twillRunId = UUID.randomUUID().toString();
            store.recordProgramStart(runId, twillRunId, Collections.emptyMap(), AppFabricTestHelper.createSourceId(sourceId.incrementAndGet()));
            store.recordProgramRunning(runId, System.currentTimeMillis(), twillRunId, AppFabricTestHelper.createSourceId(sourceId.incrementAndGet()));
            store.recordProgramSuspend(runId, AppFabricTestHelper.createSourceId(sourceId.incrementAndGet()), System.currentTimeMillis());
            // one run in stopping state
            runId = programId.run(RunIds.generate());
            store.recordProgramProvisioning(runId, Collections.emptyMap(), SINGLETON_PROFILE_MAP, AppFabricTestHelper.createSourceId(sourceId.incrementAndGet()), ARTIFACT_ID);
            store.recordProgramProvisioned(runId, 3, AppFabricTestHelper.createSourceId(sourceId.incrementAndGet()));
            twillRunId = UUID.randomUUID().toString();
            store.recordProgramStart(runId, twillRunId, Collections.emptyMap(), AppFabricTestHelper.createSourceId(sourceId.incrementAndGet()));
            store.recordProgramRunning(runId, System.currentTimeMillis(), twillRunId, AppFabricTestHelper.createSourceId(sourceId.incrementAndGet()));
            store.recordProgramStopping(runId, AppFabricTestHelper.createSourceId(sourceId.incrementAndGet()), System.currentTimeMillis(), System.currentTimeMillis() + 1000);
            // one run in each stopped state
            for (ProgramRunStatus runStatus : ProgramRunStatus.values()) {
                if (!runStatus.isEndState()) {
                    continue;
                }
                runId = programId.run(RunIds.generate());
                store.recordProgramProvisioning(runId, Collections.emptyMap(), SINGLETON_PROFILE_MAP, AppFabricTestHelper.createSourceId(sourceId.incrementAndGet()), ARTIFACT_ID);
                store.recordProgramProvisioned(runId, 3, AppFabricTestHelper.createSourceId(sourceId.incrementAndGet()));
                twillRunId = UUID.randomUUID().toString();
                store.recordProgramStart(runId, twillRunId, Collections.emptyMap(), AppFabricTestHelper.createSourceId(sourceId.incrementAndGet()));
                store.recordProgramStop(runId, System.currentTimeMillis(), runStatus, null, AppFabricTestHelper.createSourceId(sourceId.incrementAndGet()));
            }
        });
    }
    Set<ProgramRunStatus> activeStates = new HashSet<>();
    activeStates.add(ProgramRunStatus.PENDING);
    activeStates.add(ProgramRunStatus.STARTING);
    activeStates.add(ProgramRunStatus.RUNNING);
    activeStates.add(ProgramRunStatus.SUSPENDED);
    activeStates.add(ProgramRunStatus.STOPPING);
    // test the instance level method and namespace level method
    TransactionRunners.run(transactionRunner, context -> {
        AppMetadataStore store = AppMetadataStore.create(context);
        Map<ProgramId, Set<ProgramRunStatus>> allExpected = new HashMap<>();
        Map<ProgramId, Set<ProgramRunStatus>> allActual = new HashMap<>();
        // check active runs per namespace
        for (NamespaceId namespace : namespaces) {
            Map<ProgramRunId, RunRecordDetail> activeRuns = store.getActiveRuns(namespace);
            // we expect 4 runs per program, with 4 programs in each namespace
            Map<ProgramId, Set<ProgramRunStatus>> expected = new HashMap<>();
            expected.put(namespace.app(app1).mr(program1), activeStates);
            expected.put(namespace.app(app1).mr(program2), activeStates);
            expected.put(namespace.app(app2).mr(program1), activeStates);
            expected.put(namespace.app(app2).mr(program2), activeStates);
            Map<ProgramId, Set<ProgramRunStatus>> actual = new HashMap<>();
            actual.put(namespace.app(app1).mr(program1), new HashSet<>());
            actual.put(namespace.app(app1).mr(program2), new HashSet<>());
            actual.put(namespace.app(app2).mr(program1), new HashSet<>());
            actual.put(namespace.app(app2).mr(program2), new HashSet<>());
            allActual.putAll(actual);
            for (Map.Entry<ProgramRunId, RunRecordDetail> activeRun : activeRuns.entrySet()) {
                ProgramId programId = activeRun.getKey().getParent();
                Assert.assertTrue("Unexpected program returned: " + programId, actual.containsKey(activeRun.getKey().getParent()));
                actual.get(programId).add(activeRun.getValue().getStatus());
            }
            Assert.assertEquals(expected, actual);
            allExpected.putAll(expected);
        }
        // test the instance level method
        for (Map.Entry<ProgramRunId, RunRecordDetail> activeRun : store.getActiveRuns(x -> true).entrySet()) {
            ProgramId programId = activeRun.getKey().getParent();
            Assert.assertTrue("Unexpected program returned: " + programId, allActual.containsKey(activeRun.getKey().getParent()));
            allActual.get(programId).add(activeRun.getValue().getStatus());
        }
        Assert.assertEquals(allExpected, allActual);
        // test the count-all method
        Assert.assertEquals(store.getActiveRuns(x -> true).size(), store.countActiveRuns(null));
        Assert.assertEquals(store.getActiveRuns(x -> true).size(), store.countActiveRuns(100));
        Assert.assertEquals(2, store.countActiveRuns(2));
    });
    // check active runs per app
    for (ApplicationId app : apps) {
        TransactionRunners.run(transactionRunner, context -> {
            AppMetadataStore store = AppMetadataStore.create(context);
            Map<ProgramRunId, RunRecordDetail> activeRuns = store.getActiveRuns(app);
            // we expect 3 runs per program, with 2 programs in each app
            Map<ProgramId, Set<ProgramRunStatus>> expected = new HashMap<>();
            expected.put(app.mr(program1), activeStates);
            expected.put(app.mr(program2), activeStates);
            Map<ProgramId, Set<ProgramRunStatus>> actual = new HashMap<>();
            actual.put(app.mr(program1), new HashSet<>());
            actual.put(app.mr(program2), new HashSet<>());
            for (Map.Entry<ProgramRunId, RunRecordDetail> activeRun : activeRuns.entrySet()) {
                ProgramId programId = activeRun.getKey().getParent();
                Assert.assertTrue("Unexpected program returned: " + programId, actual.containsKey(activeRun.getKey().getParent()));
                actual.get(programId).add(activeRun.getValue().getStatus());
            }
            Assert.assertEquals(expected, actual);
        });
    }
    // check active runs per program
    for (ProgramId program : programs) {
        TransactionRunners.run(transactionRunner, context -> {
            AppMetadataStore store = AppMetadataStore.create(context);
            Map<ProgramRunId, RunRecordDetail> activeRuns = store.getActiveRuns(program);
            Set<ProgramRunStatus> actual = new HashSet<>();
            for (Map.Entry<ProgramRunId, RunRecordDetail> activeRun : activeRuns.entrySet()) {
                Assert.assertEquals(program, activeRun.getKey().getParent());
                actual.add(activeRun.getValue().getStatus());
            }
            Assert.assertEquals(activeStates, actual);
        });
    }
}
Also used : Arrays(java.util.Arrays) TransactionRunners(io.cdap.cdap.spi.data.transaction.TransactionRunners) NamespaceId(io.cdap.cdap.proto.id.NamespaceId) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) HashMap(java.util.HashMap) Callable(java.util.concurrent.Callable) ProgramType(io.cdap.cdap.proto.ProgramType) AtomicReference(java.util.concurrent.atomic.AtomicReference) TreeSet(java.util.TreeSet) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) LinkedHashMap(java.util.LinkedHashMap) ProgramRunId(io.cdap.cdap.proto.id.ProgramRunId) Lists(com.google.common.collect.Lists) ImmutableList(com.google.common.collect.ImmutableList) ScanApplicationsRequest(io.cdap.cdap.app.store.ScanApplicationsRequest) AppFabricTestHelper(io.cdap.cdap.internal.AppFabricTestHelper) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) Map(java.util.Map) RunId(org.apache.twill.api.RunId) SystemArguments(io.cdap.cdap.internal.app.runtime.SystemArguments) Before(org.junit.Before) SortOrder(io.cdap.cdap.spi.data.SortOrder) ImmutableMap(com.google.common.collect.ImmutableMap) RunIds(io.cdap.cdap.common.app.RunIds) Collection(java.util.Collection) ApplicationSpecification(io.cdap.cdap.api.app.ApplicationSpecification) ProgramId(io.cdap.cdap.proto.id.ProgramId) Set(java.util.Set) Test(org.junit.Test) IOException(java.io.IOException) UUID(java.util.UUID) ProgramRunStatus(io.cdap.cdap.proto.ProgramRunStatus) Collectors(java.util.stream.Collectors) TimeUnit(java.util.concurrent.TimeUnit) AtomicLong(java.util.concurrent.atomic.AtomicLong) List(java.util.List) Stream(java.util.stream.Stream) TransactionRunner(io.cdap.cdap.spi.data.transaction.TransactionRunner) ProfileId(io.cdap.cdap.proto.id.ProfileId) AllProgramsApp(io.cdap.cdap.AllProgramsApp) Assert(org.junit.Assert) ApplicationId(io.cdap.cdap.proto.id.ApplicationId) Collections(java.util.Collections) ArtifactId(io.cdap.cdap.api.artifact.ArtifactId) Specifications(io.cdap.cdap.internal.app.deploy.Specifications) TreeSet(java.util.TreeSet) HashSet(java.util.HashSet) Set(java.util.Set) HashMap(java.util.HashMap) LinkedHashMap(java.util.LinkedHashMap) ProgramId(io.cdap.cdap.proto.id.ProgramId) ProgramRunStatus(io.cdap.cdap.proto.ProgramRunStatus) NamespaceId(io.cdap.cdap.proto.id.NamespaceId) ProgramRunId(io.cdap.cdap.proto.id.ProgramRunId) ApplicationId(io.cdap.cdap.proto.id.ApplicationId) HashMap(java.util.HashMap) LinkedHashMap(java.util.LinkedHashMap) Map(java.util.Map) ImmutableMap(com.google.common.collect.ImmutableMap) HashSet(java.util.HashSet) Test(org.junit.Test)

Example 18 with ApplicationId

use of io.cdap.cdap.proto.id.ApplicationId in project cdap by caskdata.

the class AppMetadataStoreTest method testBatchApplications.

@Test
public void testBatchApplications() {
    ApplicationSpecification appSpec = Specifications.from(new AllProgramsApp());
    // Writes 20 application specs
    for (int i = 0; i < 20; i++) {
        String appName = "test" + i;
        TransactionRunners.run(transactionRunner, context -> {
            AppMetadataStore store = AppMetadataStore.create(context);
            store.writeApplication(NamespaceId.DEFAULT.getNamespace(), appName, ApplicationId.DEFAULT_VERSION, appSpec);
            store.writeApplication(NamespaceId.SYSTEM.getNamespace(), appName, ApplicationId.DEFAULT_VERSION, appSpec);
        });
    }
    TransactionRunners.run(transactionRunner, context -> {
        AppMetadataStore store = AppMetadataStore.create(context);
        long count = store.getApplicationCount();
        // System apps are not included in the count
        Assert.assertEquals(20, count);
    });
    // Batch read 30, expect to get back 20
    List<ApplicationId> appIds = new ArrayList<>();
    for (int i = 0; i < 30; i++) {
        appIds.add(NamespaceId.DEFAULT.app("test" + i));
    }
    Map<ApplicationId, ApplicationMeta> result = TransactionRunners.run(transactionRunner, context -> {
        AppMetadataStore store = AppMetadataStore.create(context);
        return store.getApplicationsForAppIds(appIds);
    });
    Assert.assertEquals(20, result.size());
    for (int i = 0; i < 20; i++) {
        Assert.assertTrue("Missing application test" + i, result.containsKey(NamespaceId.DEFAULT.app("test" + i)));
    }
}
Also used : ApplicationSpecification(io.cdap.cdap.api.app.ApplicationSpecification) ArrayList(java.util.ArrayList) AllProgramsApp(io.cdap.cdap.AllProgramsApp) ApplicationId(io.cdap.cdap.proto.id.ApplicationId) Test(org.junit.Test)

Example 19 with ApplicationId

use of io.cdap.cdap.proto.id.ApplicationId in project cdap by caskdata.

the class AppMetadataStoreTest method testScanApplicationsWithNamespace.

@Test
public void testScanApplicationsWithNamespace() {
    ApplicationSpecification appSpec = Specifications.from(new AllProgramsApp());
    // Writes 100 application specs
    int count = 100;
    for (int i = 0; i < count / 2; i++) {
        String defaultAppName = "test" + (2 * i);
        TransactionRunners.run(transactionRunner, context -> {
            AppMetadataStore store = AppMetadataStore.create(context);
            store.writeApplication(NamespaceId.DEFAULT.getNamespace(), defaultAppName, ApplicationId.DEFAULT_VERSION, appSpec);
        });
        String cdapAppName = "test" + (2 * i + 1);
        TransactionRunners.run(transactionRunner, context -> {
            AppMetadataStore store = AppMetadataStore.create(context);
            store.writeApplication(NamespaceId.CDAP.getNamespace(), cdapAppName, ApplicationId.DEFAULT_VERSION, appSpec);
        });
    }
    // Scan all apps
    Map<ApplicationId, ApplicationMeta> apps = new LinkedHashMap<>();
    TransactionRunners.run(transactionRunner, context -> {
        AppMetadataStore store = AppMetadataStore.create(context);
        store.scanApplications(ScanApplicationsRequest.builder().setNamespaceId(NamespaceId.DEFAULT).build(), entry -> {
            apps.put(entry.getKey(), entry.getValue());
            return true;
        });
    });
    Assert.assertEquals(count / 2, apps.size());
}
Also used : ApplicationSpecification(io.cdap.cdap.api.app.ApplicationSpecification) AllProgramsApp(io.cdap.cdap.AllProgramsApp) ApplicationId(io.cdap.cdap.proto.id.ApplicationId) LinkedHashMap(java.util.LinkedHashMap) Test(org.junit.Test)

Example 20 with ApplicationId

use of io.cdap.cdap.proto.id.ApplicationId in project cdap by caskdata.

the class AppMetadataStoreTest method testScanRunningInRangeWithBatch.

@Test
public void testScanRunningInRangeWithBatch() throws Exception {
    // Add some run records
    TreeSet<Long> expected = new TreeSet<>();
    for (int i = 0; i < 100; ++i) {
        ApplicationId application = NamespaceId.DEFAULT.app("app" + i);
        final ProgramId program = application.program(ProgramType.values()[i % ProgramType.values().length], "program" + i);
        final RunId runId = RunIds.generate(runIdTime.incrementAndGet());
        final ProgramRunId programRunId = program.run(runId);
        expected.add(RunIds.getTime(runId, TimeUnit.MILLISECONDS));
        // Start the program and stop it
        final int j = i;
        // A sourceId to keep incrementing for each call of app meta data store persisting
        TransactionRunners.run(transactionRunner, context -> {
            AppMetadataStore metadataStoreDataset = AppMetadataStore.create(context);
            recordProvisionAndStart(programRunId, metadataStoreDataset);
            metadataStoreDataset.recordProgramRunning(programRunId, RunIds.getTime(runId, TimeUnit.SECONDS) + 1, null, AppFabricTestHelper.createSourceId(sourceId.incrementAndGet()));
            metadataStoreDataset.recordProgramStop(programRunId, RunIds.getTime(runId, TimeUnit.SECONDS), STOP_STATUSES.get(j % STOP_STATUSES.size()), null, AppFabricTestHelper.createSourceId(sourceId.incrementAndGet()));
        });
    }
    // Run full scan
    runScan(expected, 0, Long.MAX_VALUE);
    // In all below assertions, TreeSet and metadataStore both have start time inclusive and end time exclusive.
    // Run the scan with time limit
    runScan(expected.subSet(30 * 10000L, 90 * 10000L), TimeUnit.MILLISECONDS.toSeconds(30 * 10000), TimeUnit.MILLISECONDS.toSeconds(90 * 10000));
    runScan(expected.subSet(90 * 10000L, 101 * 10000L), TimeUnit.MILLISECONDS.toSeconds(90 * 10000), TimeUnit.MILLISECONDS.toSeconds(101 * 10000));
    // After range
    runScan(expected.subSet(101 * 10000L, 200 * 10000L), TimeUnit.MILLISECONDS.toSeconds(101 * 10000), TimeUnit.MILLISECONDS.toSeconds(200 * 10000));
    // Identical start and end time
    runScan(expected.subSet(31 * 10000L, 31 * 10000L), TimeUnit.MILLISECONDS.toSeconds(31 * 10000), TimeUnit.MILLISECONDS.toSeconds(31 * 10000));
    // One unit difference between start and end time
    runScan(expected.subSet(30 * 10000L, 31 * 10000L), TimeUnit.MILLISECONDS.toSeconds(30 * 10000), TimeUnit.MILLISECONDS.toSeconds(31 * 10000));
    // Before range
    runScan(expected.subSet(1000L, 10000L), TimeUnit.MILLISECONDS.toSeconds(1000), TimeUnit.MILLISECONDS.toSeconds(10000));
}
Also used : TreeSet(java.util.TreeSet) AtomicLong(java.util.concurrent.atomic.AtomicLong) ProgramRunId(io.cdap.cdap.proto.id.ProgramRunId) ApplicationId(io.cdap.cdap.proto.id.ApplicationId) ProgramId(io.cdap.cdap.proto.id.ProgramId) ProgramRunId(io.cdap.cdap.proto.id.ProgramRunId) RunId(org.apache.twill.api.RunId) Test(org.junit.Test)

Aggregations

ApplicationId (io.cdap.cdap.proto.id.ApplicationId)789 Test (org.junit.Test)410 AppRequest (io.cdap.cdap.proto.artifact.AppRequest)279 ProgramId (io.cdap.cdap.proto.id.ProgramId)263 ApplicationManager (io.cdap.cdap.test.ApplicationManager)225 ETLStage (io.cdap.cdap.etl.proto.v2.ETLStage)223 ETLBatchConfig (io.cdap.cdap.etl.proto.v2.ETLBatchConfig)196 StructuredRecord (io.cdap.cdap.api.data.format.StructuredRecord)180 Table (io.cdap.cdap.api.dataset.table.Table)178 WorkflowManager (io.cdap.cdap.test.WorkflowManager)169 NamespaceId (io.cdap.cdap.proto.id.NamespaceId)154 Schema (io.cdap.cdap.api.data.schema.Schema)147 ArrayList (java.util.ArrayList)129 HashSet (java.util.HashSet)126 ApplicationSpecification (io.cdap.cdap.api.app.ApplicationSpecification)124 HashMap (java.util.HashMap)109 KeyValueTable (io.cdap.cdap.api.dataset.lib.KeyValueTable)107 ArtifactSummary (io.cdap.cdap.api.artifact.ArtifactSummary)88 ProgramRunId (io.cdap.cdap.proto.id.ProgramRunId)88 Path (javax.ws.rs.Path)75