Examples with NamespaceId io.cdap.cdap.proto.id.NamespaceId used on opensource projects

Example 66 with NamespaceId

use of io.cdap.cdap.proto.id.NamespaceId in project cdap by caskdata.

the class RunRecordCorrectorServiceTest method testFixProgram.

@Test
public void testFixProgram() throws Exception {
    final AtomicInteger sourceId = new AtomicInteger(0);
    // Write 10 services with starting state
    // Write 10 workers with running state
    Map<ProgramRunId, ProgramRunStatus> expectedStates = new HashMap<>();
    ArtifactId artifactId = NamespaceId.DEFAULT.artifact("testArtifact", "1.0").toApiArtifactId();
    for (int i = 0; i < 10; i++) {
        ProgramRunId serviceId = NamespaceId.DEFAULT.app("test").service("service" + i).run(randomRunId());
        store.setProvisioning(serviceId, Collections.emptyMap(), SINGLETON_PROFILE_MAP, Bytes.toBytes(sourceId.getAndIncrement()), artifactId);
        store.setProvisioned(serviceId, 0, Bytes.toBytes(sourceId.getAndIncrement()));
        store.setStart(serviceId, null, Collections.emptyMap(), Bytes.toBytes(sourceId.getAndIncrement()));
        expectedStates.put(serviceId, ProgramRunStatus.FAILED);
        ProgramRunId workerId = new NamespaceId("ns").app("test").worker("worker" + i).run(randomRunId());
        store.setProvisioning(workerId, Collections.emptyMap(), SINGLETON_PROFILE_MAP, Bytes.toBytes(sourceId.getAndIncrement()), artifactId);
        store.setProvisioned(workerId, 0, Bytes.toBytes(sourceId.getAndIncrement()));
        store.setStart(workerId, null, Collections.emptyMap(), Bytes.toBytes(sourceId.getAndIncrement()));
        store.setRunning(workerId, System.currentTimeMillis(), null, Bytes.toBytes(sourceId.getAndIncrement()));
        expectedStates.put(workerId, ProgramRunStatus.FAILED);
    }
    // Write a service with suspend state
    ProgramRunId flowId = new NamespaceId("ns").app("test").service("flow").run(randomRunId());
    store.setProvisioning(flowId, Collections.emptyMap(), SINGLETON_PROFILE_MAP, Bytes.toBytes(sourceId.getAndIncrement()), artifactId);
    store.setProvisioned(flowId, 0, Bytes.toBytes(sourceId.getAndIncrement()));
    store.setStart(flowId, null, Collections.emptyMap(), Bytes.toBytes(sourceId.getAndIncrement()));
    store.setRunning(flowId, System.currentTimeMillis(), null, Bytes.toBytes(sourceId.getAndIncrement()));
    store.setSuspend(flowId, Bytes.toBytes(sourceId.getAndIncrement()), -1);
    expectedStates.put(flowId, ProgramRunStatus.SUSPENDED);
    // Write two MR in starting state. One with workflow information, one without.
    ProgramRunId mrId = NamespaceId.DEFAULT.app("app").mr("mr").run(randomRunId());
    store.setProvisioning(mrId, Collections.emptyMap(), SINGLETON_PROFILE_MAP, Bytes.toBytes(sourceId.getAndIncrement()), artifactId);
    store.setProvisioned(mrId, 0, Bytes.toBytes(sourceId.getAndIncrement()));
    store.setStart(mrId, null, Collections.emptyMap(), Bytes.toBytes(sourceId.getAndIncrement()));
    expectedStates.put(mrId, ProgramRunStatus.FAILED);
    ProgramRunId workflowId = NamespaceId.DEFAULT.app("app").workflow("workflow").run(randomRunId());
    ProgramRunId mrInWorkflowId = workflowId.getParent().getParent().mr("mrInWorkflow").run(randomRunId());
    store.setProvisioning(mrInWorkflowId, Collections.emptyMap(), ImmutableMap.of(ProgramOptionConstants.WORKFLOW_NAME, workflowId.getProgram(), ProgramOptionConstants.WORKFLOW_RUN_ID, workflowId.getRun(), ProgramOptionConstants.WORKFLOW_NODE_ID, "mr", SystemArguments.PROFILE_NAME, ProfileId.NATIVE.getScopedName()), Bytes.toBytes(sourceId.getAndIncrement()), artifactId);
    store.setProvisioned(mrInWorkflowId, 0, Bytes.toBytes(sourceId.getAndIncrement()));
    store.setStart(mrInWorkflowId, null, ImmutableMap.of(ProgramOptionConstants.WORKFLOW_NAME, workflowId.getProgram(), ProgramOptionConstants.WORKFLOW_RUN_ID, workflowId.getRun(), ProgramOptionConstants.WORKFLOW_NODE_ID, "mr"), Bytes.toBytes(sourceId.getAndIncrement()));
    expectedStates.put(workflowId, ProgramRunStatus.STARTING);
    // Write the workflow in RUNNING state.
    store.setProvisioning(workflowId, Collections.emptyMap(), SINGLETON_PROFILE_MAP, Bytes.toBytes(sourceId.getAndIncrement()), artifactId);
    store.setProvisioned(workflowId, 0, Bytes.toBytes(sourceId.getAndIncrement()));
    store.setStart(workflowId, null, Collections.emptyMap(), Bytes.toBytes(sourceId.getAndIncrement()));
    store.setRunning(workflowId, System.currentTimeMillis(), null, Bytes.toBytes(sourceId.getAndIncrement()));
    expectedStates.put(workflowId, ProgramRunStatus.RUNNING);
    // Use a ProgramRuntimeService that only reports running state based on a set of know ids
    final Map<ProgramId, RunId> runningSet = new HashMap<>();
    ProgramRuntimeService programRuntimeService = new AbstractProgramRuntimeService(cConf, null, null, new NoOpProgramStateWriter(), null) {

        @Override
        public ProgramLiveInfo getLiveInfo(ProgramId programId) {
            return new NotRunningProgramLiveInfo(programId);
        }

        @Override
        public Map<RunId, RuntimeInfo> list(ProgramId program) {
            RunId runId = runningSet.get(program);
            if (runId != null) {
                RuntimeInfo runtimeInfo = new SimpleRuntimeInfo(null, program);
                return Collections.singletonMap(runId, runtimeInfo);
            }
            return Collections.emptyMap();
        }
    };
    // Have both flow and workflow running
    runningSet.put(flowId.getParent(), RunIds.fromString(flowId.getRun()));
    runningSet.put(workflowId.getParent(), RunIds.fromString(workflowId.getRun()));
    ProgramStateWriter programStateWriter = new NoOpProgramStateWriter() {

        @Override
        public void error(ProgramRunId programRunId, Throwable failureCause) {
            store.setStop(programRunId, System.currentTimeMillis(), ProgramRunStatus.FAILED, new BasicThrowable(failureCause), Bytes.toBytes(sourceId.getAndIncrement()));
        }
    };
    // Create a run record fixer.
    // Set the start buffer time to -1 so that it fixes right away.
    // Also use a small tx batch size to validate the batching logic.
    RunRecordCorrectorService fixer = new RunRecordCorrectorService(cConf, store, programStateWriter, programRuntimeService, namespaceAdmin, datasetFramework, -1L, 5) {
    };
    fixer.fixRunRecords();
    // Validates all expected states
    for (Map.Entry<ProgramRunId, ProgramRunStatus> entry : expectedStates.entrySet()) {
        validateExpectedState(entry.getKey(), entry.getValue());
    }
    // Remove the workflow from the running set and mark it as completed
    runningSet.remove(workflowId.getParent());
    store.setStop(workflowId, System.currentTimeMillis(), ProgramRunStatus.COMPLETED, Bytes.toBytes(sourceId.getAndIncrement()));
    fixer.fixRunRecords();
    // Both the workflow and the MR in workflow should be changed to failed state
    expectedStates.put(workflowId, ProgramRunStatus.COMPLETED);
    expectedStates.put(mrInWorkflowId, ProgramRunStatus.FAILED);
    // Validates all expected states again
    for (Map.Entry<ProgramRunId, ProgramRunStatus> entry : expectedStates.entrySet()) {
        validateExpectedState(entry.getKey(), entry.getValue());
    }
}

Example 67 with NamespaceId

use of io.cdap.cdap.proto.id.NamespaceId in project cdap by caskdata.

the class TriggerCodecTest method testContainingTrigger.

private void testContainingTrigger(ProtoTrigger proto, Trigger trigger) {
    ProgramSchedule proto1 = new ProgramSchedule("sched1", "one partition schedule", new NamespaceId("test").app("a").worker("ww"), ImmutableMap.of("prop3", "abc"), proto, ImmutableList.of());
    ProgramSchedule sched1 = new ProgramSchedule("sched1", "one partition schedule", new NamespaceId("test").app("a").worker("ww"), ImmutableMap.of("prop3", "abc"), trigger, ImmutableList.of());
    Assert.assertEquals(sched1, GSON.fromJson(GSON.toJson(proto1), ProgramSchedule.class));
}

Example 68 with NamespaceId

use of io.cdap.cdap.proto.id.NamespaceId in project cdap by caskdata.

the class NamespaceHttpHandlerTest method testNamespaceClient.

@Test
public void testNamespaceClient() throws Exception {
    // tests the NamespaceClient's ability to interact with Namespace service/handlers.
    NamespaceAdmin namespaceClient = getInjector().getInstance(NamespaceAdmin.class);
    // test setup creates two namespaces in @BeforeClass, apart from the default namespace which always exists.
    List<NamespaceMeta> namespaces = namespaceClient.list();
    Assert.assertEquals(3, namespaces.size());
    Set<NamespaceMeta> expectedNamespaces = ImmutableSet.of(NamespaceMeta.DEFAULT, TEST_NAMESPACE_META1, TEST_NAMESPACE_META2);
    for (NamespaceMeta actual : namespaces) {
        actual = new NamespaceMeta.Builder(actual).setGeneration(0L).build();
        Assert.assertTrue(expectedNamespaces.contains(actual));
    }
    NamespaceMeta namespaceMeta = namespaceClient.get(new NamespaceId(TEST_NAMESPACE1));
    namespaceMeta = new NamespaceMeta.Builder(namespaceMeta).setGeneration(0L).build();
    Assert.assertEquals(TEST_NAMESPACE_META1, namespaceMeta);
    try {
        namespaceClient.get(new NamespaceId("nonExistentNamespace"));
        Assert.fail("Did not expect namespace 'nonExistentNamespace' to exist.");
    } catch (NotFoundException expected) {
    // expected
    }
    // test create and get
    NamespaceId fooNamespace = new NamespaceId("fooNamespace");
    NamespaceMeta toCreate = new NamespaceMeta.Builder().setName(fooNamespace).build();
    namespaceClient.create(toCreate);
    NamespaceMeta receivedMeta = namespaceClient.get(fooNamespace);
    Assert.assertNotNull(receivedMeta);
    Assert.assertEquals(toCreate, new NamespaceMeta.Builder(receivedMeta).setGeneration(0L).build());
    waitForAndCheckNamespaceCount(4);
    namespaceClient.delete(fooNamespace);
    try {
        namespaceClient.get(fooNamespace);
        Assert.fail("Did not expect namespace '" + fooNamespace + "' to exist after deleting it.");
    } catch (NotFoundException expected) {
    // expected
    }
    waitForAndCheckNamespaceCount(3);
}

Example 69 with NamespaceId

use of io.cdap.cdap.proto.id.NamespaceId in project cdap by caskdata.

the class NamespaceHttpHandlerTest method testDeleteDatasetsOnly.

@Test
public void testDeleteDatasetsOnly() throws Exception {
    CConfiguration cConf = getInjector().getInstance(CConfiguration.class);
    // test deleting non-existent namespace
    assertResponseCode(200, createNamespace(NAME));
    assertResponseCode(200, getNamespace(NAME));
    NamespacePathLocator namespacePathLocator = getInjector().getInstance(NamespacePathLocator.class);
    Location nsLocation = namespacePathLocator.get(new NamespaceId(NAME));
    Assert.assertTrue(nsLocation.exists());
    DatasetFramework dsFramework = getInjector().getInstance(DatasetFramework.class);
    deploy(AppWithServices.class, 200, Constants.Gateway.API_VERSION_3_TOKEN, NAME);
    deploy(AppWithDataset.class, 200, Constants.Gateway.API_VERSION_3_TOKEN, NAME);
    DatasetId myDataset = new DatasetId(NAME, "myds");
    Assert.assertTrue(dsFramework.hasInstance(myDataset));
    Id.Program program = Id.Program.from(NAME_ID, "AppWithServices", ProgramType.SERVICE, "NoOpService");
    startProgram(program);
    waitState(program, ProgramStatus.RUNNING.name());
    boolean resetEnabled = cConf.getBoolean(Constants.Dangerous.UNRECOVERABLE_RESET);
    cConf.setBoolean(Constants.Dangerous.UNRECOVERABLE_RESET, false);
    // because reset is not enabled
    assertResponseCode(403, deleteNamespaceData(NAME));
    Assert.assertTrue(nsLocation.exists());
    cConf.setBoolean(Constants.Dangerous.UNRECOVERABLE_RESET, resetEnabled);
    // because service is running
    assertResponseCode(409, deleteNamespaceData(NAME));
    Assert.assertTrue(nsLocation.exists());
    stopProgram(program);
    waitState(program, ProgramStatus.STOPPED.name());
    assertResponseCode(200, deleteNamespaceData(NAME));
    Assert.assertTrue(nsLocation.exists());
    Assert.assertEquals(2, getAppList(NAME).size());
    Assert.assertEquals("AppWithServices", getAppDetails(NAME, "AppWithServices").get("name").getAsString());
    Assert.assertEquals(AppWithDataset.class.getSimpleName(), getAppDetails(NAME, AppWithDataset.class.getSimpleName()).get("name").getAsString());
    assertResponseCode(200, getNamespace(NAME));
    Assert.assertFalse(dsFramework.hasInstance(myDataset));
    assertResponseCode(200, deleteNamespace(NAME));
    assertResponseCode(404, getNamespace(NAME));
}

Example 70 with NamespaceId

use of io.cdap.cdap.proto.id.NamespaceId in project cdap by caskdata.

the class NamespaceHttpHandlerTest method testDeleteAll.

@Test
public void testDeleteAll() throws Exception {
    CConfiguration cConf = getInjector().getInstance(CConfiguration.class);
    // test deleting non-existent namespace
    assertResponseCode(404, deleteNamespace("doesnotexist"));
    assertResponseCode(200, createNamespace(NAME));
    assertResponseCode(200, getNamespace(NAME));
    assertResponseCode(200, createNamespace(OTHER_NAME));
    assertResponseCode(200, getNamespace(OTHER_NAME));
    NamespacePathLocator namespacePathLocator = getInjector().getInstance(NamespacePathLocator.class);
    Location nsLocation = namespacePathLocator.get(new NamespaceId(NAME));
    Assert.assertTrue(nsLocation.exists());
    DatasetFramework dsFramework = getInjector().getInstance(DatasetFramework.class);
    deploy(AppWithServices.class, 200, Constants.Gateway.API_VERSION_3_TOKEN, NAME);
    deploy(AppWithDataset.class, 200, Constants.Gateway.API_VERSION_3_TOKEN, NAME);
    deploy(AppForUnrecoverableResetTest.class, 200, Constants.Gateway.API_VERSION_3_TOKEN, OTHER_NAME);
    DatasetId myDataset = new DatasetId(NAME, "myds");
    Assert.assertTrue(dsFramework.hasInstance(myDataset));
    Id.Program program = Id.Program.from(NAME_ID, "AppWithServices", ProgramType.SERVICE, "NoOpService");
    startProgram(program);
    waitState(program, ProgramStatus.RUNNING.name());
    boolean resetEnabled = cConf.getBoolean(Constants.Dangerous.UNRECOVERABLE_RESET);
    cConf.setBoolean(Constants.Dangerous.UNRECOVERABLE_RESET, false);
    // because unrecoverable reset is disabled
    assertResponseCode(403, deleteNamespace(NAME));
    cConf.setBoolean(Constants.Dangerous.UNRECOVERABLE_RESET, resetEnabled);
    // because service is running
    assertResponseCode(409, deleteNamespace(NAME));
    Assert.assertTrue(nsLocation.exists());
    stopProgram(program);
    waitState(program, ProgramStatus.STOPPED.name());
    // delete should work now
    assertResponseCode(200, deleteNamespace(NAME));
    Assert.assertFalse(nsLocation.exists());
    Assert.assertFalse(dsFramework.hasInstance(myDataset));
    assertResponseCode(200, deleteNamespace(OTHER_NAME));
    // Create the namespace again and deploy the application containing schedules.
    // Application deployment should succeed.
    assertResponseCode(200, createNamespace(OTHER_NAME));
    deploy(AppForUnrecoverableResetTest.class, 200, Constants.Gateway.API_VERSION_3_TOKEN, OTHER_NAME);
    assertResponseCode(200, deleteNamespace(OTHER_NAME));
}