Examples with NamespaceMeta co.cask.cdap.proto.NamespaceMeta used on opensource projects

Example 16 with NamespaceMeta

use of co.cask.cdap.proto.NamespaceMeta in project cdap by caskdata.

the class IntegrationTestBase method checkSystemServices.

protected void checkSystemServices() throws TimeoutException, InterruptedException {
    Callable<Boolean> cdapAvailable = new Callable<Boolean>() {

        @Override
        public Boolean call() throws Exception {
            // first wait for all system services to be 'OK'
            if (!getMonitorClient().allSystemServicesOk()) {
                return false;
            }
            // For non-default namespaces, simply check that the dataset service is up with list().
            // If list() does not throw exception, which means the http request receives response
            // status HTTP_OK and dataset service is up, then check if default namespace exists, if so return true.
            List<NamespaceMeta> list = getNamespaceClient().list();
            if (!configuredNamespace.equals(NamespaceId.DEFAULT)) {
                return true;
            }
            // default namespace exists before integration test starts
            for (NamespaceMeta namespaceMeta : list) {
                if (namespaceMeta.getNamespaceId().equals(NamespaceId.DEFAULT)) {
                    return true;
                }
            }
            return false;
        }
    };
    String errorMessage = String.format("CDAP Services are not available. Retried for %s seconds.", SERVICE_CHECK_TIMEOUT_SECONDS);
    try {
        checkServicesWithRetry(cdapAvailable, errorMessage);
    } catch (Throwable e) {
        Throwable rootCause = Throwables.getRootCause(e);
        if (rootCause instanceof UnauthenticatedException) {
            // security is enabled, we need to get access token before checking system services
            try {
                accessToken = fetchAccessToken();
            } catch (IOException ex) {
                throw Throwables.propagate(ex);
            }
            checkServicesWithRetry(cdapAvailable, errorMessage);
        } else {
            throw Throwables.propagate(rootCause);
        }
    }
    LOG.info("CDAP Services are up and running!");
}

Example 17 with NamespaceMeta

use of co.cask.cdap.proto.NamespaceMeta in project cdap by caskdata.

the class TestFrameworkTestRun method testCrossNSMapperDatasetAccess.

@Category(SlowTests.class)
@Test
public void testCrossNSMapperDatasetAccess() throws Exception {
    NamespaceMeta inputNS = new NamespaceMeta.Builder().setName("inputNS").build();
    NamespaceMeta outputNS = new NamespaceMeta.Builder().setName("outputNS").build();
    getNamespaceAdmin().create(inputNS);
    getNamespaceAdmin().create(outputNS);
    addDatasetInstance(inputNS.getNamespaceId().dataset("table1"), "keyValueTable");
    addDatasetInstance(outputNS.getNamespaceId().dataset("table2"), "keyValueTable");
    DataSetManager<KeyValueTable> tableManager = getDataset(inputNS.getNamespaceId().dataset("table1"));
    KeyValueTable inputTable = tableManager.get();
    inputTable.write("hello", "world");
    tableManager.flush();
    ApplicationManager appManager = deployApplication(DatasetCrossNSAccessWithMAPApp.class);
    Map<String, String> argsForMR = ImmutableMap.of(DatasetCrossNSAccessWithMAPApp.INPUT_DATASET_NS, inputNS.getName(), DatasetCrossNSAccessWithMAPApp.INPUT_DATASET_NAME, "table1", DatasetCrossNSAccessWithMAPApp.OUTPUT_DATASET_NS, outputNS.getName(), DatasetCrossNSAccessWithMAPApp.OUTPUT_DATASET_NAME, "table2");
    MapReduceManager mrManager = appManager.getMapReduceManager(DatasetCrossNSAccessWithMAPApp.MAPREDUCE_PROGRAM).start(argsForMR);
    mrManager.waitForRun(ProgramRunStatus.COMPLETED, 5, TimeUnit.MINUTES);
    appManager.stopAll();
    DataSetManager<KeyValueTable> outTableManager = getDataset(outputNS.getNamespaceId().dataset("table2"));
    verifyMapperJobOutput(DatasetCrossNSAccessWithMAPApp.class, outTableManager);
}

Example 18 with NamespaceMeta

use of co.cask.cdap.proto.NamespaceMeta in project cdap by caskdata.

the class ProgramLifecycleService method retrieveProgramIdForRunRecord.

/**
   * Helper method to get {@link ProgramId} for a RunRecord for type of program
   *
   * @param programType Type of program to search
   * @param runId The target id of the {@link RunRecord} to find
   * @return the program id of the run record or {@code null} if does not exist.
   */
@Nullable
private ProgramId retrieveProgramIdForRunRecord(ProgramType programType, String runId) {
    // Get list of namespaces (borrow logic from AbstractAppFabricHttpHandler#listPrograms)
    List<NamespaceMeta> namespaceMetas = nsStore.list();
    // For each, get all programs under it
    ProgramId targetProgramId = null;
    for (NamespaceMeta nm : namespaceMetas) {
        NamespaceId namespace = Ids.namespace(nm.getName());
        Collection<ApplicationSpecification> appSpecs = store.getAllApplications(namespace);
        // For each application get the programs checked against run records
        for (ApplicationSpecification appSpec : appSpecs) {
            switch(programType) {
                case FLOW:
                    for (String programName : appSpec.getFlows().keySet()) {
                        ProgramId programId = validateProgramForRunRecord(nm.getName(), appSpec.getName(), appSpec.getAppVersion(), programType, programName, runId);
                        if (programId != null) {
                            targetProgramId = programId;
                            break;
                        }
                    }
                    break;
                case MAPREDUCE:
                    for (String programName : appSpec.getMapReduce().keySet()) {
                        ProgramId programId = validateProgramForRunRecord(nm.getName(), appSpec.getName(), appSpec.getAppVersion(), programType, programName, runId);
                        if (programId != null) {
                            targetProgramId = programId;
                            break;
                        }
                    }
                    break;
                case SPARK:
                    for (String programName : appSpec.getSpark().keySet()) {
                        ProgramId programId = validateProgramForRunRecord(nm.getName(), appSpec.getName(), appSpec.getAppVersion(), programType, programName, runId);
                        if (programId != null) {
                            targetProgramId = programId;
                            break;
                        }
                    }
                    break;
                case SERVICE:
                    for (String programName : appSpec.getServices().keySet()) {
                        ProgramId programId = validateProgramForRunRecord(nm.getName(), appSpec.getName(), appSpec.getAppVersion(), programType, programName, runId);
                        if (programId != null) {
                            targetProgramId = programId;
                            break;
                        }
                    }
                    break;
                case WORKER:
                    for (String programName : appSpec.getWorkers().keySet()) {
                        ProgramId programId = validateProgramForRunRecord(nm.getName(), appSpec.getName(), appSpec.getAppVersion(), programType, programName, runId);
                        if (programId != null) {
                            targetProgramId = programId;
                            break;
                        }
                    }
                    break;
                case WORKFLOW:
                    for (String programName : appSpec.getWorkflows().keySet()) {
                        ProgramId programId = validateProgramForRunRecord(nm.getName(), appSpec.getName(), appSpec.getAppVersion(), programType, programName, runId);
                        if (programId != null) {
                            targetProgramId = programId;
                            break;
                        }
                    }
                    break;
                case CUSTOM_ACTION:
                case WEBAPP:
                    // no-op
                    break;
                default:
                    LOG.debug("Unknown program type: " + programType.name());
                    break;
            }
            if (targetProgramId != null) {
                break;
            }
        }
        if (targetProgramId != null) {
            break;
        }
    }
    return targetProgramId;
}

Example 19 with NamespaceMeta

use of co.cask.cdap.proto.NamespaceMeta in project cdap by caskdata.

the class HBaseQueueAdmin method upgrade.

@Override
public void upgrade() throws Exception {
    int numThreads = cConf.getInt(Constants.Upgrade.UPGRADE_THREAD_POOL_SIZE);
    final ExecutorService executor = Executors.newFixedThreadPool(numThreads, new ThreadFactoryBuilder().setNameFormat("hbase-cmd-executor-%d").setDaemon(true).build());
    try {
        final Map<TableId, Future<?>> allFutures = new HashMap<>();
        // For each queue config table and queue data table in each namespace, perform an upgrade
        for (final NamespaceMeta namespaceMeta : namespaceQueryAdmin.list()) {
            impersonator.doAs(namespaceMeta.getNamespaceId(), new Callable<Void>() {

                @Override
                public Void call() throws Exception {
                    Map<TableId, Future<?>> futures = upgradeQueues(namespaceMeta, executor);
                    allFutures.putAll(futures);
                    return null;
                }
            });
        }
        // Wait for the queue upgrades to complete
        Map<TableId, Throwable> failed = waitForUpgrade(allFutures);
        if (!failed.isEmpty()) {
            for (Map.Entry<TableId, Throwable> entry : failed.entrySet()) {
                LOG.error("Failed to upgrade queue table {}", entry.getKey(), entry.getValue());
            }
            throw new Exception(String.format("Error upgrading queue tables. %s of %s failed", failed.size(), allFutures.size()));
        }
    } finally {
        // We'll have tasks pending in the executor only on an interrupt, when user wants to abort the upgrade.
        // Use shutdownNow() to interrupt the tasks and abort.
        executor.shutdownNow();
    }
}

Example 20 with NamespaceMeta

use of co.cask.cdap.proto.NamespaceMeta in project cdap by caskdata.

the class HiveExploreServiceTestRun method testQueriesCount.

@Test
public void testQueriesCount() throws Exception {
    NamespaceId testNamespace1 = new NamespaceId("testQueriesCount");
    NamespaceMeta namespaceMeta = new NamespaceMeta.Builder().setName(testNamespace1).build();
    namespaceAdmin.create(namespaceMeta);
    exploreClient.addNamespace(namespaceMeta).get();
    try {
        Assert.assertEquals(0, exploreService.getActiveQueryCount(testNamespace1));
        ListenableFuture<ExploreExecutionResult> future = exploreClient.submit(testNamespace1, "show tables");
        ExploreExecutionResult result = null;
        try {
            result = future.get();
            Assert.assertEquals(1, exploreService.getActiveQueryCount(testNamespace1));
        } finally {
            if (result != null) {
                result.close();
            }
            Assert.assertEquals(0, exploreService.getActiveQueryCount(testNamespace1));
        }
    } finally {
        exploreClient.removeNamespace(testNamespace1).get();
    }
}