Examples with MessagingService io.cdap.cdap.messaging.MessagingService used on opensource projects

Example 26 with MessagingService

use of io.cdap.cdap.messaging.MessagingService in project cdap by cdapio.

the class SparkProgramRunner method run.

@Override
public ProgramController run(Program program, ProgramOptions options) {
    LOG.trace("Starting Spark program {} with SparkProgramRunner of ClassLoader {}", program.getId(), getClass().getClassLoader());
    // Get the RunId first. It is used for the creation of the ClassLoader closing thread.
    Arguments arguments = options.getArguments();
    RunId runId = ProgramRunners.getRunId(options);
    Deque<Closeable> closeables = new LinkedList<>();
    try {
        // Extract and verify parameters
        ApplicationSpecification appSpec = program.getApplicationSpecification();
        Preconditions.checkNotNull(appSpec, "Missing application specification.");
        ProgramType processorType = program.getType();
        Preconditions.checkNotNull(processorType, "Missing processor type.");
        Preconditions.checkArgument(processorType == ProgramType.SPARK, "Only Spark process type is supported.");
        SparkSpecification spec = appSpec.getSpark().get(program.getName());
        Preconditions.checkNotNull(spec, "Missing SparkSpecification for %s", program.getName());
        String host = options.getArguments().getOption(ProgramOptionConstants.HOST);
        Preconditions.checkArgument(host != null, "No hostname is provided");
        // Get the WorkflowProgramInfo if it is started by Workflow
        WorkflowProgramInfo workflowInfo = WorkflowProgramInfo.create(arguments);
        DatasetFramework programDatasetFramework = workflowInfo == null ? datasetFramework : NameMappedDatasetFramework.createFromWorkflowProgramInfo(datasetFramework, workflowInfo, appSpec);
        // Setup dataset framework context, if required
        if (programDatasetFramework instanceof ProgramContextAware) {
            ProgramId programId = program.getId();
            ((ProgramContextAware) programDatasetFramework).setContext(new BasicProgramContext(programId.run(runId)));
        }
        PluginInstantiator pluginInstantiator = createPluginInstantiator(options, program.getClassLoader());
        if (pluginInstantiator != null) {
            closeables.addFirst(pluginInstantiator);
        }
        SparkRuntimeContext runtimeContext = new SparkRuntimeContext(new Configuration(hConf), program, options, cConf, host, txClient, programDatasetFramework, metricsCollectionService, workflowInfo, pluginInstantiator, secureStore, secureStoreManager, accessEnforcer, authenticationContext, messagingService, serviceAnnouncer, pluginFinder, locationFactory, metadataReader, metadataPublisher, namespaceQueryAdmin, fieldLineageWriter, remoteClientFactory, () -> {
        });
        closeables.addFirst(runtimeContext);
        Spark spark;
        try {
            spark = new InstantiatorFactory(false).get(TypeToken.of(program.<Spark>getMainClass())).create();
        } catch (Exception e) {
            LOG.error("Failed to instantiate Spark class for {}", spec.getClassName(), e);
            throw Throwables.propagate(e);
        }
        boolean isLocal = SparkRuntimeContextConfig.isLocal(options);
        SparkSubmitter submitter;
        // If MasterEnvironment is not available, use non-master env spark submitters
        MasterEnvironment masterEnv = MasterEnvironments.getMasterEnvironment();
        if (masterEnv != null && cConf.getBoolean(Constants.Environment.PROGRAM_SUBMISSION_MASTER_ENV_ENABLED, true)) {
            submitter = new MasterEnvironmentSparkSubmitter(cConf, locationFactory, host, runtimeContext, masterEnv);
        } else {
            submitter = isLocal ? new LocalSparkSubmitter() : new DistributedSparkSubmitter(hConf, locationFactory, host, runtimeContext, options.getArguments().getOption(Constants.AppFabric.APP_SCHEDULER_QUEUE));
        }
        Service sparkRuntimeService = new SparkRuntimeService(cConf, spark, getPluginArchive(options), runtimeContext, submitter, locationFactory, isLocal, fieldLineageWriter, masterEnv);
        sparkRuntimeService.addListener(createRuntimeServiceListener(closeables), Threads.SAME_THREAD_EXECUTOR);
        ProgramController controller = new SparkProgramController(sparkRuntimeService, runtimeContext);
        LOG.debug("Starting Spark Job. Context: {}", runtimeContext);
        if (isLocal || UserGroupInformation.isSecurityEnabled()) {
            sparkRuntimeService.start();
        } else {
            ProgramRunners.startAsUser(cConf.get(Constants.CFG_HDFS_USER), sparkRuntimeService);
        }
        return controller;
    } catch (Throwable t) {
        closeAllQuietly(closeables);
        throw Throwables.propagate(t);
    }
}

Example 27 with MessagingService

use of io.cdap.cdap.messaging.MessagingService in project cdap by cdapio.

the class TestBase method finish.

@AfterClass
public static void finish() throws Exception {
    if (--nestedStartCount != 0) {
        return;
    }
    if (previewRunnerManager instanceof Service) {
        ((Service) previewRunnerManager).stopAndWait();
    }
    previewHttpServer.stopAndWait();
    if (cConf.getBoolean(Constants.Security.Authorization.ENABLED)) {
        InstanceId instance = new InstanceId(cConf.get(Constants.INSTANCE_NAME));
        Principal principal = new Principal(System.getProperty("user.name"), Principal.PrincipalType.USER);
        accessControllerInstantiator.get().grant(Authorizable.fromEntityId(instance), principal, ImmutableSet.of(StandardPermission.UPDATE));
        accessControllerInstantiator.get().grant(Authorizable.fromEntityId(NamespaceId.DEFAULT), principal, ImmutableSet.of(StandardPermission.UPDATE));
    }
    namespaceAdmin.delete(NamespaceId.DEFAULT);
    accessControllerInstantiator.close();
    if (programScheduler instanceof Service) {
        ((Service) programScheduler).stopAndWait();
    }
    metricsCollectionService.stopAndWait();
    if (scheduler instanceof Service) {
        ((Service) scheduler).stopAndWait();
    }
    Closeables.closeQuietly(exploreClient);
    if (exploreExecutorService != null) {
        exploreExecutorService.stopAndWait();
    }
    datasetService.stopAndWait();
    dsOpService.stopAndWait();
    metadataService.stopAndWait();
    metadataSubscriberService.stopAndWait();
    Closeables.closeQuietly(metadataStorage);
    txService.stopAndWait();
    if (messagingService instanceof Service) {
        ((Service) messagingService).stopAndWait();
    }
    appFabricServer.stopAndWait();
    supportBundleInternalService.stopAndWait();
    appFabricHealthCheckService.stopAndWait();
}

Example 28 with MessagingService

use of io.cdap.cdap.messaging.MessagingService in project cdap by cdapio.

the class DefaultPreviewRunner method startUp.

@Override
protected void startUp() throws Exception {
    LOG.debug("Starting preview runner service");
    StoreDefinition.createAllTables(structuredTableAdmin);
    if (messagingService instanceof Service) {
        ((Service) messagingService).startAndWait();
    }
    dsOpExecService.startAndWait();
    datasetService.startAndWait();
    // It is recommended to initialize log appender after datasetService is started,
    // since log appender instantiates a dataset.
    logAppenderInitializer.initialize();
    LoggingContextAccessor.setLoggingContext(new ServiceLoggingContext(NamespaceId.SYSTEM.getNamespace(), Constants.Logging.COMPONENT_NAME, Constants.Service.PREVIEW_HTTP));
    Futures.allAsList(applicationLifecycleService.start(), programRuntimeService.start(), metricsCollectionService.start(), programNotificationSubscriberService.start()).get();
    Files.createDirectories(previewIdDirPath);
    // Reconcile status for abruptly terminated preview runs
    try (Stream<Path> paths = Files.walk(Paths.get(previewIdDirPath.toString()))) {
        paths.filter(Files::isRegularFile).forEach(path -> {
            try (Reader reader = Files.newBufferedReader(path, StandardCharsets.UTF_8)) {
                ProgramId programId = GSON.fromJson(reader, ProgramId.class);
                long submitTimeMillis = RunIds.getTime(programId.getApplication(), TimeUnit.MILLISECONDS);
                PreviewStatus status = new PreviewStatus(PreviewStatus.Status.KILLED_BY_EXCEEDING_MEMORY_LIMIT, submitTimeMillis, new BasicThrowable(new Exception("Preview runner container killed possibly because of out of memory. " + "Please try running preview again.")), null, null);
                previewTerminated(programId, status);
            } catch (IOException e) {
                LOG.warn("Error reading file {}. Ignoring", path, e);
            }
        });
    }
}

Example 29 with MessagingService

use of io.cdap.cdap.messaging.MessagingService in project cdap by cdapio.

the class RuntimeClientServerTest method beforeTest.

@Before
public void beforeTest() throws Exception {
    cConf = CConfiguration.create();
    cConf.set(Constants.CFG_LOCAL_DATA_DIR, TEMP_FOLDER.newFolder().getAbsolutePath());
    cConf.setBoolean(Constants.RuntimeMonitor.COMPRESSION_ENABLED, compression);
    cConf.setBoolean(Constants.AppFabric.SPARK_EVENT_LOGS_ENABLED, true);
    cConf.set(TEST_TOPIC_KEY, TEST_TOPIC);
    cConf.set(Constants.RuntimeMonitor.TOPICS_CONFIGS, Constants.Logging.TMS_TOPIC_PREFIX + ":1," + TEST_TOPIC_KEY);
    Injector injector = Guice.createInjector(new ConfigModule(cConf), RemoteAuthenticatorModules.getNoOpModule(), new InMemoryDiscoveryModule(), new LocalLocationModule(), new MessagingServerRuntimeModule().getInMemoryModules(), new AuthenticationContextModules().getNoOpModule(), new RuntimeServerModule() {

        @Override
        protected void bindRequestValidator() {
            bind(RuntimeRequestValidator.class).toInstance((programRunId, request) -> new ProgramRunInfo(ProgramRunStatus.STOPPING, null));
        }

        @Override
        protected void bindLogProcessor() {
            bind(RemoteExecutionLogProcessor.class).toInstance(payloads -> {
                // For testing purpose, we just store logs to a list
                payloads.forEachRemaining(bytes -> logEntries.add(new String(bytes, StandardCharsets.UTF_8)));
            });
        }
    }, new AbstractModule() {

        @Override
        protected void configure() {
            bind(MetricsCollectionService.class).to(NoOpMetricsCollectionService.class);
        }
    });
    messagingService = injector.getInstance(MessagingService.class);
    if (messagingService instanceof Service) {
        ((Service) messagingService).startAndWait();
    }
    messagingService.createTopic(new TopicMetadata(NamespaceId.SYSTEM.topic("topic")));
    runtimeServer = injector.getInstance(RuntimeServer.class);
    runtimeServer.startAndWait();
    runtimeClient = injector.getInstance(RuntimeClient.class);
    locationFactory = injector.getInstance(LocationFactory.class);
}

Example 30 with MessagingService

use of io.cdap.cdap.messaging.MessagingService in project cdap by cdapio.

the class RuntimeClientServiceTest method beforeTest.

@Before
public void beforeTest() throws Exception {
    CConfiguration cConf = CConfiguration.create();
    cConf.set(Constants.CFG_LOCAL_DATA_DIR, TEMP_FOLDER.newFolder().getAbsolutePath());
    cConf.set(Constants.RuntimeMonitor.TOPICS_CONFIGS, TOPIC_CONFIGS_VALUE);
    topicConfigs = RuntimeMonitors.createTopicConfigs(cConf);
    InMemoryDiscoveryService discoveryService = new InMemoryDiscoveryService();
    // Injector for the server side
    Injector injector = Guice.createInjector(new ConfigModule(cConf), new LocalLocationModule(), new MessagingServerRuntimeModule().getInMemoryModules(), new AuthenticationContextModules().getNoOpModule(), new RuntimeServerModule() {

        @Override
        protected void bindRequestValidator() {
            bind(RuntimeRequestValidator.class).toInstance((programRunId, request) -> new ProgramRunInfo(ProgramRunStatus.COMPLETED, null));
        }

        @Override
        protected void bindLogProcessor() {
            bind(RemoteExecutionLogProcessor.class).toInstance(payloads -> {
            });
        }
    }, new AbstractModule() {

        @Override
        protected void configure() {
            bind(MetricsCollectionService.class).to(NoOpMetricsCollectionService.class);
            bind(DiscoveryService.class).toInstance(discoveryService);
            bind(DiscoveryServiceClient.class).toInstance(discoveryService);
        }
    });
    messagingService = injector.getInstance(MessagingService.class);
    if (messagingService instanceof Service) {
        ((Service) messagingService).startAndWait();
    }
    runtimeServer = injector.getInstance(RuntimeServer.class);
    runtimeServer.startAndWait();
    // Injector for the client side
    clientCConf = CConfiguration.create();
    clientCConf.set(Constants.CFG_LOCAL_DATA_DIR, TEMP_FOLDER.newFolder().getAbsolutePath());
    clientCConf.set(Constants.RuntimeMonitor.TOPICS_CONFIGS, TOPIC_CONFIGS_VALUE);
    // Shorten the poll delay and grace period to speed up testing of program terminate state handling
    clientCConf.setLong(Constants.RuntimeMonitor.POLL_TIME_MS, 200);
    clientCConf.setLong(Constants.RuntimeMonitor.GRACEFUL_SHUTDOWN_MS, 3000);
    // Use smaller batch size so that fetches is broken into multiple fetches
    clientCConf.setInt(Constants.RuntimeMonitor.BATCH_SIZE, 1);
    injector = Guice.createInjector(new ConfigModule(clientCConf), RemoteAuthenticatorModules.getNoOpModule(), new MessagingServerRuntimeModule().getInMemoryModules(), new AuthenticationContextModules().getNoOpModule(), new AbstractModule() {

        @Override
        protected void configure() {
            bind(MetricsCollectionService.class).to(NoOpMetricsCollectionService.class);
            bind(DiscoveryService.class).toInstance(discoveryService);
            bind(DiscoveryServiceClient.class).toInstance(discoveryService);
            bind(ProgramRunId.class).toInstance(PROGRAM_RUN_ID);
        }
    });
    clientMessagingService = injector.getInstance(MessagingService.class);
    if (clientMessagingService instanceof Service) {
        ((Service) clientMessagingService).startAndWait();
    }
    runtimeClientService = injector.getInstance(RuntimeClientService.class);
    runtimeClientService.startAndWait();
}