Examples with ServiceManager com.google.common.util.concurrent.ServiceManager used on opensource projects

Example 6 with ServiceManager

use of com.google.common.util.concurrent.ServiceManager in project incubator-gobblin by apache.

the class GobblinYarnAppLauncher method addServices.

private void addServices() throws IOException {
    List<Service> services = Lists.newArrayList();
    if (this.securityManager.isPresent()) {
        LOGGER.info("Adding KeyManagerService since key management is enabled");
        services.add(this.securityManager.get());
    }
    if (!this.config.hasPath(GobblinYarnConfigurationKeys.LOG_COPIER_DISABLE_DRIVER_COPY) || !this.config.getBoolean(GobblinYarnConfigurationKeys.LOG_COPIER_DISABLE_DRIVER_COPY)) {
        services.add(buildLogCopier(this.config, new Path(this.sinkLogRootDir, this.applicationName + Path.SEPARATOR + this.applicationId.get().toString()), GobblinClusterUtils.getAppWorkDirPathFromConfig(this.config, this.fs, this.applicationName, this.applicationId.get().toString())));
    }
    if (config.getBoolean(ConfigurationKeys.JOB_EXECINFO_SERVER_ENABLED_KEY)) {
        LOGGER.info("Starting the job execution info server since it is enabled");
        Properties properties = ConfigUtils.configToProperties(config);
        JobExecutionInfoServer executionInfoServer = new JobExecutionInfoServer(properties);
        services.add(executionInfoServer);
        if (config.getBoolean(ConfigurationKeys.ADMIN_SERVER_ENABLED_KEY)) {
            LOGGER.info("Starting the admin UI server since it is enabled");
            services.add(ServiceBasedAppLauncher.createAdminServer(properties, executionInfoServer.getAdvertisedServerUri()));
        }
    } else if (config.getBoolean(ConfigurationKeys.ADMIN_SERVER_ENABLED_KEY)) {
        LOGGER.warn("NOT starting the admin UI because the job execution info server is NOT enabled");
    }
    if (services.size() > 0) {
        this.serviceManager = Optional.of(new ServiceManager(services));
        this.serviceManager.get().startAsync();
    } else {
        serviceManager = Optional.absent();
    }
}

Example 7 with ServiceManager

use of com.google.common.util.concurrent.ServiceManager in project cdap-ingest by caskdata.

the class FileTailerIT method mockMetricsProcessor.

private void mockMetricsProcessor(PipeManager manager) throws IOException, NoSuchMethodException, InvocationTargetException, IllegalAccessException, NoSuchFieldException {
    List<Pipe> pipeList = new ArrayList<Pipe>();
    StreamClient client = null;
    StreamWriter writer = null;
    try {
        Method method1 = manager.getClass().getDeclaredMethod("getPipeConfigs");
        method1.setAccessible(true);
        List<PipeConfiguration> pipeConfList = (List<PipeConfiguration>) method1.invoke(manager);
        for (PipeConfiguration pipeConf : pipeConfList) {
            FileTailerQueue queue = new FileTailerQueue(pipeConf.getQueueSize());
            client = pipeConf.getSinkConfiguration().getStreamClient();
            String streamName = pipeConf.getSinkConfiguration().getStreamName();
            Method method2 = manager.getClass().getDeclaredMethod("getStreamWriterForPipe", StreamClient.class, String.class);
            method2.setAccessible(true);
            writer = (StreamWriter) method2.invoke(manager, client, streamName);
            FileTailerStateProcessor stateProcessor = new FileTailerStateProcessorImpl(pipeConf.getDaemonDir(), pipeConf.getStateFile());
            FileTailerMetricsProcessor metricsProcessor = new FileTailerMetricsProcessor(pipeConf.getDaemonDir(), pipeConf.getStatisticsFile(), pipeConf.getStatisticsSleepInterval(), pipeConf.getPipeName(), pipeConf.getSourceConfiguration().getFileName()) {

                @Override
                public void onReadEventMetric(int eventSize) {
                    super.onReadEventMetric(eventSize);
                    read.incrementAndGet();
                }

                @Override
                public void onIngestEventMetric(int latency) {
                    super.onIngestEventMetric(latency);
                    ingest.incrementAndGet();
                }
            };
            pipeList.add(new Pipe(new LogTailer(pipeConf, queue, stateProcessor, metricsProcessor, null), new FileTailerSink(queue, writer, SinkStrategy.LOADBALANCE, stateProcessor, metricsProcessor, null, pipeConf.getSinkConfiguration().getPackSize()), metricsProcessor));
            client = null;
            writer = null;
        }
        Field field = manager.getClass().getDeclaredField("serviceManager");
        field.setAccessible(true);
        Field modifiersField = Field.class.getDeclaredField("modifiers");
        modifiersField.setAccessible(true);
        modifiersField.setInt(field, field.getModifiers() & ~Modifier.FINAL);
        field.set(manager, new ServiceManager(pipeList));
    } finally {
        if (client != null) {
            client.close();
        }
        if (writer != null) {
            writer.close();
        }
    }
}

Example 8 with ServiceManager

use of com.google.common.util.concurrent.ServiceManager in project cdap-ingest by caskdata.

the class PipeManager method setupPipes.

/**
 * Pipes setup
 *
 * @throws IOException in case a client stream cannot be created
 */
private ServiceManager setupPipes() throws IOException {
    StreamClient client = null;
    StreamWriter writer = null;
    try {
        List<PipeConfiguration> pipeConfList = getPipeConfigs();
        for (PipeConfiguration pipeConf : pipeConfList) {
            FileTailerQueue queue = new FileTailerQueue(pipeConf.getQueueSize());
            client = pipeConf.getSinkConfiguration().getStreamClient();
            String streamName = pipeConf.getSinkConfiguration().getStreamName();
            writer = getStreamWriterForPipe(client, streamName);
            FileTailerStateProcessor stateProcessor = new FileTailerStateProcessorImpl(pipeConf.getDaemonDir(), pipeConf.getStateFile());
            FileTailerMetricsProcessor metricsProcessor = new FileTailerMetricsProcessor(pipeConf.getDaemonDir(), pipeConf.getStatisticsFile(), pipeConf.getStatisticsSleepInterval(), pipeConf.getPipeName(), pipeConf.getSourceConfiguration().getFileName());
            pipeList.add(new Pipe(new LogTailer(pipeConf, queue, stateProcessor, metricsProcessor, null), new FileTailerSink(queue, writer, SinkStrategy.LOADBALANCE, stateProcessor, metricsProcessor, null, pipeConf.getSinkConfiguration().getPackSize()), metricsProcessor));
            client = null;
            writer = null;
        }
        return new ServiceManager(pipeList);
    } catch (ConfigurationLoadingException e) {
        throw new ConfigurationLoadingException("Error during loading configuration from file: " + confFile.getAbsolutePath() + e.getMessage());
    } finally {
        if (client != null) {
            client.close();
        }
        if (writer != null) {
            writer.close();
        }
    }
}

Example 9 with ServiceManager

use of com.google.common.util.concurrent.ServiceManager in project incubator-gobblin by apache.

the class SingleTaskRunner method initServices.

private void initServices() {
    final Properties properties = ConfigUtils.configToProperties(this.clusterConfig);
    this.taskExecutor = new TaskExecutor(properties);
    this.taskStateTracker = new GobblinHelixTaskStateTracker(properties);
    final List<Service> services = Lists.newArrayList(this.taskExecutor, this.taskStateTracker);
    this.serviceManager = new ServiceManager(services);
}

Example 10 with ServiceManager

use of com.google.common.util.concurrent.ServiceManager in project incubator-gobblin by apache.

the class ObservingFSFlowEdgeTemplateCatalogTest method testModifyFlowTemplate.

@Test
public void testModifyFlowTemplate() throws Exception {
    ObservingFSFlowEdgeTemplateCatalog catalog = new ObservingFSFlowEdgeTemplateCatalog(this.templateCatalogCfg, new ReentrantReadWriteLock());
    ServiceManager serviceManager = new ServiceManager(Lists.newArrayList(catalog));
    serviceManager.startAsync().awaitHealthy(5, TimeUnit.SECONDS);
    // Check cached flow template is returned
    FlowTemplate flowTemplate1 = catalog.getFlowTemplate(new URI(FSFlowTemplateCatalogTest.TEST_TEMPLATE_DIR_URI));
    FlowTemplate flowTemplate2 = catalog.getFlowTemplate(new URI(FSFlowTemplateCatalogTest.TEST_TEMPLATE_DIR_URI));
    Assert.assertSame(flowTemplate1, flowTemplate2);
    // Update a file flow catalog and check that the getFlowTemplate returns the new value
    Path flowConfPath = new File(new File(this.templateDir, FSFlowTemplateCatalogTest.TEST_TEMPLATE_NAME), "flow.conf").toPath();
    List<String> lines = java.nio.file.Files.readAllLines(flowConfPath);
    for (int i = 0; i < lines.size(); i++) {
        if (lines.get(i).equals("gobblin.flow.edge.input.dataset.descriptor.0.format=avro")) {
            lines.set(i, "gobblin.flow.edge.input.dataset.descriptor.0.format=any");
            break;
        }
    }
    java.nio.file.Files.write(flowConfPath, lines);
    Function testFunction = new GetFlowTemplateConfigFunction(new URI(FSFlowTemplateCatalogTest.TEST_TEMPLATE_DIR_URI), catalog, "gobblin.flow.edge.input.dataset.descriptor.0.format");
    AssertWithBackoff.create().timeoutMs(10000).assertEquals(testFunction, "any", "flow template updated");
}