Examples with ServiceDiscovered org.apache.twill.discovery.ServiceDiscovered used on opensource projects

Example 6 with ServiceDiscovered

use of org.apache.twill.discovery.ServiceDiscovered in project cdap by caskdata.

the class LocalLogAppenderResilientTest method testResilientLogging.

@Test
public void testResilientLogging() throws Exception {
    Configuration hConf = new Configuration();
    CConfiguration cConf = CConfiguration.create();
    File datasetDir = new File(tmpFolder.newFolder(), "datasetUser");
    //noinspection ResultOfMethodCallIgnored
    datasetDir.mkdirs();
    cConf.set(Constants.Dataset.Manager.OUTPUT_DIR, datasetDir.getAbsolutePath());
    cConf.set(Constants.Service.MASTER_SERVICES_BIND_ADDRESS, "localhost");
    cConf.set(Constants.Dataset.Executor.ADDRESS, "localhost");
    cConf.setInt(Constants.Dataset.Executor.PORT, Networks.getRandomPort());
    cConf.set(Constants.CFG_LOCAL_DATA_DIR, tmpFolder.newFolder().getAbsolutePath());
    Injector injector = Guice.createInjector(new ConfigModule(cConf, hConf), new IOModule(), new ZKClientModule(), new KafkaClientModule(), new DiscoveryRuntimeModule().getInMemoryModules(), new NonCustomLocationUnitTestModule().getModule(), new DataFabricModules().getInMemoryModules(), new DataSetsModules().getStandaloneModules(), new DataSetServiceModules().getInMemoryModules(), new TransactionMetricsModule(), new ExploreClientModule(), new LoggingModules().getInMemoryModules(), new NamespaceClientRuntimeModule().getInMemoryModules(), new AuthorizationTestModule(), new AuthorizationEnforcementModule().getInMemoryModules(), new AuthenticationContextModules().getMasterModule(), new AbstractModule() {

        @Override
        protected void configure() {
            bind(UGIProvider.class).to(UnsupportedUGIProvider.class);
            bind(OwnerAdmin.class).to(NoOpOwnerAdmin.class);
        }
    });
    TransactionManager txManager = injector.getInstance(TransactionManager.class);
    txManager.startAndWait();
    DatasetOpExecutorService opExecutorService = injector.getInstance(DatasetOpExecutorService.class);
    opExecutorService.startAndWait();
    // Start the logging before starting the service.
    LoggingContextAccessor.setLoggingContext(new FlowletLoggingContext("TRL_ACCT_1", "APP_1", "FLOW_1", "FLOWLET_1", "RUN", "INSTANCE"));
    String logBaseDir = "trl-log/log_files_" + new Random(System.currentTimeMillis()).nextLong();
    cConf.set(LoggingConfiguration.LOG_BASE_DIR, logBaseDir);
    cConf.setInt(LoggingConfiguration.LOG_MAX_FILE_SIZE_BYTES, 20 * 1024);
    final LogAppender appender = injector.getInstance(LocalLogAppender.class);
    new LogAppenderInitializer(appender).initialize("TestResilientLogging");
    int failureMsgCount = 3;
    final CountDownLatch failureLatch = new CountDownLatch(failureMsgCount);
    LoggerContext loggerContext = (LoggerContext) LoggerFactory.getILoggerFactory();
    loggerContext.getStatusManager().add(new StatusListener() {

        @Override
        public void addStatusEvent(Status status) {
            if (status.getLevel() != Status.ERROR || status.getOrigin() != appender) {
                return;
            }
            Throwable cause = status.getThrowable();
            if (cause != null) {
                Throwable rootCause = Throwables.getRootCause(cause);
                if (rootCause instanceof ServiceUnavailableException) {
                    String serviceName = ((ServiceUnavailableException) rootCause).getServiceName();
                    if (Constants.Service.DATASET_MANAGER.equals(serviceName)) {
                        failureLatch.countDown();
                    }
                }
            }
        }
    });
    Logger logger = LoggerFactory.getLogger("TestResilientLogging");
    for (int i = 0; i < failureMsgCount; ++i) {
        Exception e1 = new Exception("Test Exception1");
        Exception e2 = new Exception("Test Exception2", e1);
        logger.warn("Test log message " + i + " {} {}", "arg1", "arg2", e2);
    }
    // Wait for the three failure to append to happen
    // The wait time has to be > 3 seconds because DatasetServiceClient has 1 second timeout on discovery
    failureLatch.await(5, TimeUnit.SECONDS);
    // Start dataset service, wait for it to be discoverable
    DatasetService dsService = injector.getInstance(DatasetService.class);
    dsService.startAndWait();
    final CountDownLatch startLatch = new CountDownLatch(1);
    DiscoveryServiceClient discoveryClient = injector.getInstance(DiscoveryServiceClient.class);
    discoveryClient.discover(Constants.Service.DATASET_MANAGER).watchChanges(new ServiceDiscovered.ChangeListener() {

        @Override
        public void onChange(ServiceDiscovered serviceDiscovered) {
            if (!Iterables.isEmpty(serviceDiscovered)) {
                startLatch.countDown();
            }
        }
    }, Threads.SAME_THREAD_EXECUTOR);
    startLatch.await(5, TimeUnit.SECONDS);
    // Do some more logging after the service is started.
    for (int i = 5; i < 10; ++i) {
        Exception e1 = new Exception("Test Exception1");
        Exception e2 = new Exception("Test Exception2", e1);
        logger.warn("Test log message " + i + " {} {}", "arg1", "arg2", e2);
    }
    appender.stop();
    // Verify - we should have at least 5 events.
    LoggingContext loggingContext = new FlowletLoggingContext("TRL_ACCT_1", "APP_1", "FLOW_1", "", "RUN", "INSTANCE");
    FileLogReader logTail = injector.getInstance(FileLogReader.class);
    LoggingTester.LogCallback logCallback1 = new LoggingTester.LogCallback();
    logTail.getLogPrev(loggingContext, ReadRange.LATEST, 10, Filter.EMPTY_FILTER, logCallback1);
    List<LogEvent> allEvents = logCallback1.getEvents();
    Assert.assertTrue(allEvents.toString(), allEvents.size() >= 5);
    // Finally - stop all services
    Services.chainStop(dsService, opExecutorService, txManager);
}

Example 7 with ServiceDiscovered

use of org.apache.twill.discovery.ServiceDiscovered in project cdap by caskdata.

the class TransactionServiceManager method isServiceAvailable.

@Override
public boolean isServiceAvailable() {
    try {
        ServiceDiscovered discovered = discoveryServiceClient.discover(serviceName);
        Discoverable discoverable = new RandomEndpointStrategy(discovered).pick(discoveryTimeout, TimeUnit.SECONDS);
        if (discoverable == null) {
            return false;
        }
        return txClient.status().equals(Constants.Monitor.STATUS_OK);
    } catch (IllegalArgumentException e) {
        return false;
    } catch (Exception e) {
        LOG.warn("Unable to ping {} : Reason {} ", serviceName, e.getMessage());
        return false;
    }
}

Example 8 with ServiceDiscovered

use of org.apache.twill.discovery.ServiceDiscovered in project cdap by caskdata.

the class RouterServiceLookup method discover.

private EndpointStrategy discover(RouteDestination routeDestination) throws ExecutionException {
    LOG.debug("Looking up service name {}", routeDestination);
    // If its a user service, then use DistributionEndpoint Strategy
    String serviceName = routeDestination.getServiceName();
    ServiceDiscovered serviceDiscovered = discoveryServiceClient.discover(serviceName);
    EndpointStrategy endpointStrategy = ServiceDiscoverable.isServiceDiscoverable(serviceName) ? new UserServiceEndpointStrategy(serviceDiscovered, routeStore, ServiceDiscoverable.getId(serviceName), fallbackStrategy, routeDestination.getVersion()) : new RandomEndpointStrategy(serviceDiscovered);
    if (endpointStrategy.pick(300L, TimeUnit.MILLISECONDS) == null) {
        LOG.debug("Discoverable endpoint {} not found", routeDestination);
    }
    return endpointStrategy;
}

Example 9 with ServiceDiscovered

use of org.apache.twill.discovery.ServiceDiscovered in project cdap by caskdata.

the class DatasetService method startUp.

@Override
protected void startUp() throws Exception {
    LOG.info("Starting DatasetService...");
    typeService.startAndWait();
    opExecutorClient.startAndWait();
    httpService.startAndWait();
    // setting watch for ops executor service that we need to be running to operate correctly
    ServiceDiscovered discover = discoveryServiceClient.discover(Constants.Service.DATASET_EXECUTOR);
    opExecutorDiscovered = SettableFuture.create();
    opExecutorServiceWatch = discover.watchChanges(new ServiceDiscovered.ChangeListener() {

        @Override
        public void onChange(ServiceDiscovered serviceDiscovered) {
            if (!Iterables.isEmpty(serviceDiscovered)) {
                LOG.info("Discovered {} service", Constants.Service.DATASET_EXECUTOR);
                opExecutorDiscovered.set(serviceDiscovered);
            }
        }
    }, MoreExecutors.sameThreadExecutor());
    for (DatasetMetricsReporter metricsReporter : metricReporters) {
        metricsReporter.start();
    }
}

Example 10 with ServiceDiscovered

use of org.apache.twill.discovery.ServiceDiscovered in project cdap by caskdata.

the class DatasetServiceTestBase method getPort.

private synchronized int getPort() {
    int attempts = 0;
    while (port < 0 && attempts++ < 10) {
        ServiceDiscovered discovered = discoveryServiceClient.discover(Constants.Service.DATASET_MANAGER);
        if (!discovered.iterator().hasNext()) {
            Uninterruptibles.sleepUninterruptibly(1, TimeUnit.SECONDS);
            continue;
        }
        port = discovered.iterator().next().getSocketAddress().getPort();
    }
    return port;
}