use of io.pravega.segmentstore.server.host.stat.AutoScaleMonitor in project pravega by pravega.
the class ServiceStarter method start.
// endregion
// region Service Operation
public void start() throws Exception {
Exceptions.checkNotClosed(this.closed, this);
healthServiceManager = new HealthServiceManager(serviceConfig.getHealthCheckInterval());
healthServiceManager.start();
log.info("Initializing HealthService ...");
MetricsConfig metricsConfig = builderConfig.getConfig(MetricsConfig::builder);
if (metricsConfig.isEnableStatistics()) {
log.info("Initializing metrics provider ...");
MetricsProvider.initialize(metricsConfig);
statsProvider = MetricsProvider.getMetricsProvider();
statsProvider.start();
}
log.info("Initializing ZooKeeper Client ...");
this.zkClient = createZKClient();
log.info("Initializing Service Builder ...");
this.serviceBuilder.initialize();
log.info("Creating StreamSegmentService ...");
StreamSegmentStore service = this.serviceBuilder.createStreamSegmentService();
log.info("Creating TableStoreService ...");
TableStore tableStoreService = this.serviceBuilder.createTableStoreService();
log.info("Creating Segment Stats recorder ...");
autoScaleMonitor = new AutoScaleMonitor(service, builderConfig.getConfig(AutoScalerConfig::builder));
AutoScalerConfig autoScalerConfig = builderConfig.getConfig(AutoScalerConfig::builder);
TokenVerifierImpl tokenVerifier = null;
if (autoScalerConfig.isAuthEnabled()) {
tokenVerifier = new TokenVerifierImpl(autoScalerConfig.getTokenSigningKey());
}
// Log the configuration
log.info(serviceConfig.toString());
log.info(autoScalerConfig.toString());
this.listener = new PravegaConnectionListener(this.serviceConfig.isEnableTls(), this.serviceConfig.isEnableTlsReload(), this.serviceConfig.getListeningIPAddress(), this.serviceConfig.getListeningPort(), service, tableStoreService, autoScaleMonitor.getStatsRecorder(), autoScaleMonitor.getTableSegmentStatsRecorder(), tokenVerifier, this.serviceConfig.getCertFile(), this.serviceConfig.getKeyFile(), this.serviceConfig.isReplyWithStackTraceOnError(), serviceBuilder.getLowPriorityExecutor(), this.serviceConfig.getTlsProtocolVersion(), healthServiceManager);
this.listener.startListening();
log.info("PravegaConnectionListener started successfully.");
if (serviceConfig.isEnableAdminGateway()) {
this.adminListener = new AdminConnectionListener(this.serviceConfig.isEnableTls(), this.serviceConfig.isEnableTlsReload(), this.serviceConfig.getListeningIPAddress(), this.serviceConfig.getAdminGatewayPort(), service, tableStoreService, tokenVerifier, this.serviceConfig.getCertFile(), this.serviceConfig.getKeyFile(), this.serviceConfig.getTlsProtocolVersion(), healthServiceManager);
this.adminListener.startListening();
log.info("AdminConnectionListener started successfully.");
}
log.info("StreamSegmentService started.");
healthServiceManager.register(new ZKHealthContributor(zkClient));
healthServiceManager.register(new CacheManagerHealthContributor(serviceBuilder.getCacheManager()));
healthServiceManager.register(new SegmentContainerRegistryHealthContributor(serviceBuilder.getSegmentContainerRegistry()));
if (this.serviceConfig.isRestServerEnabled()) {
log.info("Initializing RESTServer ...");
List<Object> resources = new ArrayList<>();
resources.add(new HealthImpl(new AuthHandlerManager(serviceConfig.getRestServerConfig()), healthServiceManager.getEndpoint()));
MetricsProvider.getMetricsProvider().prometheusResource().ifPresent(resources::add);
restServer = new RESTServer(serviceConfig.getRestServerConfig(), Set.copyOf(resources));
restServer.startAsync();
restServer.awaitRunning();
}
}
use of io.pravega.segmentstore.server.host.stat.AutoScaleMonitor in project pravega by pravega.
the class InProcessMockClientAdapter method startUp.
// endregion
// region ClientAdapterBase Implementation
@Override
protected void startUp() throws Exception {
int segmentStorePort = this.testConfig.getSegmentStorePort(0);
val store = getStreamSegmentStore();
this.autoScaleMonitor = new AutoScaleMonitor(store, AutoScalerConfig.builder().build());
this.listener = new PravegaConnectionListener(false, false, "localhost", segmentStorePort, store, getTableStore(), autoScaleMonitor.getStatsRecorder(), TableSegmentStatsRecorder.noOp(), new PassingTokenVerifier(), null, null, false, NoOpScheduledExecutor.get(), SecurityConfigDefaults.TLS_PROTOCOL_VERSION);
this.listener.startListening();
this.streamManager = new MockStreamManager(SCOPE, LISTENING_ADDRESS, segmentStorePort);
this.streamManager.createScope(SCOPE);
this.kvtManager = new MockKVTManager(this.streamManager);
super.startUp();
}
use of io.pravega.segmentstore.server.host.stat.AutoScaleMonitor in project pravega by pravega.
the class EndToEndAutoScaleUpWithTxnTest method main.
public static void main(String[] args) throws Exception {
try {
@Cleanup TestingServer zkTestServer = new TestingServerStarter().start();
int port = Config.SERVICE_PORT;
@Cleanup ControllerWrapper controllerWrapper = new ControllerWrapper(zkTestServer.getConnectString(), port);
Controller controller = controllerWrapper.getController();
controllerWrapper.getControllerService().createScope(NameUtils.INTERNAL_SCOPE_NAME, 0L).get();
@Cleanup ConnectionFactory connectionFactory = new SocketConnectionFactoryImpl(ClientConfig.builder().build());
@Cleanup ClientFactoryImpl internalCF = new ClientFactoryImpl(NameUtils.INTERNAL_SCOPE_NAME, controller, connectionFactory);
@Cleanup("shutdownNow") val executor = ExecutorServiceHelpers.newScheduledThreadPool(1, "test");
@Cleanup ServiceBuilder serviceBuilder = ServiceBuilder.newInMemoryBuilder(ServiceBuilderConfig.getDefaultConfig());
serviceBuilder.initialize();
StreamSegmentStore store = serviceBuilder.createStreamSegmentService();
TableStore tableStore = serviceBuilder.createTableStoreService();
@Cleanup AutoScaleMonitor autoScaleMonitor = new AutoScaleMonitor(store, internalCF, AutoScalerConfig.builder().with(AutoScalerConfig.MUTE_IN_SECONDS, 0).with(AutoScalerConfig.COOLDOWN_IN_SECONDS, 0).build());
@Cleanup PravegaConnectionListener server = new PravegaConnectionListener(false, false, "localhost", 12345, store, tableStore, autoScaleMonitor.getStatsRecorder(), autoScaleMonitor.getTableSegmentStatsRecorder(), null, null, null, true, serviceBuilder.getLowPriorityExecutor(), Config.TLS_PROTOCOL_VERSION.toArray(new String[Config.TLS_PROTOCOL_VERSION.size()]));
server.startListening();
controllerWrapper.awaitRunning();
controllerWrapper.getControllerService().createScope("test", 0L).get();
controller.createStream("test", "test", CONFIG).get();
@Cleanup MockClientFactory clientFactory = new MockClientFactory("test", controller, internalCF.getConnectionPool());
// Mocking pravega service by putting scale up and scale down requests for the stream
EventWriterConfig writerConfig = EventWriterConfig.builder().transactionTimeoutTime(30000).build();
TransactionalEventStreamWriter<String> test = clientFactory.createTransactionalEventWriter("writer", "test", new UTF8StringSerializer(), writerConfig);
// region Successful commit tests
Transaction<String> txn1 = test.beginTxn();
txn1.writeEvent("1");
txn1.flush();
Map<Double, Double> map = new HashMap<>();
map.put(0.0, 1.0 / 3.0);
map.put(1.0 / 3.0, 2.0 / 3.0);
map.put(2.0 / 3.0, 1.0);
Stream stream = new StreamImpl("test", "test");
controller.startScale(stream, Collections.singletonList(0L), map).get();
Transaction<String> txn2 = test.beginTxn();
txn2.writeEvent("2");
txn2.flush();
txn2.commit();
txn1.commit();
Thread.sleep(1000);
@Cleanup ReaderGroupManager readerGroupManager = new ReaderGroupManagerImpl("test", controller, clientFactory);
readerGroupManager.createReaderGroup("readergrp", ReaderGroupConfig.builder().stream("test/test").build());
final EventStreamReader<String> reader = clientFactory.createReader("1", "readergrp", new JavaSerializer<>(), ReaderConfig.builder().build());
String event1 = reader.readNextEvent(SECONDS.toMillis(60)).getEvent();
String event2 = reader.readNextEvent(SECONDS.toMillis(60)).getEvent();
assert event1.equals("1");
assert event2.equals("2");
final AtomicBoolean done = new AtomicBoolean(false);
startWriter(test, done);
Retry.withExpBackoff(10, 10, 100, 10000).retryingOn(NotDoneException.class).throwingOn(RuntimeException.class).runAsync(() -> controller.getCurrentSegments("test", "test").thenAccept(streamSegments -> {
if (streamSegments.getSegments().stream().anyMatch(x -> NameUtils.getEpoch(x.getSegmentId()) > 5)) {
System.err.println("Success");
log.info("Success");
System.exit(0);
} else {
throw new NotDoneException();
}
}), executor).exceptionally(e -> {
System.err.println("Failure");
log.error("Failure");
System.exit(1);
return null;
}).get();
} catch (Throwable e) {
System.err.print("Test failed with exception: " + e.getMessage());
log.error("Test failed with exception: {}", e);
System.exit(-1);
}
System.exit(0);
}
use of io.pravega.segmentstore.server.host.stat.AutoScaleMonitor in project pravega by pravega.
the class MetricsTest method setup.
@Before
public void setup() throws Exception {
final int controllerPort = TestUtils.getAvailableListenPort();
final String serviceHost = "localhost";
final int servicePort = TestUtils.getAvailableListenPort();
final int containerCount = 4;
// 1. Start Metrics service
log.info("Initializing metrics provider ...");
MetricsConfig metricsConfig = MetricsConfig.builder().with(MetricsConfig.ENABLE_STATISTICS, true).with(MetricsConfig.ENABLE_STATSD_REPORTER, false).build();
metricsConfig.setDynamicCacheEvictionDuration(Duration.ofSeconds(2));
MetricsProvider.initialize(metricsConfig);
statsProvider = MetricsProvider.getMetricsProvider();
statsProvider.startWithoutExporting();
log.info("Metrics Stats provider is started");
// 2. Start ZK
this.zkTestServer = new TestingServerStarter().start();
// 3. Start Pravega SegmentStore service.
serviceBuilder = ServiceBuilder.newInMemoryBuilder(ServiceBuilderConfig.getDefaultConfig());
serviceBuilder.initialize();
StreamSegmentStore store = serviceBuilder.createStreamSegmentService();
monitor = new AutoScaleMonitor(store, AutoScalerConfig.builder().build());
TableStore tableStore = serviceBuilder.createTableStoreService();
this.server = new PravegaConnectionListener(false, false, "localhost", servicePort, store, tableStore, monitor.getStatsRecorder(), monitor.getTableSegmentStatsRecorder(), new PassingTokenVerifier(), null, null, true, this.serviceBuilder.getLowPriorityExecutor(), SecurityConfigDefaults.TLS_PROTOCOL_VERSION);
this.server.startListening();
// 4. Start Pravega Controller service
this.controllerWrapper = new ControllerWrapper(zkTestServer.getConnectString(), false, controllerPort, serviceHost, servicePort, containerCount);
this.controllerWrapper.awaitRunning();
this.controller = controllerWrapper.getController();
}
use of io.pravega.segmentstore.server.host.stat.AutoScaleMonitor in project pravega by pravega.
the class StreamMetricsTest method setup.
@Before
public void setup() throws Exception {
controllerPort = TestUtils.getAvailableListenPort();
final String serviceHost = "localhost";
final int servicePort = TestUtils.getAvailableListenPort();
final int containerCount = 4;
// 1. Start Metrics service
log.info("Initializing metrics provider ...");
MetricsConfig metricsConfig = MetricsConfig.builder().with(MetricsConfig.ENABLE_STATISTICS, true).with(MetricsConfig.ENABLE_STATSD_REPORTER, false).build();
metricsConfig.setDynamicCacheEvictionDuration(Duration.ofSeconds(60));
MetricsProvider.initialize(metricsConfig);
statsProvider = MetricsProvider.getMetricsProvider();
statsProvider.startWithoutExporting();
log.info("Metrics Stats provider is started");
// 2. Start ZK
this.zkTestServer = new TestingServerStarter().start();
// 3. Start Pravega SegmentStore service.
serviceBuilder = ServiceBuilder.newInMemoryBuilder(ServiceBuilderConfig.getDefaultConfig());
serviceBuilder.initialize();
StreamSegmentStore store = serviceBuilder.createStreamSegmentService();
monitor = new AutoScaleMonitor(store, AutoScalerConfig.builder().build());
TableStore tableStore = serviceBuilder.createTableStoreService();
this.server = new PravegaConnectionListener(false, false, "localhost", servicePort, store, tableStore, monitor.getStatsRecorder(), monitor.getTableSegmentStatsRecorder(), new PassingTokenVerifier(), null, null, true, this.serviceBuilder.getLowPriorityExecutor(), SecurityConfigDefaults.TLS_PROTOCOL_VERSION);
this.server.startListening();
// 4. Start Pravega Controller service
this.controllerWrapper = new ControllerWrapper(zkTestServer.getConnectString(), false, controllerPort, serviceHost, servicePort, containerCount);
this.controllerWrapper.awaitRunning();
this.controller = controllerWrapper.getController();
}
Aggregations