use of com.github.ambry.commons.SSLFactory in project ambry by linkedin.
the class AccountTool method getRouter.
/**
* Create a {@link Router} for {@link RouterStore}.
* @return The {@link Router}.
* @throws Exception Any unexpected exception.
*/
private Router getRouter() throws Exception {
// Create a HelixAccountService for the router.
AccountServiceFactory accountServiceFactory = Utils.getObj("com.github.ambry.account.HelixAccountServiceFactory", verifiableProperties, registry);
accountService = (HelixAccountService) accountServiceFactory.getAccountService();
ClusterMapConfig clusterMapConfig = new ClusterMapConfig(verifiableProperties);
ClusterAgentsFactory clusterAgentsFactory = Utils.getObj(clusterMapConfig.clusterMapClusterAgentsFactory, clusterMapConfig, hardwareLayout, partitionLayout);
clusterMap = clusterAgentsFactory.getClusterMap();
SSLFactory sslFactory = getSSLFactoryIfRequired();
// Create a NonBlockingRouter.
RouterFactory routerFactory = Utils.getObj("com.github.ambry.router.NonBlockingRouterFactory", verifiableProperties, clusterMap, new LoggingNotificationSystem(), sslFactory, accountService);
router = routerFactory.getRouter();
return router;
}
use of com.github.ambry.commons.SSLFactory in project ambry by linkedin.
the class ServerAdminTool method main.
/**
* Runs the server admin tool
* @param args associated arguments.
* @throws Exception
*/
public static void main(String[] args) throws Exception {
VerifiableProperties verifiableProperties = ToolUtils.getVerifiableProperties(args);
ServerAdminToolConfig config = new ServerAdminToolConfig(verifiableProperties);
ClusterMapConfig clusterMapConfig = new ClusterMapConfig(verifiableProperties);
ClusterMap clusterMap = ((ClusterAgentsFactory) Utils.getObj(clusterMapConfig.clusterMapClusterAgentsFactory, clusterMapConfig, config.hardwareLayoutFilePath, config.partitionLayoutFilePath)).getClusterMap();
SSLFactory sslFactory = !clusterMapConfig.clusterMapSslEnabledDatacenters.isEmpty() ? SSLFactory.getNewInstance(new SSLConfig(verifiableProperties)) : null;
ServerAdminTool serverAdminTool = new ServerAdminTool(clusterMap, sslFactory, verifiableProperties);
File file = new File(config.dataOutputFilePath);
if (!file.exists() && !file.createNewFile()) {
throw new IllegalStateException("Could not create " + file);
}
FileOutputStream outputFileStream = new FileOutputStream(config.dataOutputFilePath);
DataNodeId dataNodeId = clusterMap.getDataNodeId(config.hostname, config.port);
if (dataNodeId == null) {
throw new IllegalArgumentException("Could not find a data node corresponding to " + config.hostname + ":" + config.port);
}
switch(config.typeOfOperation) {
case GetBlobProperties:
BlobId blobId = new BlobId(config.blobId, clusterMap);
Pair<ServerErrorCode, BlobProperties> bpResponse = serverAdminTool.getBlobProperties(dataNodeId, blobId, config.getOption, clusterMap);
if (bpResponse.getFirst() == ServerErrorCode.No_Error) {
LOGGER.info("Blob properties for {} from {}: {}", blobId, dataNodeId, bpResponse.getSecond());
} else {
LOGGER.error("Failed to get blob properties for {} from {} with option {}. Error code is {}", blobId, dataNodeId, config.getOption, bpResponse.getFirst());
}
break;
case GetUserMetadata:
blobId = new BlobId(config.blobId, clusterMap);
Pair<ServerErrorCode, ByteBuffer> umResponse = serverAdminTool.getUserMetadata(dataNodeId, blobId, config.getOption, clusterMap);
if (umResponse.getFirst() == ServerErrorCode.No_Error) {
writeBufferToFile(umResponse.getSecond(), outputFileStream);
LOGGER.info("User metadata for {} from {} written to {}", blobId, dataNodeId, config.dataOutputFilePath);
} else {
LOGGER.error("Failed to get user metadata for {} from {} with option {}. Error code is {}", blobId, dataNodeId, config.getOption, umResponse.getFirst());
}
break;
case GetBlob:
blobId = new BlobId(config.blobId, clusterMap);
Pair<ServerErrorCode, BlobData> bResponse = serverAdminTool.getBlob(dataNodeId, blobId, config.getOption, clusterMap);
if (bResponse.getFirst() == ServerErrorCode.No_Error) {
LOGGER.info("Blob type of {} from {} is {}", blobId, dataNodeId, bResponse.getSecond().getBlobType());
ByteBuf buffer = bResponse.getSecond().content();
try {
writeByteBufToFile(buffer, outputFileStream);
} finally {
buffer.release();
}
LOGGER.info("Blob data for {} from {} written to {}", blobId, dataNodeId, config.dataOutputFilePath);
} else {
LOGGER.error("Failed to get blob data for {} from {} with option {}. Error code is {}", blobId, dataNodeId, config.getOption, bResponse.getFirst());
}
break;
case TriggerCompaction:
if (config.partitionIds.length > 0 && !config.partitionIds[0].isEmpty()) {
for (String partitionIdStr : config.partitionIds) {
PartitionId partitionId = getPartitionIdFromStr(partitionIdStr, clusterMap);
ServerErrorCode errorCode = serverAdminTool.triggerCompaction(dataNodeId, partitionId);
if (errorCode == ServerErrorCode.No_Error) {
LOGGER.info("Compaction has been triggered for {} on {}", partitionId, dataNodeId);
} else {
LOGGER.error("From {}, received server error code {} for trigger compaction request on {}", dataNodeId, errorCode, partitionId);
}
}
} else {
LOGGER.error("There were no partitions provided to trigger compaction on");
}
break;
case RequestControl:
if (config.partitionIds.length > 0 && !config.partitionIds[0].isEmpty()) {
for (String partitionIdStr : config.partitionIds) {
PartitionId partitionId = getPartitionIdFromStr(partitionIdStr, clusterMap);
sendRequestControlRequest(serverAdminTool, dataNodeId, partitionId, config.requestTypeToControl, config.enableState);
}
} else {
LOGGER.info("No partition list provided. Requesting enable status of {} to be set to {} on all partitions", config.requestTypeToControl, config.enableState);
sendRequestControlRequest(serverAdminTool, dataNodeId, null, config.requestTypeToControl, config.enableState);
}
break;
case ReplicationControl:
List<String> origins = Collections.emptyList();
if (config.origins.length > 0 && !config.origins[0].isEmpty()) {
origins = Arrays.asList(config.origins);
}
if (config.partitionIds.length > 0 && !config.partitionIds[0].isEmpty()) {
for (String partitionIdStr : config.partitionIds) {
PartitionId partitionId = getPartitionIdFromStr(partitionIdStr, clusterMap);
sendReplicationControlRequest(serverAdminTool, dataNodeId, partitionId, origins, config.enableState);
}
} else {
LOGGER.info("No partition list provided. Requesting enable status for replication from {} to be set to {} on " + "all partitions", origins.isEmpty() ? "all DCs" : origins, config.enableState);
sendReplicationControlRequest(serverAdminTool, dataNodeId, null, origins, config.enableState);
}
break;
case CatchupStatus:
if (config.partitionIds.length > 0 && !config.partitionIds[0].isEmpty()) {
for (String partitionIdStr : config.partitionIds) {
PartitionId partitionId = getPartitionIdFromStr(partitionIdStr, clusterMap);
Pair<ServerErrorCode, Boolean> response = serverAdminTool.isCaughtUp(dataNodeId, partitionId, config.acceptableLagInBytes, config.numReplicasCaughtUpPerPartition);
if (response.getFirst() == ServerErrorCode.No_Error) {
LOGGER.info("Replicas are {} within {} bytes for {}", response.getSecond() ? "" : "NOT", config.acceptableLagInBytes, partitionId);
} else {
LOGGER.error("From {}, received server error code {} for request for catchup status of {}", dataNodeId, response.getFirst(), partitionId);
}
}
} else {
Pair<ServerErrorCode, Boolean> response = serverAdminTool.isCaughtUp(dataNodeId, null, config.acceptableLagInBytes, config.numReplicasCaughtUpPerPartition);
if (response.getFirst() == ServerErrorCode.No_Error) {
LOGGER.info("Replicas are {} within {} bytes for all partitions", response.getSecond() ? "" : "NOT", config.acceptableLagInBytes);
} else {
LOGGER.error("From {}, received server error code {} for request for catchup status of all partitions", dataNodeId, response.getFirst());
}
}
break;
case BlobStoreControl:
if (config.partitionIds.length > 0 && !config.partitionIds[0].isEmpty()) {
for (String partitionIdStr : config.partitionIds) {
PartitionId partitionId = getPartitionIdFromStr(partitionIdStr, clusterMap);
sendBlobStoreControlRequest(serverAdminTool, dataNodeId, partitionId, config.numReplicasCaughtUpPerPartition, config.storeControlRequestType);
}
} else {
LOGGER.error("There were no partitions provided to be controlled (Start/Stop)");
}
break;
default:
throw new IllegalStateException("Recognized but unsupported operation: " + config.typeOfOperation);
}
serverAdminTool.close();
outputFileStream.close();
clusterMap.close();
System.out.println("Server admin tool is safely closed");
System.exit(0);
}
use of com.github.ambry.commons.SSLFactory in project ambry by linkedin.
the class BlockingChannelConnectionPool method initializeSSLSocketFactory.
private void initializeSSLSocketFactory() throws Exception {
try {
SSLFactory sslFactory = SSLFactory.getNewInstance(sslConfig);
SSLContext sslContext = sslFactory.getSSLContext();
this.sslSocketFactory = sslContext.getSocketFactory();
this.sslSocketFactoryClientInitializationCount.inc();
} catch (Exception e) {
this.sslSocketFactoryClientInitializationErrorCount.inc();
logger.error("SSLSocketFactory Client Initialization Error ", e);
throw e;
}
}
use of com.github.ambry.commons.SSLFactory in project ambry by linkedin.
the class RestServerMain method main.
public static void main(String[] args) {
final RestServer restServer;
int exitCode = 0;
ClusterMap clusterMap = null;
try {
InvocationOptions options = new InvocationOptions(args);
Properties properties = Utils.loadProps(options.serverPropsFilePath);
VerifiableProperties verifiableProperties = new VerifiableProperties(properties);
ClusterMapConfig clusterMapConfig = new ClusterMapConfig(verifiableProperties);
ClusterAgentsFactory clusterAgentsFactory = Utils.getObj(clusterMapConfig.clusterMapClusterAgentsFactory, clusterMapConfig, options.hardwareLayoutFilePath, options.partitionLayoutFilePath);
clusterMap = clusterAgentsFactory.getClusterMap();
SSLFactory sslFactory = getSSLFactoryIfRequired(verifiableProperties);
logger.info("Bootstrapping RestServer");
restServer = new RestServer(verifiableProperties, clusterMap, new LoggingNotificationSystem(), sslFactory);
// attach shutdown handler to catch control-c
Runtime.getRuntime().addShutdownHook(new Thread(() -> {
logger.info("Received shutdown signal. Shutting down RestServer");
restServer.shutdown();
}));
restServer.start();
restServer.awaitShutdown();
} catch (Exception e) {
logger.error("Exception during bootstrap of RestServer", e);
exitCode = 1;
} finally {
if (clusterMap != null) {
clusterMap.close();
}
}
logger.info("Exiting RestServerMain");
System.exit(exitCode);
}
use of com.github.ambry.commons.SSLFactory in project ambry by linkedin.
the class AmbryServer method startup.
public void startup() throws InstantiationException {
try {
logger.info("starting");
clusterParticipants = clusterAgentsFactory.getClusterParticipants();
logger.info("Setting up JMX.");
long startTime = SystemTime.getInstance().milliseconds();
reporter = reporterFactory != null ? reporterFactory.apply(registry) : JmxReporter.forRegistry(registry).build();
reporter.start();
logger.info("creating configs");
NetworkConfig networkConfig = new NetworkConfig(properties);
StoreConfig storeConfig = new StoreConfig(properties);
DiskManagerConfig diskManagerConfig = new DiskManagerConfig(properties);
ServerConfig serverConfig = new ServerConfig(properties);
ReplicationConfig replicationConfig = new ReplicationConfig(properties);
ConnectionPoolConfig connectionPoolConfig = new ConnectionPoolConfig(properties);
SSLConfig sslConfig = new SSLConfig(properties);
ClusterMapConfig clusterMapConfig = new ClusterMapConfig(properties);
StatsManagerConfig statsConfig = new StatsManagerConfig(properties);
CloudConfig cloudConfig = new CloudConfig(properties);
// verify the configs
properties.verify();
scheduler = Utils.newScheduler(serverConfig.serverSchedulerNumOfthreads, false);
// mismatch in sealed/stopped replica lists that maintained by each participant.
if (clusterParticipants != null && clusterParticipants.size() > 1 && serverConfig.serverParticipantsConsistencyCheckerPeriodSec > 0) {
consistencyChecker = new ParticipantsConsistencyChecker(clusterParticipants, metrics);
logger.info("Scheduling participants consistency checker with a period of {} secs", serverConfig.serverParticipantsConsistencyCheckerPeriodSec);
consistencyCheckerScheduler = Utils.newScheduler(1, "consistency-checker-", false);
consistencyCheckerTask = consistencyCheckerScheduler.scheduleAtFixedRate(consistencyChecker, 0, serverConfig.serverParticipantsConsistencyCheckerPeriodSec, TimeUnit.SECONDS);
}
logger.info("checking if node exists in clustermap host {} port {}", networkConfig.hostName, networkConfig.port);
DataNodeId nodeId = clusterMap.getDataNodeId(networkConfig.hostName, networkConfig.port);
if (nodeId == null) {
throw new IllegalArgumentException("The node " + networkConfig.hostName + ":" + networkConfig.port + "is not present in the clustermap. Failing to start the datanode");
}
AccountServiceFactory accountServiceFactory = Utils.getObj(serverConfig.serverAccountServiceFactory, properties, registry);
AccountService accountService = accountServiceFactory.getAccountService();
StoreKeyFactory storeKeyFactory = Utils.getObj(storeConfig.storeKeyFactory, clusterMap);
// In most cases, there should be only one participant in the clusterParticipants list. If there are more than one
// and some components require sole participant, the first one in the list will be primary participant.
storageManager = new StorageManager(storeConfig, diskManagerConfig, scheduler, registry, storeKeyFactory, clusterMap, nodeId, new BlobStoreHardDelete(), clusterParticipants, time, new BlobStoreRecovery(), accountService);
storageManager.start();
SSLFactory sslFactory = new NettySslHttp2Factory(sslConfig);
if (clusterMapConfig.clusterMapEnableHttp2Replication) {
connectionPool = new Http2BlockingChannelPool(sslFactory, new Http2ClientConfig(properties), new Http2ClientMetrics(registry));
} else {
connectionPool = new BlockingChannelConnectionPool(connectionPoolConfig, sslConfig, clusterMapConfig, registry);
}
connectionPool.start();
StoreKeyConverterFactory storeKeyConverterFactory = Utils.getObj(serverConfig.serverStoreKeyConverterFactory, properties, registry);
Predicate<MessageInfo> skipPredicate = new ReplicationSkipPredicate(accountService, replicationConfig);
replicationManager = new ReplicationManager(replicationConfig, clusterMapConfig, storeConfig, storageManager, storeKeyFactory, clusterMap, scheduler, nodeId, connectionPool, registry, notificationSystem, storeKeyConverterFactory, serverConfig.serverMessageTransformer, clusterParticipants.get(0), skipPredicate);
replicationManager.start();
if (replicationConfig.replicationEnabledWithVcrCluster) {
logger.info("Creating Helix cluster spectator for cloud to store replication.");
vcrClusterSpectator = _vcrClusterAgentsFactory.getVcrClusterSpectator(cloudConfig, clusterMapConfig);
cloudToStoreReplicationManager = new CloudToStoreReplicationManager(replicationConfig, clusterMapConfig, storeConfig, storageManager, storeKeyFactory, clusterMap, scheduler, nodeId, connectionPool, registry, notificationSystem, storeKeyConverterFactory, serverConfig.serverMessageTransformer, vcrClusterSpectator, clusterParticipants.get(0));
cloudToStoreReplicationManager.start();
}
logger.info("Creating StatsManager to publish stats");
accountStatsMySqlStore = statsConfig.enableMysqlReport ? (AccountStatsMySqlStore) new AccountStatsMySqlStoreFactory(properties, clusterMapConfig, registry).getAccountStatsStore() : null;
statsManager = new StatsManager(storageManager, clusterMap.getReplicaIds(nodeId), registry, statsConfig, time, clusterParticipants.get(0), accountStatsMySqlStore, accountService);
if (serverConfig.serverStatsPublishLocalEnabled) {
statsManager.start();
}
ArrayList<Port> ports = new ArrayList<Port>();
ports.add(new Port(networkConfig.port, PortType.PLAINTEXT));
if (nodeId.hasSSLPort()) {
ports.add(new Port(nodeId.getSSLPort(), PortType.SSL));
}
networkServer = new SocketServer(networkConfig, sslConfig, registry, ports);
FindTokenHelper findTokenHelper = new FindTokenHelper(storeKeyFactory, replicationConfig);
requests = new AmbryServerRequests(storageManager, networkServer.getRequestResponseChannel(), clusterMap, nodeId, registry, metrics, findTokenHelper, notificationSystem, replicationManager, storeKeyFactory, serverConfig, storeKeyConverterFactory, statsManager, clusterParticipants.get(0));
requestHandlerPool = new RequestHandlerPool(serverConfig.serverRequestHandlerNumOfThreads, networkServer.getRequestResponseChannel(), requests);
networkServer.start();
// Start netty http2 server
if (nodeId.hasHttp2Port()) {
NettyConfig nettyConfig = new NettyConfig(properties);
NettyMetrics nettyMetrics = new NettyMetrics(registry);
Http2ServerMetrics http2ServerMetrics = new Http2ServerMetrics(registry);
Http2ClientConfig http2ClientConfig = new Http2ClientConfig(properties);
logger.info("Http2 port {} is enabled. Starting HTTP/2 service.", nodeId.getHttp2Port());
NettyServerRequestResponseChannel requestResponseChannel = new NettyServerRequestResponseChannel(networkConfig.queuedMaxRequests, http2ServerMetrics);
AmbryServerRequests ambryServerRequestsForHttp2 = new AmbryServerRequests(storageManager, requestResponseChannel, clusterMap, nodeId, registry, metrics, findTokenHelper, notificationSystem, replicationManager, storeKeyFactory, serverConfig, storeKeyConverterFactory, statsManager, clusterParticipants.get(0));
requestHandlerPoolForHttp2 = new RequestHandlerPool(serverConfig.serverRequestHandlerNumOfThreads, requestResponseChannel, ambryServerRequestsForHttp2);
NioServerFactory nioServerFactory = new StorageServerNettyFactory(nodeId.getHttp2Port(), requestResponseChannel, sslFactory, nettyConfig, http2ClientConfig, metrics, nettyMetrics, http2ServerMetrics, serverSecurityService);
nettyHttp2Server = nioServerFactory.getNioServer();
nettyHttp2Server.start();
}
// Other code
List<AmbryStatsReport> ambryStatsReports = new ArrayList<>();
Set<String> validStatsTypes = new HashSet<>();
for (StatsReportType type : StatsReportType.values()) {
validStatsTypes.add(type.toString());
}
if (serverConfig.serverStatsPublishReportEnabled) {
serverConfig.serverStatsReportsToPublish.forEach(e -> {
if (validStatsTypes.contains(e)) {
ambryStatsReports.add(new AmbryStatsReportImpl(serverConfig.serverQuotaStatsAggregateIntervalInMinutes, StatsReportType.valueOf(e)));
}
});
}
if (vcrClusterSpectator != null) {
vcrClusterSpectator.spectate();
}
Callback<StatsSnapshot> accountServiceCallback = new AccountServiceCallback(accountService);
for (ClusterParticipant clusterParticipant : clusterParticipants) {
clusterParticipant.participate(ambryStatsReports, accountStatsMySqlStore, accountServiceCallback);
}
if (nettyInternalMetrics != null) {
nettyInternalMetrics.start();
logger.info("NettyInternalMetric starts");
}
logger.info("started");
long processingTime = SystemTime.getInstance().milliseconds() - startTime;
metrics.serverStartTimeInMs.update(processingTime);
logger.info("Server startup time in Ms {}", processingTime);
} catch (Exception e) {
logger.error("Error during startup", e);
throw new InstantiationException("failure during startup " + e);
}
}
Aggregations