Examples with ClusterAgentsFactory com.github.ambry.clustermap.ClusterAgentsFactory used on opensource projects

Example 6 with ClusterAgentsFactory

use of com.github.ambry.clustermap.ClusterAgentsFactory in project ambry by linkedin.

the class StoreCopier method main.

public static void main(String[] args) throws Exception {
    VerifiableProperties properties = ToolUtils.getVerifiableProperties(args);
    CopierConfig config = new CopierConfig(properties);
    StoreConfig storeConfig = new StoreConfig(properties);
    ClusterMapConfig clusterMapConfig = new ClusterMapConfig(properties);
    ClusterAgentsFactory clusterAgentsFactory = Utils.getObj(clusterMapConfig.clusterMapClusterAgentsFactory, clusterMapConfig, config.hardwareLayoutFilePath, config.partitionLayoutFilePath);
    try (ClusterMap clusterMap = clusterAgentsFactory.getClusterMap()) {
        StoreKeyFactory storeKeyFactory = Utils.getObj(storeConfig.storeKeyFactory, clusterMap);
        File srcDir = new File(config.srcStoreDirPath);
        File tgtDir = new File(config.tgtStoreDirPath);
        StoreMetrics metrics = new StoreMetrics(clusterMap.getMetricRegistry());
        DiskSpaceAllocator diskSpaceAllocator = new DiskSpaceAllocator(false, null, 0, new StorageManagerMetrics(clusterMap.getMetricRegistry()));
        try (StoreCopier storeCopier = new StoreCopier("src", srcDir, tgtDir, config.storeCapacity, config.fetchSizeInBytes, storeConfig, metrics, storeKeyFactory, new DiskIOScheduler(null), diskSpaceAllocator, Collections.emptyList(), SystemTime.getInstance())) {
            storeCopier.copy(new StoreFindTokenFactory(storeKeyFactory).getNewFindToken());
        }
    }
}

Example 7 with ClusterAgentsFactory

use of com.github.ambry.clustermap.ClusterAgentsFactory in project ambry by linkedin.

the class ConcurrencyTestTool method main.

public static void main(String[] args) throws Exception {
    InvocationOptions options = new InvocationOptions(args);
    Properties properties = Utils.loadProps(options.routerPropsFilePath);
    ToolUtils.addClusterMapProperties(properties);
    ClusterMapConfig clusterMapConfig = new ClusterMapConfig(new VerifiableProperties(properties));
    ClusterMap clusterMap = ((ClusterAgentsFactory) Utils.getObj(clusterMapConfig.clusterMapClusterAgentsFactory, clusterMapConfig, options.hardwareLayoutFilePath, options.partitionLayoutFilePath)).getClusterMap();
    PutGetHelperFactory putGetHelperFactory = null;
    if (options.putGetHelperFactoryStr.equals(ROUTER_PUT_GET_HELPER)) {
        putGetHelperFactory = Utils.getObj(options.putGetHelperFactoryStr, properties, clusterMap, options.maxBlobSizeInBytes, options.minBlobSizeInBytes);
    } else if (options.putGetHelperFactoryStr.equals(SERVER_PUT_GET_HELPER)) {
        putGetHelperFactory = Utils.getObj(options.putGetHelperFactoryStr, properties, options.hostName, options.port, clusterMap, options.maxBlobSizeInBytes, options.minBlobSizeInBytes);
    }
    PutGetHelper putGetHelper = putGetHelperFactory.getPutGetHelper();
    ConcurrencyTestTool concurrencyTestTool = new ConcurrencyTestTool();
    concurrencyTestTool.startTest(putGetHelper, options.maxParallelPutCount, options.parallelGetCount, options.totalPutBlobCount, options.maxGetCountPerBlob, options.burstCountForGet, options.maxFailuresPerPutBatchToStopPuts, options.maxFailuresPerGetBatchToStopGets, options.deleteAndValidate, options.sleepTimeBetweenBatchPutsInMs, options.sleepTimeBetweenBatchGetsInMs, options.measurementIntervalInSecs);
}

Example 8 with ClusterAgentsFactory

use of com.github.ambry.clustermap.ClusterAgentsFactory in project ambry by linkedin.

the class RestServerMain method main.

public static void main(String[] args) {
    final RestServer restServer;
    int exitCode = 0;
    ClusterMap clusterMap = null;
    try {
        InvocationOptions options = new InvocationOptions(args);
        Properties properties = Utils.loadProps(options.serverPropsFilePath);
        VerifiableProperties verifiableProperties = new VerifiableProperties(properties);
        ClusterMapConfig clusterMapConfig = new ClusterMapConfig(verifiableProperties);
        ClusterAgentsFactory clusterAgentsFactory = Utils.getObj(clusterMapConfig.clusterMapClusterAgentsFactory, clusterMapConfig, options.hardwareLayoutFilePath, options.partitionLayoutFilePath);
        clusterMap = clusterAgentsFactory.getClusterMap();
        SSLFactory sslFactory = getSSLFactoryIfRequired(verifiableProperties);
        logger.info("Bootstrapping RestServer");
        restServer = new RestServer(verifiableProperties, clusterMap, new LoggingNotificationSystem(), sslFactory);
        // attach shutdown handler to catch control-c
        Runtime.getRuntime().addShutdownHook(new Thread(() -> {
            logger.info("Received shutdown signal. Shutting down RestServer");
            restServer.shutdown();
        }));
        restServer.start();
        restServer.awaitShutdown();
    } catch (Exception e) {
        logger.error("Exception during bootstrap of RestServer", e);
        exitCode = 1;
    } finally {
        if (clusterMap != null) {
            clusterMap.close();
        }
    }
    logger.info("Exiting RestServerMain");
    System.exit(exitCode);
}

Example 9 with ClusterAgentsFactory

use of com.github.ambry.clustermap.ClusterAgentsFactory in project ambry by linkedin.

the class ServerTestUtil method endToEndCloudBackupTest.

/**
 * Tests blobs put to dataNode can be backed up by {@link com.github.ambry.cloud.VcrReplicationManager}.
 * @param cluster the {@link MockCluster} of dataNodes.
 * @param zkConnectString ZK endpoint to establish VCR cluster
 * @param vcrClusterName the name of VCR cluster
 * @param dataNode the datanode where blobs are originally put.
 * @param clientSSLConfig the {@link SSLConfig}.
 * @param clientSSLSocketFactory the {@link SSLSocketFactory}.
 * @param notificationSystem the {@link MockNotificationSystem} to track blobs event in {@link MockCluster}.
 * @param vcrSSLProps SSL related properties for VCR. Can be {@code null}.
 * @param doTtlUpdate Do ttlUpdate request if {@code true}.
 */
static void endToEndCloudBackupTest(MockCluster cluster, String zkConnectString, String vcrClusterName, DataNodeId dataNode, SSLConfig clientSSLConfig, SSLSocketFactory clientSSLSocketFactory, MockNotificationSystem notificationSystem, Properties vcrSSLProps, boolean doTtlUpdate) throws Exception {
    int blobBackupCount = 10;
    int blobSize = 100;
    int userMetaDataSize = 100;
    ClusterAgentsFactory clusterAgentsFactory = cluster.getClusterAgentsFactory();
    // Send blobs to DataNode
    byte[] userMetadata = new byte[userMetaDataSize];
    byte[] data = new byte[blobSize];
    short accountId = Utils.getRandomShort(TestUtils.RANDOM);
    short containerId = Utils.getRandomShort(TestUtils.RANDOM);
    long ttl = doTtlUpdate ? TimeUnit.DAYS.toMillis(1) : Utils.Infinite_Time;
    BlobProperties properties = new BlobProperties(blobSize, "serviceid1", null, null, false, ttl, cluster.time.milliseconds(), accountId, containerId, false, null, null, null);
    TestUtils.RANDOM.nextBytes(userMetadata);
    TestUtils.RANDOM.nextBytes(data);
    Port port;
    if (clientSSLConfig == null) {
        port = new Port(dataNode.getPort(), PortType.PLAINTEXT);
    } else {
        port = new Port(dataNode.getSSLPort(), PortType.SSL);
    }
    ConnectedChannel channel = getBlockingChannelBasedOnPortType(port, "localhost", clientSSLSocketFactory, clientSSLConfig);
    channel.connect();
    CountDownLatch latch = new CountDownLatch(1);
    DirectSender runnable = new DirectSender(cluster, channel, blobBackupCount, data, userMetadata, properties, null, latch);
    Thread threadToRun = new Thread(runnable);
    threadToRun.start();
    assertTrue("Did not put all blobs in 2 minutes", latch.await(2, TimeUnit.MINUTES));
    // TODO: remove this temp fix after fixing race condition in MockCluster/MockNotificationSystem
    Thread.sleep(3000);
    List<BlobId> blobIds = runnable.getBlobIds();
    for (BlobId blobId : blobIds) {
        notificationSystem.awaitBlobCreations(blobId.getID());
        if (doTtlUpdate) {
            updateBlobTtl(channel, blobId, cluster.time.milliseconds());
        }
    }
    HelixControllerManager helixControllerManager = VcrTestUtil.populateZkInfoAndStartController(zkConnectString, vcrClusterName, cluster.getClusterMap());
    // Start the VCR and CloudBackupManager
    Properties props = VcrTestUtil.createVcrProperties(dataNode.getDatacenterName(), vcrClusterName, zkConnectString, 12310, 12410, 12510, vcrSSLProps);
    LatchBasedInMemoryCloudDestination latchBasedInMemoryCloudDestination = new LatchBasedInMemoryCloudDestination(blobIds, clusterAgentsFactory.getClusterMap());
    CloudDestinationFactory cloudDestinationFactory = new LatchBasedInMemoryCloudDestinationFactory(latchBasedInMemoryCloudDestination);
    VcrServer vcrServer = VcrTestUtil.createVcrServer(new VerifiableProperties(props), clusterAgentsFactory, notificationSystem, cloudDestinationFactory);
    vcrServer.startup();
    // Waiting for backup done
    assertTrue("Did not backup all blobs in 2 minutes", latchBasedInMemoryCloudDestination.awaitUpload(2, TimeUnit.MINUTES));
    Map<String, CloudBlobMetadata> cloudBlobMetadataMap = latchBasedInMemoryCloudDestination.getBlobMetadata(blobIds);
    for (BlobId blobId : blobIds) {
        CloudBlobMetadata cloudBlobMetadata = cloudBlobMetadataMap.get(blobId.toString());
        assertNotNull("cloudBlobMetadata should not be null", cloudBlobMetadata);
        assertEquals("AccountId mismatch", accountId, cloudBlobMetadata.getAccountId());
        assertEquals("ContainerId mismatch", containerId, cloudBlobMetadata.getContainerId());
        assertEquals("Expiration time mismatch", Utils.Infinite_Time, cloudBlobMetadata.getExpirationTime());
    // TODO: verify other metadata and blob data
    }
    vcrServer.shutdown();
    helixControllerManager.syncStop();
}

Example 10 with ClusterAgentsFactory

use of com.github.ambry.clustermap.ClusterAgentsFactory in project ambry by linkedin.

the class AmbryMain method main.

public static void main(String[] args) {
    final AmbryServer ambryServer;
    int exitCode = 0;
    try {
        InvocationOptions options = new InvocationOptions(args);
        Properties properties = Utils.loadProps(options.serverPropsFilePath);
        VerifiableProperties verifiableProperties = new VerifiableProperties(properties);
        ClusterMapConfig clusterMapConfig = new ClusterMapConfig(verifiableProperties);
        CloudConfig cloudConfig = new CloudConfig(verifiableProperties);
        ClusterAgentsFactory clusterAgentsFactory = Utils.getObj(clusterMapConfig.clusterMapClusterAgentsFactory, clusterMapConfig, options.hardwareLayoutFilePath, options.partitionLayoutFilePath);
        logger.info("Bootstrapping AmbryServer");
        VcrClusterAgentsFactory vcrClusterAgentsFactory = Utils.getObj(cloudConfig.vcrClusterAgentsFactoryClass);
        ambryServer = new AmbryServer(verifiableProperties, clusterAgentsFactory, vcrClusterAgentsFactory, SystemTime.getInstance());
        // attach shutdown handler to catch control-c
        Runtime.getRuntime().addShutdownHook(new Thread() {

            public void run() {
                logger.info("Received shutdown signal. Shutting down AmbryServer");
                ambryServer.shutdown();
            }
        });
        ambryServer.startup();
        ambryServer.awaitShutdown();
    } catch (Exception e) {
        logger.error("Exception during bootstrap of AmbryServer", e);
        exitCode = 1;
    }
    logger.info("Exiting AmbryMain");
    System.exit(exitCode);
}