Examples with FindToken com.github.ambry.replication.FindToken used on opensource projects

Example 11 with FindToken

use of com.github.ambry.replication.FindToken in project ambry by linkedin.

the class AzureIntegrationTest method testFindEntriesSince.

/**
 * Test findEntriesSince with specified cloud token factory.
 * @param replicationCloudTokenFactory the factory to use.
 * @throws Exception on error
 */
private void testFindEntriesSince(String replicationCloudTokenFactory) throws Exception {
    logger.info("Testing findEntriesSince with {}", replicationCloudTokenFactory);
    testProperties.setProperty(ReplicationConfig.REPLICATION_CLOUD_TOKEN_FACTORY, replicationCloudTokenFactory);
    VerifiableProperties verifiableProperties = new VerifiableProperties(testProperties);
    ReplicationConfig replicationConfig = new ReplicationConfig(verifiableProperties);
    FindTokenFactory findTokenFactory = new FindTokenHelper(null, replicationConfig).getFindTokenFactoryFromReplicaType(ReplicaType.CLOUD_BACKED);
    azureDest = (AzureCloudDestination) new AzureCloudDestinationFactory(verifiableProperties, new MetricRegistry(), clusterMap).getCloudDestination();
    cleanup();
    PartitionId partitionId = new MockPartitionId(testPartition, MockClusterMap.DEFAULT_PARTITION_CLASS);
    String partitionPath = String.valueOf(testPartition);
    // Upload some blobs with different upload times
    int blobCount = 90;
    int chunkSize = 1000;
    int maxTotalSize = 20000;
    int expectedNumQueries = (blobCount * chunkSize) / maxTotalSize + 1;
    long now = System.currentTimeMillis();
    long startTime = now - TimeUnit.DAYS.toMillis(7);
    for (int j = 0; j < blobCount; j++) {
        BlobId blobId = new BlobId(BLOB_ID_V6, BlobIdType.NATIVE, dataCenterId, accountId, containerId, partitionId, false, BlobDataType.DATACHUNK);
        InputStream inputStream = getBlobInputStream(chunkSize);
        CloudBlobMetadata cloudBlobMetadata = new CloudBlobMetadata(blobId, startTime, Utils.Infinite_Time, chunkSize, CloudBlobMetadata.EncryptionOrigin.VCR, vcrKmsContext, cryptoAgentFactory, chunkSize, (short) 0);
        cloudBlobMetadata.setUploadTime(startTime + j * 1000);
        assertTrue("Expected upload to return true", uploadBlobWithRetry(blobId, chunkSize, cloudBlobMetadata, inputStream, cloudRequestAgent, azureDest));
    }
    FindToken findToken = findTokenFactory.getNewFindToken();
    // Call findEntriesSince in a loop until no new entries are returned
    FindResult findResult;
    int numQueries = 0;
    int totalBlobsReturned = 0;
    do {
        findResult = findEntriesSinceWithRetry(partitionPath, findToken, maxTotalSize);
        findToken = findResult.getUpdatedFindToken();
        if (!findResult.getMetadataList().isEmpty()) {
            numQueries++;
        }
        totalBlobsReturned += findResult.getMetadataList().size();
    } while (!noMoreFindSinceEntries(findResult, findToken));
    assertEquals("Wrong number of queries", expectedNumQueries, numQueries);
    assertEquals("Wrong number of blobs", blobCount, totalBlobsReturned);
    assertEquals("Wrong byte count", blobCount * chunkSize, findToken.getBytesRead());
    cleanup();
}

Example 12 with FindToken

use of com.github.ambry.replication.FindToken in project ambry by linkedin.

the class DumpReplicaTokenTool method dumpReplicaToken.

/**
 * Dumps replica token file
 * @throws Exception
 */
private void dumpReplicaToken() throws Exception {
    logger.info("Dumping replica token file {}", fileToRead);
    DataInputStream stream = new DataInputStream(new FileInputStream(fileToRead));
    short version = stream.readShort();
    switch(version) {
        case 0:
            int Crc_Size = 8;
            StoreKeyFactory storeKeyFactory = Utils.getObj("com.github.ambry.commons.BlobIdFactory", clusterMap);
            FindTokenFactory findTokenFactory = Utils.getObj("com.github.ambry.store.StoreFindTokenFactory", storeKeyFactory);
            while (stream.available() > Crc_Size) {
                // read partition id
                PartitionId partitionId = clusterMap.getPartitionIdFromStream(stream);
                // read remote node host name
                String hostname = Utils.readIntString(stream);
                // read remote replica path
                String replicaPath = Utils.readIntString(stream);
                // read remote port
                int port = stream.readInt();
                // read total bytes read from local store
                long totalBytesReadFromLocalStore = stream.readLong();
                // read replica token
                FindToken token = findTokenFactory.getFindToken(stream);
                logger.info("partitionId {} hostname {} replicaPath {} port {} totalBytesReadFromLocalStore {} token {}", partitionId, hostname, replicaPath, port, totalBytesReadFromLocalStore, token);
            }
            logger.info("crc {}", stream.readLong());
            break;
        default:
            logger.error("Version {} unsupported ", version);
    }
}