Examples with StoreKeyFactory com.github.ambry.store.StoreKeyFactory used on opensource projects

Example 26 with StoreKeyFactory

use of com.github.ambry.store.StoreKeyFactory in project ambry by linkedin.

the class ServerHardDeleteTest method ensureCleanupTokenCatchesUp.

/**
 * Waits and ensures that the hard delete cleanup token catches up to the expected token value.
 * @param path the path to the cleanup token.
 * @param mockClusterMap the {@link MockClusterMap} being used for the cluster.
 * @param expectedTokenValue the expected value that the cleanup token should contain. Until this value is reached,
 *                           the method will keep reopening the file and read the value or until a predefined
 *                           timeout is reached.
 * @throws Exception if there were any I/O errors or the sleep gets interrupted.
 */
void ensureCleanupTokenCatchesUp(String path, MockClusterMap mockClusterMap, long expectedTokenValue) throws Exception {
    final int TIMEOUT = 10000;
    File cleanupTokenFile = new File(path, "cleanuptoken");
    StoreFindToken endToken;
    long parsedTokenValue = -1;
    long endTime = SystemTime.getInstance().milliseconds() + TIMEOUT;
    do {
        if (cleanupTokenFile.exists()) {
            /* The cleanup token format is as follows:
           --
           token_version
           startTokenForRecovery
           endTokenForRecovery
           numBlobsInRange
           pause flag
           --
           blob1_blobReadOptions {version, offset, sz, ttl, key}
           blob2_blobReadOptions
           ....
           blobN_blobReadOptions
           --
           length_of_blob1_messageStoreRecoveryInfo
           blob1_messageStoreRecoveryInfo {headerVersion, userMetadataVersion, userMetadataSize, blobRecordVersion,
            blobType, blobStreamSize}
           length_of_blob2_messageStoreRecoveryInfo
           blob2_messageStoreRecoveryInfo
           ....
           length_of_blobN_messageStoreRecoveryInfo
           blobN_messageStoreRecoveryInfo
           crc
           ---
         */
            CrcInputStream crcStream = new CrcInputStream(new FileInputStream(cleanupTokenFile));
            DataInputStream stream = new DataInputStream(crcStream);
            try {
                short version = stream.readShort();
                Assert.assertEquals(version, HardDeleter.Cleanup_Token_Version_V1);
                StoreKeyFactory storeKeyFactory = Utils.getObj("com.github.ambry.commons.BlobIdFactory", mockClusterMap);
                FindTokenFactory factory = Utils.getObj("com.github.ambry.store.StoreFindTokenFactory", storeKeyFactory);
                factory.getFindToken(stream);
                endToken = (StoreFindToken) factory.getFindToken(stream);
                Offset endTokenOffset = endToken.getOffset();
                parsedTokenValue = endTokenOffset == null ? -1 : endTokenOffset.getOffset();
                boolean pauseFlag = stream.readByte() == (byte) 1;
                int num = stream.readInt();
                List<StoreKey> storeKeyList = new ArrayList<StoreKey>(num);
                for (int i = 0; i < num; i++) {
                    // Read BlobReadOptions
                    short blobReadOptionsVersion = stream.readShort();
                    switch(blobReadOptionsVersion) {
                        case 1:
                            Offset.fromBytes(stream);
                            stream.readLong();
                            stream.readLong();
                            StoreKey key = storeKeyFactory.getStoreKey(stream);
                            storeKeyList.add(key);
                            break;
                        default:
                            Assert.assertFalse(true);
                    }
                }
                for (int i = 0; i < num; i++) {
                    int length = stream.readInt();
                    short headerVersion = stream.readShort();
                    short userMetadataVersion = stream.readShort();
                    int userMetadataSize = stream.readInt();
                    short blobRecordVersion = stream.readShort();
                    if (blobRecordVersion == MessageFormatRecord.Blob_Version_V2) {
                        short blobType = stream.readShort();
                    }
                    long blobStreamSize = stream.readLong();
                    StoreKey key = storeKeyFactory.getStoreKey(stream);
                    Assert.assertTrue(storeKeyList.get(i).equals(key));
                }
                long crc = crcStream.getValue();
                Assert.assertEquals(crc, stream.readLong());
                Thread.sleep(1000);
            } finally {
                stream.close();
            }
        }
    } while (SystemTime.getInstance().milliseconds() < endTime && parsedTokenValue < expectedTokenValue);
    Assert.assertEquals(expectedTokenValue, parsedTokenValue);
}

Example 27 with StoreKeyFactory

use of com.github.ambry.store.StoreKeyFactory in project ambry by linkedin.

the class BlobValidator method main.

/**
 * Runs the BlobValidator
 * @param args associated arguments.
 * @throws Exception
 */
public static void main(String[] args) throws Exception {
    VerifiableProperties verifiableProperties = ToolUtils.getVerifiableProperties(args);
    BlobValidatorConfig config = new BlobValidatorConfig(verifiableProperties);
    ClusterMapConfig clusterMapConfig = new ClusterMapConfig(verifiableProperties);
    ClusterMap clusterMap = ((ClusterAgentsFactory) Utils.getObj(clusterMapConfig.clusterMapClusterAgentsFactory, clusterMapConfig, config.hardwareLayoutFilePath, config.partitionLayoutFilePath)).getClusterMap();
    List<BlobId> blobIds = getBlobIds(config, clusterMap);
    SSLFactory sslFactory = !clusterMapConfig.clusterMapSslEnabledDatacenters.isEmpty() ? SSLFactory.getNewInstance(new SSLConfig(verifiableProperties)) : null;
    StoreKeyFactory storeKeyFactory = new BlobIdFactory(clusterMap);
    BlobValidator validator = new BlobValidator(clusterMap, config.replicasToContactPerSec, sslFactory, verifiableProperties);
    LOGGER.info("Validation starting");
    switch(config.typeOfOperation) {
        case ValidateBlobOnAllReplicas:
            Map<BlobId, List<String>> mismatchDetailsMap = validator.validateBlobsOnAllReplicas(blobIds, config.getOption, clusterMap, storeKeyFactory);
            logMismatches(mismatchDetailsMap);
            break;
        case ValidateBlobOnDatacenter:
            if (config.datacenter.isEmpty() || !clusterMap.hasDatacenter(config.datacenter)) {
                throw new IllegalArgumentException("Please provide a valid datacenter");
            }
            mismatchDetailsMap = validator.validateBlobsOnDatacenter(config.datacenter, blobIds, config.getOption, clusterMap, storeKeyFactory);
            logMismatches(mismatchDetailsMap);
            break;
        case ValidateBlobOnReplica:
            DataNodeId dataNodeId = clusterMap.getDataNodeId(config.hostname, config.port);
            if (dataNodeId == null) {
                throw new IllegalArgumentException("Could not find a data node corresponding to " + config.hostname + ":" + config.port);
            }
            List<ServerErrorCode> validErrorCodes = Arrays.asList(ServerErrorCode.No_Error, ServerErrorCode.Blob_Deleted, ServerErrorCode.Blob_Expired);
            Map<BlobId, ServerErrorCode> blobIdToErrorCode = validator.validateBlobsOnReplica(dataNodeId, blobIds, config.getOption, clusterMap, storeKeyFactory);
            for (Map.Entry<BlobId, ServerErrorCode> entry : blobIdToErrorCode.entrySet()) {
                ServerErrorCode errorCode = entry.getValue();
                if (!validErrorCodes.contains(errorCode)) {
                    LOGGER.error("[{}] received error code: {}", entry.getKey(), errorCode);
                }
            }
            break;
        default:
            throw new IllegalStateException("Recognized but unsupported operation: " + config.typeOfOperation);
    }
    LOGGER.info("Validation complete");
    validator.close();
    clusterMap.close();
}