Examples with VoldemortException voldemort.VoldemortException used on opensource projects

Example 6 with VoldemortException

use of voldemort.VoldemortException in project voldemort by voldemort.

the class R2Store method getVersions.

@Override
public List<Version> getVersions(ByteArray key) {
    List<Version> resultList = new ArrayList<Version>();
    String base64Key = RestUtils.encodeVoldemortKey(key.get());
    RestRequestBuilder rb = null;
    try {
        rb = new RestRequestBuilder(new URI(this.restBootstrapURL + "/" + getName() + "/" + base64Key));
        String timeoutStr = Long.toString(this.config.getTimeoutConfig().getOperationTimeout(VoldemortOpCode.GET_VERSION_OP_CODE));
        rb.setHeader(RestMessageHeaders.X_VOLD_GET_VERSION, "true");
        RestResponse response = fetchGetResponse(rb, timeoutStr);
        final ByteString entity = response.getEntity();
        if (entity != null) {
            resultList = parseGetVersionResponse(entity);
        } else {
            if (logger.isDebugEnabled()) {
                logger.debug("Did not get any response!");
            }
        }
    } catch (ExecutionException e) {
        if (e.getCause() instanceof RestException) {
            RestException exception = (RestException) e.getCause();
            if (logger.isDebugEnabled()) {
                logger.debug("REST EXCEPTION STATUS : " + exception.getResponse().getStatus());
            }
        } else {
            throw new VoldemortException("Unknown HTTP request execution exception: " + e.getMessage(), e);
        }
    } catch (InterruptedException e) {
        if (logger.isDebugEnabled()) {
            logger.debug("Operation interrupted : " + e.getMessage(), e);
        }
        throw new VoldemortException("Operation interrupted exception: " + e.getMessage(), e);
    } catch (URISyntaxException e) {
        throw new VoldemortException("Illegal HTTP URL" + e.getMessage(), e);
    }
    return resultList;
}

Example 7 with VoldemortException

use of voldemort.VoldemortException in project voldemort by voldemort.

the class R2Store method getAll.

@Override
public Map<ByteArray, List<Versioned<byte[]>>> getAll(Iterable<ByteArray> keys, Map<ByteArray, byte[]> transforms) throws VoldemortException {
    Map<ByteArray, List<Versioned<byte[]>>> resultMap = new HashMap<ByteArray, List<Versioned<byte[]>>>();
    int numberOfKeys = 0;
    try {
        Iterator<ByteArray> it = keys.iterator();
        StringBuilder keyArgs = null;
        while (it.hasNext()) {
            ByteArray key = it.next();
            String base64Key = RestUtils.encodeVoldemortKey(key.get());
            if (keyArgs == null) {
                keyArgs = new StringBuilder();
                keyArgs.append(base64Key);
            } else {
                keyArgs.append("," + base64Key);
            }
            numberOfKeys++;
        }
        // TODO a common way to handle getAll with any number of keys
        if (numberOfKeys == 1) {
            List<Versioned<byte[]>> resultList = new ArrayList<Versioned<byte[]>>();
            it = keys.iterator();
            ByteArray key = it.next();
            byte[] singleKeyTransforms = null;
            if (transforms != null) {
                singleKeyTransforms = transforms.get(key);
            }
            resultList = this.get(key, singleKeyTransforms);
            resultMap.put(key, resultList);
        } else {
            RestRequestBuilder rb = new RestRequestBuilder(new URI(this.restBootstrapURL + "/" + getName() + "/" + keyArgs.toString()));
            rb.setMethod(GET);
            rb.setHeader("Accept", MULTIPART_CONTENT_TYPE);
            String timeoutStr = Long.toString(this.config.getTimeoutConfig().getOperationTimeout(VoldemortOpCode.GET_ALL_OP_CODE));
            rb.setHeader(RestMessageHeaders.X_VOLD_REQUEST_TIMEOUT_MS, timeoutStr);
            rb.setHeader(RestMessageHeaders.X_VOLD_REQUEST_ORIGIN_TIME_MS, String.valueOf(System.currentTimeMillis()));
            if (this.routingTypeCode != null) {
                rb.setHeader(RestMessageHeaders.X_VOLD_ROUTING_TYPE_CODE, this.routingTypeCode);
            }
            if (this.zoneId != INVALID_ZONE_ID) {
                rb.setHeader(RestMessageHeaders.X_VOLD_ZONE_ID, String.valueOf(this.zoneId));
            }
            RestRequest request = rb.build();
            Future<RestResponse> f = client.restRequest(request);
            // This will block
            RestResponse response = f.get();
            // Parse the response
            final ByteString entity = response.getEntity();
            String contentType = response.getHeader(CONTENT_TYPE);
            if (entity != null) {
                if (contentType.equalsIgnoreCase(MULTIPART_CONTENT_TYPE)) {
                    resultMap = parseGetAllResults(entity);
                } else {
                    if (logger.isDebugEnabled()) {
                        logger.debug("Did not receive a multipart response");
                    }
                }
            } else {
                if (logger.isDebugEnabled()) {
                    logger.debug("Did not get any response!");
                }
            }
        }
    } catch (ExecutionException e) {
        if (e.getCause() instanceof RestException) {
            RestException exception = (RestException) e.getCause();
            if (logger.isDebugEnabled()) {
                logger.debug("REST EXCEPTION STATUS : " + exception.getResponse().getStatus());
            }
        } else {
            throw new VoldemortException("Unknown HTTP request execution exception: " + e.getMessage(), e);
        }
    } catch (InterruptedException e) {
        if (logger.isDebugEnabled()) {
            logger.debug("Operation interrupted : " + e.getMessage(), e);
        }
        throw new VoldemortException("Operation interrupted exception: " + e.getMessage(), e);
    } catch (URISyntaxException e) {
        throw new VoldemortException("Illegal HTTP URL" + e.getMessage(), e);
    }
    return resultMap;
}

Example 8 with VoldemortException

use of voldemort.VoldemortException in project voldemort by voldemort.

the class HdfsFetcherTest method testCheckSumMetadata.

public void testCheckSumMetadata() throws Exception {
    // Generate 0_0.[index | data] and their corresponding metadata
    File testSourceDirectory = TestUtils.createTempDir();
    File testDestinationDirectory = TestUtils.createTempDir();
    // Missing metadata file
    File indexFile = new File(testSourceDirectory, "0_0.index");
    FileUtils.writeByteArrayToFile(indexFile, TestUtils.randomBytes(100));
    File dataFile = new File(testSourceDirectory, "0_0.data");
    FileUtils.writeByteArrayToFile(dataFile, TestUtils.randomBytes(400));
    HdfsFetcher fetcher = new HdfsFetcher();
    File fetchedFile = fetcher.fetch(testSourceDirectory.getAbsolutePath(), testDestinationDirectory.getAbsolutePath() + "1");
    assertNotNull(fetchedFile);
    assertEquals(fetchedFile.getAbsolutePath(), testDestinationDirectory.getAbsolutePath() + "1");
    // Write bad metadata file
    File metadataFile = new File(testSourceDirectory, ".metadata");
    FileUtils.writeByteArrayToFile(metadataFile, TestUtils.randomBytes(100));
    try {
        fetchedFile = fetcher.fetch(testSourceDirectory.getAbsolutePath(), testDestinationDirectory.getAbsolutePath() + "2");
        fail("Should have thrown an exception since metadata file is corrupt");
    } catch (VoldemortException e) {
    }
    metadataFile.delete();
    // Missing metadata checksum type
    metadataFile = new File(testSourceDirectory, ".metadata");
    ReadOnlyStorageMetadata metadata = new ReadOnlyStorageMetadata();
    metadata.add(ReadOnlyStorageMetadata.FORMAT, ReadOnlyStorageFormat.READONLY_V2.getCode());
    FileUtils.writeStringToFile(metadataFile, metadata.toJsonString());
    fetchedFile = fetcher.fetch(testSourceDirectory.getAbsolutePath(), testDestinationDirectory.getAbsolutePath() + "3");
    assertNotNull(fetchedFile);
    assertEquals(fetchedFile.getAbsolutePath(), testDestinationDirectory.getAbsolutePath() + "3");
    metadataFile.delete();
    // Incorrect checksum type + missing checksum
    metadata.add(ReadOnlyStorageMetadata.CHECKSUM_TYPE, "blah");
    FileUtils.writeStringToFile(metadataFile, metadata.toJsonString());
    fetchedFile = fetcher.fetch(testSourceDirectory.getAbsolutePath(), testDestinationDirectory.getAbsolutePath() + "4");
    assertNotNull(fetchedFile);
    assertEquals(fetchedFile.getAbsolutePath(), testDestinationDirectory.getAbsolutePath() + "4");
    metadataFile.delete();
    // Incorrect metadata checksum
    metadata.add(ReadOnlyStorageMetadata.CHECKSUM_TYPE, CheckSum.toString(CheckSumType.MD5));
    metadata.add(ReadOnlyStorageMetadata.CHECKSUM, "1234");
    FileUtils.writeStringToFile(metadataFile, metadata.toJsonString());
    fetchedFile = fetcher.fetch(testSourceDirectory.getAbsolutePath(), testDestinationDirectory.getAbsolutePath() + "5");
    assertNull(fetchedFile);
    metadataFile.delete();
    // Correct metadata checksum - MD5
    metadata.add(ReadOnlyStorageMetadata.CHECKSUM, new String(Hex.encodeHex(CheckSumTests.calculateCheckSum(testSourceDirectory.listFiles(), CheckSumType.MD5))));
    FileUtils.writeStringToFile(metadataFile, metadata.toJsonString());
    fetchedFile = fetcher.fetch(testSourceDirectory.getAbsolutePath(), testDestinationDirectory.getAbsolutePath() + "6");
    assertNotNull(fetchedFile);
    assertEquals(fetchedFile.getAbsolutePath(), testDestinationDirectory.getAbsolutePath() + "6");
    // Correct metadata checksum - ADLER32
    metadata.add(ReadOnlyStorageMetadata.CHECKSUM_TYPE, CheckSum.toString(CheckSumType.ADLER32));
    metadata.add(ReadOnlyStorageMetadata.CHECKSUM, new String(Hex.encodeHex(CheckSumTests.calculateCheckSum(testSourceDirectory.listFiles(), CheckSumType.ADLER32))));
    FileUtils.writeStringToFile(metadataFile, metadata.toJsonString());
    fetchedFile = fetcher.fetch(testSourceDirectory.getAbsolutePath(), testDestinationDirectory.getAbsolutePath() + "7");
    assertNotNull(fetchedFile);
    assertEquals(fetchedFile.getAbsolutePath(), testDestinationDirectory.getAbsolutePath() + "7");
    // Correct metadata checksum - CRC32
    metadata.add(ReadOnlyStorageMetadata.CHECKSUM_TYPE, CheckSum.toString(CheckSumType.CRC32));
    metadata.add(ReadOnlyStorageMetadata.CHECKSUM, new String(Hex.encodeHex(CheckSumTests.calculateCheckSum(testSourceDirectory.listFiles(), CheckSumType.CRC32))));
    FileUtils.writeStringToFile(metadataFile, metadata.toJsonString());
    fetchedFile = fetcher.fetch(testSourceDirectory.getAbsolutePath(), testDestinationDirectory.getAbsolutePath() + "8");
    assertNotNull(fetchedFile);
    assertEquals(fetchedFile.getAbsolutePath(), testDestinationDirectory.getAbsolutePath() + "8");
}

Example 9 with VoldemortException

use of voldemort.VoldemortException in project voldemort by voldemort.

the class ReadOnlyTestUtils method unGunzipFile.

/**
     * Un gzip the compressedFile to the given decompressedFile
     * 
     * @param compressedFile
     * @param decompressedFile
     */
public static void unGunzipFile(String compressedFile, String decompressedFile) {
    byte[] buffer = new byte[1024];
    try {
        FileSystem fs = FileSystem.getLocal(new Configuration());
        FSDataInputStream fileIn = fs.open(new Path(compressedFile));
        GZIPInputStream gZIPInputStream = new GZIPInputStream(fileIn);
        FileOutputStream fileOutputStream = new FileOutputStream(decompressedFile);
        int bytes_read;
        while ((bytes_read = gZIPInputStream.read(buffer)) > 0) {
            fileOutputStream.write(buffer, 0, bytes_read);
        }
        gZIPInputStream.close();
        fileOutputStream.close();
    } catch (IOException ex) {
        throw new VoldemortException("Got IOException while trying to un-gzip file: " + compressedFile, ex);
    }
}

Example 10 with VoldemortException

use of voldemort.VoldemortException in project voldemort by voldemort.

the class HadoopUtils method getHadoopFileSystem.

public static FileSystem getHadoopFileSystem(VoldemortConfig voldemortConfig, String sourceFileUrl) throws Exception {
    final Path source = new Path(sourceFileUrl);
    final Configuration config = getConfiguration(voldemortConfig, source);
    final int maxAttempts = voldemortConfig.getReadOnlyFetchRetryCount();
    final String keytabPath = voldemortConfig.getReadOnlyKeytabPath();
    FileSystem fs = null;
    for (int attempt = 1; attempt <= maxAttempts; attempt++) {
        try {
            if (keytabPath.length() > 0) {
                // but we will redo it if an AuthenticationException is caught below.
                synchronized (HadoopUtils.class) {
                    long timeSinceLastLogin = System.currentTimeMillis() - lastLoginTime;
                    // 2- To prevent NPEs if the currentHadoopUser is reset to null in the catch block.
                    if (currentHadoopUser == null || timeSinceLastLogin > voldemortConfig.getReadOnlyLoginIntervalMs()) {
                        if (!new File(keytabPath).exists()) {
                            logger.error("Invalid keytab file path. Please provide a valid keytab path");
                            throw new VoldemortException("Error in getting Hadoop filesystem. Invalid keytab file path.");
                        }
                        UserGroupInformation.setConfiguration(config);
                        UserGroupInformation.loginUserFromKeytab(voldemortConfig.getReadOnlyKerberosUser(), keytabPath);
                        currentHadoopUser = UserGroupInformation.getCurrentUser();
                        lastLoginTime = System.currentTimeMillis();
                        logger.info("I have logged in as " + currentHadoopUser.getUserName());
                    } else {
                        // FileSystem caching is disabled. If enabled, the code has a known bug
                        // FileSystem returns the cached object per scheme, authority and user
                        // This causes the FileSystem object to be shared among multiple fetches/lock
                        // But each owner closes the FileSystem at the end and surprising others still using it.
                        // reloginFromKeytab() will not actually do anything unless the token is close to expiring.
                        currentHadoopUser.reloginFromKeytab();
                    }
                }
            }
            fs = source.getFileSystem(config);
            // Just a small operation to make sure the FileSystem instance works.
            fs.exists(source);
            break;
        } catch (VoldemortException e) {
            IOUtils.closeQuietly(fs);
            // We only intend to catch and retry Hadoop-related exceptions, not Voldemort ones.
            throw e;
        } catch (Exception e) {
            IOUtils.closeQuietly(fs);
            if (ExceptionUtils.recursiveClassEquals(e, AuthenticationException.class)) {
                logger.info("Got an AuthenticationException from HDFS. " + "Will retry to login from scratch, on next attempt.", e);
                synchronized (HadoopUtils.class) {
                    // Synchronized to prevent NPEs in the other synchronized block, above.
                    currentHadoopUser = null;
                }
            }
            if (attempt < maxAttempts) {
                // We may need to sleep
                long retryDelayMs = voldemortConfig.getReadOnlyFetchRetryDelayMs();
                if (retryDelayMs > 0) {
                    // Doing random back off so that all nodes do not end up swarming the KDC infra
                    long randomDelay = (long) (Math.random() * retryDelayMs + retryDelayMs);
                    logger.error("Could not get a valid Filesystem object on attempt # " + attempt + " / " + maxAttempts + ". Trying again in " + randomDelay + " ms.");
                    try {
                        Thread.sleep(randomDelay);
                    } catch (InterruptedException ie) {
                        logger.error("Fetcher interrupted while waiting to retry", ie);
                        Thread.currentThread().interrupt();
                    }
                }
            } else {
                throw e;
            }
        }
    }
    return fs;
}