Examples with SamzaException org.apache.samza.SamzaException used on opensource projects

Example 26 with SamzaException

use of org.apache.samza.SamzaException in project samza by apache.

the class HdfsSystemConsumer method getPartitionDescriptor.

private List<String> getPartitionDescriptor(SystemStreamPartition systemStreamPartition) {
    String streamName = systemStreamPartition.getStream();
    Partition partition = systemStreamPartition.getPartition();
    try {
        return cachedPartitionDescriptorMap.get(streamName).get(partition);
    } catch (ExecutionException e) {
        throw new SamzaException("Failed to obtain descriptor for " + systemStreamPartition, e);
    }
}

Example 27 with SamzaException

use of org.apache.samza.SamzaException in project samza by apache.

the class TestDirectoryPartitioner method testInvalidDirectoryUpdating.

@Test
public void testInvalidDirectoryUpdating() {
    // the update is invalid when at least one old file is removed
    List<FileMetadata> testList = new ArrayList<>();
    int NUM_INPUT = 6;
    String[] inputFiles = { "part-001.avro", "part-002.avro", "part-003.avro", "part-005.avro", "part-004.avro", "part-006.avro" };
    long[] fileLength = { 150582, 138132, 214005, 205738, 158273, 982345 };
    for (int i = 0; i < NUM_INPUT; i++) {
        testList.add(new FileMetadata(inputFiles[i], fileLength[i]));
    }
    String whiteList = ".*";
    String blackList = "";
    String groupPattern = "";
    int EXPECTED_NUM_PARTITION = 6;
    int[][] EXPECTED_PARTITIONING = { { 0 }, { 1 }, { 2 }, { 3 }, { 4 }, { 5 } };
    DirectoryPartitioner directoryPartitioner = new DirectoryPartitioner(whiteList, blackList, groupPattern, new TestFileSystemAdapter(testList));
    Map<Partition, SystemStreamPartitionMetadata> metadataMap = directoryPartitioner.getPartitionMetadataMap("hdfs", null);
    Assert.assertEquals(EXPECTED_NUM_PARTITION, metadataMap.size());
    Map<Partition, List<String>> descriporMap = directoryPartitioner.getPartitionDescriptor("hdfs");
    verifyPartitionDescriptor(inputFiles, EXPECTED_PARTITIONING, EXPECTED_NUM_PARTITION, descriporMap);
    String[] updatedInputFiles = { "part-001.avro", "part-002.avro", "part-003.avro", "part-005.avro", // remove part-004 and replace it with 007
    "part-007.avro", "part-006.avro" };
    long[] updatedFileLength = { 150582, 138132, 214005, 205738, 158273, 982345 };
    testList.clear();
    for (int i = 0; i < NUM_INPUT; i++) {
        testList.add(new FileMetadata(updatedInputFiles[i], updatedFileLength[i]));
    }
    directoryPartitioner = new DirectoryPartitioner(whiteList, blackList, groupPattern, new TestFileSystemAdapter(testList));
    try {
        directoryPartitioner.getPartitionMetadataMap("hdfs", descriporMap);
        Assert.fail("Expect exception thrown from getting metadata. Should not reach this point.");
    } catch (SamzaException e) {
    // expect exception to be thrown
    }
}

Example 28 with SamzaException

use of org.apache.samza.SamzaException in project samza by apache.

the class StreamAppender method setupSystem.

protected void setupSystem() {
    config = getConfig();
    SystemFactory systemFactory = null;
    Log4jSystemConfig log4jSystemConfig = new Log4jSystemConfig(config);
    if (streamName == null) {
        streamName = getStreamName(log4jSystemConfig.getJobName(), log4jSystemConfig.getJobId());
    }
    String systemName = log4jSystemConfig.getSystemName();
    String systemFactoryName = log4jSystemConfig.getSystemFactory(systemName);
    if (systemFactoryName != null) {
        systemFactory = Util.<SystemFactory>getObj(systemFactoryName);
    } else {
        throw new SamzaException("Could not figure out \"" + systemName + "\" system factory for log4j StreamAppender to use");
    }
    setSerde(log4jSystemConfig, systemName, streamName);
    systemProducer = systemFactory.getProducer(systemName, config, new MetricsRegistryMap());
    systemStream = new SystemStream(systemName, streamName);
    systemProducer.register(SOURCE);
    systemProducer.start();
    log.info(SOURCE + " has been registered in " + systemName + ". So all the logs will be sent to " + streamName + " in " + systemName + ". Logs are partitioned by " + key);
}

Example 29 with SamzaException

use of org.apache.samza.SamzaException in project samza by apache.

the class JobsClient method queryJobStatusServers.

/**
   *
   * This method initiates http get request to the job status servers sequentially,
   * returns the first response from an job status server that returns a 2xx code(success response).
   * When a job status server is down or returns a error response, it tries to reach out to
   * the next job status server in the sequence, to complete the http get request.
   *
   * @param requestUrlBuilder to build the request url, given job status server base url.
   * @param <T> return type of the http get response.
   * @return the response from any one of the job status server.
   * @throws Exception when all the job status servers are unavailable.
   *
   */
private <T> T queryJobStatusServers(Function<String, String> requestUrlBuilder, TypeReference<T> typeReference) {
    SamzaException fetchException = null;
    for (String jobStatusServer : jobStatusServers) {
        String requestUrl = requestUrlBuilder.apply(jobStatusServer);
        try {
            ObjectMapper objectMapper = new ObjectMapper();
            byte[] response = httpGet(requestUrl);
            return objectMapper.readValue(response, typeReference);
        } catch (Exception e) {
            String exceptionMessage = String.format("Exception in http get request from url: %s.", requestUrl);
            LOG.error(exceptionMessage, e);
            fetchException = new SamzaException(exceptionMessage, e);
        }
    }
    throw fetchException;
}

Example 30 with SamzaException

use of org.apache.samza.SamzaException in project samza by apache.

the class JobsClient method httpGet.

/**
   * This method initiates http get request on the request url and returns the
   * response returned from the http get.
   * @param requestUrl url on which the http get request has to be performed.
   * @return the http get response.
   * @throws IOException if there are problems with the http get request.
   */
private byte[] httpGet(String requestUrl) throws IOException {
    GetMethod getMethod = new GetMethod(requestUrl);
    try {
        int responseCode = httpClient.executeMethod(getMethod);
        LOG.debug("Received response code: {} for the get request on the url: {}", responseCode, requestUrl);
        byte[] response = getMethod.getResponseBody();
        if (responseCode != HttpStatus.SC_OK) {
            throw new SamzaException(String.format("Received response code: %s for get request on: %s, with message: %s.", responseCode, requestUrl, StringUtils.newStringUtf8(response)));
        }
        return response;
    } finally {
        getMethod.releaseConnection();
    }
}