Examples with SamzaException org.apache.samza.SamzaException used on opensource projects

Example 21 with SamzaException

use of org.apache.samza.SamzaException in project samza by apache.

the class DirectoryPartitioner method validateAndGetOriginalFilteredFiles.

/*
    * This class holds the assumption that the directory remains immutable.
    * If the directory does changes:
    * ignore new files showing up in the directory based on an old version of partition descriptor;
    * throw {@link org.apache.samza.SamzaException} if at least one old file doesn't exist anymore
    */
private List<FileMetadata> validateAndGetOriginalFilteredFiles(List<FileMetadata> newFileList, Map<Partition, List<String>> existingPartitionDescriptor) {
    assert newFileList != null;
    assert existingPartitionDescriptor != null;
    Set<String> oldFileSet = new HashSet<>();
    existingPartitionDescriptor.values().forEach(oldFileSet::addAll);
    Set<String> newFileSet = new HashSet<>();
    newFileList.forEach(file -> newFileSet.add(file.getPath()));
    if (!newFileSet.containsAll(oldFileSet)) {
        throw new SamzaException("The list of new files is not a super set of the old files. diff = " + oldFileSet.removeAll(newFileSet));
    }
    Iterator<FileMetadata> iterator = newFileList.iterator();
    while (iterator.hasNext()) {
        FileMetadata file = iterator.next();
        if (!oldFileSet.contains(file.getPath())) {
            iterator.remove();
        }
    }
    return newFileList;
}

Example 22 with SamzaException

use of org.apache.samza.SamzaException in project samza by apache.

the class HdfsFileSystemAdapter method getAllFiles.

public List<FileMetadata> getAllFiles(String streamName) {
    List<FileMetadata> ret = new ArrayList<>();
    try {
        Path streamPath = new Path(streamName);
        FileSystem fileSystem = streamPath.getFileSystem(new Configuration());
        FileStatus[] fileStatuses = fileSystem.listStatus(streamPath);
        for (FileStatus fileStatus : fileStatuses) {
            ret.add(new FileMetadata(fileStatus.getPath().toString(), fileStatus.getLen()));
        }
    } catch (IOException e) {
        LOG.error("Failed to get the list of files for " + streamName, e);
        throw new SamzaException(e);
    }
    return ret;
}

Example 23 with SamzaException

use of org.apache.samza.SamzaException in project samza by apache.

the class AvroFileHdfsReader method seek.

@Override
public void seek(String singleFileOffset) {
    try {
        // See comments for AvroFileCheckpoint to understand the behavior below
        AvroFileCheckpoint checkpoint = new AvroFileCheckpoint(singleFileOffset);
        if (checkpoint.isStartingOffset()) {
            // seek to the beginning of the first block
            fileReader.sync(0);
            curBlockStart = fileReader.previousSync();
            curRecordOffset = 0;
            return;
        }
        fileReader.seek(checkpoint.getBlockStart());
        for (int i = 0; i < checkpoint.getRecordOffset(); i++) {
            if (fileReader.hasNext()) {
                fileReader.next();
            }
        }
        curBlockStart = checkpoint.getBlockStart();
        curRecordOffset = checkpoint.getRecordOffset();
    } catch (IOException e) {
        throw new SamzaException(e);
    }
}

Example 24 with SamzaException

use of org.apache.samza.SamzaException in project samza by apache.

the class TestZkUtils method testPublishNewJobModel.

@Test
public void testPublishNewJobModel() {
    ZkKeyBuilder keyBuilder = new ZkKeyBuilder("test");
    String root = keyBuilder.getRootPath();
    zkClient.deleteRecursive(root);
    String version = "1";
    String oldVersion = "0";
    zkUtils.makeSurePersistentPathsExists(new String[] { root, keyBuilder.getJobModelPathPrefix(), keyBuilder.getJobModelVersionPath() });
    zkUtils.publishJobModelVersion(oldVersion, version);
    Assert.assertEquals(version, zkUtils.getJobModelVersion());
    String newerVersion = Long.toString(Long.valueOf(version) + 1);
    zkUtils.publishJobModelVersion(version, newerVersion);
    Assert.assertEquals(newerVersion, zkUtils.getJobModelVersion());
    try {
        //invalid new version
        zkUtils.publishJobModelVersion(oldVersion, "10");
        Assert.fail("publish invalid version should've failed");
    } catch (SamzaException e) {
    // expected
    }
    // create job model
    Map<String, String> configMap = new HashMap<>();
    Map<String, ContainerModel> containers = new HashMap<>();
    MapConfig config = new MapConfig(configMap);
    JobModel jobModel = new JobModel(config, containers);
    zkUtils.publishJobModel(version, jobModel);
    Assert.assertEquals(jobModel, zkUtils.getJobModel(version));
}

Example 25 with SamzaException

use of org.apache.samza.SamzaException in project samza by apache.

the class TestTaskFactoryUtil method testFinalizeTaskFactory.

@Test
public void testFinalizeTaskFactory() throws NoSuchFieldException, IllegalAccessException {
    Object mockFactory = mock(Object.class);
    try {
        TaskFactoryUtil.finalizeTaskFactory(mockFactory, true, null);
        fail("Should have failed with validation");
    } catch (SamzaException se) {
    // expected
    }
    StreamTaskFactory mockStreamFactory = mock(StreamTaskFactory.class);
    Object retFactory = TaskFactoryUtil.finalizeTaskFactory(mockStreamFactory, true, null);
    assertEquals(retFactory, mockStreamFactory);
    ExecutorService mockThreadPool = mock(ExecutorService.class);
    retFactory = TaskFactoryUtil.finalizeTaskFactory(mockStreamFactory, false, mockThreadPool);
    assertTrue(retFactory instanceof AsyncStreamTaskFactory);
    assertTrue(((AsyncStreamTaskFactory) retFactory).createInstance() instanceof AsyncStreamTaskAdapter);
    AsyncStreamTaskAdapter taskAdapter = (AsyncStreamTaskAdapter) ((AsyncStreamTaskFactory) retFactory).createInstance();
    Field executorSrvFld = AsyncStreamTaskAdapter.class.getDeclaredField("executor");
    executorSrvFld.setAccessible(true);
    ExecutorService executor = (ExecutorService) executorSrvFld.get(taskAdapter);
    assertEquals(executor, mockThreadPool);
    AsyncStreamTaskFactory mockAsyncStreamFactory = mock(AsyncStreamTaskFactory.class);
    try {
        TaskFactoryUtil.finalizeTaskFactory(mockAsyncStreamFactory, true, null);
        fail("Should have failed");
    } catch (SamzaException se) {
    // expected
    }
    retFactory = TaskFactoryUtil.finalizeTaskFactory(mockAsyncStreamFactory, false, null);
    assertEquals(retFactory, mockAsyncStreamFactory);
}