Examples with State org.apache.gobblin.configuration.State used on opensource projects

Example 61 with State

use of org.apache.gobblin.configuration.State in project incubator-gobblin by apache.

the class BaseDataPublisherTest method testBogusMetadataMerger.

/**
 * Test that an exception is properly thrown if we configure a merger that doesn't actually implement
 * MetadataMerger
 */
@Test(expectedExceptions = IllegalArgumentException.class)
public void testBogusMetadataMerger() throws IOException {
    State s = buildDefaultState(1);
    s.setProp(ConfigurationKeys.DATA_PUBLISH_WRITER_METADATA_KEY, "true");
    s.setProp(ConfigurationKeys.DATA_PUBLISH_WRITER_METADATA_MERGER_NAME_KEY, "java.lang.String");
    s.setProp(ConfigurationKeys.DATA_PUBLISHER_METADATA_STR, "foobar");
    WorkUnitState wuState = new WorkUnitState();
    addStateToWorkunit(s, wuState);
    BaseDataPublisher publisher = new BaseDataPublisher(s);
    publisher.publishMetadata(Collections.singletonList(wuState));
}

Example 62 with State

use of org.apache.gobblin.configuration.State in project incubator-gobblin by apache.

the class BaseDataPublisherTest method testWithPartitionKey.

@Test
public void testWithPartitionKey() throws IOException {
    File publishPath = Files.createTempDir();
    try {
        File part1 = new File(publishPath, "1-2-3-4");
        part1.mkdir();
        File part2 = new File(publishPath, "5-6-7-8");
        part2.mkdir();
        State s = buildDefaultState(1);
        String md = new GlobalMetadata().toJson();
        s.removeProp(ConfigurationKeys.DATA_PUBLISHER_METADATA_OUTPUT_DIR);
        s.setProp(ConfigurationKeys.DATA_PUBLISH_WRITER_METADATA_KEY, "true");
        s.setProp(ConfigurationKeys.WRITER_METADATA_KEY, md);
        s.setProp(ConfigurationKeys.DATA_PUBLISHER_FINAL_DIR, publishPath.getAbsolutePath());
        s.setProp(ConfigurationKeys.DATA_PUBLISHER_APPEND_EXTRACT_TO_FINAL_DIR, "false");
        s.setProp(ConfigurationKeys.DATA_PUBLISHER_METADATA_OUTPUT_FILE, "metadata.json");
        WorkUnitState wuState1 = new WorkUnitState();
        wuState1.setProp(ConfigurationKeys.WRITER_PARTITION_PATH_KEY, "1-2-3-4");
        wuState1.setProp(ConfigurationKeys.WRITER_METADATA_KEY, md);
        addStateToWorkunit(s, wuState1);
        WorkUnitState wuState2 = new WorkUnitState();
        wuState2.setProp(ConfigurationKeys.WRITER_PARTITION_PATH_KEY, "5-6-7-8");
        wuState2.setProp(ConfigurationKeys.WRITER_METADATA_KEY, md);
        addStateToWorkunit(s, wuState2);
        BaseDataPublisher publisher = new BaseDataPublisher(s);
        publisher.publishMetadata(ImmutableList.of(wuState1, wuState2));
        Assert.assertTrue(new File(part1, "metadata.json").exists());
        Assert.assertTrue(new File(part2, "metadata.json").exists());
    } finally {
        FileUtils.deleteDirectory(publishPath);
    }
}

Example 63 with State

use of org.apache.gobblin.configuration.State in project incubator-gobblin by apache.

the class FsDataWriter method getFinalState.

@Override
public State getFinalState() {
    State state = new State();
    state.setProp("RecordsWritten", recordsWritten());
    try {
        state.setProp("BytesWritten", bytesWritten());
    } catch (Exception exception) {
    // If Writer fails to return bytesWritten, it might not be implemented, or implemented incorrectly.
    // Omit property instead of failing.
    }
    return state;
}

Example 64 with State

use of org.apache.gobblin.configuration.State in project incubator-gobblin by apache.

the class HiveWritableHdfsDataWriterBuilder method build.

@SuppressWarnings("deprecation")
@Override
public DataWriter<Writable> build() throws IOException {
    Preconditions.checkNotNull(this.destination);
    Preconditions.checkArgument(!Strings.isNullOrEmpty(this.writerId));
    State properties = this.destination.getProperties();
    if (!properties.contains(WRITER_WRITABLE_CLASS) || !properties.contains(WRITER_OUTPUT_FORMAT_CLASS)) {
        HiveSerDeWrapper serializer = HiveSerDeWrapper.getSerializer(properties);
        properties.setProp(WRITER_WRITABLE_CLASS, serializer.getSerDe().getSerializedClass().getName());
        properties.setProp(WRITER_OUTPUT_FORMAT_CLASS, serializer.getOutputFormatClassName());
    }
    return new HiveWritableHdfsDataWriter(this, properties);
}

Example 65 with State

use of org.apache.gobblin.configuration.State in project incubator-gobblin by apache.

the class FsRenameCommitStepTest method setUp.

@BeforeClass
public void setUp() throws IOException {
    this.fs = FileSystem.getLocal(new Configuration());
    this.fs.delete(new Path(ROOT_DIR), true);
    Path dir1 = new Path(ROOT_DIR, "dir1");
    Path dir2 = new Path(ROOT_DIR, "dir2");
    this.fs.mkdirs(dir1);
    this.fs.mkdirs(dir2);
    Path src = new Path(dir1, "file");
    Path dst = new Path(dir2, "file");
    this.fs.createNewFile(src);
    this.step = (FsRenameCommitStep) new FsRenameCommitStep.Builder<>().from(src).to(dst).withProps(new State()).build();
}