Examples with State org.apache.gobblin.configuration.State used on opensource projects

Example 91 with State

use of org.apache.gobblin.configuration.State in project incubator-gobblin by apache.

the class DefaultLimiterFactoryTest method testNewLimiter.

@Test
public void testNewLimiter() {
    State stateWithRateLimitDeprecatedKeys = new State();
    stateWithRateLimitDeprecatedKeys.setProp(DefaultLimiterFactory.EXTRACT_LIMIT_TYPE_KEY, BaseLimiterType.RATE_BASED);
    stateWithRateLimitDeprecatedKeys.setProp(DefaultLimiterFactory.EXTRACT_LIMIT_RATE_LIMIT_KEY_DEP, "10");
    stateWithRateLimitDeprecatedKeys.setProp(DefaultLimiterFactory.EXTRACT_LIMIT_RATE_LIMIT_TIMEUNIT_KEY_DEP, TimeUnit.MINUTES);
    Limiter rateLimiterFromDeprecatedKeys = DefaultLimiterFactory.newLimiter(stateWithRateLimitDeprecatedKeys);
    Assert.assertTrue(rateLimiterFromDeprecatedKeys instanceof RateBasedLimiter);
    Assert.assertTrue(stateWithRateLimitDeprecatedKeys.contains(DefaultLimiterFactory.EXTRACT_LIMIT_RATE_LIMIT_KEY));
    Assert.assertFalse(stateWithRateLimitDeprecatedKeys.contains(DefaultLimiterFactory.EXTRACT_LIMIT_RATE_LIMIT_KEY_DEP));
    Assert.assertTrue(stateWithRateLimitDeprecatedKeys.contains(DefaultLimiterFactory.EXTRACT_LIMIT_RATE_LIMIT_TIMEUNIT_KEY));
    Assert.assertFalse(stateWithRateLimitDeprecatedKeys.contains(DefaultLimiterFactory.EXTRACT_LIMIT_RATE_LIMIT_TIMEUNIT_KEY_DEP));
    State stateWithTimeLimitDeprecatedKeys = new State();
    stateWithTimeLimitDeprecatedKeys.setProp(DefaultLimiterFactory.EXTRACT_LIMIT_TYPE_KEY, BaseLimiterType.TIME_BASED);
    stateWithTimeLimitDeprecatedKeys.setProp(DefaultLimiterFactory.EXTRACT_LIMIT_TIME_LIMIT_KEY_DEP, "10");
    stateWithTimeLimitDeprecatedKeys.setProp(DefaultLimiterFactory.EXTRACT_LIMIT_TIME_LIMIT_TIMEUNIT_KEY_DEP, TimeUnit.MINUTES);
    Limiter timeLimiterFromDeprecatedKeys = DefaultLimiterFactory.newLimiter(stateWithTimeLimitDeprecatedKeys);
    Assert.assertTrue(timeLimiterFromDeprecatedKeys instanceof TimeBasedLimiter);
    Assert.assertTrue(stateWithTimeLimitDeprecatedKeys.contains(DefaultLimiterFactory.EXTRACT_LIMIT_TIME_LIMIT_KEY));
    Assert.assertFalse(stateWithTimeLimitDeprecatedKeys.contains(DefaultLimiterFactory.EXTRACT_LIMIT_TIME_LIMIT_KEY_DEP));
    Assert.assertTrue(stateWithTimeLimitDeprecatedKeys.contains(DefaultLimiterFactory.EXTRACT_LIMIT_TIME_LIMIT_TIMEUNIT_KEY));
    Assert.assertFalse(stateWithTimeLimitDeprecatedKeys.contains(DefaultLimiterFactory.EXTRACT_LIMIT_TIME_LIMIT_TIMEUNIT_KEY_DEP));
    State stateWithNewKeys = new State();
    stateWithNewKeys.setProp(DefaultLimiterFactory.EXTRACT_LIMIT_TYPE_KEY, BaseLimiterType.RATE_BASED);
    stateWithNewKeys.setProp(DefaultLimiterFactory.EXTRACT_LIMIT_RATE_LIMIT_KEY_DEP, "10");
    stateWithNewKeys.setProp(DefaultLimiterFactory.EXTRACT_LIMIT_RATE_LIMIT_TIMEUNIT_KEY_DEP, TimeUnit.MINUTES);
    Limiter rateLimiterFromNewKeys = DefaultLimiterFactory.newLimiter(stateWithNewKeys);
    Assert.assertTrue(rateLimiterFromNewKeys instanceof RateBasedLimiter);
    Assert.assertTrue(stateWithNewKeys.contains(DefaultLimiterFactory.EXTRACT_LIMIT_RATE_LIMIT_KEY));
    Assert.assertFalse(stateWithNewKeys.contains(DefaultLimiterFactory.EXTRACT_LIMIT_RATE_LIMIT_KEY_DEP));
    Assert.assertTrue(stateWithNewKeys.contains(DefaultLimiterFactory.EXTRACT_LIMIT_RATE_LIMIT_TIMEUNIT_KEY));
    Assert.assertFalse(stateWithNewKeys.contains(DefaultLimiterFactory.EXTRACT_LIMIT_RATE_LIMIT_TIMEUNIT_KEY_DEP));
}

Example 92 with State

use of org.apache.gobblin.configuration.State in project incubator-gobblin by apache.

the class WriterUtilsTest method testGetDataPublisherFinalOutputDir.

@Test
public void testGetDataPublisherFinalOutputDir() {
    State state = new State();
    state.setProp(ConfigurationKeys.DATA_PUBLISHER_FINAL_DIR, TEST_DATA_PUBLISHER_FINAL_DIR);
    state.setProp(ConfigurationKeys.WRITER_FILE_PATH, TEST_WRITER_FILE_PATH);
    Assert.assertEquals(WriterUtils.getDataPublisherFinalDir(state, 0, 0), new Path(TEST_DATA_PUBLISHER_FINAL_DIR, TEST_WRITER_FILE_PATH));
    state.setProp(ConfigurationKeys.DATA_PUBLISHER_FINAL_DIR + ".0", TEST_DATA_PUBLISHER_FINAL_DIR);
    state.setProp(ConfigurationKeys.WRITER_FILE_PATH + ".0", TEST_WRITER_FILE_PATH);
    Assert.assertEquals(WriterUtils.getDataPublisherFinalDir(state, 2, 0), new Path(TEST_DATA_PUBLISHER_FINAL_DIR, TEST_WRITER_FILE_PATH));
    state.setProp(ConfigurationKeys.DATA_PUBLISHER_FINAL_DIR + ".1", TEST_DATA_PUBLISHER_FINAL_DIR);
    state.setProp(ConfigurationKeys.WRITER_FILE_PATH + ".1", TEST_WRITER_FILE_PATH);
    Assert.assertEquals(WriterUtils.getDataPublisherFinalDir(state, 2, 1), new Path(TEST_DATA_PUBLISHER_FINAL_DIR, TEST_WRITER_FILE_PATH));
}

Example 93 with State

use of org.apache.gobblin.configuration.State in project incubator-gobblin by apache.

the class DatasetUtilsTest method testGetDatasetSpecificState.

@Test
public void testGetDatasetSpecificState() {
    String dataset1 = "testDataset1";
    String dataset2 = "testDataset2";
    String dataset3 = "testDataset3";
    String testKey1 = "testKey1";
    String testValue1 = "testValue1";
    SourceState state = new SourceState();
    state.setProp(DatasetUtils.DATASET_SPECIFIC_PROPS, "[{\"dataset\" : \"" + dataset1 + "\", \"" + testKey1 + "\" : \"" + testValue1 + "\"}, {\"dataset\" : \"" + dataset2 + "\", \"" + testKey1 + "\" : \"" + testValue1 + "\"}]");
    Map<String, State> datasetSpecificStateMap = DatasetUtils.getDatasetSpecificProps(Lists.newArrayList(dataset1, dataset3), state);
    State dataset1ExpectedState = new State();
    dataset1ExpectedState.setProp(testKey1, testValue1);
    Assert.assertEquals(datasetSpecificStateMap.get(dataset1), dataset1ExpectedState);
    Assert.assertNull(datasetSpecificStateMap.get(dataset2));
    Assert.assertNull(datasetSpecificStateMap.get(dataset3));
}

Example 94 with State

use of org.apache.gobblin.configuration.State in project incubator-gobblin by apache.

the class DatasetUtilsTest method testGetDatasetSpecificStateWithRegex.

@Test
public void testGetDatasetSpecificStateWithRegex() {
    String dataset1 = "testDataset1";
    String dataset2 = "testDataset2";
    String dataset3 = "otherTestDataset1";
    String testKey1 = "testKey1";
    String testValue1 = "testValue1";
    SourceState state = new SourceState();
    state.setProp(DatasetUtils.DATASET_SPECIFIC_PROPS, "[{\"dataset\" : \"testDataset.*\", \"" + testKey1 + "\" : \"" + testValue1 + "\"}]");
    Map<String, State> datasetSpecificStateMap = DatasetUtils.getDatasetSpecificProps(Lists.newArrayList(dataset1, dataset2, dataset3), state);
    State dataset1ExpectedState = new State();
    dataset1ExpectedState.setProp(testKey1, testValue1);
    State dataset2ExpectedState = new State();
    dataset2ExpectedState.setProp(testKey1, testValue1);
    Assert.assertEquals(datasetSpecificStateMap.get(dataset1), dataset1ExpectedState);
    Assert.assertEquals(datasetSpecificStateMap.get(dataset2), dataset2ExpectedState);
    Assert.assertNull(datasetSpecificStateMap.get(dataset3));
}

Example 95 with State

use of org.apache.gobblin.configuration.State in project incubator-gobblin by apache.

the class AvroHdfsDataWriterTest method setUp.

@BeforeClass
public void setUp() throws Exception {
    // Making the staging and/or output dirs if necessary
    File stagingDir = new File(TestConstants.TEST_STAGING_DIR);
    File outputDir = new File(TestConstants.TEST_OUTPUT_DIR);
    if (!stagingDir.exists()) {
        stagingDir.mkdirs();
    }
    if (!outputDir.exists()) {
        outputDir.mkdirs();
    }
    this.schema = new Schema.Parser().parse(TestConstants.AVRO_SCHEMA);
    this.filePath = TestConstants.TEST_EXTRACT_NAMESPACE.replaceAll("\\.", "/") + "/" + TestConstants.TEST_EXTRACT_TABLE + "/" + TestConstants.TEST_EXTRACT_ID + "_" + TestConstants.TEST_EXTRACT_PULL_TYPE;
    properties = new State();
    properties.setProp(ConfigurationKeys.WRITER_BUFFER_SIZE, ConfigurationKeys.DEFAULT_BUFFER_SIZE);
    properties.setProp(ConfigurationKeys.WRITER_FILE_SYSTEM_URI, TestConstants.TEST_FS_URI);
    properties.setProp(ConfigurationKeys.WRITER_STAGING_DIR, TestConstants.TEST_STAGING_DIR);
    properties.setProp(ConfigurationKeys.WRITER_OUTPUT_DIR, TestConstants.TEST_OUTPUT_DIR);
    properties.setProp(ConfigurationKeys.WRITER_FILE_PATH, this.filePath);
    properties.setProp(ConfigurationKeys.WRITER_FILE_NAME, TestConstants.TEST_FILE_NAME);
    // Build a writer to write test records
    this.writer = new AvroDataWriterBuilder().writeTo(Destination.of(Destination.DestinationType.HDFS, properties)).writeInFormat(WriterOutputFormat.AVRO).withWriterId(TestConstants.TEST_WRITER_ID).withSchema(this.schema).withBranches(1).forBranch(0).build();
}