Examples with Extractor org.apache.gobblin.source.extractor.Extractor used on opensource projects

Example 1 with Extractor

use of org.apache.gobblin.source.extractor.Extractor in project incubator-gobblin by apache.

the class SequentialTestSource method getExtractor.

@Override
public Extractor<String, Object> getExtractor(WorkUnitState state) throws IOException {
    Config config = ConfigFactory.parseProperties(state.getProperties());
    configureIfNeeded(config);
    final LongWatermark lowWatermark = state.getWorkunit().getLowWatermark(LongWatermark.class);
    final WorkUnitState workUnitState = state;
    final int index = state.getPropAsInt(WORK_UNIT_INDEX);
    final TestBatchExtractor extractor = new TestBatchExtractor(index, lowWatermark, numRecordsPerExtract, sleepTimePerRecord, workUnitState);
    if (!streaming) {
        return extractor;
    } else {
        return (Extractor) new TestStreamingExtractor(extractor);
    }
}

Example 2 with Extractor

use of org.apache.gobblin.source.extractor.Extractor in project incubator-gobblin by apache.

the class GoogleDriveSourceTest method testGetExtractor.

public void testGetExtractor() throws IOException {
    @SuppressWarnings("rawtypes") GoogleDriveSource source = new GoogleDriveSource<>();
    GoogleDriveFsHelper fsHelper = mock(GoogleDriveFsHelper.class);
    source.fsHelper = fsHelper;
    Extractor extractor = source.getExtractor(new WorkUnitState());
    Assert.assertTrue(extractor instanceof GoogleDriveExtractor);
}

Example 3 with Extractor

use of org.apache.gobblin.source.extractor.Extractor in project incubator-gobblin by apache.

the class LimiterStopEventTest method testGetLimiterStopMetadataCase0.

@Test
public void testGetLimiterStopMetadataCase0() throws InterruptedException {
    Properties properties = new Properties();
    String key1 = "topic";
    String key2 = "partition.id";
    String key3 = "others";
    String keyList = Joiner.on(',').join(key1, key2);
    properties.setProperty(LimiterConfigurationKeys.LIMITER_REPORT_KEY_LIST, keyList);
    properties.setProperty(key1, "1111");
    properties.setProperty(key2, "1111");
    Extractor extractor = mock(Extractor.class);
    Limiter limiter = mock(Limiter.class);
    TaskState taskState = mock(TaskState.class);
    WorkUnit workUnit = mock(WorkUnit.class);
    Mockito.when(taskState.getWorkunit()).thenReturn(workUnit);
    Mockito.when(taskState.getJobId()).thenReturn("123");
    Mockito.when(taskState.getTaskAttemptId()).thenReturn(Optional.of("555"));
    Mockito.when(taskState.getTaskId()).thenReturn("888");
    Mockito.when(limiter.acquirePermits(1)).thenReturn(null);
    Mockito.when(taskState.getProp(ConfigurationKeys.DATASET_URN_KEY, ConfigurationKeys.DEFAULT_DATASET_URN)).thenReturn("file://xyz");
    Mockito.when(workUnit.getProperties()).thenReturn(properties);
    LimitingExtractorDecorator<String, String> decorator = new LimitingExtractorDecorator<>(extractor, limiter, taskState);
    try {
        Method method = LimitingExtractorDecorator.class.getDeclaredMethod("getLimiterStopMetadata");
        method.setAccessible(true);
        ImmutableMap<String, String> metaData = (ImmutableMap<String, String>) method.invoke(decorator);
        Assert.assertEquals(metaData.containsKey(key1), true);
        Assert.assertEquals(metaData.containsKey(key2), true);
        Assert.assertEquals(metaData.containsKey(key3), false);
    } catch (Exception e) {
        Assert.fail();
    }
}

Example 4 with Extractor

use of org.apache.gobblin.source.extractor.Extractor in project incubator-gobblin by apache.

the class LimiterStopEventTest method testGetLimiterStopMetadataCase3.

@Test
public void testGetLimiterStopMetadataCase3() throws InterruptedException {
    Properties properties = new Properties();
    String key1 = "topic";
    String key2 = "partition.id";
    String keyList = Joiner.on(',').join(key1, key2);
    String subKey1 = key2 + "....";
    String subKey2 = key2 + "##fjpaierbng;";
    String subKey3 = key2 + "x[n  sdf";
    String subKey4 = key2 + "";
    properties.setProperty(LimiterConfigurationKeys.LIMITER_REPORT_KEY_LIST, keyList);
    properties.setProperty(subKey1, "1111");
    properties.setProperty(subKey2, "1111");
    properties.setProperty(subKey3, "1111");
    properties.setProperty(subKey4, "1111");
    properties.setProperty(key1, "1111");
    properties.setProperty(key2, "1111");
    Extractor extractor = mock(Extractor.class);
    Limiter limiter = mock(Limiter.class);
    TaskState taskState = mock(TaskState.class);
    WorkUnit workUnit = mock(WorkUnit.class);
    Mockito.when(taskState.getWorkunit()).thenReturn(workUnit);
    Mockito.when(taskState.getJobId()).thenReturn("123");
    Mockito.when(taskState.getTaskAttemptId()).thenReturn(Optional.of("555"));
    Mockito.when(taskState.getTaskId()).thenReturn("888");
    Mockito.when(limiter.acquirePermits(1)).thenReturn(null);
    Mockito.when(taskState.getProp(ConfigurationKeys.DATASET_URN_KEY, ConfigurationKeys.DEFAULT_DATASET_URN)).thenReturn("file://xyz");
    Mockito.when(workUnit.getProperties()).thenReturn(properties);
    LimitingExtractorDecorator<String, String> decorator = new LimitingExtractorDecorator<>(extractor, limiter, taskState);
    try {
        Method method = LimitingExtractorDecorator.class.getDeclaredMethod("getLimiterStopMetadata");
        method.setAccessible(true);
        ImmutableMap<String, String> metaData = (ImmutableMap<String, String>) method.invoke(decorator);
        Assert.assertEquals(metaData.containsKey(key1), true);
        Assert.assertEquals(metaData.containsKey(key2), true);
        Assert.assertEquals(metaData.containsKey(subKey1), true);
        Assert.assertEquals(metaData.containsKey(subKey2), true);
        Assert.assertEquals(metaData.containsKey(subKey3), true);
        Assert.assertEquals(metaData.containsKey(subKey4), true);
    } catch (Exception e) {
        Assert.fail();
    }
}

Example 5 with Extractor

use of org.apache.gobblin.source.extractor.Extractor in project incubator-gobblin by apache.

the class LimiterStopEventTest method testGetLimiterStopMetadataCase1.

@Test
public void testGetLimiterStopMetadataCase1() throws InterruptedException {
    Properties properties = new Properties();
    String key1 = "topic";
    String key2 = "partition.id";
    String keyList = Joiner.on(',').join(key1, key2);
    String subKey1 = key2 + ".0";
    String subKey2 = key2 + ".1";
    String subKey3 = key2 + ".2";
    String subKey4 = key2 + ".3";
    String subKey5 = "partition";
    properties.setProperty(LimiterConfigurationKeys.LIMITER_REPORT_KEY_LIST, keyList);
    properties.setProperty(subKey1, "1111");
    properties.setProperty(subKey2, "1111");
    properties.setProperty(subKey3, "1111");
    properties.setProperty(subKey4, "1111");
    Extractor extractor = mock(Extractor.class);
    Limiter limiter = mock(Limiter.class);
    TaskState taskState = mock(TaskState.class);
    WorkUnit workUnit = mock(WorkUnit.class);
    Mockito.when(taskState.getWorkunit()).thenReturn(workUnit);
    Mockito.when(taskState.getJobId()).thenReturn("123");
    Mockito.when(taskState.getTaskAttemptId()).thenReturn(Optional.of("555"));
    Mockito.when(taskState.getTaskId()).thenReturn("888");
    Mockito.when(limiter.acquirePermits(1)).thenReturn(null);
    Mockito.when(taskState.getProp(ConfigurationKeys.DATASET_URN_KEY, ConfigurationKeys.DEFAULT_DATASET_URN)).thenReturn("file://xyz");
    Mockito.when(workUnit.getProperties()).thenReturn(properties);
    LimitingExtractorDecorator<String, String> decorator = new LimitingExtractorDecorator<>(extractor, limiter, taskState);
    try {
        Method method = LimitingExtractorDecorator.class.getDeclaredMethod("getLimiterStopMetadata");
        method.setAccessible(true);
        ImmutableMap<String, String> metaData = (ImmutableMap<String, String>) method.invoke(decorator);
        Assert.assertEquals(metaData.containsKey(subKey1), true);
        Assert.assertEquals(metaData.containsKey(subKey2), true);
        Assert.assertEquals(metaData.containsKey(subKey3), true);
        Assert.assertEquals(metaData.containsKey(subKey4), true);
        Assert.assertEquals(metaData.containsKey(subKey5), false);
    } catch (Exception e) {
        Assert.fail();
    }
}