Examples with Either org.apache.gobblin.util.Either used on opensource projects

Example 6 with Either

use of org.apache.gobblin.util.Either in project incubator-gobblin by apache.

the class MockedKafkaJobMonitor method parseJobSpec.

@Override
public Collection<Either<JobSpec, URI>> parseJobSpec(byte[] message) throws IOException {
    try {
        String messageString = new String(message, Charsets.UTF_8);
        List<Either<JobSpec, URI>> jobSpecs = Lists.newArrayList();
        for (String oneInstruction : SPLITTER_COMMA.split(messageString)) {
            List<String> tokens = SPLITTER_COLON.splitToList(oneInstruction);
            if (tokens.get(0).equals(REMOVE)) {
                URI uri = new URI(tokens.get(1));
                jobSpecs.add(Either.<JobSpec, URI>right(uri));
            } else {
                URI uri = new URI(tokens.get(0));
                String version = tokens.get(1);
                JobSpec jobSpec = new JobSpec.Builder(uri).withConfig(ConfigFactory.empty()).withVersion(version).build();
                jobSpecs.add(Either.<JobSpec, URI>left(jobSpec));
            }
        }
        return jobSpecs;
    } catch (URISyntaxException use) {
        throw new IOException(use);
    }
}

Example 7 with Either

use of org.apache.gobblin.util.Either in project incubator-gobblin by apache.

the class SLAEventKafkaJobMonitorTest method testParseJobSpec.

@Test
public void testParseJobSpec() throws Exception {
    SLAEventKafkaJobMonitor monitor = new SLAEventKafkaJobMonitor("topic", null, new URI("/base/URI"), HighLevelConsumerTest.getSimpleConfig(Optional.of(KafkaJobMonitor.KAFKA_JOB_MONITOR_PREFIX)), new NoopSchemaVersionWriter(), Optional.<Pattern>absent(), Optional.<Pattern>absent(), this.templateURI, ImmutableMap.of("metadataKey1", "key1"));
    monitor.buildMetricsContextAndMetrics();
    GobblinTrackingEvent event = createSLAEvent("DatasetPublish", new URI("/data/myDataset"), ImmutableMap.of("metadataKey1", "value1", "key1", "value2"));
    Collection<Either<JobSpec, URI>> jobSpecs = monitor.parseJobSpec(event);
    Assert.assertEquals(jobSpecs.size(), 1);
    JobSpec jobSpec = (JobSpec) jobSpecs.iterator().next().get();
    Assert.assertEquals(jobSpec.getUri(), new URI("/base/URI/data/myDataset"));
    Assert.assertEquals(jobSpec.getTemplateURI().get(), templateURI);
    // should insert configuration from metadata
    Assert.assertEquals(jobSpec.getConfig().getString("key1"), "value1");
    monitor.shutdownMetrics();
}

Example 8 with Either

use of org.apache.gobblin.util.Either in project incubator-gobblin by apache.

the class SLAEventKafkaJobMonitorTest method testFilterByName.

@Test
public void testFilterByName() throws Exception {
    SLAEventKafkaJobMonitor monitor = new SLAEventKafkaJobMonitor("topic", null, new URI("/base/URI"), HighLevelConsumerTest.getSimpleConfig(Optional.of(KafkaJobMonitor.KAFKA_JOB_MONITOR_PREFIX)), new NoopSchemaVersionWriter(), Optional.<Pattern>absent(), Optional.of(Pattern.compile("^accept.*")), this.templateURI, ImmutableMap.<String, String>of());
    monitor.buildMetricsContextAndMetrics();
    GobblinTrackingEvent event;
    Collection<Either<JobSpec, URI>> jobSpecs;
    event = createSLAEvent("acceptthis", new URI("/data/myDataset"), Maps.<String, String>newHashMap());
    jobSpecs = monitor.parseJobSpec(event);
    Assert.assertEquals(jobSpecs.size(), 1);
    Assert.assertEquals(monitor.getRejectedEvents().getCount(), 0);
    event = createSLAEvent("donotacceptthis", new URI("/data/myDataset"), Maps.<String, String>newHashMap());
    jobSpecs = monitor.parseJobSpec(event);
    Assert.assertEquals(jobSpecs.size(), 0);
    Assert.assertEquals(monitor.getRejectedEvents().getCount(), 1);
    monitor.shutdownMetrics();
}

Example 9 with Either

use of org.apache.gobblin.util.Either in project incubator-gobblin by apache.

the class IteratorExecutor method logAndThrowFailures.

/**
 * Log failures in the output of {@link #executeAndGetResults()}, and also propagate exception to upper layer.
 * @param results output of {@link #executeAndGetResults()}
 * @param useLogger logger to log the messages into.
 * @param atMost will log at most this many errors.
 */
public static <T> void logAndThrowFailures(List<Either<T, ExecutionException>> results, Logger useLogger, int atMost) {
    Logger actualLogger = useLogger == null ? log : useLogger;
    Iterator<Either<T, ExecutionException>> it = results.iterator();
    int printed = 0;
    ExecutionException exc = null;
    while (it.hasNext()) {
        Either<T, ExecutionException> nextResult = it.next();
        if (nextResult instanceof Either.Right) {
            exc = ((Either.Right<T, ExecutionException>) nextResult).getRight();
            actualLogger.error("Iterator executor failure.", exc);
            printed++;
            if (printed >= atMost) {
                break;
            }
        }
    }
    /**
     * Throw any exception that Executor ran into.
     */
    if (printed > 0) {
        throw new RuntimeException(exc);
    }
}

Example 10 with Either

use of org.apache.gobblin.util.Either in project incubator-gobblin by apache.

the class KafkaAvroJobMonitorTest method testSimple.

@Test
public void testSimple() throws Exception {
    TestKafkaAvroJobMonitor monitor = new TestKafkaAvroJobMonitor(GobblinTrackingEvent.SCHEMA$, new NoopSchemaVersionWriter());
    monitor.buildMetricsContextAndMetrics();
    AvroSerializer<GobblinTrackingEvent> serializer = new AvroBinarySerializer<>(GobblinTrackingEvent.SCHEMA$, new NoopSchemaVersionWriter());
    GobblinTrackingEvent event = new GobblinTrackingEvent(0L, "namespace", "event", Maps.<String, String>newHashMap());
    Collection<Either<JobSpec, URI>> results = monitor.parseJobSpec(serializer.serializeRecord(event));
    Assert.assertEquals(results.size(), 1);
    Assert.assertEquals(monitor.events.size(), 1);
    Assert.assertEquals(monitor.events.get(0), event);
    monitor.shutdownMetrics();
}