Examples with AvroJobSpec org.apache.gobblin.runtime.job_spec.AvroJobSpec used on opensource projects

Example 1 with AvroJobSpec

use of org.apache.gobblin.runtime.job_spec.AvroJobSpec in project incubator-gobblin by apache.

the class AvroJobSpecKafkaJobMonitor method parseJobSpec.

/**
 * Creates a {@link JobSpec} or {@link URI} from the {@link AvroJobSpec} record.
 * @param record the record as an {@link AvroJobSpec}
 * @return a {@link JobSpec} or {@link URI} wrapped in a {@link Collection} of {@link Either}
 */
@Override
public Collection<Either<JobSpec, URI>> parseJobSpec(AvroJobSpec record) {
    JobSpec.Builder jobSpecBuilder = JobSpec.builder(record.getUri());
    Properties props = new Properties();
    props.putAll(record.getProperties());
    jobSpecBuilder.withJobCatalogURI(record.getUri()).withVersion(record.getVersion()).withDescription(record.getDescription()).withConfigAsProperties(props);
    if (!record.getTemplateUri().isEmpty()) {
        try {
            jobSpecBuilder.withTemplate(new URI(record.getTemplateUri()));
        } catch (URISyntaxException e) {
            log.error("could not parse template URI " + record.getTemplateUri());
        }
    }
    String verbName = record.getMetadata().get(VERB_KEY);
    Verb verb = Verb.valueOf(verbName);
    JobSpec jobSpec = jobSpecBuilder.build();
    log.info("Parsed job spec " + jobSpec.toString());
    if (verb == Verb.ADD || verb == Verb.UPDATE) {
        return Lists.newArrayList(Either.<JobSpec, URI>left(jobSpec));
    } else {
        return Lists.newArrayList(Either.<JobSpec, URI>right(jobSpec.getUri()));
    }
}

Example 2 with AvroJobSpec

use of org.apache.gobblin.runtime.job_spec.AvroJobSpec in project incubator-gobblin by apache.

the class SimpleKafkaSpecProducer method deleteSpec.

@Override
public Future<?> deleteSpec(URI deletedSpecURI) {
    AvroJobSpec avroJobSpec = AvroJobSpec.newBuilder().setUri(deletedSpecURI.toString()).setMetadata(ImmutableMap.of(VERB_KEY, SpecExecutor.Verb.DELETE.name())).build();
    log.info("Deleting Spec: " + deletedSpecURI + " using Kafka.");
    return getKafkaProducer().write(_serializer.serializeRecord(avroJobSpec), WriteCallback.EMPTY);
}

Example 3 with AvroJobSpec

use of org.apache.gobblin.runtime.job_spec.AvroJobSpec in project incubator-gobblin by apache.

the class SimpleKafkaSpecProducer method convertToAvroJobSpec.

private AvroJobSpec convertToAvroJobSpec(Spec spec, SpecExecutor.Verb verb) {
    if (spec instanceof JobSpec) {
        JobSpec jobSpec = (JobSpec) spec;
        AvroJobSpec.Builder avroJobSpecBuilder = AvroJobSpec.newBuilder();
        avroJobSpecBuilder.setUri(jobSpec.getUri().toString()).setVersion(jobSpec.getVersion()).setDescription(jobSpec.getDescription()).setProperties(Maps.fromProperties(jobSpec.getConfigAsProperties())).setMetadata(ImmutableMap.of(VERB_KEY, verb.name()));
        if (jobSpec.getTemplateURI().isPresent()) {
            avroJobSpecBuilder.setTemplateUri(jobSpec.getTemplateURI().get().toString());
        }
        return avroJobSpecBuilder.build();
    } else {
        throw new RuntimeException("Unsupported spec type " + spec.getClass());
    }
}

Example 4 with AvroJobSpec

use of org.apache.gobblin.runtime.job_spec.AvroJobSpec in project incubator-gobblin by apache.

the class AvroJobSpecDeserializer method configure.

@Override
public void configure(Map<String, ?> configs, boolean isKey) {
    InputStream dummyInputStream = new ByteArrayInputStream(new byte[0]);
    _decoder = DecoderFactory.get().binaryDecoder(dummyInputStream, null);
    _reader = new SpecificDatumReader<AvroJobSpec>(AvroJobSpec.SCHEMA$);
    _versionWriter = new FixedSchemaVersionWriter();
}

Example 5 with AvroJobSpec

use of org.apache.gobblin.runtime.job_spec.AvroJobSpec in project incubator-gobblin by apache.

the class SimpleKafkaSpecConsumer method changedSpecs.

@Override
public Future<? extends List<Pair<SpecExecutor.Verb, Spec>>> changedSpecs() {
    List<Pair<SpecExecutor.Verb, Spec>> changesSpecs = new ArrayList<>();
    initializeWatermarks();
    this.currentPartitionIdx = -1;
    while (!allPartitionsFinished()) {
        if (currentPartitionFinished()) {
            moveToNextPartition();
            continue;
        }
        if (this.messageIterator == null || !this.messageIterator.hasNext()) {
            try {
                this.messageIterator = fetchNextMessageBuffer();
            } catch (Exception e) {
                log.error(String.format("Failed to fetch next message buffer for partition %s. Will skip this partition.", getCurrentPartition()), e);
                moveToNextPartition();
                continue;
            }
            if (this.messageIterator == null || !this.messageIterator.hasNext()) {
                moveToNextPartition();
                continue;
            }
        }
        while (!currentPartitionFinished()) {
            if (!this.messageIterator.hasNext()) {
                break;
            }
            KafkaConsumerRecord nextValidMessage = this.messageIterator.next();
            // until we get to x.
            if (nextValidMessage.getOffset() < _nextWatermark.get(this.currentPartitionIdx)) {
                continue;
            }
            _nextWatermark.set(this.currentPartitionIdx, nextValidMessage.getNextOffset());
            try {
                final AvroJobSpec record;
                if (nextValidMessage instanceof ByteArrayBasedKafkaRecord) {
                    record = decodeRecord((ByteArrayBasedKafkaRecord) nextValidMessage);
                } else if (nextValidMessage instanceof DecodeableKafkaRecord) {
                    record = ((DecodeableKafkaRecord<?, AvroJobSpec>) nextValidMessage).getValue();
                } else {
                    throw new IllegalStateException("Unsupported KafkaConsumerRecord type. The returned record can either be ByteArrayBasedKafkaRecord" + " or DecodeableKafkaRecord");
                }
                JobSpec.Builder jobSpecBuilder = JobSpec.builder(record.getUri());
                Properties props = new Properties();
                props.putAll(record.getProperties());
                jobSpecBuilder.withJobCatalogURI(record.getUri()).withVersion(record.getVersion()).withDescription(record.getDescription()).withConfigAsProperties(props);
                if (!record.getTemplateUri().isEmpty()) {
                    jobSpecBuilder.withTemplate(new URI(record.getTemplateUri()));
                }
                String verbName = record.getMetadata().get(VERB_KEY);
                SpecExecutor.Verb verb = SpecExecutor.Verb.valueOf(verbName);
                changesSpecs.add(new ImmutablePair<SpecExecutor.Verb, Spec>(verb, jobSpecBuilder.build()));
            } catch (Throwable t) {
                log.error("Could not decode record at partition " + this.currentPartitionIdx + " offset " + nextValidMessage.getOffset());
            }
        }
    }
    return new CompletedFuture(changesSpecs, null);
}