Examples with OTransformer com.orientechnologies.orient.etl.transformer.OTransformer used on opensource projects

Example 1 with OTransformer

use of com.orientechnologies.orient.etl.transformer.OTransformer in project orientdb by orientechnologies.

the class OETLProcessor method end.

protected void end() {
    for (OTransformer t : transformers) t.end();
    if (source != null)
        source.end();
    extractor.end();
    loader.end();
    for (OBlock t : endBlocks) {
        t.begin();
        t.execute();
        t.end();
    }
    elapsed = System.currentTimeMillis() - startTime;
    if (dumpTask != null) {
        dumpTask.cancel();
    }
    out(LOG_LEVELS.INFO, "END ETL PROCESSOR");
    dumpProgress();
}

Example 2 with OTransformer

use of com.orientechnologies.orient.etl.transformer.OTransformer in project orientdb by orientechnologies.

the class OETLPipeline method execute.

protected Object execute(final OExtractedItem source) {
    int retry = 0;
    do {
        try {
            Object current = source.payload;
            context.setVariable("extractedNum", source.num);
            context.setVariable("extractedPayload", source.payload);
            for (OTransformer t : transformers) {
                current = t.transform(current);
                if (current == null) {
                    if (logLevel == DEBUG) {
                        OLogManager.instance().warn(this, "Transformer [%s] returned null, skip rest of pipeline execution", t);
                        break;
                    }
                }
            }
            if (current != null)
                // LOAD
                loader.load(this, current, context);
            return current;
        } catch (ONeedRetryException e) {
            loader.rollback(this);
            retry++;
            processor.out(INFO, "Error in pipeline execution, retry = %d/%d (exception=%s)", retry, maxRetries, e);
        } catch (OETLProcessHaltedException e) {
            processor.out(ERROR, "Pipeline execution halted");
            processor.getStats().incrementErrors();
            loader.rollback(this);
            throw e;
        } catch (Exception e) {
            processor.out(ERROR, "Error in Pipeline execution: %s", e);
            processor.getStats().incrementErrors();
            if (!haltOnError) {
                return null;
            }
            loader.rollback(this);
            throw OException.wrapException(new OETLProcessHaltedException("Halt"), e);
        }
    } while (retry < maxRetries);
    return this;
}

Example 3 with OTransformer

use of com.orientechnologies.orient.etl.transformer.OTransformer in project orientdb by orientechnologies.

the class OETLProcessor method analyzeFlow.

protected void analyzeFlow() {
    if (extractor == null)
        throw new OConfigurationException("extractor is null");
    if (loader == null)
        throw new OConfigurationException("loader is null");
    OETLComponent lastComponent = extractor;
    for (OTransformer t : transformers) {
        checkTypeCompatibility(t, lastComponent);
        lastComponent = t;
    }
    checkTypeCompatibility(loader, lastComponent);
}

Example 4 with OTransformer

use of com.orientechnologies.orient.etl.transformer.OTransformer in project orientdb by orientechnologies.

the class OETLProcessor method configureTransformers.

private void configureTransformers(Collection<ODocument> iTransformers, OCommandContext iContext) throws IllegalAccessException, InstantiationException {
    transformers = new ArrayList<OTransformer>();
    if (iTransformers != null) {
        for (ODocument t : iTransformers) {
            String name = t.fieldNames()[0];
            final OTransformer tr = factory.getTransformer(name);
            transformers.add(tr);
            configureComponent(tr, t.<ODocument>field(name), iContext);
        }
    }
}