Search in sources :

Example 26 with TalendRuntimeException

use of org.talend.daikon.exception.TalendRuntimeException in project components by Talend.

the class NormalizeUtils method generateNormalizedRecord.

/**
 * Generate a new Record which contains the normalized value `outputValue`.
 */
public static GenericRecord generateNormalizedRecord(IndexedRecord inputRecord, Schema inputSchema, Schema outputSchema, String[] pathToNormalize, int pathIterator, Object outputValue) {
    GenericRecordBuilder outputRecord = new GenericRecordBuilder(outputSchema);
    for (Schema.Field field : inputSchema.getFields()) {
        if (outputSchema.getField(field.name()) != null) {
            // The column was existing on the input record, we forward it to the output record.
            Object inputValue = inputRecord.get(inputSchema.getField(field.name()).pos());
            if ((pathToNormalize.length > 0) && (pathIterator < pathToNormalize.length) && (field.name().equals(pathToNormalize[pathIterator]))) {
                // if we are on a object or a list (aka the element to normalize), we save it to the output.
                if (inputValue instanceof GenericData.Record) {
                    // use recursivity to reach the element to normalize
                    Schema inputChildSchema = getChildSchemaAsRecord(inputSchema, field);
                    Schema outputChildSchema = getChildSchemaAsRecord(outputSchema, field);
                    Object childRecord = generateNormalizedRecord((IndexedRecord) inputValue, inputChildSchema, outputChildSchema, pathToNormalize, ++pathIterator, outputValue);
                    outputRecord.set(field.name(), childRecord);
                } else if (inputValue instanceof List) {
                    // not available.
                    if (pathIterator == pathToNormalize.length - 1) {
                        outputRecord.set(field.name(), outputValue);
                    } else {
                        throw new TalendRuntimeException(CommonErrorCodes.UNEXPECTED_ARGUMENT, new Throwable(String.format("You cannot reach element to normalize inside a list.")));
                    }
                } else {
                    // Reach the element to normalize => duplicate it.
                    if (pathIterator == pathToNormalize.length - 1) {
                        outputRecord.set(field.name(), outputValue);
                    } else {
                        // it should be done on duplicateRecord.
                        throw new TalendRuntimeException(CommonErrorCodes.UNEXPECTED_ARGUMENT, new Throwable(String.format("Accessing to leaf element of the field %s should be done on duplicate record", field.name())));
                    }
                }
            } else {
                // We just have to duplicate everything.
                if (inputValue instanceof GenericData.Record) {
                    // Use a specific function to duplicate elements.
                    Schema inputChildSchema = getChildSchemaAsRecord(inputSchema, field);
                    Schema outputChildSchema = getChildSchemaAsRecord(outputSchema, field);
                    Object childRecord = duplicateRecord((IndexedRecord) inputValue, inputChildSchema, outputChildSchema);
                    outputRecord.set(field.name(), childRecord);
                } else {
                    outputRecord.set(field.name(), inputValue);
                }
            }
        } else {
            throw new TalendRuntimeException(CommonErrorCodes.UNEXPECTED_ARGUMENT, new Throwable(String.format("The field %s is not present on the input schema", field.name())));
        }
    }
    return outputRecord.build();
}
Also used : TalendRuntimeException(org.talend.daikon.exception.TalendRuntimeException) Schema(org.apache.avro.Schema) GenericRecordBuilder(org.apache.avro.generic.GenericRecordBuilder) GenericRecord(org.apache.avro.generic.GenericRecord) IndexedRecord(org.apache.avro.generic.IndexedRecord) List(java.util.List) ArrayList(java.util.ArrayList)

Example 27 with TalendRuntimeException

use of org.talend.daikon.exception.TalendRuntimeException in project components by Talend.

the class NormalizeUtils method getInputFields.

/**
 * Get the sub-fields from inputRecord of the field columnName.
 *
 * @return list contains the sub-fields from inputRecord of the field columnName
 */
public static List<Object> getInputFields(IndexedRecord inputRecord, String columnName) {
    ArrayList<Object> inputFields = new ArrayList<Object>();
    String[] path = columnName.split("\\.");
    Schema schema = inputRecord.getSchema();
    for (Integer i = 0; i < path.length; i++) {
        // The column was existing on the input record, we forward it to the output record.
        if (schema.getField(path[i]) == null) {
            throw new TalendRuntimeException(CommonErrorCodes.UNEXPECTED_ARGUMENT, new Throwable(String.format("The field %s is not present on the input record", columnName)));
        }
        Object inputValue = inputRecord.get(schema.getField(path[i]).pos());
        // or directly a value.
        if (inputValue instanceof GenericData.Record) {
            // If we are on a record, we need to recursively do the process
            inputRecord = (IndexedRecord) inputValue;
            // record, so we need to get the true sub-schema
            if (schema.getField(path[i]).schema().getType().equals(Schema.Type.RECORD)) {
                schema = schema.getField(path[i]).schema();
                if (i == path.length - 1) {
                    inputFields.add(inputValue);
                }
            } else if (schema.getField(path[i]).schema().getType().equals(Schema.Type.UNION)) {
                if (i == path.length - 1) {
                    inputFields.add(inputValue);
                    break;
                }
                for (Schema childSchema : schema.getField(path[i]).schema().getTypes()) {
                    if (childSchema.getType().equals(Schema.Type.RECORD)) {
                        schema = childSchema;
                        break;
                    }
                }
            }
        } else if (inputValue instanceof List) {
            for (int j = 0; j < ((List) inputValue).size(); j++) {
                inputFields.add(((List) inputValue).get(j));
            }
            break;
        } else {
            if (i == path.length - 1) {
                inputFields.add(inputValue);
            }
        }
    }
    return inputFields;
}
Also used : TalendRuntimeException(org.talend.daikon.exception.TalendRuntimeException) Schema(org.apache.avro.Schema) ArrayList(java.util.ArrayList) GenericRecord(org.apache.avro.generic.GenericRecord) IndexedRecord(org.apache.avro.generic.IndexedRecord) List(java.util.List) ArrayList(java.util.ArrayList)

Example 28 with TalendRuntimeException

use of org.talend.daikon.exception.TalendRuntimeException in project components by Talend.

the class TSalesforceInputProperties method validateGuessSchema.

public ValidationResult validateGuessSchema() {
    ValidationResultMutable validationResult = new ValidationResultMutable();
    try (SandboxedInstance sandboxedInstance = getSandboxedInstance(SOURCE_OR_SINK_CLASS)) {
        SalesforceRuntimeSourceOrSink salesforceSourceOrSink = (SalesforceRuntimeSourceOrSink) sandboxedInstance.getInstance();
        salesforceSourceOrSink.initialize(null, this);
        Schema schema = ((SalesforceSchemaHelper<Schema>) salesforceSourceOrSink).guessSchema(query.getValue());
        module.main.schema.setValue(schema);
        validationResult.setStatus(ValidationResult.Result.OK);
    } catch (TalendRuntimeException tre) {
        String errorMessage = getI18nMessage("errorMessage.validateGuessSchemaSoqlError", tre.getMessage());
        validationResult.setStatus(ValidationResult.Result.ERROR).setMessage(errorMessage);
    } catch (RuntimeException e1) {
        String errorMessage = getI18nMessage("errorMessage.validateGuessSchemaRuntimeError", e1.getMessage());
        validationResult.setStatus(ValidationResult.Result.ERROR).setMessage(errorMessage);
    } catch (IOException e2) {
        String errorMessage = getI18nMessage("errorMessage.validateGuessSchemaConnectionError", e2.getMessage());
        validationResult.setStatus(ValidationResult.Result.ERROR).setMessage(errorMessage);
    }
    return validationResult;
}
Also used : SandboxedInstance(org.talend.daikon.sandbox.SandboxedInstance) SalesforceDefinition.getSandboxedInstance(org.talend.components.salesforce.SalesforceDefinition.getSandboxedInstance) TalendRuntimeException(org.talend.daikon.exception.TalendRuntimeException) SalesforceSchemaHelper(org.talend.components.salesforce.schema.SalesforceSchemaHelper) TalendRuntimeException(org.talend.daikon.exception.TalendRuntimeException) Schema(org.apache.avro.Schema) ValidationResultMutable(org.talend.daikon.properties.ValidationResultMutable) SalesforceRuntimeSourceOrSink(org.talend.components.salesforce.common.SalesforceRuntimeSourceOrSink) IOException(java.io.IOException)

Aggregations

TalendRuntimeException (org.talend.daikon.exception.TalendRuntimeException)28 IndexedRecord (org.apache.avro.generic.IndexedRecord)10 IOException (java.io.IOException)8 Test (org.junit.Test)7 Pipeline (org.apache.beam.sdk.Pipeline)6 Schema (org.apache.avro.Schema)5 Path (org.apache.hadoop.fs.Path)5 ConvertToIndexedRecord (org.talend.components.adapter.beam.transform.ConvertToIndexedRecord)5 ArrayList (java.util.ArrayList)4 List (java.util.List)4 SimpleFileIOOutputProperties (org.talend.components.simplefileio.output.SimpleFileIOOutputProperties)4 BufferedWriter (java.io.BufferedWriter)3 OutputStream (java.io.OutputStream)3 OutputStreamWriter (java.io.OutputStreamWriter)3 ObjectMapper (com.fasterxml.jackson.databind.ObjectMapper)2 DateTimeException (java.time.DateTimeException)2 LocalDateTime (java.time.LocalDateTime)2 GenericRecord (org.apache.avro.generic.GenericRecord)2 StringUtils (org.apache.commons.lang.StringUtils)2 SandboxedInstance (org.talend.daikon.sandbox.SandboxedInstance)2