Examples with Record org.apache.avro.generic.GenericData.Record used on opensource projects

Example 31 with Record

use of org.apache.avro.generic.GenericData.Record in project components by Talend.

the class MarketoSOAPClient method convertLeadActivityRecords.

private List<IndexedRecord> convertLeadActivityRecords(List<ActivityRecord> activityRecords, Schema schema, Map<String, String> mappings) {
    List<IndexedRecord> results = new ArrayList<>();
    for (ActivityRecord input : activityRecords) {
        IndexedRecord record = new Record(schema);
        for (Field f : schema.getFields()) {
            // find matching marketo column name
            String col = mappings.get(f.name());
            if (col == null) {
                LOG.warn("[convertLeadActivityRecords] Couldn't find mapping for column {}.", f.name());
                continue;
            }
            switch(col) {
                case FIELD_ID:
                    record.put(f.pos(), input.getId() != null ? input.getId().getValue() : null);
                    break;
                case FIELD_MARKETO_GUID:
                    record.put(f.pos(), input.getMarketoGUID());
                    break;
                case FIELD_ACTIVITY_DATE_TIME:
                    record.put(f.pos(), input.getActivityDateTime() != null ? input.getActivityDateTime().toGregorianCalendar().getTimeInMillis() : null);
                    break;
                case FIELD_ACTIVITY_TYPE:
                    record.put(f.pos(), input.getActivityType());
                    break;
                case FIELD_MKTG_ASSET_NAME:
                    record.put(f.pos(), input.getMktgAssetName());
                    break;
                case FIELD_MKT_PERSON_ID:
                    record.put(f.pos(), input.getMktPersonId());
                    break;
                case FIELD_CAMPAIGN:
                    record.put(f.pos(), input.getCampaign() != null ? input.getCampaign().getValue() : null);
                    break;
                case FIELD_FOREIGN_SYS_ID:
                    record.put(f.pos(), input.getForeignSysId() != null ? input.getForeignSysId().getValue() : null);
                    break;
                case FIELD_PERSON_NAME:
                    record.put(f.pos(), input.getPersonName() != null ? input.getPersonName().getValue() : null);
                    break;
                case FIELD_ORG_NAME:
                    record.put(f.pos(), input.getOrgName() != null ? input.getOrgName().getValue() : null);
                    break;
                case FIELD_FOREIGN_SYS_ORG_ID:
                    record.put(f.pos(), input.getForeignSysOrgId() != null ? input.getForeignSysOrgId().getValue() : null);
                    break;
                default:
                    if (!input.getActivityAttributes().isNil()) {
                        for (Attribute attr : input.getActivityAttributes().getValue().getAttributes()) {
                            if (attr.getAttrName().equals(col)) {
                                record.put(f.pos(), attr.getAttrValue());
                            }
                        }
                    }
            }
        }
        results.add(record);
    }
    return results;
}

Example 32 with Record

use of org.apache.avro.generic.GenericData.Record in project components by Talend.

the class MarketoOutputWriter method fillRecord.

public IndexedRecord fillRecord(SyncStatus status, Schema schema, IndexedRecord record) {
    Boolean isDynamic = Boolean.FALSE;
    Schema currentSchema = schema;
    if (AvroUtils.isIncludeAllFields(schema)) {
        isDynamic = true;
        if (dynamicSchema == null) {
            dynamicSchema = MarketoSourceOrSink.mergeDynamicSchemas(record.getSchema(), schema);
        }
        currentSchema = dynamicSchema;
    }
    IndexedRecord outRecord = new Record(currentSchema);
    for (Field f : currentSchema.getFields()) {
        switch(f.name()) {
            case FIELD_LEAD_ID:
            case FIELD_ID_SOAP:
            case FIELD_ID_REST:
            case FIELD_CAMPAIGN_ID:
                // when the request failed, get it from input record
                if (status.getId() == null) {
                    try {
                        outRecord.put(currentSchema.getField(f.name()).pos(), record.get(inputSchema.getField(f.name()).pos()));
                    } catch (NullPointerException e) {
                        LOG.error("Could not find field `{}` in schema : {}.", f.name(), e.getMessage());
                    }
                } else {
                    outRecord.put(f.pos(), status.getId());
                }
                break;
            case FIELD_SUCCESS:
                outRecord.put(f.pos(), Boolean.parseBoolean(status.getStatus()));
                break;
            case FIELD_STATUS:
                outRecord.put(f.pos(), status.getStatus());
                break;
            case FIELD_ERROR_MSG:
            case FIELD_REASON:
                outRecord.put(f.pos(), status.getAvailableReason());
                break;
            case FIELD_MARKETO_GUID:
                outRecord.put(f.pos(), status.getMarketoGUID());
                break;
            case FIELD_SEQ:
                outRecord.put(f.pos(), status.getSeq());
                break;
            default:
                if (isDynamic) {
                    outRecord.put(currentSchema.getField(f.name()).pos(), record.get(f.pos()));
                } else {
                    outRecord.put(currentSchema.getField(f.name()).pos(), record.get(inputSchema.getField(f.name()).pos()));
                }
        }
    }
    return outRecord;
}

Example 33 with Record

use of org.apache.avro.generic.GenericData.Record in project components by Talend.

the class MarketoClientCustomObjectsTestIT method testDeleteCustomObjectsByIdField.

@Test
public void testDeleteCustomObjectsByIdField() throws Exception {
    MarketoSyncResult rs = createCustomObjectRecords("");
    assertTrue(rs.isSuccess());
    List<String> mktoIds = new ArrayList<>();
    for (SyncStatus sr : rs.getRecords()) {
        mktoIds.add(sr.getMarketoGUID());
    }
    // 
    oprops.customObjectName.setValue(TEST_CO_NAME_SMARTPHONE);
    Schema s = // 
    SchemaBuilder.record("sn").fields().name(FIELD_CO_MARKETO_GUID).type().stringType().noDefault().endRecord();
    List<IndexedRecord> records = new ArrayList<>();
    IndexedRecord r1;
    for (String m : mktoIds) {
        r1 = new Record(s);
        r1.put(0, m);
        records.add(r1);
    }
    MarketoSource source = new MarketoSource();
    oprops.customObjectDeleteBy.setValue(CustomObjectDeleteBy.idField);
    source.initialize(null, oprops);
    MarketoRESTClient client = (MarketoRESTClient) source.getClientService(null);
    rs = client.deleteCustomObjects(oprops, records);
    assertTrue(rs.isSuccess());
    List<SyncStatus> changes = rs.getRecords();
    assertEquals(TEST_SMARTPHONE_MODELS.length, changes.size());
    for (SyncStatus r : changes) {
        assertNotNull(r);
        assertNotNull(r.getSeq());
        assertEquals(STATUS_DELETED, r.getStatus());
        LOG.debug("r = {}.", r);
    }
}

Example 34 with Record

use of org.apache.avro.generic.GenericData.Record in project components by Talend.

the class GoogleDrivePutReader method start.

@Override
public boolean start() throws IOException {
    super.start();
    String localFilePath = properties.localFilePath.getValue();
    String destinationFolderId = properties.destinationFolderAccessMethod.getValue().equals(AccessMethod.Id) ? properties.destinationFolder.getValue() : utils.getFolderId(properties.destinationFolder.getValue(), false);
    GoogleDrivePutParameters p = new GoogleDrivePutParameters(destinationFolderId, properties.fileName.getValue(), properties.overwrite.getValue(), localFilePath);
    sentFile = utils.putResource(p);
    record = new Record(properties.schemaMain.schema.getValue());
    record.put(0, java.nio.file.Files.readAllBytes(Paths.get(localFilePath)));
    record.put(1, sentFile.getParents().get(0));
    record.put(2, sentFile.getId());
    result.totalCount++;
    result.successCount++;
    return true;
}

Example 35 with Record

use of org.apache.avro.generic.GenericData.Record in project components by Talend.

the class GoogleDrivePutWriter method write.

@Override
public void write(Object object) throws IOException {
    if (object == null) {
        return;
    }
    IndexedRecord input = (IndexedRecord) object;
    Object data = input.get(0);
    LOG.debug("data [{}] {}.", data.getClass().getCanonicalName(), data.toString());
    byte[] bytes = null;
    if (data instanceof byte[]) {
        bytes = (byte[]) data;
    } else {
        bytes = data.toString().getBytes();
    }
    // 
    String destinationFolderId = properties.destinationFolderAccessMethod.getValue().equals(AccessMethod.Id) ? properties.destinationFolder.getValue() : utils.getFolderId(properties.destinationFolder.getValue(), false);
    GoogleDrivePutParameters p = new GoogleDrivePutParameters(destinationFolderId, properties.fileName.getValue(), properties.overwrite.getValue(), bytes);
    sentFile = utils.putResource(p);
    // 
    IndexedRecord record = new Record(properties.schemaMain.schema.getValue());
    record.put(0, bytes);
    // TODO should return this values in outOfBandRecord
    record.put(1, sentFile.getParents().get(0));
    record.put(2, sentFile.getId());
    cleanWrites();
    result.successCount++;
    result.totalCount++;
    successfulWrites.add(record);
}