Examples with WorkUnitState org.apache.gobblin.configuration.WorkUnitState used on opensource projects

Example 41 with WorkUnitState

use of org.apache.gobblin.configuration.WorkUnitState in project incubator-gobblin by apache.

the class NoopPublisher method publishData.

/**
 * Publish the data for the given tasks.
 *
 * @param states
 */
@Override
public void publishData(Collection<? extends WorkUnitState> states) throws IOException {
    for (WorkUnitState state : states) {
        if (state.getWorkingState() == WorkUnitState.WorkingState.SUCCESSFUL) {
            state.setWorkingState(WorkUnitState.WorkingState.COMMITTED);
            log.info("Marking state committed");
        }
    }
}

Example 42 with WorkUnitState

use of org.apache.gobblin.configuration.WorkUnitState in project incubator-gobblin by apache.

the class CsvToJsonConverterV2Test method convertOutput.

public void convertOutput() throws IOException {
    JsonParser parser = new JsonParser();
    JsonElement jsonElement = parser.parse(new InputStreamReader(getClass().getResourceAsStream("/converter/csv/schema_with_10_fields.json")));
    JsonArray outputSchema = jsonElement.getAsJsonArray();
    CSVParser csvParser = new CSVParser();
    String[] inputRecord = csvParser.parseLine(row10Cols);
    CsvToJsonConverterV2 converter = new CsvToJsonConverterV2();
    converter.init(new WorkUnitState());
    JsonObject actual = converter.createOutput(outputSchema, inputRecord);
    JsonObject expected = parser.parse(new InputStreamReader(getClass().getResourceAsStream("/converter/csv/10_fields.json"))).getAsJsonObject();
    Assert.assertEquals(expected, actual);
    converter.close();
}

Example 43 with WorkUnitState

use of org.apache.gobblin.configuration.WorkUnitState in project incubator-gobblin by apache.

the class CsvToJsonConverterV2Test method convertOutputAddingNull.

public void convertOutputAddingNull() throws IOException, DataConversionException {
    JsonParser parser = new JsonParser();
    JsonElement jsonElement = parser.parse(new InputStreamReader(getClass().getResourceAsStream("/converter/csv/schema_with_11_fields.json")));
    JsonArray outputSchema = jsonElement.getAsJsonArray();
    CSVParser csvParser = new CSVParser();
    String[] inputRecord = csvParser.parseLine(row11Cols);
    CsvToJsonConverterV2 converter = new CsvToJsonConverterV2();
    WorkUnitState wuState = new WorkUnitState();
    wuState.setProp(CsvToJsonConverterV2.CUSTOM_ORDERING, "0,1,-1,3,4,5,6,7,8,9,10");
    converter.init(wuState);
    JsonObject actual = converter.convertRecord(outputSchema, inputRecord, wuState).iterator().next();
    JsonObject expected = parser.parse(new InputStreamReader(getClass().getResourceAsStream("/converter/csv/11_fields_with_null.json"))).getAsJsonObject();
    Assert.assertEquals(expected, actual);
    converter.close();
}

Example 44 with WorkUnitState

use of org.apache.gobblin.configuration.WorkUnitState in project incubator-gobblin by apache.

the class AvroFieldsPickConverterTest method testFieldsPick.

@Test
public void testFieldsPick() throws Exception {
    Schema inputSchema = new Schema.Parser().parse(getClass().getResourceAsStream("/converter/fieldPickInput.avsc"));
    WorkUnitState workUnitState = new WorkUnitState();
    workUnitState.setProp(ConfigurationKeys.CONVERTER_AVRO_FIELD_PICK_FIELDS, "name,favorite_number,favorite_color");
    try (AvroFieldsPickConverter converter = new AvroFieldsPickConverter()) {
        Schema converted = converter.convertSchema(inputSchema, workUnitState);
        Schema expected = new Schema.Parser().parse(getClass().getResourceAsStream("/converter/fieldPickExpected.avsc"));
        JSONAssert.assertEquals(expected.toString(), converted.toString(), false);
    }
}

Example 45 with WorkUnitState

use of org.apache.gobblin.configuration.WorkUnitState in project incubator-gobblin by apache.

the class AvroToRestJsonEntryConverterTest method testEqualConversion.

public void testEqualConversion() throws DataConversionException, IOException, JSONException {
    JsonParser parser = new JsonParser();
    String expectedResourceKey = "/sobject/user/John";
    String expectedJsonStr = "{ \"name\" : \"John\", \"favorite_number\" : 9, \"favorite_color\" : \"blue\", \"date_of_birth\" : 1462387756716, \"last_modified\" : 0, \"created\" : 1462387756716, \"address\" : {\"city\" : \"Mountain view\", \"street_number\" : 2029 } }";
    RestEntry<JsonObject> expected = new RestEntry<JsonObject>(expectedResourceKey, parser.parse(expectedJsonStr).getAsJsonObject());
    WorkUnitState workUnitState = new WorkUnitState();
    workUnitState.setProp(AvroToRestJsonEntryConverter.CONVERTER_AVRO_REST_ENTRY_RESOURCE_KEY, "/sobject/user/${name}");
    testConversion(expected, workUnitState);
}