Examples with FieldInfo org.apache.apex.malhar.lib.util.FieldInfo used on opensource projects

Example 26 with FieldInfo

use of org.apache.apex.malhar.lib.util.FieldInfo in project apex-malhar by apache.

the class JdbcPOJOPollInputOperator method setFieldInfosItem.

/**
 * Function to initialize the list of {@link FieldInfo} externally from configuration/properties file.
 * Example entry in the properties/configuration file:
 *   <property>
 *     <name>dt.operator.JdbcPOJOInput.fieldInfosItem[0]</name>
 *     <value>
 *      {
 *       "columnName":"ID",
 *       "pojoFieldExpression": "id",
 *       "type":"INTEGER"
 *      }
 *     </value>
 *   </property>
 * @param index is the index in the list which is to be initialized.
 * @param value is the JSON String with appropriate mappings for {@link FieldInfo}.
 */
public void setFieldInfosItem(int index, String value) {
    try {
        JSONObject jo = new JSONObject(value);
        FieldInfo fieldInfo = new FieldInfo(jo.getString("columnName"), jo.getString("pojoFieldExpression"), FieldInfo.SupportType.valueOf(jo.getString("type")));
        final int need = index - fieldInfos.size() + 1;
        for (int i = 0; i < need; i++) {
            fieldInfos.add(null);
        }
        fieldInfos.set(index, fieldInfo);
    } catch (Exception e) {
        throw new RuntimeException("Exception in setting FieldInfo " + value + " " + e.getMessage());
    }
}

Example 27 with FieldInfo

use of org.apache.apex.malhar.lib.util.FieldInfo in project apex-malhar by apache.

the class RedisPOJOOperatorTest method testInputOperator.

@Test
public void testInputOperator() throws IOException {
    @SuppressWarnings("unused") Class<?> clazz = org.codehaus.janino.CompilerFactory.class;
    this.operatorStore = new RedisStore();
    this.testStore = new RedisStore();
    testStore.connect();
    ScanParams params = new ScanParams();
    params.count(100);
    Map<String, String> value = new HashMap<String, String>();
    value.put("Column1", "abc");
    value.put("Column2", "1");
    Map<String, String> value1 = new HashMap<String, String>();
    value1.put("Column1", "def");
    value1.put("Column2", "2");
    Map<String, String> value2 = new HashMap<String, String>();
    value2.put("Column1", "ghi");
    value2.put("Column2", "3");
    testStore.put("test_abc_in", value);
    testStore.put("test_def_in", value1);
    testStore.put("test_ghi_in", value2);
    try {
        LocalMode lma = LocalMode.newInstance();
        DAG dag = lma.getDAG();
        RedisPOJOInputOperator inputOperator = dag.addOperator("input", new RedisPOJOInputOperator());
        final ObjectCollectorModule collector = dag.addOperator("collector", new ObjectCollectorModule());
        ArrayList<FieldInfo> fields = new ArrayList<FieldInfo>();
        fields.add(new FieldInfo("Column1", "stringValue", SupportType.STRING));
        fields.add(new FieldInfo("Column2", "intValue", SupportType.INTEGER));
        inputOperator.setDataColumns(fields);
        inputOperator.setOutputClass(TestClass.class.getName());
        inputOperator.setStore(operatorStore);
        dag.addStream("stream", inputOperator.outputPort, collector.inputPort);
        final LocalMode.Controller lc = lma.getController();
        new Thread("LocalClusterController") {

            @Override
            public void run() {
                long startTms = System.currentTimeMillis();
                long timeout = 10000L;
                try {
                    Thread.sleep(1000);
                    while (System.currentTimeMillis() - startTms < timeout) {
                        if (ObjectCollectorModule.resultMap.size() < 3) {
                            Thread.sleep(10);
                        } else {
                            break;
                        }
                    }
                } catch (InterruptedException ex) {
                // 
                }
                lc.shutdown();
            }
        }.start();
        lc.run();
        Assert.assertTrue(ObjectCollectorModule.resultMap.containsKey("test_abc_in"));
        Assert.assertTrue(ObjectCollectorModule.resultMap.containsKey("test_def_in"));
        Assert.assertTrue(ObjectCollectorModule.resultMap.containsKey("test_ghi_in"));
        TestClass a = (TestClass) ObjectCollectorModule.resultMap.get("test_abc_in");
        Assert.assertNotNull(a);
        Assert.assertEquals("abc", a.stringValue);
        Assert.assertEquals("1", a.intValue.toString());
    } finally {
        for (KeyValPair<String, String> entry : CollectorModule.resultMap) {
            testStore.remove(entry.getKey());
        }
        testStore.disconnect();
    }
}

Example 28 with FieldInfo

use of org.apache.apex.malhar.lib.util.FieldInfo in project apex-malhar by apache.

the class MaxPerKeyExamples method addInputFieldInfos.

/**
 * Method to set field info for {@link JdbcPOJOInputOperator}.
 * @return
 */
private List<FieldInfo> addInputFieldInfos() {
    List<FieldInfo> fieldInfos = Lists.newArrayList();
    fieldInfos.add(new FieldInfo("MONTH", "month", FieldInfo.SupportType.INTEGER));
    fieldInfos.add(new FieldInfo("DAY", "day", FieldInfo.SupportType.INTEGER));
    fieldInfos.add(new FieldInfo("YEAR", "year", FieldInfo.SupportType.INTEGER));
    fieldInfos.add(new FieldInfo("MEANTEMP", "meanTemp", FieldInfo.SupportType.DOUBLE));
    return fieldInfos;
}

Example 29 with FieldInfo

use of org.apache.apex.malhar.lib.util.FieldInfo in project apex-malhar by apache.

the class CassandraOperatorTest method testCassandraOutputOperator.

@Test
public void testCassandraOutputOperator() {
    TestOutputOperator outputOperator = setupForOutputOperatorTest();
    List<FieldInfo> fieldInfos = Lists.newArrayList();
    fieldInfos.add(new FieldInfo("id", "id", null));
    fieldInfos.add(new FieldInfo("age", "age", null));
    fieldInfos.add(new FieldInfo("doubleValue", "doubleValue", null));
    fieldInfos.add(new FieldInfo("floatValue", "floatValue", null));
    fieldInfos.add(new FieldInfo("last_visited", "last_visited", null));
    fieldInfos.add(new FieldInfo("lastname", "lastname", null));
    fieldInfos.add(new FieldInfo("list1", "list1", null));
    fieldInfos.add(new FieldInfo("map1", "map1", null));
    fieldInfos.add(new FieldInfo("set1", "set1", null));
    fieldInfos.add(new FieldInfo("test", "test", null));
    outputOperator.setFieldInfos(fieldInfos);
    outputOperator.setup(context);
    outputOperator.input.setup(tpc);
    outputOperator.activate(context);
    List<TestPojo> events = Lists.newArrayList();
    for (int i = 0; i < 3; i++) {
        Set<Integer> set = new HashSet<Integer>();
        set.add(i);
        List<Integer> list = new ArrayList<Integer>();
        list.add(i);
        Map<String, Integer> map = new HashMap<String, Integer>();
        map.put("key" + i, i);
        events.add(new TestPojo(UUID.randomUUID(), i, "abclast", true, i, 2.0, set, list, map, new Date(System.currentTimeMillis())));
    }
    outputOperator.beginWindow(0);
    for (TestPojo event : events) {
        outputOperator.input.process(event);
    }
    outputOperator.endWindow();
    Assert.assertEquals("rows in db", 3, outputOperator.getNumOfEventsInStore());
    outputOperator.getEventsInStore();
}

Example 30 with FieldInfo

use of org.apache.apex.malhar.lib.util.FieldInfo in project apex-malhar by apache.

the class CassandraOperatorTest method testupdateQueryWithParameters.

@Test
public void testupdateQueryWithParameters() throws InterruptedException {
    UUID id = UUID.fromString("94ab597c-a5ff-4997-8343-68993d446b14");
    TestPojo testPojo = new TestPojo(id, 20, "Laura", true, 10, 2.0, new HashSet<Integer>(), new ArrayList<Integer>(), null, new Date(System.currentTimeMillis()));
    String insert = "INSERT INTO " + KEYSPACE + "." + TABLE_NAME + " (ID, age, lastname, test, floatValue, doubleValue)" + " VALUES (94ab597c-a5ff-4997-8343-68993d446b14, 20, 'Laura', true, 10, 2.0);";
    session.execute(insert);
    String recordsQuery = "SELECT * from " + TABLE_NAME + ";";
    ResultSet resultSetRecords = session.execute(recordsQuery);
    Row row = resultSetRecords.iterator().next();
    Assert.assertEquals("Updated last name", "Laura", row.getString("lastname"));
    // wait till cassandra writes the record
    Thread.sleep(1000);
    // update record
    String updateLastName = "Laurel";
    String updateQuery = "update " + KEYSPACE + "." + TABLE_NAME + " set lastname='" + updateLastName + "' where id=?";
    // set specific files required by update command in order as per query
    List<FieldInfo> fieldInfos = Lists.newArrayList();
    fieldInfos.add(new FieldInfo("id", "id", null));
    // reset the operator to run new query
    TestOutputOperator outputOperator = setupForOutputOperatorTest();
    outputOperator.setQuery(updateQuery);
    outputOperator.setFieldInfos(fieldInfos);
    outputOperator.setup(context);
    outputOperator.input.setup(tpc);
    outputOperator.activate(context);
    outputOperator.beginWindow(1);
    outputOperator.input.process(testPojo);
    outputOperator.endWindow();
    recordsQuery = "SELECT * from " + TABLE_NAME + ";";
    resultSetRecords = session.execute(recordsQuery);
    row = resultSetRecords.iterator().next();
    Assert.assertEquals("Updated last name", updateLastName, row.getString("lastname"));
}