Examples with Schema edu.uci.ics.textdb.api.schema.Schema used on opensource projects

Example 6 with Schema

use of edu.uci.ics.textdb.api.schema.Schema in project textdb by TextDB.

the class SimilarityJoinPredicate method generateOutputSchema.

@Override
public Schema generateOutputSchema(Schema innerOperatorSchema, Schema outerOperatorSchema) throws DataFlowException {
    List<Attribute> outputAttributeList = new ArrayList<>();
    // add _ID field first
    outputAttributeList.add(SchemaConstants._ID_ATTRIBUTE);
    for (Attribute attr : innerOperatorSchema.getAttributes()) {
        String attrName = attr.getAttributeName();
        AttributeType attrType = attr.getAttributeType();
        // ignore _id, spanList, and payload
        if (attrName.equals(SchemaConstants._ID) || attrName.equals(SchemaConstants.SPAN_LIST) || attrName.equals(SchemaConstants.PAYLOAD)) {
            continue;
        }
        outputAttributeList.add(new Attribute(INNER_PREFIX + attrName, attrType));
    }
    for (Attribute attr : outerOperatorSchema.getAttributes()) {
        String attrName = attr.getAttributeName();
        AttributeType attrType = attr.getAttributeType();
        // ignore _id, spanList, and payload
        if (attrName.equals(SchemaConstants._ID) || attrName.equals(SchemaConstants.SPAN_LIST) || attrName.equals(SchemaConstants.PAYLOAD)) {
            continue;
        }
        outputAttributeList.add(new Attribute(OUTER_PREFIX + attrName, attrType));
    }
    // add spanList field
    outputAttributeList.add(SchemaConstants.SPAN_LIST_ATTRIBUTE);
    // add payload field if one of them contains payload
    if (innerOperatorSchema.containsField(SchemaConstants.PAYLOAD) || outerOperatorSchema.containsField(SchemaConstants.PAYLOAD)) {
        outputAttributeList.add(SchemaConstants.PAYLOAD_ATTRIBUTE);
    }
    return new Schema(outputAttributeList.stream().toArray(Attribute[]::new));
}

Example 7 with Schema

use of edu.uci.ics.textdb.api.schema.Schema in project textdb by TextDB.

the class ExcelSinkTest method attributeTypeTest.

@Test
public // writing 10000 tuples
void attributeTypeTest() throws Exception {
    ArrayList<String> attributeNames = new ArrayList<>();
    attributeNames.add(TestConstants.FIRST_NAME);
    attributeNames.add(TestConstants.LAST_NAME);
    attributeNames.add(TestConstants.AGE);
    attributeNames.add(TestConstants.HEIGHT);
    attributeNames.add(TestConstants.DATE_OF_BIRTH);
    attributeNames.add(TestConstants.DESCRIPTION);
    // Prepare Schema
    Attribute[] schemaAttributes = new Attribute[TestConstants.ATTRIBUTES_PEOPLE.length];
    for (int count = 0; count < schemaAttributes.length; count++) {
        schemaAttributes[count] = TestConstants.ATTRIBUTES_PEOPLE[count];
    }
    // Prepare 10000 tuples as a tupleList
    int testSize = 10000;
    Random rand = new Random();
    List<Tuple> tupleList = new ArrayList<Tuple>();
    for (int i = 0; i < testSize; i++) {
        IField[] fields = { new StringField(getRandomString()), new StringField(getRandomString()), new IntegerField(rand.nextInt()), new DoubleField(rand.nextDouble() * rand.nextInt()), new DateField(getRandomDate()), new TextField(getRandomString()) };
        tupleList.add(new Tuple(new Schema(schemaAttributes), fields));
    }
    assert (tupleList.size() == testSize);
    IOperator inputOperator = Mockito.mock(IOperator.class);
    Mockito.when(inputOperator.getOutputSchema()).thenReturn(new Schema(schemaAttributes));
    OngoingStubbing<Tuple> stubbing = Mockito.when(inputOperator.getNextTuple());
    for (Tuple t : tupleList) {
        stubbing = stubbing.thenReturn(t);
    }
    stubbing = stubbing.thenReturn(null);
    // excel writing test
    excelSink = new ExcelSink(new ExcelSinkPredicate());
    excelSink.setInputOperator(inputOperator);
    excelSink.open();
    excelSink.collectAllTuples();
    excelSink.close();
    Files.deleteIfExists(Paths.get(excelSink.getFilePath()));
}

Example 8 with Schema

use of edu.uci.ics.textdb.api.schema.Schema in project textdb by TextDB.

the class FileSourceOperatorTest method test2.

/*
     * Test FileSourceOperator with a Directory.
     * Optional parameters are all set to default. (only list files directly in this folder)
     * 
     * Only the files directly under this directory will be used.
     *     
     * expected results: test1.txt and test2.txt will be included.
     */
@Test
public void test2() throws Exception {
    String attrName = "content";
    Schema schema = new Schema(new Attribute(attrName, AttributeType.TEXT));
    FileSourcePredicate predicate = new FileSourcePredicate(tempFolderPath.toString(), attrName);
    FileSourceOperator fileSource = new FileSourceOperator(predicate);
    Tuple tuple;
    ArrayList<Tuple> exactResults = new ArrayList<>();
    fileSource.open();
    while ((tuple = fileSource.getNextTuple()) != null) {
        exactResults.add(tuple);
    }
    fileSource.close();
    List<Tuple> expectedResults = Arrays.asList(new Tuple(schema, new TextField(tempFile1String)), new Tuple(schema, new TextField(tempFile2String)));
    Assert.assertTrue(TestUtils.equals(expectedResults, exactResults));
}

Example 9 with Schema

use of edu.uci.ics.textdb.api.schema.Schema in project textdb by TextDB.

the class FileSourceOperatorTest method test6.

/*
     * Test FileSourceOperator with custom extensions.
     * 
     * Specify "tmp" as q valid extension. Only test3.tmp should be read.
     */
@Test
public void test6() throws Exception {
    String attrName = "content";
    Schema schema = new Schema(new Attribute(attrName, AttributeType.TEXT));
    FileSourcePredicate predicate = new FileSourcePredicate(tempFile3Path.toString(), attrName, null, null, Arrays.asList("tmp"));
    FileSourceOperator fileSource = new FileSourceOperator(predicate);
    Tuple tuple;
    ArrayList<Tuple> exactResults = new ArrayList<>();
    fileSource.open();
    while ((tuple = fileSource.getNextTuple()) != null) {
        exactResults.add(tuple);
    }
    fileSource.close();
    List<Tuple> expectedResults = Arrays.asList(new Tuple(schema, new TextField(tempFile3String)));
    Assert.assertTrue(TestUtils.equals(expectedResults, exactResults));
}

Example 10 with Schema

use of edu.uci.ics.textdb.api.schema.Schema in project textdb by TextDB.

the class FileSourceOperatorTest method test1.

/*
     * Test FileSourceOperator with a single "txt" file.
     * Optional parameters are all set to default.
     */
@Test
public void test1() throws Exception {
    String attrName = "content";
    Schema schema = new Schema(new Attribute(attrName, AttributeType.TEXT));
    FileSourcePredicate predicate = new FileSourcePredicate(tempFile1Path.toString(), attrName);
    FileSourceOperator fileSource = new FileSourceOperator(predicate);
    Tuple tuple;
    ArrayList<Tuple> exactResults = new ArrayList<>();
    fileSource.open();
    while ((tuple = fileSource.getNextTuple()) != null) {
        exactResults.add(tuple);
    }
    fileSource.close();
    List<Tuple> expectedResults = Arrays.asList(new Tuple(schema, new TextField(tempFile1String)));
    Assert.assertTrue(TestUtils.equals(expectedResults, exactResults));
}