use of edu.uci.ics.texera.api.constants.SchemaConstants in project textdb by TextDB.
the class JoinDistancePredicate method generateIntersectionSchema.
/**
* Create outputSchema, which is the intersection of innerOperator's schema and outerOperator's schema.
* The attributes have to be exactly the same (name and type) to be intersected.
*
* InnerOperator's attributes and outerOperator's attributes must:
* both contain the attributes to be joined.
* both contain "_ID" attribute.
* both contain "spanList" attribute.
*
* @return outputSchema
*/
private Schema generateIntersectionSchema(Schema innerOperatorSchema, Schema outerOperatorSchema) throws DataflowException {
List<Attribute> innerAttributes = innerOperatorSchema.getAttributes();
List<Attribute> outerAttributes = outerOperatorSchema.getAttributes();
List<Attribute> intersectionAttributes = innerAttributes.stream().filter(attr -> outerAttributes.contains(attr)).collect(Collectors.toList());
Schema intersectionSchema = new Schema(intersectionAttributes.stream().toArray(Attribute[]::new));
// check if output schema contain necessary attributes
if (intersectionSchema.getAttributes().isEmpty()) {
throw new DataflowException("inner operator and outer operator don't share any common attributes");
} else if (!intersectionSchema.containsAttribute(this.joinAttributeName)) {
throw new DataflowException("inner operator or outer operator doesn't contain join attribute");
} else if (!intersectionSchema.containsAttribute(SchemaConstants._ID)) {
throw new DataflowException("inner operator or outer operator doesn't contain _ID attribute");
} else if (!intersectionSchema.containsAttribute(SchemaConstants.SPAN_LIST)) {
throw new DataflowException("inner operator or outer operator doesn't contain spanList attribute");
}
// check if join attribute is TEXT or STRING
AttributeType joinAttrType = intersectionSchema.getAttribute(this.joinAttributeName).getType();
if (joinAttrType != AttributeType.TEXT && joinAttrType != AttributeType.STRING) {
throw new DataflowException(String.format("Join attribute %s must be either TEXT or STRING.", this.joinAttributeName));
}
return intersectionSchema;
}
use of edu.uci.ics.texera.api.constants.SchemaConstants in project textdb by TextDB.
the class JoinDistancePredicate method joinTuples.
/**
* This method is called by the Join operator to perform the join on the
* tuples passed.
*
* @return New Tuple containing the result of join operation.
*/
@Override
public Tuple joinTuples(Tuple innerTuple, Tuple outerTuple, Schema outputSchema) throws Exception {
List<Span> newJoinSpanList = new ArrayList<>();
/*
* We expect the values of all fields to be the same for innerTuple and outerTuple.
* We only checks _ID field, and field to be joined, since they are crucial to join operator.
* For other fields, we use the value from innerTuple.
* check if the _ID fields are the same
*/
if (!compareField(innerTuple, outerTuple, SchemaConstants._ID)) {
return null;
}
// check if the fields to be joined are the same
if (!compareField(innerTuple, outerTuple, this.joinAttributeName)) {
return null;
}
/*
* If either/both tuples have no span information, return null.
* Check using try/catch if both the tuples have span information.
* If not return null; so we can process next tuple.
*/
ListField<Span> spanFieldOfInnerTuple = innerTuple.getField(SchemaConstants.SPAN_LIST);
ListField<Span> spanFieldOfOuterTuple = outerTuple.getField(SchemaConstants.SPAN_LIST);
List<Span> innerSpanList = null;
List<Span> outerSpanList = null;
// ListField
if (spanFieldOfInnerTuple.getClass().equals(ListField.class)) {
innerSpanList = spanFieldOfInnerTuple.getValue();
}
if (spanFieldOfOuterTuple.getClass().equals(ListField.class)) {
outerSpanList = spanFieldOfOuterTuple.getValue();
}
Iterator<Span> outerSpanIter = outerSpanList.iterator();
// the ones specified in the JoinPredicate during "sort merge"?)
while (outerSpanIter.hasNext()) {
Span outerSpan = outerSpanIter.next();
// If not return null.
if (!outerSpan.getAttributeName().equals(this.joinAttributeName)) {
continue;
}
Iterator<Span> innerSpanIter = innerSpanList.iterator();
while (innerSpanIter.hasNext()) {
Span innerSpan = innerSpanIter.next();
if (!innerSpan.getAttributeName().equals(this.joinAttributeName)) {
continue;
}
Integer threshold = this.getThreshold();
if (Math.abs(outerSpan.getStart() - innerSpan.getStart()) <= threshold && Math.abs(outerSpan.getEnd() - innerSpan.getEnd()) <= threshold) {
Integer newSpanStartIndex = Math.min(innerSpan.getStart(), outerSpan.getStart());
Integer newSpanEndIndex = Math.max(innerSpan.getEnd(), outerSpan.getEnd());
String attributeName = this.joinAttributeName;
String fieldValue = (String) innerTuple.getField(attributeName).getValue();
String newFieldValue = fieldValue.substring(newSpanStartIndex, newSpanEndIndex);
String spanKey = outerSpan.getKey() + "_" + innerSpan.getKey();
Span newSpan = new Span(attributeName, newSpanStartIndex, newSpanEndIndex, spanKey, newFieldValue);
newJoinSpanList.add(newSpan);
}
}
}
if (newJoinSpanList.isEmpty()) {
return null;
}
// create output fields based on innerTuple's value
List<Attribute> outputAttrList = outputSchema.getAttributes();
List<IField> outputFields = outputAttrList.stream().filter(attr -> !attr.equals(SchemaConstants.SPAN_LIST_ATTRIBUTE)).map(attr -> attr.getName()).map(attributeName -> innerTuple.getField(attributeName, IField.class)).collect(Collectors.toList());
outputFields.add(new ListField<>(newJoinSpanList));
return new Tuple(outputSchema, outputFields.stream().toArray(IField[]::new));
}
use of edu.uci.ics.texera.api.constants.SchemaConstants in project textdb by TextDB.
the class FileSourceOperator method open.
@Override
public void open() throws TexeraException {
if (cursor != CLOSED) {
return;
}
cursor = OPENED;
try {
List<String> columnNames = null;
if (predicate.getFileFormat() != null && predicate.getFileFormat() == FileSourcePredicate.FileFormat.CSV_WITH_HEADER) {
Optional<String> header = Files.lines(pathList.get(0)).findFirst();
if (header.isPresent()) {
columnNames = Arrays.stream(header.get().split(predicate.getColumnDelimiter())).collect(Collectors.toList());
}
}
if (predicate.getColumnDelimiter() != null) {
Optional<String> firstLine = Files.lines(pathList.get(0)).findFirst();
if (firstLine.isPresent()) {
columnNames = IntStream.range(0, firstLine.get().split(predicate.getColumnDelimiter()).length).map(i -> i + 1).mapToObj(i -> "c" + i).collect(Collectors.toList());
}
}
if (columnNames == null) {
columnNames = Collections.singletonList("c1");
}
List<Attribute> attributes = columnNames.stream().map(name -> new Attribute(name, AttributeType.TEXT)).collect(Collectors.toList());
this.outputSchema = new Schema.Builder().add(SchemaConstants._ID_ATTRIBUTE).add(attributes).build();
} catch (IOException e) {
throw new DataflowException(e);
}
}
use of edu.uci.ics.texera.api.constants.SchemaConstants in project textdb by TextDB.
the class CSVSink method open.
@Override
public void open() throws TexeraException {
if (cursor != CLOSED) {
return;
}
inputOperator.open();
inputSchema = inputOperator.getOutputSchema();
outputSchema = new Schema(inputSchema.getAttributes().stream().filter(attr -> !attr.getName().equalsIgnoreCase(SchemaConstants._ID)).filter(attr -> !attr.getName().equalsIgnoreCase(SchemaConstants.PAYLOAD)).filter(attr -> !attr.getType().equals(AttributeType.LIST)).toArray(Attribute[]::new));
DateFormat df = new SimpleDateFormat("yyyyMMdd-HHmmss");
fileName = df.format(new Date()) + ".csv";
File file = new File(csvIndexDirectory.resolve(fileName).toString());
try {
if (Files.notExists(csvIndexDirectory)) {
Files.createDirectories(csvIndexDirectory);
}
csvWriter = new CSVWriter(new FileWriter(file));
} catch (IOException e) {
throw new DataflowException(e);
}
// write csv headers
List<String> attributeNames = outputSchema.getAttributeNames();
csvWriter.writeNext(attributeNames.stream().toArray(String[]::new));
cursor = OPENED;
}
use of edu.uci.ics.texera.api.constants.SchemaConstants in project textdb by TextDB.
the class ExcelSink method open.
@Override
public void open() throws TexeraException {
if (cursor != CLOSED) {
return;
}
inputOperator.open();
inputSchema = inputOperator.getOutputSchema();
outputSchema = new Schema(inputSchema.getAttributes().stream().filter(attr -> !attr.getName().equalsIgnoreCase(SchemaConstants._ID)).filter(attr -> !attr.getName().equalsIgnoreCase(SchemaConstants.PAYLOAD)).filter(attr -> !attr.getType().equals(AttributeType.LIST)).toArray(Attribute[]::new));
wb = new XSSFWorkbook();
DateFormat df = new SimpleDateFormat("yyyyMMdd-HHmmss");
fileName = df.format(new Date()) + ".xlsx";
try {
if (Files.notExists(excelIndexDirectory)) {
Files.createDirectories(excelIndexDirectory);
}
fileOut = new FileOutputStream(excelIndexDirectory.resolve(fileName).toString());
} catch (IOException e) {
throw new DataflowException(e);
}
sheet = wb.createSheet("new sheet");
Row row = sheet.createRow(0);
List<String> attributeNames = outputSchema.getAttributeNames();
for (int i = 0; i < attributeNames.size(); i++) {
String attributeName = attributeNames.get(i);
row.createCell(i).setCellValue(attributeName);
}
cursor = OPENED;
}
Aggregations