use of eu.esdihumboldt.hale.common.align.transformation.service.TransformationService in project hale by halestudio.
the class OrientInstanceService method performTransformation.
/**
* Perform the transformation
*
* @return if the transformation was successful
*/
protected boolean performTransformation() {
final TransformationService ts = getTransformationService();
if (ts == null) {
log.userError("No transformation service available");
return false;
}
final AtomicBoolean transformationFinished = new AtomicBoolean(false);
final AtomicBoolean transformationCanceled = new AtomicBoolean(false);
IRunnableWithProgress op = new IRunnableWithProgress() {
@Override
public void run(IProgressMonitor monitor) throws InvocationTargetException, InterruptedException {
try {
Alignment alignment = getAlignmentService().getAlignment();
if (alignment.getActiveTypeCells().isEmpty()) {
// early exit if there are no type relations
return;
}
// determine if there are any active type cells w/o source
boolean transformEmpty = false;
for (Cell cell : alignment.getActiveTypeCells()) {
if (cell.getSource() == null || cell.getSource().isEmpty()) {
transformEmpty = true;
break;
}
}
InstanceCollection sources = getInstances(DataSet.SOURCE);
if (!transformEmpty && sources.isEmpty()) {
return;
}
HaleOrientInstanceSink sink = new HaleOrientInstanceSink(transformed, true);
TransformationReport report;
ATransaction trans = log.begin("Instance transformation");
try {
report = ts.transform(alignment, sources, sink, HaleUI.getServiceProvider(), new ProgressMonitorIndicator(monitor));
// publish report
ReportService rs = PlatformUI.getWorkbench().getService(ReportService.class);
rs.addReport(report);
} finally {
try {
sink.close();
} catch (IOException e) {
// ignore
}
trans.end();
}
} finally {
// remember if canceled
if (monitor.isCanceled()) {
transformationCanceled.set(true);
}
// transformation finished
transformationFinished.set(true);
}
}
};
try {
ThreadProgressMonitor.runWithProgressDialog(op, ts.isCancelable());
} catch (Throwable e) {
log.error("Error starting transformation process", e);
}
// wait for transformation to complete
HaleUI.waitFor(transformationFinished);
return !transformationCanceled.get();
}
use of eu.esdihumboldt.hale.common.align.transformation.service.TransformationService in project hale by halestudio.
the class Transformation method transform.
/**
* Transform the given instances, according to the given alignment.
*
* @param sources the collection of source instances
* @param targetSink the target sink
* @param exportJob the export job
* @param validationJob the validation job, may be <code>null</code>
* @param alignment the alignment, may not be changed outside this method
* @param sourceSchema the source schema
* @param reportHandler the report handler
* @param serviceProvider the service provider in the transformation context
* @param processId the identifier for the transformation process, may be
* <code>null</code> if grouping the jobs to a job family is not
* necessary
* @return the future representing the successful completion of the
* transformation (note that a successful completion doesn't
* necessary mean there weren't any internal transformation errors)
*/
public static ListenableFuture<Boolean> transform(InstanceCollection sources, final TransformationSink targetSink, final ExportJob exportJob, final ValidationJob validationJob, final Alignment alignment, SchemaSpace sourceSchema, final ReportHandler reportHandler, final ServiceProvider serviceProvider, final Object processId) {
final SettableFuture<Boolean> result = SettableFuture.create();
final InstanceCollection sourceToUse;
// Check whether to create a temporary database or not.
// Currently do not create a temporary DB is there are Retypes/Creates
// only.
boolean useTempDatabase = false;
final LocalOrientDB db;
for (Cell cell : alignment.getActiveTypeCells()) if (!isStreamingTypeTransformation(cell.getTransformationIdentifier())) {
useTempDatabase = true;
break;
}
// Create temporary database if necessary.
if (useTempDatabase) {
// create db
File tmpDir = Files.createTempDir();
db = new LocalOrientDB(tmpDir);
tmpDir.deleteOnExit();
// get instance collection
// sourceToUse = new BrowseOrientInstanceCollection(db, sourceSchema, DataSet.SOURCE);
// only yield instances that were actually inserted
// this is also done in OrientInstanceService
// TODO make configurable?
sourceToUse = FilteredInstanceCollection.applyFilter(new BrowseOrientInstanceCollection(db, sourceSchema, DataSet.SOURCE), new Filter() {
@Override
public boolean match(Instance instance) {
if (instance instanceof OInstance) {
return ((OInstance) instance).isInserted();
}
return true;
}
});
} else {
sourceToUse = new StatsCountInstanceCollection(sources, reportHandler);
db = null;
}
// create transformation job
final AbstractTransformationJob transformJob = new AbstractTransformationJob("Transformation") {
/**
* @see org.eclipse.core.runtime.jobs.Job#run(org.eclipse.core.runtime.IProgressMonitor)
*/
@Override
protected IStatus run(IProgressMonitor monitor) {
TransformationService transformationService = HalePlatform.getService(TransformationService.class);
TransformationReport report = transformationService.transform(alignment, sourceToUse, targetSink, serviceProvider, new ProgressMonitorIndicator(monitor));
try {
// publish report
reportHandler.publishReport(report);
if (report.isSuccess()) {
return Status.OK_STATUS;
} else {
return ERROR_STATUS;
}
} finally {
// and may lead to the transformation report being lost
if (monitor.isCanceled()) {
targetSink.done(true);
return Status.CANCEL_STATUS;
} else {
targetSink.done(false);
}
}
}
};
// set process IDs to group jobs in a job family
if (processId != null) {
transformJob.setProcessId(processId);
exportJob.setProcessId(processId);
if (validationJob != null) {
validationJob.setProcessId(processId);
}
}
exportJob.setUser(true);
// the jobs should cancel each other
transformJob.addJobChangeListener(new JobChangeAdapter() {
@Override
public void done(IJobChangeEvent event) {
if (!event.getResult().isOK()) {
// log transformation job error (because it otherwise gets
// lost)
String msg = "Error during transformation";
if (event.getResult().getMessage() != null) {
msg = ": " + event.getResult().getMessage();
}
log.error(msg, event.getResult().getException());
// failing transformation is done by cancelling the export
exportJob.cancel();
}
if (db != null) {
db.delete();
}
}
});
// after export is done, validation should run
exportJob.addJobChangeListener(new JobChangeAdapter() {
@Override
public void done(IJobChangeEvent event) {
if (!event.getResult().isOK()) {
transformJob.cancel();
// failure
failure(result, event);
} else {
if (validationJob == null) {
// success
result.set(true);
} else {
// schedule the validation job
validationJob.schedule();
}
}
}
});
// validation ends the process
if (validationJob != null) {
validationJob.addJobChangeListener(new JobChangeAdapter() {
@Override
public void done(IJobChangeEvent event) {
if (!event.getResult().isOK()) {
// failure
failure(result, event);
} else {
// success
result.set(true);
}
}
});
}
if (useTempDatabase) {
// Initialize instance index with alignment
InstanceIndexService indexService = serviceProvider.getService(InstanceIndexService.class);
indexService.addPropertyMappings(alignment.getActiveTypeCells(), serviceProvider);
// run store instance job first...
Job storeJob = new StoreInstancesJob("Load source instances into temporary database", db, sources, serviceProvider, reportHandler, true) {
@Override
protected void onComplete() {
// onComplete is also called if monitor is cancelled...
}
@Override
public boolean belongsTo(Object family) {
if (processId == null) {
return super.belongsTo(family);
}
return AbstractTransformationJob.createFamily(processId).equals(family);
}
};
// and schedule jobs on successful completion
storeJob.addJobChangeListener(new JobChangeAdapter() {
@Override
public void done(IJobChangeEvent event) {
if (event.getResult().isOK()) {
exportJob.schedule();
transformJob.schedule();
} else {
failure(result, event);
}
}
});
storeJob.schedule();
} else {
// otherwise feed InstanceProcessors directly from the
// InstanceCollection...
// TODO Implement differently, not w/ PseudoInstanceReference which
// will cause memory problems
// final InstanceProcessingExtension ext = new InstanceProcessingExtension(
// serviceProvider);
// final List<InstanceProcessor> processors = ext.getInstanceProcessors();
//
// ResourceIterator<Instance> it = sourceToUse.iterator();
// try {
// while (it.hasNext()) {
// Instance instance = it.next();
//
// ResolvableInstanceReference resolvableRef = new ResolvableInstanceReference(
// new PseudoInstanceReference(instance), sourceToUse);
// processors.forEach(p -> p.process(instance, resolvableRef));
//
// }
// } finally {
// it.close();
// }
// ...and schedule jobs
exportJob.schedule();
transformJob.schedule();
}
return result;
}
use of eu.esdihumboldt.hale.common.align.transformation.service.TransformationService in project hale by halestudio.
the class InlineTransformation method evaluate.
@Override
protected Object evaluate(String transformationIdentifier, TransformationEngine engine, ListMultimap<String, PropertyValue> variables, String resultName, PropertyEntityDefinition resultProperty, Map<String, String> executionParameters, TransformationLog log) throws TransformationException, NoResultException {
List<PropertyValue> sources = variables.get(null);
if (sources.isEmpty()) {
throw new NoResultException("No source available to transform");
}
PropertyValue source = sources.get(0);
Object sourceValue = source.getValue();
if (sourceValue == null) {
throw new NoResultException("Source value is null");
}
if (!(sourceValue instanceof Instance)) {
throw new TransformationException("Sources for inline transformation must be instances");
}
Instance sourceInstance = (Instance) sourceValue;
TypeDefinition sourceType = sourceInstance.getDefinition();
// get the original alignment
Alignment orgAlignment = getExecutionContext().getAlignment();
MutableAlignment alignment = new DefaultAlignment(orgAlignment);
// identify relevant type cell(s)
MutableCell queryCell = new DefaultCell();
ListMultimap<String, Type> sourceEntities = ArrayListMultimap.create();
sourceEntities.put(null, new DefaultType(new TypeEntityDefinition(sourceType, SchemaSpaceID.SOURCE, null)));
queryCell.setSource(sourceEntities);
ListMultimap<String, Type> targetEntities = ArrayListMultimap.create();
targetEntities.put(null, new DefaultType(new TypeEntityDefinition(resultProperty.getDefinition().getPropertyType(), SchemaSpaceID.TARGET, null)));
queryCell.setTarget(targetEntities);
Collection<? extends Cell> candidates = alignment.getTypeCells(queryCell);
if (candidates.isEmpty()) {
log.error(log.createMessage("No type transformations found for inline transformation", null));
throw new NoResultException();
}
// filter alignment -> only keep relevant type relations
List<Cell> allTypeCells = new ArrayList<>(alignment.getTypeCells());
for (Cell cell : allTypeCells) {
// remove cell
alignment.removeCell(cell);
if (!cell.getTransformationMode().equals(TransformationMode.disabled)) {
// only readd if not disabled
MutableCell copy = new DefaultCell(cell);
if (candidates.contains(cell)) {
// readd as active
copy.setTransformationMode(TransformationMode.active);
} else {
// readd as passive
copy.setTransformationMode(TransformationMode.passive);
}
alignment.addCell(copy);
}
}
// prepare transformation input/output
DefaultInstanceCollection sourceInstances = new DefaultInstanceCollection();
sourceInstances.add(sourceInstance);
DefaultInstanceSink target = new DefaultInstanceSink();
// run transformation
TransformationService ts = getExecutionContext().getService(TransformationService.class);
if (ts == null) {
throw new TransformationException("Transformation service not available for inline transformation");
}
ProgressIndicator progressIndicator = new LogProgressIndicator();
TransformationReport report = ts.transform(alignment, sourceInstances, new ThreadSafeInstanceSink<InstanceSink>(target), getExecutionContext(), progressIndicator);
// copy report messages
log.importMessages(report);
if (!report.isSuccess()) {
// copy report messages
log.importMessages(report);
throw new TransformationException("Inline transformation failed");
}
// extract result
List<Instance> targetList = target.getInstances();
if (targetList.isEmpty()) {
log.error(log.createMessage("Inline transformation yielded no result", null));
throw new NoResultException("No result from inline transformation");
}
if (targetList.size() > 1) {
log.error(log.createMessage("Inline transformation yielded multiple results, only first result is used", null));
}
return targetList.get(0);
}
Aggregations