use of org.apache.hop.pipeline.transform.TransformMeta in project hop by apache.
the class InjectDataSetIntoTransformExtensionPoint method callExtensionPoint.
@Override
public void callExtensionPoint(ILogChannel log, IVariables variables, final IPipelineEngine<PipelineMeta> pipeline) throws HopException {
if (!(pipeline instanceof LocalPipelineEngine)) {
throw new HopPluginException("Unit tests can only run using a local pipeline engine type");
}
final PipelineMeta pipelineMeta = pipeline.getPipelineMeta();
boolean dataSetEnabled = "Y".equalsIgnoreCase(pipeline.getVariable(DataSetConst.VAR_RUN_UNIT_TEST));
if (log.isDetailed()) {
log.logDetailed("Data Set enabled? " + dataSetEnabled);
}
if (!dataSetEnabled) {
return;
}
String unitTestName = pipeline.getVariable(DataSetConst.VAR_UNIT_TEST_NAME);
if (log.isDetailed()) {
log.logDetailed("Unit test name: " + unitTestName);
}
try {
IHopMetadataProvider metadataProvider = pipelineMeta.getMetadataProvider();
//
if (StringUtil.isEmpty(unitTestName)) {
return;
}
PipelineUnitTest unitTest = metadataProvider.getSerializer(PipelineUnitTest.class).load(unitTestName);
if (unitTest == null) {
if (log.isDetailed()) {
log.logDetailed("Unit test '" + unitTestName + "' could not be found");
}
return;
}
//
for (final TransformMeta transformMeta : pipeline.getPipelineMeta().getTransforms()) {
String transformName = transformMeta.getName();
PipelineUnitTestSetLocation inputLocation = unitTest.findInputLocation(transformName);
if (inputLocation != null && StringUtils.isNotEmpty(inputLocation.getDataSetName())) {
String inputDataSetName = inputLocation.getDataSetName();
log.logDetailed("Data Set location found for transform '" + transformName + "' and data set " + inputDataSetName);
// We need to inject data from the data set with the specified name into the transform
//
injectDataSetIntoTransform((LocalPipelineEngine) pipeline, inputDataSetName, metadataProvider, transformMeta, inputLocation);
}
// How about capturing rows for golden data review?
//
PipelineUnitTestSetLocation goldenLocation = unitTest.findGoldenLocation(transformName);
if (goldenLocation != null) {
String goldenDataSetName = goldenLocation.getDataSetName();
if (!StringUtil.isEmpty(goldenDataSetName)) {
log.logDetailed("Capturing rows for validation at pipeline end, transform='" + transformMeta.getName() + "', golden set '" + goldenDataSetName);
final RowCollection rowCollection = new RowCollection();
// Create a row collection map if it's missing...
//
@SuppressWarnings("unchecked") Map<String, RowCollection> collectionMap = (Map<String, RowCollection>) pipeline.getExtensionDataMap().get(DataSetConst.ROW_COLLECTION_MAP);
if (collectionMap == null) {
collectionMap = new HashMap<>();
pipeline.getExtensionDataMap().put(DataSetConst.ROW_COLLECTION_MAP, collectionMap);
}
// Keep the map for safe keeping...
//
collectionMap.put(transformMeta.getName(), rowCollection);
// We'll capture the rows from this one and then evaluate them after execution...
//
IEngineComponent component = pipeline.findComponent(transformMeta.getName(), 0);
component.addRowListener(new RowAdapter() {
@Override
public void rowReadEvent(IRowMeta rowMeta, Object[] row) throws HopTransformException {
if (rowCollection.getRowMeta() == null) {
rowCollection.setRowMeta(rowMeta);
}
rowCollection.getRows().add(row);
}
});
}
}
}
} catch (Throwable e) {
throw new HopException("Unable to inject data set rows", e);
}
}
use of org.apache.hop.pipeline.transform.TransformMeta in project hop by apache.
the class PipelineMetaModifier method getTestPipeline.
public PipelineMeta getTestPipeline(ILogChannel log, IVariables variables, IHopMetadataProvider metadataProvider) throws HopException {
// OK, so now replace an input transform with a data set attached with an Injector transform...
// However, we don't want to have the user see this so we need to copy pipeline.pipelineMeta
// first...
//
// Clone seems to has problems so we'll take the long (XML) way around...
//
InputStream stream;
try {
stream = new ByteArrayInputStream(pipelineMeta.getXml(variables).getBytes(Const.XML_ENCODING));
} catch (UnsupportedEncodingException e) {
throw new HopException("Encoding error", e);
}
PipelineMeta copyPipelineMeta = new PipelineMeta(stream, metadataProvider, true, variables);
// Pass the metadata references...
//
copyPipelineMeta.setMetadataProvider(pipelineMeta.getMetadataProvider());
//
for (PipelineUnitTestDatabaseReplacement dbReplacement : unitTest.getDatabaseReplacements()) {
String sourceDatabaseName = variables.resolve(dbReplacement.getOriginalDatabaseName());
String replacementDatabaseName = variables.resolve(dbReplacement.getReplacementDatabaseName());
DatabaseMeta sourceDatabaseMeta = copyPipelineMeta.findDatabase(sourceDatabaseName);
DatabaseMeta replacementDatabaseMeta = copyPipelineMeta.findDatabase(replacementDatabaseName);
if (sourceDatabaseMeta == null) {
throw new HopException("Unable to find source database connection '" + sourceDatabaseName + "', can not be replaced");
}
if (replacementDatabaseMeta == null) {
throw new HopException("Unable to find replacement database connection '" + replacementDatabaseName + "', can not be used to replace");
}
if (log.isDetailed()) {
log.logDetailed("Replaced database connection '" + sourceDatabaseName + "' with connection '" + replacementDatabaseName + "'");
}
sourceDatabaseMeta.replaceMeta(replacementDatabaseMeta);
}
// Replace all transforms with an Input Data Set marker with an Injector
// Replace all transforms with a Golden Data Set marker with a Dummy
// Apply the tweaks to the transforms:
// - Bypass : replace with Dummy
// - Remove : remove transform and all connected hops.
//
// Loop over the original pipeline to allow us to safely modify the copy
//
List<TransformMeta> transforms = pipelineMeta.getTransforms();
for (TransformMeta transform : transforms) {
TransformMeta transformMeta = copyPipelineMeta.findTransform(transform.getName());
PipelineUnitTestSetLocation inputLocation = unitTest.findInputLocation(transformMeta.getName());
PipelineUnitTestSetLocation goldenLocation = unitTest.findGoldenLocation(transformMeta.getName());
PipelineUnitTestTweak transformTweak = unitTest.findTweak(transformMeta.getName());
//
if (inputLocation != null) {
handleInputDataSet(log, inputLocation, unitTest, pipelineMeta, transformMeta, metadataProvider);
}
//
if (goldenLocation != null) {
handleGoldenDataSet(log, goldenLocation, transformMeta, metadataProvider);
}
if (transformTweak != null && transformTweak.getTweak() != null) {
switch(transformTweak.getTweak()) {
case NONE:
break;
case REMOVE_TRANSFORM:
handleTweakRemoveTransform(log, copyPipelineMeta, transformMeta);
break;
case BYPASS_TRANSFORM:
handleTweakBypassTransform(log, transformMeta);
break;
default:
break;
}
}
}
return copyPipelineMeta;
}
use of org.apache.hop.pipeline.transform.TransformMeta in project hop by apache.
the class WriteToDataSetExtensionPoint method callExtensionPoint.
@Override
public void callExtensionPoint(ILogChannel log, IVariables variables, IPipelineEngine<PipelineMeta> pipeline) throws HopException {
final PipelineMeta pipelineMeta = pipeline.getPipelineMeta();
boolean writeToDataSet = "Y".equalsIgnoreCase(pipeline.getVariable(DataSetConst.VAR_WRITE_TO_DATASET));
if (!writeToDataSet) {
return;
}
pipeline.addExecutionFinishedListener(engine -> {
// Remove the flag when done.
// We don't want to write to the data set every time we run
//
pipeline.setVariable(DataSetConst.VAR_WRITE_TO_DATASET, null);
// Prevent memory leaking as well
//
WriteToDataSetExtensionPoint.transformsMap.remove(pipelineMeta.getName());
WriteToDataSetExtensionPoint.mappingsMap.remove(pipelineMeta.getName());
WriteToDataSetExtensionPoint.setsMap.remove(pipelineMeta.getName());
});
try {
IHopMetadataProvider metadataProvider = pipelineMeta.getMetadataProvider();
if (metadataProvider == null) {
// Nothing to do here, we can't reference data sets.
return;
}
//
for (final TransformMeta transformMeta : pipeline.getPipelineMeta().getTransforms()) {
// We might want to pass the data from this transform into a data set all by itself...
// For this we want to attach a row listener which writes the data.
//
TransformMeta injectMeta = transformsMap.get(pipelineMeta.getName());
if (injectMeta != null && injectMeta.equals(transformMeta)) {
final List<SourceToTargetMapping> mappings = mappingsMap.get(pipelineMeta.getName());
final DataSet dataSet = setsMap.get(pipelineMeta.getName());
if (mappings != null && dataSet != null) {
passTransformRowsToDataSet(pipeline, pipelineMeta, transformMeta, mappings, dataSet);
}
}
}
} catch (Throwable e) {
throw new HopException("Unable to pass rows to data set", e);
}
}
use of org.apache.hop.pipeline.transform.TransformMeta in project hop by apache.
the class HopNeo4jPerspective method openTransform.
private void openTransform(Session session, String name, String type, String id) {
LogChannel.UI.logDetailed("Open transform : " + id + ", name : " + name + ", type: " + type);
Map<String, Object> params = new HashMap<>();
params.put("subjectName", name);
params.put("subjectType", type);
params.put("subjectId", id);
StringBuilder cypher = new StringBuilder();
cypher.append(// TRANSFORM
"MATCH(e:Execution { name : $subjectName, type : $subjectType, id : $subjectId } )");
cypher.append(// Transform
"-[:EXECUTION_OF_TRANSFORM]->(t:Transform { name : $subjectName } )");
cypher.append("-[:TRANSFORM_OF_PIPELINE]->(p:Pipeline) ");
cypher.append("RETURN p.filename, t.name ");
String[] names = session.readTransaction(tx -> {
Result statementResult = tx.run(cypher.toString(), params);
if (!statementResult.hasNext()) {
statementResult.consume();
// No file found
return null;
}
Record record = statementResult.next();
statementResult.consume();
String filename = LoggingCore.getStringValue(record, 0);
String transformName = LoggingCore.getStringValue(record, 1);
return new String[] { filename, transformName };
});
if (names == null) {
return;
}
String filename = names[0];
String transformName = names[1];
if (StringUtils.isEmpty(filename)) {
return;
}
try {
hopGui.fileDelegate.fileOpen(filename);
if (StringUtils.isEmpty(transformName)) {
return;
}
HopDataOrchestrationPerspective perspective = HopGui.getDataOrchestrationPerspective();
IHopFileTypeHandler typeHandler = perspective.getActiveFileTypeHandler();
if (typeHandler == null || !(typeHandler instanceof HopGuiPipelineGraph)) {
return;
}
HopGuiPipelineGraph graph = (HopGuiPipelineGraph) typeHandler;
PipelineMeta pipelineMeta = graph.getPipelineMeta();
TransformMeta transformMeta = pipelineMeta.findTransform(transformName);
if (transformMeta == null) {
return;
}
pipelineMeta.unselectAll();
transformMeta.setSelected(true);
graph.editTransform(pipelineMeta, transformMeta);
} catch (Exception e) {
new ErrorDialog(hopGui.getShell(), BaseMessages.getString(PKG, "Neo4jPerspectiveDialog.OpeningTransform.Dialog.Header"), BaseMessages.getString(PKG, "Neo4jPerspectiveDialog.OpeningTransform.Dialog.Message"), e);
}
}
use of org.apache.hop.pipeline.transform.TransformMeta in project hop by apache.
the class UserDefinedJavaClassDialog method open.
@Override
public String open() {
Shell parent = getParent();
shell = new Shell(parent, SWT.DIALOG_TRIM | SWT.RESIZE | SWT.MAX | SWT.MIN);
props.setLook(shell);
setShellImage(shell, input);
lsMod = e -> input.setChanged();
changed = input.hasChanged();
FormLayout formLayout = new FormLayout();
formLayout.marginWidth = Const.FORM_MARGIN;
formLayout.marginHeight = Const.FORM_MARGIN;
shell.setLayout(formLayout);
shell.setText(BaseMessages.getString(PKG, "UserDefinedJavaClassDialog.Shell.Title"));
int middle = props.getMiddlePct();
margin = props.getMargin();
// Buttons go at the very bottom
//
wOk = new Button(shell, SWT.PUSH);
wOk.setText(BaseMessages.getString(PKG, "System.Button.OK"));
wOk.addListener(SWT.Selection, e -> ok());
Button wTest = new Button(shell, SWT.PUSH);
wTest.setText(BaseMessages.getString(PKG, "UserDefinedJavaClassDialog.TestClass.Button"));
wTest.addListener(SWT.Selection, e -> test());
wCancel = new Button(shell, SWT.PUSH);
wCancel.setText(BaseMessages.getString(PKG, "System.Button.Cancel"));
wCancel.addListener(SWT.Selection, e -> cancel());
setButtonPositions(new Button[] { wOk, wTest, wCancel }, margin, null);
// Filename line
wlTransformName = new Label(shell, SWT.RIGHT);
wlTransformName.setText(BaseMessages.getString(PKG, "UserDefinedJavaClassDialog.TransformName.Label"));
props.setLook(wlTransformName);
fdlTransformName = new FormData();
fdlTransformName.left = new FormAttachment(0, 0);
fdlTransformName.right = new FormAttachment(middle, -margin);
fdlTransformName.top = new FormAttachment(0, margin);
wlTransformName.setLayoutData(fdlTransformName);
wTransformName = new Text(shell, SWT.SINGLE | SWT.LEFT | SWT.BORDER);
wTransformName.setText(transformName);
props.setLook(wTransformName);
wTransformName.addModifyListener(lsMod);
fdTransformName = new FormData();
fdTransformName.left = new FormAttachment(middle, 0);
fdTransformName.top = new FormAttachment(0, margin);
fdTransformName.right = new FormAttachment(100, 0);
wTransformName.setLayoutData(fdTransformName);
SashForm wSash = new SashForm(shell, SWT.VERTICAL);
// Top sash form
//
Composite wTop = new Composite(wSash, SWT.NONE);
props.setLook(wTop);
FormLayout topLayout = new FormLayout();
topLayout.marginWidth = Const.FORM_MARGIN;
topLayout.marginHeight = Const.FORM_MARGIN;
wTop.setLayout(topLayout);
// Script line
Label wlScriptFunctions = new Label(wTop, SWT.NONE);
wlScriptFunctions.setText(BaseMessages.getString(PKG, "UserDefinedJavaClassDialog.ClassesAndSnippits.Label"));
props.setLook(wlScriptFunctions);
FormData fdlScriptFunctions = new FormData();
fdlScriptFunctions.left = new FormAttachment(0, 0);
fdlScriptFunctions.top = new FormAttachment(0, 0);
wlScriptFunctions.setLayoutData(fdlScriptFunctions);
// Tree View Test
wTree = new Tree(wTop, SWT.BORDER | SWT.H_SCROLL | SWT.V_SCROLL);
props.setLook(wTree);
FormData fdlTree = new FormData();
fdlTree.left = new FormAttachment(0, 0);
fdlTree.top = new FormAttachment(wlScriptFunctions, margin);
fdlTree.right = new FormAttachment(20, 0);
fdlTree.bottom = new FormAttachment(100, -margin);
wTree.setLayoutData(fdlTree);
// Script line
Label wlScript = new Label(wTop, SWT.NONE);
wlScript.setText(BaseMessages.getString(PKG, "UserDefinedJavaClassDialog.Class.Label"));
props.setLook(wlScript);
FormData fdlScript = new FormData();
fdlScript.left = new FormAttachment(wTree, margin);
fdlScript.top = new FormAttachment(0, 0);
wlScript.setLayoutData(fdlScript);
wlPosition = new Label(wTop, SWT.NONE);
wlPosition.setText(BaseMessages.getString(PKG, "UserDefinedJavaClassDialog.Position.Label", 1, 1));
props.setLook(wlPosition);
FormData fdlPosition = new FormData();
fdlPosition.left = new FormAttachment(wTree, margin);
fdlPosition.right = new FormAttachment(30, 0);
fdlPosition.bottom = new FormAttachment(100, 0);
wlPosition.setLayoutData(fdlPosition);
folder = new CTabFolder(wTop, SWT.BORDER | SWT.RESIZE);
folder.setUnselectedImageVisible(true);
folder.setUnselectedCloseVisible(true);
FormData fdScript = new FormData();
fdScript.left = new FormAttachment(wTree, margin);
fdScript.top = new FormAttachment(wlScript, margin);
fdScript.right = new FormAttachment(100, -5);
fdScript.bottom = new FormAttachment(wlPosition, -margin);
folder.setLayoutData(fdScript);
Text wlHelpLabel = new Text(wTop, SWT.V_SCROLL | SWT.LEFT);
wlHelpLabel.setEditable(false);
wlHelpLabel.setText("Help");
props.setLook(wlHelpLabel);
FormData fdHelpLabel = new FormData();
fdHelpLabel.left = new FormAttachment(wlPosition, margin);
fdHelpLabel.top = new FormAttachment(folder, margin);
fdHelpLabel.right = new FormAttachment(100, -5);
fdHelpLabel.bottom = new FormAttachment(100, 0);
wlHelpLabel.setLayoutData(fdHelpLabel);
wlHelpLabel.setVisible(false);
FormData fdTop = new FormData();
fdTop.left = new FormAttachment(0, 0);
fdTop.top = new FormAttachment(0, 0);
fdTop.right = new FormAttachment(100, 0);
fdTop.bottom = new FormAttachment(100, 0);
wTop.setLayoutData(fdTop);
//
// Add a tab folder for the parameters and various input and output
// streams
//
wTabFolder = new CTabFolder(wSash, SWT.BORDER);
props.setLook(wTabFolder, Props.WIDGET_STYLE_TAB);
wTabFolder.setUnselectedCloseVisible(false);
FormData fdTabFolder = new FormData();
fdTabFolder.left = new FormAttachment(0, 0);
fdTabFolder.right = new FormAttachment(100, 0);
fdTabFolder.top = new FormAttachment(0, 0);
fdTabFolder.bottom = new FormAttachment(wOk, -2 * margin);
wTabFolder.setLayoutData(fdTabFolder);
// The Fields tab...
//
addFieldsTab();
// The parameters
//
addParametersTab();
prevTransformNames = pipelineMeta.getPrevTransformNames(transformMeta);
nextTransformNames = pipelineMeta.getNextTransformNames(transformMeta);
// OK, add another tab for the input settings...
//
addInfoTab();
addTargetTab();
// Select the fields tab...
//
wTabFolder.setSelection(fieldsTab);
FormData fdSash = new FormData();
fdSash.left = new FormAttachment(0, 0);
fdSash.top = new FormAttachment(wTransformName, 0);
fdSash.right = new FormAttachment(100, 0);
fdSash.bottom = new FormAttachment(wOk, -2 * margin);
wSash.setLayoutData(fdSash);
wSash.setWeights(75, 25);
wTree.addListener(SWT.MouseDoubleClick, this::treeDblClick);
folder.addCTabFolder2Listener(new CTabFolder2Adapter() {
@Override
public void close(CTabFolderEvent event) {
CTabItem cItem = (CTabItem) event.item;
event.doit = false;
if (cItem != null && folder.getItemCount() > 1) {
MessageBox messageBox = new MessageBox(shell, SWT.ICON_QUESTION | SWT.NO | SWT.YES);
messageBox.setText(BaseMessages.getString(PKG, "UserDefinedJavaClassDialog.DeleteItem.Label"));
messageBox.setMessage(BaseMessages.getString(PKG, "UserDefinedJavaClassDialog.ConfirmDeleteItem.Label", cItem.getText()));
switch(messageBox.open()) {
case SWT.YES:
modifyTabTree(cItem, TabActions.DELETE_ITEM);
event.doit = true;
break;
default:
break;
}
}
}
});
cMenu = new Menu(shell, SWT.POP_UP);
buildingFolderMenu();
tMenu = new Menu(shell, SWT.POP_UP);
buildingTreeMenu();
// Adding the Default Transform Class Item to the Tree
wTreeClassesItem = new TreeItem(wTree, SWT.NULL);
wTreeClassesItem.setImage(guiResource.getImageFolder());
wTreeClassesItem.setText(BaseMessages.getString(PKG, "UserDefinedJavaClassDialog.Classes.Label"));
getData();
// Adding the Rest (Functions, InputItems, etc.) to the Tree
buildSnippitsTree();
// Input Fields
itemInput = new TreeItem(wTree, SWT.NULL);
itemInput.setImage(guiResource.getImageInput());
itemInput.setText(BaseMessages.getString(PKG, "UserDefinedJavaClassDialog.InputFields.Label"));
itemInput.setData("Field Helpers");
// Info Fields
itemInfo = new TreeItem(wTree, SWT.NULL);
itemInfo.setImage(guiResource.getImageInput());
itemInfo.setText(BaseMessages.getString(PKG, "UserDefinedJavaClassDialog.InfoFields.Label"));
itemInfo.setData("Field Helpers");
// Output Fields
itemOutput = new TreeItem(wTree, SWT.NULL);
itemOutput.setImage(guiResource.getImageOutput());
itemOutput.setText(BaseMessages.getString(PKG, "UserDefinedJavaClassDialog.OutputFields.Label"));
itemOutput.setData("Field Helpers");
// Display waiting message for input
TreeItem itemWaitFieldsIn = new TreeItem(itemInput, SWT.NULL);
itemWaitFieldsIn.setText(BaseMessages.getString(PKG, "UserDefinedJavaClassDialog.GettingFields.Label"));
itemWaitFieldsIn.setForeground(guiResource.getColorDirectory());
itemInput.setExpanded(true);
// Display waiting message for info
TreeItem itemWaitFieldsInfo = new TreeItem(itemInfo, SWT.NULL);
itemWaitFieldsInfo.setText(BaseMessages.getString(PKG, "UserDefinedJavaClassDialog.GettingFields.Label"));
itemWaitFieldsInfo.setForeground(guiResource.getColorDirectory());
itemInfo.setExpanded(true);
// Display waiting message for output
TreeItem itemWaitFieldsOut = new TreeItem(itemOutput, SWT.NULL);
itemWaitFieldsOut.setText(BaseMessages.getString(PKG, "UserDefinedJavaClassDialog.GettingFields.Label"));
itemWaitFieldsOut.setForeground(guiResource.getColorDirectory());
itemOutput.setExpanded(true);
//
// Search the fields in the background
//
final Runnable runnable = () -> {
TransformMeta transformMeta = pipelineMeta.findTransform(transformName);
if (transformMeta != null) {
try {
inputRowMeta = pipelineMeta.getPrevTransformFields(variables, transformMeta);
infoRowMeta = pipelineMeta.getPrevInfoFields(variables, transformMeta);
outputRowMeta = pipelineMeta.getThisTransformFields(variables, transformMeta, null, inputRowMeta.clone());
populateFieldsTree();
} catch (HopException e) {
log.logError(BaseMessages.getString(PKG, "System.Dialog.GetFieldsFailed.Message"), e);
}
}
};
new Thread(runnable).start();
addRenameToTreeScriptItems();
input.setChanged(changed);
// Create the drag source on the tree
DragSource ds = new DragSource(wTree, DND.DROP_MOVE);
ds.setTransfer(new Transfer[] { TextTransfer.getInstance() });
ds.addDragListener(new DragSourceAdapter() {
@Override
public void dragStart(DragSourceEvent event) {
boolean doit = false;
TreeItem item = wTree.getSelection()[0];
// Allow dragging snippits and field helpers
if (item != null && item.getParentItem() != null) {
if ("Snippits Category".equals(item.getParentItem().getData()) && !"Snippits Category".equals(item.getData())) {
doit = true;
} else if ("Field Helpers".equals(item.getParentItem().getData())) {
doit = true;
} else if (item.getParentItem().getParentItem() != null && "Field Helpers".equals(item.getParentItem().getParentItem().getData())) {
doit = true;
}
}
event.doit = doit;
}
@Override
public void dragSetData(DragSourceEvent event) {
// Set the data to be the first selected item's data
event.data = wTree.getSelection()[0].getData();
}
});
BaseDialog.defaultShellHandling(shell, c -> ok(), c -> cancel());
return transformName;
}
Aggregations