use of org.pentaho.di.trans.steps.tableinput.TableInputMeta in project pdi-dataservice-server-plugin by pentaho.
the class TableInputParameterGeneration method getSQL.
private String getSQL(StepInterface stepInterface) throws PushDownOptimizationException {
TableInput tableInput;
if (stepInterface instanceof TableInput) {
tableInput = (TableInput) stepInterface;
} else {
throw new PushDownOptimizationException("Unable to push down to push down to type " + stepInterface.getClass());
}
final TableInputMeta tableInputMeta = (TableInputMeta) tableInput.getStepMeta().getStepMetaInterface();
return tableInputMeta.getSQL();
}
use of org.pentaho.di.trans.steps.tableinput.TableInputMeta in project data-access by pentaho.
the class TableInputTransformGenerator method createInputStep.
protected StepMeta createInputStep(TransMeta transMeta) {
TableInputMeta inputMeta = new TableInputMeta();
inputMeta.setDatabaseMeta(sourceDatabaseMeta);
inputMeta.setExecuteEachInputRow(false);
inputMeta.setRowLimit(Integer.toString(rowLimit));
inputMeta.setSQL(sql);
inputMeta.setVariableReplacementActive(false);
inputMeta.setLazyConversionActive(false);
// inputMeta.setTargetSteps(null);
StepMeta inputStepMeta = new StepMeta(TABLE_INPUT, TABLE_INPUT, inputMeta);
inputStepMeta.setStepErrorMeta(new StepErrorMeta(transMeta, inputStepMeta));
transMeta.addStep(inputStepMeta);
final FileTransformStats stats = getTransformStats();
StepErrorMeta inputErrorMeta = new StepErrorMeta(transMeta, inputStepMeta) {
public void addErrorRowData(Object[] row, int startIndex, long nrErrors, String errorDescriptions, String fieldNames, String errorCodes) {
StringBuffer sb = new StringBuffer();
sb.append("Rejected Row: ");
for (Object rowData : row) {
sb.append(rowData);
sb.append(", ");
}
sb.append("\r\n");
stats.getErrors().add(sb.toString() + errorDescriptions);
super.addErrorRowData(row, startIndex, nrErrors, errorDescriptions, fieldNames, errorCodes);
}
};
StepMeta outputDummyStepMeta = addDummyStep(transMeta, "InputErrorDummy");
inputErrorMeta.setTargetStep(outputDummyStepMeta);
inputErrorMeta.setEnabled(true);
inputStepMeta.setStepErrorMeta(inputErrorMeta);
return inputStepMeta;
}
use of org.pentaho.di.trans.steps.tableinput.TableInputMeta in project pentaho-kettle by pentaho.
the class TransProfileFactory method generateTransformation.
public TransMeta generateTransformation(LoggingObjectInterface parentLoggingInterface) throws KettleException {
PluginRegistry registry = PluginRegistry.getInstance();
// Get the list of fields from the table...
//
tableLayout = getTableFields(parentLoggingInterface);
// Now start building the transformation...
//
TransMeta transMeta = new TransMeta(databaseMeta);
transMeta.addDatabase(databaseMeta);
// Create a step to read the content of the table
// Read the data from the database table...
// For now we read it all, later we add options to only read the first X rows
//
TableInputMeta readMeta = new TableInputMeta();
readMeta.setSQL("SELECT * FROM " + schemaTable);
readMeta.setDatabaseMeta(databaseMeta);
StepMeta read = new StepMeta(registry.getPluginId(StepPluginType.class, readMeta), "Read data", readMeta);
read.setLocation(50, 50);
read.setDraw(true);
transMeta.addStep(read);
// Grab the data types too
//
// Now calculate the requested statistics for all fields...
// TODO: create configuration possibility
// For now, just do : min, max, sum, count, avg, std dev. (7)
//
int[] numericCalculations = new int[] { GroupByMeta.TYPE_GROUP_MIN, GroupByMeta.TYPE_GROUP_MAX, GroupByMeta.TYPE_GROUP_SUM, GroupByMeta.TYPE_GROUP_COUNT_ALL, GroupByMeta.TYPE_GROUP_AVERAGE, GroupByMeta.TYPE_GROUP_STANDARD_DEVIATION };
int[] stringCalculations = new int[] { GroupByMeta.TYPE_GROUP_MIN, GroupByMeta.TYPE_GROUP_MAX, GroupByMeta.TYPE_GROUP_COUNT_ALL };
int[] dateCalculations = new int[] { GroupByMeta.TYPE_GROUP_MIN, GroupByMeta.TYPE_GROUP_MAX, GroupByMeta.TYPE_GROUP_COUNT_ALL };
int[] booleanCalculations = new int[] { GroupByMeta.TYPE_GROUP_MIN, GroupByMeta.TYPE_GROUP_MAX, GroupByMeta.TYPE_GROUP_COUNT_ALL };
// Run it through the "group by" step without a grouping.
// Later, we can use the UnivariateStats plugin/step perhaps.
//
GroupByMeta statsMeta = new GroupByMeta();
int nrNumeric = 0;
int nrDates = 0;
int nrStrings = 0;
int nrBooleans = 0;
for (ValueMetaInterface valueMeta : tableLayout.getValueMetaList()) {
if (valueMeta.isNumeric()) {
nrNumeric++;
}
if (valueMeta.isDate()) {
nrDates++;
}
if (valueMeta.isString()) {
nrStrings++;
}
if (valueMeta.isBoolean()) {
nrBooleans++;
}
}
int nrCalculations = nrNumeric * numericCalculations.length + nrDates * dateCalculations.length + nrStrings * stringCalculations.length + nrBooleans * booleanCalculations.length;
statsMeta.allocate(0, nrCalculations);
int calcIndex = 0;
for (int i = 0; i < tableLayout.size(); i++) {
ValueMetaInterface valueMeta = tableLayout.getValueMeta(i);
//
if (valueMeta.isNumeric()) {
// CHECKSTYLE:LineLength:OFF
for (int c = 0; c < numericCalculations.length; c++) {
statsMeta.getAggregateField()[calcIndex] = valueMeta.getName() + "(" + GroupByMeta.getTypeDesc(numericCalculations[c]) + ")";
statsMeta.getSubjectField()[calcIndex] = valueMeta.getName();
statsMeta.getAggregateType()[calcIndex] = numericCalculations[c];
calcIndex++;
}
}
//
if (valueMeta.isString()) {
// CHECKSTYLE:LineLength:OFF
for (int c = 0; c < stringCalculations.length; c++) {
statsMeta.getAggregateField()[calcIndex] = valueMeta.getName() + "(" + GroupByMeta.getTypeDesc(stringCalculations[c]) + ")";
statsMeta.getSubjectField()[calcIndex] = valueMeta.getName();
statsMeta.getAggregateType()[calcIndex] = stringCalculations[c];
calcIndex++;
}
}
//
if (valueMeta.isDate()) {
for (int c = 0; c < dateCalculations.length; c++) {
statsMeta.getAggregateField()[calcIndex] = valueMeta.getName() + "(" + GroupByMeta.getTypeDesc(dateCalculations[c]) + ")";
statsMeta.getSubjectField()[calcIndex] = valueMeta.getName();
statsMeta.getAggregateType()[calcIndex] = dateCalculations[c];
calcIndex++;
}
}
//
if (valueMeta.isBoolean()) {
for (int c = 0; c < booleanCalculations.length; c++) {
statsMeta.getAggregateField()[calcIndex] = valueMeta.getName() + "(" + GroupByMeta.getTypeDesc(booleanCalculations[c]) + ")";
statsMeta.getSubjectField()[calcIndex] = valueMeta.getName();
statsMeta.getAggregateType()[calcIndex] = booleanCalculations[c];
calcIndex++;
}
}
}
StepMeta calc = new StepMeta(registry.getPluginId(StepPluginType.class, statsMeta), "Calc", statsMeta);
calc.setLocation(250, 50);
calc.setDraw(true);
transMeta.addStep(calc);
TransHopMeta hop = new TransHopMeta(read, calc);
transMeta.addTransHop(hop);
DummyTransMeta dummyMeta = new DummyTransMeta();
StepMeta result = new StepMeta(registry.getPluginId(StepPluginType.class, dummyMeta), RESULT_STEP_NAME, dummyMeta);
result.setLocation(450, 50);
result.setDraw(true);
transMeta.addStep(result);
TransHopMeta hop2 = new TransHopMeta(calc, result);
transMeta.addTransHop(hop2);
return transMeta;
}
use of org.pentaho.di.trans.steps.tableinput.TableInputMeta in project pentaho-kettle by pentaho.
the class TransMetaConverterTest method lazyConversionTurnedOff.
@Test
public void lazyConversionTurnedOff() throws KettleException {
KettleEnvironment.init();
TransMeta transMeta = new TransMeta();
CsvInputMeta csvInputMeta = new CsvInputMeta();
csvInputMeta.setLazyConversionActive(true);
StepMeta csvInput = new StepMeta("Csv", csvInputMeta);
transMeta.addStep(csvInput);
TableInputMeta tableInputMeta = new TableInputMeta();
tableInputMeta.setLazyConversionActive(true);
StepMeta tableInput = new StepMeta("Table", tableInputMeta);
transMeta.addStep(tableInput);
Transformation trans = TransMetaConverter.convert(transMeta);
TransMeta cloneMeta;
String transMetaXml = (String) trans.getConfig().get(TransMetaConverter.TRANS_META_CONF_KEY);
Document doc;
try {
doc = XMLHandler.loadXMLString(transMetaXml);
Node stepNode = XMLHandler.getSubNode(doc, "transformation");
cloneMeta = new TransMeta(stepNode, null);
} catch (KettleXMLException | KettleMissingPluginsException e) {
throw new RuntimeException(e);
}
assertThat(((CsvInputMeta) cloneMeta.findStep("Csv").getStepMetaInterface()).isLazyConversionActive(), is(false));
assertThat(((TableInputMeta) cloneMeta.findStep("Table").getStepMetaInterface()).isLazyConversionActive(), is(false));
}
use of org.pentaho.di.trans.steps.tableinput.TableInputMeta in project pentaho-kettle by pentaho.
the class RepositoryTestBase method testTransformations.
/**
* save(trans) loadTransformation() exists() getTransformationLock() lockTransformation() unlockTransformation()
* getTransformationID() getTransformationObjects() getTransformationNames()
*/
@Ignore
@Test
public void testTransformations() throws Exception {
ILockService service = (ILockService) repository.getService(ILockService.class);
RepositoryDirectoryInterface rootDir = initRepo();
String uniqueTransName = EXP_TRANS_NAME.concat(EXP_DBMETA_NAME);
TransMeta transMeta = createTransMeta(EXP_DBMETA_NAME);
// Create a database association
DatabaseMeta dbMeta = createDatabaseMeta(EXP_DBMETA_NAME);
repository.save(dbMeta, VERSION_COMMENT_V1, null);
TableInputMeta tableInputMeta = new TableInputMeta();
tableInputMeta.setDatabaseMeta(dbMeta);
transMeta.addStep(new StepMeta(EXP_TRANS_STEP_1_NAME, tableInputMeta));
RepositoryDirectoryInterface transDir = rootDir.findDirectory(DIR_TRANSFORMATIONS);
repository.save(transMeta, VERSION_COMMENT_V1, null);
deleteStack.push(transMeta);
assertNotNull(transMeta.getObjectId());
ObjectRevision version = transMeta.getObjectRevision();
assertNotNull(version);
assertTrue(hasVersionWithComment(transMeta, VERSION_COMMENT_V1));
assertTrue(repository.exists(uniqueTransName, transDir, RepositoryObjectType.TRANSFORMATION));
TransMeta fetchedTrans = repository.loadTransformation(uniqueTransName, transDir, null, false, null);
assertEquals(uniqueTransName, fetchedTrans.getName());
assertEquals(EXP_TRANS_DESC, fetchedTrans.getDescription());
assertEquals(EXP_TRANS_EXTENDED_DESC, fetchedTrans.getExtendedDescription());
assertEquals(transDir.getPath(), fetchedTrans.getRepositoryDirectory().getPath());
assertEquals(EXP_TRANS_VERSION, fetchedTrans.getTransversion());
assertEquals(EXP_TRANS_STATUS, fetchedTrans.getTransstatus());
assertEquals(EXP_TRANS_CREATED_USER, fetchedTrans.getCreatedUser());
assertEquals(EXP_TRANS_CREATED_DATE, fetchedTrans.getCreatedDate());
assertEquals(EXP_TRANS_MOD_USER, fetchedTrans.getModifiedUser());
assertEquals(EXP_TRANS_MOD_DATE, fetchedTrans.getModifiedDate());
assertEquals(1, fetchedTrans.listParameters().length);
assertEquals(EXP_TRANS_PARAM_1_DEF, fetchedTrans.getParameterDefault(EXP_TRANS_PARAM_1_NAME));
assertEquals(EXP_TRANS_PARAM_1_DESC, fetchedTrans.getParameterDescription(EXP_TRANS_PARAM_1_NAME));
// Test reference to database connection
String[] transformations = repository.getTransformationsUsingDatabase(dbMeta.getObjectId());
assertNotNull(transformations);
assertEquals(1, transformations.length);
assertTrue(transformations[0].contains(fetchedTrans.getName()));
// TransLogTable transLogTable = fetchedTrans.getTransLogTable();
// TODO mlowery why doesn't this work?
// assertEquals(EXP_TRANS_LOG_TABLE_CONN_NAME, transLogTable.getConnectionName());
// assertEquals(EXP_TRANS_LOG_TABLE_INTERVAL, transLogTable.getLogInterval());
// assertEquals(EXP_TRANS_LOG_TABLE_SCHEMA_NAME, transLogTable.getSchemaName());
// assertEquals(EXP_TRANS_LOG_TABLE_SIZE_LIMIT, transLogTable.getLogSizeLimit());
// assertEquals(EXP_TRANS_LOG_TABLE_TABLE_NAME, transLogTable.getTableName());
// assertEquals(EXP_TRANS_LOG_TABLE_TIMEOUT_IN_DAYS, transLogTable.getTimeoutInDays());
// PerformanceLogTable perfLogTable = fetchedTrans.getPerformanceLogTable();
// TODO mlowery why doesn't this work?
// assertEquals(EXP_TRANS_LOG_TABLE_CONN_NAME, perfLogTable.getConnectionName());
// assertEquals(EXP_TRANS_LOG_TABLE_INTERVAL, perfLogTable.getLogInterval());
// assertEquals(EXP_TRANS_LOG_TABLE_SCHEMA_NAME, perfLogTable.getSchemaName());
// assertEquals(EXP_TRANS_LOG_TABLE_TABLE_NAME, perfLogTable.getTableName());
// assertEquals(EXP_TRANS_LOG_TABLE_TIMEOUT_IN_DAYS, perfLogTable.getTimeoutInDays());
// ChannelLogTable channelLogTable = fetchedTrans.getChannelLogTable();
// TODO mlowery why doesn't this work?
// assertEquals(EXP_TRANS_LOG_TABLE_CONN_NAME, channelLogTable.getConnectionName());
// assertEquals(EXP_TRANS_LOG_TABLE_SCHEMA_NAME, channelLogTable.getSchemaName());
// assertEquals(EXP_TRANS_LOG_TABLE_TABLE_NAME, channelLogTable.getTableName());
// assertEquals(EXP_TRANS_LOG_TABLE_TIMEOUT_IN_DAYS, channelLogTable.getTimeoutInDays());
// StepLogTable stepLogTable = fetchedTrans.getStepLogTable();
// TODO mlowery why doesn't this work?
// assertEquals(EXP_TRANS_LOG_TABLE_CONN_NAME, stepLogTable.getConnectionName());
// assertEquals(EXP_TRANS_LOG_TABLE_SCHEMA_NAME, stepLogTable.getSchemaName());
// assertEquals(EXP_TRANS_LOG_TABLE_TABLE_NAME, stepLogTable.getTableName());
// assertEquals(EXP_TRANS_LOG_TABLE_TIMEOUT_IN_DAYS, stepLogTable.getTimeoutInDays());
assertEquals(EXP_DBMETA_NAME, fetchedTrans.getMaxDateConnection().getName());
assertEquals(EXP_TRANS_MAX_DATE_TABLE, fetchedTrans.getMaxDateTable());
assertEquals(EXP_TRANS_MAX_DATE_FIELD, fetchedTrans.getMaxDateField());
assertEquals(EXP_TRANS_MAX_DATE_OFFSET, fetchedTrans.getMaxDateOffset(), 0);
assertEquals(EXP_TRANS_MAX_DATE_DIFF, fetchedTrans.getMaxDateDifference(), 0);
assertEquals(EXP_TRANS_SIZE_ROWSET, fetchedTrans.getSizeRowset());
// TODO mlowery why don't next two sleep fields work?
// assertEquals(EXP_TRANS_SLEEP_TIME_EMPTY, fetchedTrans.getSleepTimeEmpty());
// assertEquals(EXP_TRANS_SLEEP_TIME_FULL, fetchedTrans.getSleepTimeFull());
assertEquals(EXP_TRANS_USING_UNIQUE_CONN, fetchedTrans.isUsingUniqueConnections());
assertEquals(EXP_TRANS_FEEDBACK_SHOWN, fetchedTrans.isFeedbackShown());
assertEquals(EXP_TRANS_FEEDBACK_SIZE, fetchedTrans.getFeedbackSize());
assertEquals(EXP_TRANS_USING_THREAD_PRIORITY_MGMT, fetchedTrans.isUsingThreadPriorityManagment());
assertEquals(EXP_TRANS_SHARED_OBJECTS_FILE, fetchedTrans.getSharedObjectsFile());
assertEquals(EXP_TRANS_CAPTURE_STEP_PERF_SNAPSHOTS, fetchedTrans.isCapturingStepPerformanceSnapShots());
assertEquals(EXP_TRANS_STEP_PERF_CAP_DELAY, fetchedTrans.getStepPerformanceCapturingDelay());
// TODO mlowery why doesn't this work?
// assertEquals(1, fetchedTrans.getDependencies().size());
// assertEquals(EXP_DBMETA_NAME, fetchedTrans.getDependency(0).getDatabase().getName());
// assertEquals(EXP_TRANS_DEP_TABLE_NAME, fetchedTrans.getDependency(0).getTablename());
// assertEquals(EXP_TRANS_DEP_FIELD_NAME, fetchedTrans.getDependency(0).getFieldname());
assertEquals(3, fetchedTrans.getSteps().size());
assertEquals(EXP_TRANS_STEP_1_NAME, fetchedTrans.getStep(0).getName());
assertEquals(EXP_TRANS_STEP_ERROR_META_1_ENABLED, fetchedTrans.getStep(0).getStepErrorMeta().isEnabled());
assertEquals(EXP_TRANS_STEP_ERROR_META_1_NR_ERRORS_VALUE_NAME, fetchedTrans.getStep(0).getStepErrorMeta().getNrErrorsValuename());
assertEquals(EXP_TRANS_STEP_ERROR_META_1_DESC_VALUE_NAME, fetchedTrans.getStep(0).getStepErrorMeta().getErrorDescriptionsValuename());
assertEquals(EXP_TRANS_STEP_ERROR_META_1_FIELDS_VALUE_NAME, fetchedTrans.getStep(0).getStepErrorMeta().getErrorFieldsValuename());
assertEquals(EXP_TRANS_STEP_ERROR_META_1_CODES_VALUE_NAME, fetchedTrans.getStep(0).getStepErrorMeta().getErrorCodesValuename());
assertEquals(EXP_TRANS_STEP_ERROR_META_1_MAX_ERRORS, fetchedTrans.getStep(0).getStepErrorMeta().getMaxErrors());
assertEquals(EXP_TRANS_STEP_ERROR_META_1_MAX_PERCENT_ERRORS, fetchedTrans.getStep(0).getStepErrorMeta().getMaxPercentErrors());
assertEquals(EXP_TRANS_STEP_ERROR_META_1_MIN_PERCENT_ROWS, fetchedTrans.getStep(0).getStepErrorMeta().getMinPercentRows());
assertEquals(EXP_TRANS_STEP_2_NAME, fetchedTrans.getStep(1).getName());
assertEquals(EXP_TRANS_STEP_1_NAME, fetchedTrans.getTransHop(0).getFromStep().getName());
assertEquals(EXP_TRANS_STEP_2_NAME, fetchedTrans.getTransHop(0).getToStep().getName());
assertEquals(1, transMeta.getSlaveStepCopyPartitionDistribution().getOriginalPartitionSchemas().size());
assertTrue(transMeta.getSlaveStepCopyPartitionDistribution().getOriginalPartitionSchemas().get(0).getName().startsWith(EXP_PART_SCHEMA_NAME));
assertTrue(-1 != transMeta.getSlaveStepCopyPartitionDistribution().getPartition(EXP_SLAVE_NAME, EXP_PART_SCHEMA_NAME, 0));
assertEquals(EXP_TRANS_SLAVE_TRANSFORMATION, transMeta.isSlaveTransformation());
TransMeta transMetaById = repository.loadTransformation(transMeta.getObjectId(), null);
assertEquals(fetchedTrans, transMetaById);
assertNull(service.getTransformationLock(transMeta.getObjectId()));
service.lockTransformation(transMeta.getObjectId(), EXP_TRANS_LOCK_MSG);
assertEquals(EXP_TRANS_LOCK_MSG, service.getTransformationLock(transMeta.getObjectId()).getMessage());
assertEquals(getDate(new Date()), getDate(service.getTransformationLock(transMeta.getObjectId()).getLockDate()));
assertEquals(EXP_LOGIN_PLUS_TENANT, service.getTransformationLock(transMeta.getObjectId()).getLogin());
// TODO mlowery currently PUR lock only stores "login"; why do we need username too?
// assertEquals(EXP_USERNAME, repository.getTransformationLock(transMeta.getObjectId()).getUsername());
assertEquals(transMeta.getObjectId(), service.getTransformationLock(transMeta.getObjectId()).getObjectId());
service.unlockTransformation(transMeta.getObjectId());
assertNull(service.getTransformationLock(transMeta.getObjectId()));
transMeta.setDescription(EXP_TRANS_DESC_V2);
repository.save(transMeta, VERSION_COMMENT_V2, null);
assertTrue(hasVersionWithComment(transMeta, VERSION_COMMENT_V2));
fetchedTrans = repository.loadTransformation(uniqueTransName, transDir, null, false, null);
assertEquals(EXP_TRANS_DESC_V2, fetchedTrans.getDescription());
fetchedTrans = repository.loadTransformation(uniqueTransName, transDir, null, false, VERSION_LABEL_V1);
assertEquals(EXP_TRANS_DESC, fetchedTrans.getDescription());
transMetaById = repository.loadTransformation(transMeta.getObjectId(), VERSION_LABEL_V1);
assertEquals(fetchedTrans, transMetaById);
assertEquals(transMeta.getObjectId(), repository.getTransformationID(uniqueTransName, transDir));
assertEquals(1, repository.getTransformationObjects(transDir.getObjectId(), false).size());
assertEquals(1, repository.getTransformationObjects(transDir.getObjectId(), true).size());
assertEquals(transMeta.getName(), repository.getTransformationObjects(transDir.getObjectId(), false).get(0).getName());
assertEquals(1, repository.getTransformationNames(transDir.getObjectId(), false).length);
assertEquals(1, repository.getTransformationNames(transDir.getObjectId(), true).length);
assertEquals(transMeta.getName(), repository.getTransformationNames(transDir.getObjectId(), false)[0]);
repository.deleteTransformation(transMeta.getObjectId());
assertFalse(repository.exists(uniqueTransName, transDir, RepositoryObjectType.TRANSFORMATION));
assertEquals(0, repository.getTransformationObjects(transDir.getObjectId(), false).size());
assertEquals(1, repository.getTransformationObjects(transDir.getObjectId(), true).size());
assertEquals(transMeta.getName(), repository.getTransformationObjects(transDir.getObjectId(), true).get(0).getName());
assertEquals(0, repository.getTransformationNames(transDir.getObjectId(), false).length);
assertEquals(1, repository.getTransformationNames(transDir.getObjectId(), true).length);
assertEquals(transMeta.getName(), repository.getTransformationNames(transDir.getObjectId(), true)[0]);
}
Aggregations