Examples with Record org.talend.dataquality.matchmerge.Record used on opensource projects

Example 1 with Record

use of org.talend.dataquality.matchmerge.Record in project tdq-studio-se by Talend.

the class MatchAnalysisExecutor method execute.

/*
     * (non-Javadoc)
     * 
     * @see org.talend.dq.analysis.IAnalysisExecutor#execute(org.talend.dataquality.analysis.Analysis)
     */
public ReturnCode execute(Analysis analysis) {
    assert analysis != null;
    // --- preconditions
    ReturnCode rc = AnalysisExecutorHelper.check(analysis);
    if (!rc.isOk()) {
        AnalysisExecutorHelper.setExecutionInfoInAnalysisResult(analysis, rc.isOk(), rc.getMessage());
        return rc;
    }
    // --- creation time
    final long startime = AnalysisExecutorHelper.setExecutionDateInAnalysisResult(analysis);
    EList<Indicator> indicators = analysis.getResults().getIndicators();
    RecordMatchingIndicator recordMatchingIndicator = null;
    BlockKeyIndicator blockKeyIndicator = null;
    for (Indicator ind : indicators) {
        if (ind instanceof RecordMatchingIndicator) {
            recordMatchingIndicator = (RecordMatchingIndicator) ind;
        } else if (ind instanceof BlockKeyIndicator) {
            blockKeyIndicator = (BlockKeyIndicator) ind;
        }
    }
    if (recordMatchingIndicator == null || blockKeyIndicator == null) {
        rc.setOk(Boolean.FALSE);
        // $NON-NLS-1$
        rc.setMessage(Messages.getString("MatchAnalysisExecutor.noIndicators"));
        AnalysisExecutorHelper.setExecutionInfoInAnalysisResult(analysis, rc.isOk(), rc.getMessage());
        return rc;
    }
    List<ModelElement> anlayzedElements = analysis.getContext().getAnalysedElements();
    if (anlayzedElements == null || anlayzedElements.size() == 0) {
        rc.setOk(Boolean.FALSE);
        // $NON-NLS-1$
        rc.setMessage(Messages.getString("MatchAnalysisExecutor.EmptyAnalyzedElement"));
        AnalysisExecutorHelper.setExecutionInfoInAnalysisResult(analysis, rc.isOk(), rc.getMessage());
        return rc;
    }
    // TDQ-9664 msjian: check the store on disk path.
    Boolean isStoreOnDisk = TaggedValueHelper.getValueBoolean(SQLExecutor.STORE_ON_DISK_KEY, analysis);
    if (isStoreOnDisk) {
        String tempDataPath = TaggedValueHelper.getValueString(SQLExecutor.TEMP_DATA_DIR, analysis);
        File file = new File(tempDataPath);
        if (!file.exists() || !file.isDirectory()) {
            rc.setOk(Boolean.FALSE);
            // $NON-NLS-1$
            rc.setMessage(Messages.getString("MatchAnalysisExecutor.InvalidPath", file.getPath()));
            AnalysisExecutorHelper.setExecutionInfoInAnalysisResult(analysis, rc.isOk(), rc.getMessage());
            return rc;
        }
    }
    // TDQ-9664~
    Map<MetadataColumn, String> columnMap = getColumn2IndexMap(anlayzedElements);
    ISQLExecutor sqlExecutor = getSQLExectutor(analysis, recordMatchingIndicator, columnMap);
    if (sqlExecutor == null) {
        rc.setOk(Boolean.FALSE);
        // $NON-NLS-1$
        rc.setMessage(Messages.getString("MatchAnalysisExecutor.noSqlExecutor"));
        AnalysisExecutorHelper.setExecutionInfoInAnalysisResult(analysis, rc.isOk(), rc.getMessage());
        return rc;
    }
    if (getMonitor() != null) {
        getMonitor().worked(20);
    }
    // Set schema for match key.
    TypedReturnCode<MatchGroupResultConsumer> returnCode = new TypedReturnCode<MatchGroupResultConsumer>();
    MetadataColumn[] completeColumnSchema = AnalysisRecordGroupingUtils.getCompleteColumnSchema(columnMap);
    String[] colSchemaString = new String[completeColumnSchema.length];
    int idx = 0;
    for (MetadataColumn metadataCol : completeColumnSchema) {
        colSchemaString[idx++] = metadataCol.getName();
    }
    recordMatchingIndicator.setMatchRowSchema(colSchemaString);
    recordMatchingIndicator.reset();
    MatchGroupResultConsumer matchResultConsumer = createMatchGroupResultConsumer(recordMatchingIndicator);
    if (sqlExecutor.isStoreOnDisk()) {
        // need to execute the query
        try {
            sqlExecutor.executeQuery(analysis.getContext().getConnection(), analysis.getContext().getAnalysedElements());
        } catch (SQLException e) {
            log.error(e, e);
            rc.setOk(Boolean.FALSE);
            rc.setMessage(e.getMessage());
            AnalysisExecutorHelper.setExecutionInfoInAnalysisResult(analysis, rc.isOk(), rc.getMessage());
            return rc;
        }
        try {
            TypedReturnCode<Object> result = StoreOnDiskUtils.getDefault().executeWithStoreOnDisk(columnMap, recordMatchingIndicator, blockKeyIndicator, sqlExecutor.getStoreOnDiskHandler(), matchResultConsumer);
            if (result != null) {
                returnCode.setObject((MatchGroupResultConsumer) result.getObject());
                returnCode.setOk(result.isOk());
                returnCode.setMessage(result.getMessage());
            }
        } catch (Exception e) {
            log.error(e, e);
            returnCode.setMessage(e.getMessage());
            returnCode.setOk(false);
        }
    } else {
        // Added TDQ-9320 , use the result set iterator to replace the list of result in the memory.
        try {
            Iterator<Record> resultSetIterator = sqlExecutor.getResultSetIterator(analysis.getContext().getConnection(), anlayzedElements);
            BlockAndMatchManager bAndmManager = new BlockAndMatchManager(resultSetIterator, matchResultConsumer, columnMap, recordMatchingIndicator, blockKeyIndicator);
            bAndmManager.run();
        } catch (SQLException e) {
            log.error(e, e);
            rc.setOk(Boolean.FALSE);
            rc.setMessage(e.getMessage());
            AnalysisExecutorHelper.setExecutionInfoInAnalysisResult(analysis, rc.isOk(), rc.getMessage());
            return rc;
        } catch (BusinessException e) {
            log.error(e, e);
            rc.setOk(Boolean.FALSE);
            rc.setMessage(e.getMessage());
            AnalysisExecutorHelper.setExecutionInfoInAnalysisResult(analysis, rc.isOk(), rc.getMessage());
            return rc;
        }
    }
    if (!returnCode.isOk()) {
        rc.setOk(returnCode.isOk());
        rc.setMessage(returnCode.getMessage());
    }
    if (getMonitor() != null) {
        getMonitor().worked(20);
    }
    if (isLowMemory) {
        // $NON-NLS-1$
        rc.setMessage(Messages.getString("Evaluator.OutOfMomory", usedMemory));
    }
    // nodify the master page
    refreshTableWithMatchFullResult(analysis);
    // --- set metadata information of analysis
    AnalysisExecutorHelper.setExecutionInfoInAnalysisResult(analysis, rc.isOk(), rc.getMessage());
    // --- compute execution duration
    if (this.continueRun()) {
        long endtime = System.currentTimeMillis();
        final ExecutionInformations resultMetadata = analysis.getResults().getResultMetadata();
        resultMetadata.setExecutionDuration((int) (endtime - startime));
        resultMetadata.setOutThreshold(false);
    }
    if (getMonitor() != null) {
        getMonitor().worked(20);
    }
    return rc;
}

Example 2 with Record

use of org.talend.dataquality.matchmerge.Record in project tdq-studio-se by Talend.

the class FileCSVReaderTest method testNext.

/**
 * Test method for {@link org.talend.cwm.db.connection.file.FileCSVReader#next()}.
 */
@Test
public void testNext() {
    try {
        File file = new File("./file.txt");
        if (file.exists()) {
            file.delete();
        }
        if (file.createNewFile()) {
            PrintWriter out = new PrintWriter(new BufferedWriter(new FileWriter(file)));
            out.println("id,name,email");
            out.println("1,tom,a@b.cn");
            out.println("2,jerry,x@b.cn");
            out.println("3,bob,top@b.cn");
            out.close();
            DelimitedFileConnection delimitedFileconnection = ConnectionFactory.eINSTANCE.createDelimitedFileConnection();
            delimitedFileconnection.setFilePath(file.getAbsolutePath());
            delimitedFileconnection.setLimitValue("0");
            delimitedFileconnection.setHeaderValue("1");
            delimitedFileconnection.setFieldSeparatorValue(",");
            List<ModelElement> analysisElementList = new ArrayList<ModelElement>();
            MetadataTable table = ConnectionFactory.eINSTANCE.createMetadataTable();
            table.setName("table");
            MetadataColumn id = ConnectionFactory.eINSTANCE.createMetadataColumn();
            id.setName("id");
            id.setLabel("id");
            id.setTable(table);
            MetadataColumn name = ConnectionFactory.eINSTANCE.createMetadataColumn();
            name.setName("name");
            name.setLabel("name");
            name.setTable(table);
            MetadataColumn email = ConnectionFactory.eINSTANCE.createMetadataColumn();
            email.setName("email");
            email.setLabel("email");
            email.setTable(table);
            analysisElementList.add(id);
            analysisElementList.add(name);
            analysisElementList.add(email);
            FileCSVReader fileCSVReader = new FileCSVReader(file, delimitedFileconnection, analysisElementList);
            String fileContent = "";
            int i = 0;
            while (fileCSVReader.hasNext()) {
                i++;
                Record record = fileCSVReader.next();
                List<Attribute> attributes = record.getAttributes();
                fileContent += i + ":";
                for (Attribute attribute : attributes) {
                    fileContent += "[" + attribute.getLabel() + "," + attribute.getValue() + "]";
                }
            }
            fileCSVReader.close();
            Assert.assertEquals("1:[id,1][name,tom][email,a@b.cn]2:[id,2][name,jerry][email,x@b.cn]3:[id,3][name,bob][email,top@b.cn]", fileContent);
        }
        // delete the temp file
        if (file.exists()) {
            file.delete();
        }
    } catch (FileNotFoundException e) {
        e.printStackTrace();
    } catch (IOException e) {
        e.printStackTrace();
    }
}