Examples with ImportFileParser au.gov.asd.tac.constellation.plugins.importexport.delimited.parser.ImportFileParser used on opensource projects

Example 1 with ImportFileParser

use of au.gov.asd.tac.constellation.plugins.importexport.delimited.parser.ImportFileParser in project constellation by constellation-app.

the class DelimitedSourcePane method addFile.

private void addFile(final DelimitedImportController importController) {
    final FileChooser fileChooser = new FileChooser();
    fileChooser.setInitialDirectory(defaultDirectory);
    final ImportFileParser parser = DelimitedSourcePane.this.importFileParserComboBox.getSelectionModel().getSelectedItem();
    if (parser != null) {
        final ExtensionFilter extensionFilter = parser.getExtensionFilter();
        if (extensionFilter != null) {
            fileChooser.getExtensionFilters().add(extensionFilter);
            fileChooser.setSelectedExtensionFilter(extensionFilter);
        }
    }
    final List<File> newFiles = fileChooser.showOpenMultipleDialog(DelimitedSourcePane.this.getScene().getWindow());
    if (newFiles != null) {
        if (!newFiles.isEmpty()) {
            defaultDirectory = newFiles.get(0).getParentFile();
            DelimitedSourcePane.this.importFileParserComboBox.setDisable(true);
        }
        final ObservableList<File> files = FXCollections.observableArrayList(fileListView.getItems());
        final StringBuilder sb = new StringBuilder();
        final String alertText = "The following files could not be parsed and have been excluded from the import set:\n";
        sb.append(alertText);
        for (final File file : newFiles) {
            // prevent adding the same file again
            if (fileAlreadyAdded(file)) {
                continue;
            }
            // Attempt to parse/preview, if a failure is detected don't add the file to the set of files to import.
            try {
                if (parser != null) {
                    parser.preview(new InputSource(file), null, PREVIEW_LIMIT);
                    files.add(file);
                }
            } catch (final IOException ex) {
                // Append the name of each file that could not be imported.
                sb.append("\n");
                sb.append(file.getName());
                LOGGER.log(Level.INFO, "Unable to parse the file {0}, " + "excluding from import set.", new Object[] { file.toString() });
                LOGGER.log(Level.WARNING, ex.toString());
            }
        }
        // If file names have been appended to sb, then some files could not be imported, so notify user.
        if (!sb.toString().equals(alertText)) {
            NotifyDisplayer.displayAlert("Import from File", "Invalid file(s) found", sb.toString(), Alert.AlertType.WARNING);
        }
        fileListView.setItems(files);
        if (!newFiles.isEmpty()) {
            fileListView.getSelectionModel().select(newFiles.get(0));
            fileListView.requestFocus();
        }
        final ObservableList<File> selectedFiles = fileListView.getSelectionModel().getSelectedItems();
        importController.setFiles(files, selectedFiles.isEmpty() ? null : selectedFiles.get(0));
        importController.validateFileStructure(newFiles);
    }
}

Example 2 with ImportFileParser

use of au.gov.asd.tac.constellation.plugins.importexport.delimited.parser.ImportFileParser in project constellation by constellation-app.

the class DelimitedImportController method processImport.

@Override
public void processImport() throws PluginException {
    final List<ImportDefinition> definitions = configurationPane.createDefinitions(isFilesIncludeHeadersEnabled());
    final Graph importGraph = currentDestination.getGraph();
    final boolean schema = schemaInitialised;
    final List<File> importFiles = new ArrayList<>(files);
    final ImportFileParser parser = importFileParser;
    if (currentDestination instanceof SchemaDestination) {
        final GraphManagerListener graphManagerListener = new GraphManagerListener() {

            private boolean opened = false;

            @Override
            public void graphOpened(Graph graph) {
            // Do nothing
            }

            @Override
            public void graphClosed(Graph graph) {
            // Do nothing
            }

            @Override
            public synchronized void newActiveGraph(final Graph graph) {
                if (graph == importGraph && !opened) {
                    opened = true;
                    GraphManager.getDefault().removeGraphManagerListener(this);
                    PluginExecutor.startWith(ImportExportPluginRegistry.IMPORT_DELIMITED, false).set(ImportDelimitedPlugin.DEFINITIONS_PARAMETER_ID, definitions).set(ImportDelimitedPlugin.PARSER_PARAMETER_ID, parser).set(ImportDelimitedPlugin.FILES_PARAMETER_ID, importFiles).set(ImportDelimitedPlugin.SCHEMA_PARAMETER_ID, schema).set(ImportDelimitedPlugin.PARSER_PARAMETER_IDS_PARAMETER_ID, currentParameters).set(ImportDelimitedPlugin.FILES_INCLUDE_HEADERS_PARAMETER_ID, filesIncludeHeaders).followedBy(ArrangementPluginRegistry.GRID_COMPOSITE).followedBy(InteractiveGraphPluginRegistry.RESET_VIEW).executeWriteLater(importGraph);
                }
            }
        };
        GraphManager.getDefault().addGraphManagerListener(graphManagerListener);
        GraphOpener.getDefault().openGraph(importGraph, "graph");
    } else {
        PluginExecutor.startWith(ImportExportPluginRegistry.IMPORT_DELIMITED, false).set(ImportDelimitedPlugin.DEFINITIONS_PARAMETER_ID, definitions).set(ImportDelimitedPlugin.PARSER_PARAMETER_ID, parser).set(ImportDelimitedPlugin.FILES_PARAMETER_ID, importFiles).set(ImportDelimitedPlugin.SCHEMA_PARAMETER_ID, schema).set(ImportDelimitedPlugin.FILES_INCLUDE_HEADERS_PARAMETER_ID, filesIncludeHeaders).followedBy(ArrangementPluginRegistry.GRID_COMPOSITE).followedBy(InteractiveGraphPluginRegistry.RESET_VIEW).executeWriteLater(importGraph);
    }
}

Example 3 with ImportFileParser

use of au.gov.asd.tac.constellation.plugins.importexport.delimited.parser.ImportFileParser in project constellation by constellation-app.

the class ImportDelimitedIO method loadParameterFile.

private static void loadParameterFile(final DelimitedImportController importController, final File delimIoDir, final String templName) {
    try {
        final ObjectMapper mapper = new ObjectMapper();
        final JsonNode root = mapper.readTree(new File(delimIoDir, FilenameEncoder.encode(templName) + FileExtensionConstants.JSON));
        final JsonNode source = root.get(SOURCE);
        final String parser = source.get(PARSER).textValue();
        final ImportFileParser ifp = ImportFileParser.getParser(parser);
        if (!importController.getImportFileParser().getLabel().equals(parser)) {
            final String message = String.format("Template is for a different file Parser '%s'.", parser);
            NotifyDisplayer.displayAlert(LOAD_TEMPLATE, "File Parser Mismatch", message, Alert.AlertType.ERROR);
            return;
        }
        importController.setImportFileParser(ifp);
        final boolean schemaInit = source.get(SCHEMA_INIT).booleanValue();
        importController.setSchemaInitialised(schemaInit);
        final boolean filesIncludeHeaders = source.get(FILES_INCLUDE_HEADERS).booleanValue();
        importController.setfilesIncludeHeaders(filesIncludeHeaders);
        final boolean showAllSchemaAttributes = source.get(SHOW_ALL_SCHEMA_ATTRIBUTES) != null && source.get(SHOW_ALL_SCHEMA_ATTRIBUTES).booleanValue();
        importController.setShowAllSchemaAttributes(showAllSchemaAttributes);
        final String destination = source.get(DESTINATION).textValue();
        final SchemaFactory schemaFactory = SchemaFactoryUtilities.getSchemaFactory(destination);
        if (schemaFactory != null) {
            importController.setDestination(new SchemaDestination(schemaFactory));
            final List<ImportDefinition> definitions = new ArrayList<>();
            final ArrayNode definitionsArray = (ArrayNode) root.withArray(DEFINITIONS);
            for (final JsonNode definitionNode : definitionsArray) {
                final int firstRow = definitionNode.get(FIRST_ROW).intValue();
                final RowFilter filter = new RowFilter();
                if (definitionNode.has(FILTER)) {
                    final JsonNode filterNode = definitionNode.get(FILTER);
                    final String script = filterNode.get(SCRIPT).textValue();
                    final JsonNode columnsArray = filterNode.withArray(COLUMNS);
                    final ArrayList<String> columns = new ArrayList<>();
                    for (final JsonNode column : columnsArray) {
                        columns.add(column.textValue());
                    }
                    filter.setScript(script);
                    filter.setColumns(columns.toArray(new String[columns.size()]));
                }
                final ImportDefinition impdef = new ImportDefinition("", firstRow, filter);
                final JsonNode attributesNode = definitionNode.get(ATTRIBUTES);
                for (final AttributeType attrType : AttributeType.values()) {
                    final ArrayNode columnArray = (ArrayNode) attributesNode.withArray(attrType.toString());
                    for (final JsonNode column : columnArray) {
                        final String columnLabel = column.get(COLUMN_LABEL).textValue();
                        final String label = column.get(ATTRIBUTE_LABEL).textValue();
                        if (!importController.hasAttribute(attrType.getElementType(), label)) {
                            // Manually created attribute.
                            final String type = column.get(ATTRIBUTE_TYPE).textValue();
                            final String descr = column.get(ATTRIBUTE_DESCRIPTION).textValue();
                            final NewAttribute a = new NewAttribute(attrType.getElementType(), type, label, descr);
                            importController.createManualAttribute(a);
                        }
                        final Attribute attribute = importController.getAttribute(attrType.getElementType(), label);
                        final AttributeTranslator translator = AttributeTranslator.getTranslator(column.get(TRANSLATOR).textValue());
                        final String args = column.get(TRANSLATOR_ARGS).textValue();
                        final String defaultValue = column.get(DEFAULT_VALUE).textValue();
                        final PluginParameters params = translator.createParameters();
                        translator.setParameterValues(params, args);
                        final ImportAttributeDefinition iad = new ImportAttributeDefinition(columnLabel, defaultValue, attribute, translator, params);
                        impdef.addDefinition(attrType, iad);
                    }
                }
                definitions.add(impdef);
            }
            importController.setClearManuallyAdded(false);
            try {
                ((DelimitedImportPane) importController.getStage()).update(importController, definitions);
            } finally {
                importController.setClearManuallyAdded(true);
            }
        } else {
            final String message = String.format("Can't find schema factory '%s'", destination);
            NotifyDisplayer.displayAlert(LOAD_TEMPLATE, "Destination Schema Error", message, Alert.AlertType.ERROR);
        }
    } catch (final IOException ex) {
        LOGGER.log(Level.SEVERE, ex.getLocalizedMessage(), ex);
    }
}

Example 4 with ImportFileParser

use of au.gov.asd.tac.constellation.plugins.importexport.delimited.parser.ImportFileParser in project constellation by constellation-app.

the class ImportDelimitedPlugin method edit.

@Override
protected void edit(final GraphWriteMethods graph, final PluginInteraction interaction, final PluginParameters parameters) throws InterruptedException, PluginException {
    final ImportFileParser parser = (ImportFileParser) parameters.getParameters().get(PARSER_PARAMETER_ID).getObjectValue();
    // files will be a list of file which extends from object type
    @SuppressWarnings("unchecked") final List<File> files = (List<File>) parameters.getParameters().get(FILES_PARAMETER_ID).getObjectValue();
    // definitions will be a list of import defintions which extends from object type
    @SuppressWarnings("unchecked") final List<ImportDefinition> definitions = (List<ImportDefinition>) parameters.getParameters().get(DEFINITIONS_PARAMETER_ID).getObjectValue();
    final boolean initialiseWithSchema = parameters.getParameters().get(SCHEMA_PARAMETER_ID).getBooleanValue();
    final PluginParameters parserParameters = (PluginParameters) parameters.getParameters().get(PARSER_PARAMETER_IDS_PARAMETER_ID).getObjectValue();
    final boolean filesIncludeHeaders = parameters.getParameters().get(FILES_INCLUDE_HEADERS_PARAMETER_ID).getBooleanValue();
    boolean positionalAtrributesExist = false;
    final List<String> validFiles = new ArrayList<>();
    final List<String> emptyFiles = new ArrayList<>();
    final List<String> invalidFiles = new ArrayList<>();
    final List<String> emptyRunConfigs = new ArrayList<>();
    int totalRows = 0;
    int totalImportedRows = 0;
    int dataSize = 0;
    // a minimum) defined
    for (final ImportDefinition definition : definitions) {
        if (definition.getDefinitions(AttributeType.SOURCE_VERTEX).isEmpty() && definition.getDefinitions(AttributeType.DESTINATION_VERTEX).isEmpty()) {
            emptyRunConfigs.add(definition.getDefinitionName());
        }
    }
    for (final File file : files) {
        interaction.setProgress(0, 0, "Reading File: " + file.getName(), true);
        List<String[]> data = null;
        int importedRowsPerFile = 0;
        try {
            data = parser.parse(new InputSource(file), parserParameters);
            dataSize = filesIncludeHeaders ? data.size() - 1 : data.size();
            totalRows = totalRows + Integer.max(0, dataSize);
            if (dataSize > 0) {
                if (validFiles.isEmpty()) {
                    validFiles.add(file.getName().concat(" (").concat(Integer.toString(dataSize)).concat(" rows)"));
                } else {
                    validFiles.add(file.getName().concat(" (").concat(Integer.toString(dataSize)).concat(")"));
                }
            } else {
                emptyFiles.add(file.getName());
            }
        } catch (FileNotFoundException ex) {
            final String errorMsg = file.getPath() + " could not be found. Ignoring file during import.";
            LOGGER.log(Level.INFO, errorMsg);
            invalidFiles.add(file.getName());
        } catch (IOException ex) {
            final String errorMsg = file.getPath() + " could not be parsed. Removing file during import.";
            LOGGER.log(Level.INFO, errorMsg);
            invalidFiles.add(file.getName());
        }
        if (data != null) {
            for (final ImportDefinition definition : definitions) {
                if (definition.getDefinitions(AttributeType.SOURCE_VERTEX).isEmpty()) {
                    // Process destination vertexes if defintions are defined, otherwise there is nothing to do.
                    if (!definition.getDefinitions(AttributeType.DESTINATION_VERTEX).isEmpty()) {
                        importedRowsPerFile += processVertices(definition, graph, data, AttributeType.DESTINATION_VERTEX, initialiseWithSchema, interaction, file.getName());
                    }
                } else if (definition.getDefinitions(AttributeType.DESTINATION_VERTEX).isEmpty()) {
                    // Source defintions exist, but no destination definitions exist. Process the source definitions.
                    importedRowsPerFile += processVertices(definition, graph, data, AttributeType.SOURCE_VERTEX, initialiseWithSchema, interaction, file.getName());
                } else {
                    // Both source and destination defintions exist, process them.
                    importedRowsPerFile += processTransactions(definition, graph, data, initialiseWithSchema, interaction, file.getName());
                }
                // Determine if a positional attribute has been defined, if so update the overall flag
                final boolean isPositional = attributeDefintionIsPositional(definition.getDefinitions(AttributeType.SOURCE_VERTEX), definition.getDefinitions(AttributeType.DESTINATION_VERTEX));
                positionalAtrributesExist = (positionalAtrributesExist || isPositional);
            }
        }
        totalImportedRows += importedRowsPerFile;
        LOGGER.log(Level.INFO, "Imported {0} rows of data from file {1} containing {2} total rows", new Object[] { importedRowsPerFile, file.getPath(), dataSize });
    }
    displaySummaryAlert(graph.getVertexCount() + graph.getTransactionCount(), totalImportedRows, validFiles, emptyFiles, invalidFiles, emptyRunConfigs);
    ConstellationLoggerHelper.importPropertyBuilder(this, GraphRecordStoreUtilities.getVertices(graph, false, false, false).getAll(GraphRecordStoreUtilities.SOURCE + VisualConcept.VertexAttribute.LABEL), files, ConstellationLoggerHelper.SUCCESS);
    LOGGER.log(Level.INFO, "Auto arrangement use={0}", (!positionalAtrributesExist));
    // the graph. This does mean some nodes could sit on top of each other if multiple nodes have the same coordinates.
    if (!positionalAtrributesExist) {
        interaction.setProgress(1, 1, "Arranging", true);
        graph.validateKey(GraphElementType.VERTEX, true);
        graph.validateKey(GraphElementType.TRANSACTION, true);
        // unfortunately need to arrange with pendants and uncollide because grid arranger works based on selection
        final VertexListInclusionGraph vlGraph = new VertexListInclusionGraph(graph, AbstractInclusionGraph.Connections.NONE, new ArrayList<>());
        PluginExecutor.startWith(ArrangementPluginRegistry.GRID_COMPOSITE).followedBy(ArrangementPluginRegistry.PENDANTS).followedBy(ArrangementPluginRegistry.UNCOLLIDE).followedBy(InteractiveGraphPluginRegistry.RESET_VIEW).executeNow(vlGraph.getInclusionGraph());
        vlGraph.retrieveCoords();
    }
}