Examples with Files org.apache.flink.shaded.guava30.com.google.common.io.Files used on opensource projects

Example 6 with Files

use of org.apache.flink.shaded.guava30.com.google.common.io.Files in project flink by apache.

the class UnalignedCheckpointStressITCase method discoverRetainedCheckpoint.

private File discoverRetainedCheckpoint() throws Exception {
    // structure: root/attempt/checkpoint/_metadata
    File rootDir = temporaryFolder.getRoot();
    Path checkpointDir = null;
    for (int i = 0; i <= 1000 && checkpointDir == null; i++) {
        Thread.sleep(5);
        try (Stream<Path> files = Files.walk(Paths.get(rootDir.getPath()))) {
            checkpointDir = files.filter(Files::isRegularFile).filter(path -> path.endsWith("_metadata")).map(path -> path.getParent()).sorted(Comparator.comparingInt(UnalignedCheckpointStressITCase::getCheckpointNumberFromPath)).reduce((first, second) -> second).orElse(null);
        }
    }
    if (checkpointDir == null) {
        List<Path> files = Files.walk(Paths.get(rootDir.getPath())).collect(Collectors.toList());
        throw new IllegalStateException("Failed to find _metadata file among " + files);
    }
    return checkpointDir.toFile();
}

Example 7 with Files

use of org.apache.flink.shaded.guava30.com.google.common.io.Files in project alluxio by Alluxio.

the class PinCommandMultipleMediaIntegrationTest method pinToMediumForceEviction.

@Test
public void pinToMediumForceEviction() throws Exception {
    FileSystem fileSystem = sLocalAlluxioClusterResource.get().getClient();
    FileSystemShell fsShell = new FileSystemShell(ServerConfiguration.global());
    AlluxioURI filePathA = new AlluxioURI("/testFileA");
    AlluxioURI dirPath = new AlluxioURI("/testDirA");
    AlluxioURI filePathB = new AlluxioURI(dirPath.getPath() + "/testFileB");
    AlluxioURI filePathC = new AlluxioURI("/testFileC");
    int fileSize = SIZE_BYTES / 2;
    FileSystemTestUtils.createByteFile(fileSystem, filePathA, WritePType.CACHE_THROUGH, fileSize);
    assertTrue(fileExists(fileSystem, filePathA));
    assertEquals(0, fsShell.run("pin", filePathA.toString(), "MEM"));
    URIStatus status = fileSystem.getStatus(filePathA);
    assertTrue(status.isPinned());
    assertTrue(status.getPinnedMediumTypes().contains("MEM"));
    fileSystem.createDirectory(dirPath);
    assertEquals(0, fsShell.run("pin", dirPath.toString(), "MEM"));
    FileSystemTestUtils.createByteFile(fileSystem, filePathB, WritePType.CACHE_THROUGH, fileSize);
    assertTrue(fileExists(fileSystem, filePathB));
    URIStatus statusB = fileSystem.getStatus(filePathB);
    assertTrue(statusB.isPinned());
    assertTrue(statusB.getPinnedMediumTypes().contains("MEM"));
    FileSystemTestUtils.createByteFile(fileSystem, filePathC, WritePType.THROUGH, fileSize);
    assertEquals(100, fileSystem.getStatus(filePathA).getInAlluxioPercentage());
    assertEquals(100, fileSystem.getStatus(filePathB).getInAlluxioPercentage());
    assertEquals(0, fileSystem.getStatus(filePathC).getInAlluxioPercentage());
    assertEquals(0, fsShell.run("pin", filePathC.toString(), "SSD"));
    // Verify files are replicated into the correct tier through job service
    CommonUtils.waitFor("File being loaded", () -> sJobCluster.getMaster().getJobMaster().listDetailed().stream().anyMatch(x -> x.getStatus().equals(Status.COMPLETED) && x.getName().equals("Replicate") && x.getAffectedPaths().contains(filePathC.getPath())), sWaitOptions);
    assertEquals(100, fileSystem.getStatus(filePathC).getInAlluxioPercentage());
    assertEquals(Constants.MEDIUM_MEM, fileSystem.getStatus(filePathA).getFileBlockInfos().get(0).getBlockInfo().getLocations().get(0).getMediumType());
    assertEquals(Constants.MEDIUM_MEM, fileSystem.getStatus(filePathB).getFileBlockInfos().get(0).getBlockInfo().getLocations().get(0).getMediumType());
    assertEquals(Constants.MEDIUM_SSD, fileSystem.getStatus(filePathC).getFileBlockInfos().get(0).getBlockInfo().getLocations().get(0).getMediumType());
    assertEquals(0, fsShell.run("unpin", filePathA.toString()));
    assertEquals(0, fsShell.run("pin", filePathC.toString(), "MEM"));
    status = fileSystem.getStatus(filePathA);
    assertFalse(status.isPinned());
    assertTrue(status.getPinnedMediumTypes().isEmpty());
    // Verify files are migrated from another tier into the correct tier through job service
    // Also verify that eviction works
    CommonUtils.waitFor("File being moved", () -> sJobCluster.getMaster().getJobMaster().listDetailed().stream().anyMatch(x -> x.getStatus().equals(Status.COMPLETED) && x.getName().equals("Move") && x.getAffectedPaths().contains(filePathC.getPath())), sWaitOptions);
    assertEquals(0, fileSystem.getStatus(filePathA).getInAlluxioPercentage());
    assertEquals(Constants.MEDIUM_MEM, fileSystem.getStatus(filePathC).getFileBlockInfos().get(0).getBlockInfo().getLocations().get(0).getMediumType());
}

Example 8 with Files

use of org.apache.flink.shaded.guava30.com.google.common.io.Files in project java by wavefrontHQ.

the class QueueExporterTest method testQueueExporter.

@Test
public void testQueueExporter() throws Exception {
    File file = new File(File.createTempFile("proxyTestConverter", null).getPath() + ".queue");
    file.deleteOnExit();
    String bufferFile = file.getAbsolutePath();
    TaskQueueFactory taskQueueFactory = new TaskQueueFactoryImpl(bufferFile, false, false, 128);
    EntityPropertiesFactory entityPropFactory = new DefaultEntityPropertiesFactoryForTesting();
    QueueExporter qe = new QueueExporter(bufferFile, "2878", bufferFile + "-output", false, taskQueueFactory, entityPropFactory);
    BufferedWriter mockedWriter = EasyMock.createMock(BufferedWriter.class);
    reset(mockedWriter);
    HandlerKey key = HandlerKey.of(ReportableEntityType.POINT, "2878");
    TaskQueue<LineDelimitedDataSubmissionTask> queue = taskQueueFactory.getTaskQueue(key, 0);
    queue.clear();
    UUID proxyId = UUID.randomUUID();
    LineDelimitedDataSubmissionTask task = new LineDelimitedDataSubmissionTask(null, proxyId, new DefaultEntityPropertiesForTesting(), queue, "wavefront", ReportableEntityType.POINT, "2878", ImmutableList.of("item1", "item2", "item3"), () -> 12345L);
    task.enqueue(QueueingReason.RETRY);
    LineDelimitedDataSubmissionTask task2 = new LineDelimitedDataSubmissionTask(null, proxyId, new DefaultEntityPropertiesForTesting(), queue, "wavefront", ReportableEntityType.POINT, "2878", ImmutableList.of("item4", "item5"), () -> 12345L);
    task2.enqueue(QueueingReason.RETRY);
    mockedWriter.write("item1");
    mockedWriter.newLine();
    mockedWriter.write("item2");
    mockedWriter.newLine();
    mockedWriter.write("item3");
    mockedWriter.newLine();
    mockedWriter.write("item4");
    mockedWriter.newLine();
    mockedWriter.write("item5");
    mockedWriter.newLine();
    TaskQueue<EventDataSubmissionTask> queue2 = taskQueueFactory.getTaskQueue(HandlerKey.of(ReportableEntityType.EVENT, "2888"), 0);
    queue2.clear();
    EventDataSubmissionTask eventTask = new EventDataSubmissionTask(null, proxyId, new DefaultEntityPropertiesForTesting(), queue2, "2888", ImmutableList.of(new Event(ReportEvent.newBuilder().setStartTime(123456789L * 1000).setEndTime(123456789L * 1000 + 1).setName("Event name for testing").setHosts(ImmutableList.of("host1", "host2")).setDimensions(ImmutableMap.of("multi", ImmutableList.of("bar", "baz"))).setAnnotations(ImmutableMap.of("severity", "INFO")).setTags(ImmutableList.of("tag1")).build()), new Event(ReportEvent.newBuilder().setStartTime(123456789L * 1000).setEndTime(123456789L * 1000 + 1).setName("Event name for testing").setHosts(ImmutableList.of("host1", "host2")).setAnnotations(ImmutableMap.of("severity", "INFO")).build())), () -> 12345L);
    eventTask.enqueue(QueueingReason.RETRY);
    mockedWriter.write("@Event 123456789000 123456789001 \"Event name for testing\" " + "\"host\"=\"host1\" \"host\"=\"host2\" \"severity\"=\"INFO\" \"multi\"=\"bar\" " + "\"multi\"=\"baz\" \"tag\"=\"tag1\"");
    mockedWriter.newLine();
    mockedWriter.write("@Event 123456789000 123456789001 \"Event name for testing\" " + "\"host\"=\"host1\" \"host\"=\"host2\" \"severity\"=\"INFO\"");
    mockedWriter.newLine();
    TaskQueue<SourceTagSubmissionTask> queue3 = taskQueueFactory.getTaskQueue(HandlerKey.of(ReportableEntityType.SOURCE_TAG, "2898"), 0);
    queue3.clear();
    SourceTagSubmissionTask sourceTagTask = new SourceTagSubmissionTask(null, new DefaultEntityPropertiesForTesting(), queue3, "2898", new SourceTag(ReportSourceTag.newBuilder().setOperation(SourceOperationType.SOURCE_TAG).setAction(SourceTagAction.SAVE).setSource("testSource").setAnnotations(ImmutableList.of("newtag1", "newtag2")).build()), () -> 12345L);
    sourceTagTask.enqueue(QueueingReason.RETRY);
    mockedWriter.write("@SourceTag action=save source=\"testSource\" \"newtag1\" \"newtag2\"");
    mockedWriter.newLine();
    expectLastCall().once();
    replay(mockedWriter);
    assertEquals(2, queue.size());
    qe.processQueue(queue, mockedWriter);
    assertEquals(0, queue.size());
    assertEquals(1, queue2.size());
    qe.processQueue(queue2, mockedWriter);
    assertEquals(0, queue2.size());
    assertEquals(1, queue3.size());
    qe.processQueue(queue3, mockedWriter);
    assertEquals(0, queue3.size());
    verify(mockedWriter);
    List<String> files = ConcurrentShardedQueueFile.listFiles(bufferFile, ".spool").stream().map(x -> x.replace(bufferFile + ".", "")).collect(Collectors.toList());
    assertEquals(3, files.size());
    assertTrue(files.contains("points.2878.0.spool_0000"));
    assertTrue(files.contains("events.2888.0.spool_0000"));
    assertTrue(files.contains("sourceTags.2898.0.spool_0000"));
    HandlerKey k1 = HandlerKey.of(ReportableEntityType.POINT, "2878");
    HandlerKey k2 = HandlerKey.of(ReportableEntityType.EVENT, "2888");
    HandlerKey k3 = HandlerKey.of(ReportableEntityType.SOURCE_TAG, "2898");
    files = ConcurrentShardedQueueFile.listFiles(bufferFile, ".spool");
    Set<HandlerKey> hk = QueueExporter.getValidHandlerKeys(files, "all");
    assertEquals(3, hk.size());
    assertTrue(hk.contains(k1));
    assertTrue(hk.contains(k2));
    assertTrue(hk.contains(k3));
    hk = QueueExporter.getValidHandlerKeys(files, "2878, 2898");
    assertEquals(2, hk.size());
    assertTrue(hk.contains(k1));
    assertTrue(hk.contains(k3));
    hk = QueueExporter.getValidHandlerKeys(files, "2888");
    assertEquals(1, hk.size());
    assertTrue(hk.contains(k2));
}

Example 9 with Files

use of org.apache.flink.shaded.guava30.com.google.common.io.Files in project mzmine2 by mzmine.

the class KovatsIndexExtractionDialog method addDialogComponents.

@Override
protected void addDialogComponents() {
    super.addDialogComponents();
    mainPanel.removeAll();
    mainPanel.getParent().remove(mainPanel);
    ddlKovats = new DelayedDocumentListener(e -> updateKovatsList());
    DelayedDocumentListener ddlUpdateChart = new DelayedDocumentListener(e -> updateChart());
    newMainPanel = new JPanel(new MigLayout("fill", "[right][grow,fill]", ""));
    getContentPane().add(newMainPanel, BorderLayout.SOUTH);
    JPanel pnCenter = new JPanel(new BorderLayout());
    getContentPane().add(pnCenter, BorderLayout.CENTER);
    pnChart = new JPanel(new BorderLayout());
    pnCenter.add(pnChart, BorderLayout.CENTER);
    Box sizedummy = new Box(BoxLayout.X_AXIS);
    sizedummy.setMinimumSize(new Dimension(200, 450));
    sizedummy.setPreferredSize(new Dimension(200, 450));
    pnChart.add(sizedummy, BorderLayout.CENTER);
    // left: Kovats: min max and list
    JPanel west = new JPanel(new BorderLayout());
    newMainPanel.add(west);
    // add min max
    JPanel pnKovatsParam = new JPanel(new FlowLayout(FlowLayout.CENTER, 5, 5));
    west.add(pnKovatsParam, BorderLayout.NORTH);
    minc = (IntegerComponent) getComponentForParameter(KovatsIndexExtractionParameters.minKovats);
    maxc = (IntegerComponent) getComponentForParameter(KovatsIndexExtractionParameters.maxKovats);
    minc.addDocumentListener(ddlKovats);
    maxc.addDocumentListener(ddlKovats);
    pnKovatsParam.add(new JLabel("Min carbon:"));
    pnKovatsParam.add(minc);
    pnKovatsParam.add(new JLabel("Max carbon:"));
    pnKovatsParam.add(maxc);
    // kovats list
    JPanel pnKovatsSelect = new JPanel(new BorderLayout());
    west.add(pnKovatsSelect, BorderLayout.CENTER);
    comboKovats = (MultiChoiceComponent) getComponentForParameter(KovatsIndexExtractionParameters.kovats);
    comboKovats.addValueChangeListener(() -> handleKovatsSelectionChange());
    pnKovatsSelect.add(comboKovats, BorderLayout.CENTER);
    // center: Chart and parameters
    JPanel center = new JPanel(new BorderLayout());
    newMainPanel.add(center);
    // all parameters on peak pick panel
    JPanel pnSouth = new JPanel(new BorderLayout());
    center.add(pnSouth, BorderLayout.SOUTH);
    JPanel pnPeakPick = new JPanel(new MigLayout("", "[right][]", ""));
    pnSouth.add(pnPeakPick, BorderLayout.CENTER);
    valuesComponent = (StringComponent) getComponentForParameter(KovatsIndexExtractionParameters.pickedKovatsValues);
    MZRangeComponent mzc = (MZRangeComponent) getComponentForParameter(KovatsIndexExtractionParameters.mzRange);
    RTRangeComponent rtc = (RTRangeComponent) getComponentForParameter(KovatsIndexExtractionParameters.rtRange);
    DoubleComponent noisec = (DoubleComponent) getComponentForParameter(KovatsIndexExtractionParameters.noiseLevel);
    DoubleComponent edgeRatioC = (DoubleComponent) getComponentForParameter(KovatsIndexExtractionParameters.ratioTopEdge);
    valuesComponent.addDocumentListener(new DelayedDocumentListener(e -> kovatsValuesChanged()));
    valuesComponent.setLayout(new GridLayout(1, 1));
    pnCenter.add(valuesComponent, BorderLayout.SOUTH);
    JPanel pnButtonFlow = new JPanel();
    pnPeakPick.add(pnButtonFlow, "cell 0 0 2 1");
    JButton btnUpdateChart = new JButton("Update chart");
    btnUpdateChart.addActionListener(e -> updateChart());
    pnButtonFlow.add(btnUpdateChart);
    JButton btnPickRT = new JButton("Pick peaks");
    btnPickRT.addActionListener(e -> pickRetentionTimes());
    pnButtonFlow.add(btnPickRT);
    JButton btnSaveFile = new JButton("Save to file");
    btnSaveFile.setToolTipText("Save Kovats index file");
    btnSaveFile.addActionListener(e -> saveToFile());
    pnButtonFlow.add(btnSaveFile);
    JButton btnLoad = new JButton("Load");
    btnLoad.setToolTipText("Load Kovats index file");
    btnLoad.addActionListener(e -> loadFile());
    pnButtonFlow.add(btnLoad);
    JButton btnCombineFiles = new JButton("Combine files");
    btnCombineFiles.setToolTipText("Select multiple Kovats index files to be combined into one");
    btnCombineFiles.addActionListener(e -> combineFiles());
    pnButtonFlow.add(btnCombineFiles);
    // add combo for raw data file
    dataFiles = MZmineCore.getProjectManager().getCurrentProject().getDataFiles();
    comboDataFileName = new JComboBox<RawDataFile>(dataFiles);
    comboDataFileName2 = new JComboBox<RawDataFile>(dataFiles);
    cbSecondRaw = new JCheckBox();
    initRawDataFileSelection();
    pnPeakPick.add(new JLabel("Raw data file(s)"), "cell 0 1");
    pnPeakPick.add(comboDataFileName);
    cbSecondRaw.addItemListener(e -> useSecondDataFile(cbSecondRaw.isSelected()));
    pnPeakPick.add(cbSecondRaw, "cell 0 2");
    pnPeakPick.add(comboDataFileName2);
    // direct alkane selection < CxH2x+1 >
    JPanel pnAlkaneSelect = new JPanel();
    Dimension dim = new Dimension(SIZE, SIZE);
    JButton btnPrevAlkane = new JButton(iconPrev);
    btnPrevAlkane.addActionListener(e -> setMzRangeByAlkane(-1));
    btnPrevAlkane.setPreferredSize(dim);
    btnPrevAlkane.setMaximumSize(dim);
    JButton btnNextAlkane = new JButton(iconNext);
    btnNextAlkane.addActionListener(e -> setMzRangeByAlkane(1));
    btnNextAlkane.setPreferredSize(dim);
    btnNextAlkane.setMaximumSize(dim);
    lbCurrentAlkane = new JLabel("");
    cbCurrentAlkaneSubH = new JCheckBox("-H");
    cbCurrentAlkaneSubH.setSelected(true);
    cbCurrentAlkaneSubH.addItemListener(e -> setMzRangeByAlkane(0));
    pnAlkaneSelect.add(btnPrevAlkane);
    pnAlkaneSelect.add(lbCurrentAlkane);
    pnAlkaneSelect.add(btnNextAlkane);
    pnAlkaneSelect.add(cbCurrentAlkaneSubH);
    pnPeakPick.add(pnAlkaneSelect, "cell 1 3");
    pnPeakPick.add(new JLabel("m/z range"), "cell 0 4");
    pnPeakPick.add(mzc);
    pnPeakPick.add(new JLabel(KovatsIndexExtractionParameters.rtRange.getName()), "cell 0 5");
    pnPeakPick.add(rtc);
    pnPeakPick.add(new JLabel(KovatsIndexExtractionParameters.noiseLevel.getName()), "cell 0 6");
    pnPeakPick.add(noisec);
    pnPeakPick.add(new JLabel(KovatsIndexExtractionParameters.ratioTopEdge.getName()), "cell 0 7");
    pnPeakPick.add(edgeRatioC);
    // add listeners
    comboDataFileName.addItemListener(e -> updateChart());
    mzc.addDocumentListener(ddlUpdateChart);
    rtc.addDocumentListener(ddlUpdateChart);
    // show
    revalidate();
    updateMinimumSize();
    pack();
    updateChart();
}

Example 10 with Files

use of org.apache.flink.shaded.guava30.com.google.common.io.Files in project mzmine2 by mzmine.

the class KovatsIndexExtractionDialog method initRawDataFileSelection.

/**
 * Init raw data selection to last used raw data file or "kovats" or "dro"
 */
private void initRawDataFileSelection() {
    if (dataFiles != null && dataFiles.length <= 0)
        return;
    RawDataFilesSelection select = parameterSet.getParameter(KovatsIndexExtractionParameters.dataFiles).getValue();
    RawDataFile[] raw = null;
    // set to parameters files - if they exist in this project
    if (select != null && select.getMatchingRawDataFiles().length > 0) {
        RawDataFile[] exists = Arrays.stream(select.getMatchingRawDataFiles()).filter(r -> Arrays.stream(dataFiles).anyMatch(d -> r.getName().equals(d.getName()))).toArray(RawDataFile[]::new);
        if (exists.length > 0)
            raw = exists;
    }
    if (raw == null) {
        // find kovats or dro file
        // first use all kovats named files and then dro (max 2)
        RawDataFile[] kovats = Arrays.stream(dataFiles).filter(d -> d.getName().toLowerCase().contains("kovats")).toArray(RawDataFile[]::new);
        RawDataFile[] dro = Arrays.stream(dataFiles).filter(d -> d.getName().toLowerCase().contains("dro")).toArray(RawDataFile[]::new);
        // maximum of two files are chosen (0,1,2)
        int size = Math.min(2, kovats.length + dro.length);
        raw = new RawDataFile[size];
        for (int i = 0; i < raw.length; i++) {
            if (i < kovats.length)
                raw[i] = kovats[i];
            else
                raw[i] = dro[i - kovats.length];
        }
    }
    if (raw.length > 0) {
        selectedDataFile = raw;
        comboDataFileName.setSelectedItem(selectedDataFile[0]);
        if (raw.length > 1) {
            comboDataFileName2.setSelectedItem(selectedDataFile[1]);
        }
    }
}