/*
 * The MIT License
 *
 * Copyright (c) 2016 The Broad Institute
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 * THE SOFTWARE.
 */
package htsjdk.samtools;

import com.google.common.jimfs.Configuration;
import com.google.common.jimfs.Jimfs;
import htsjdk.HtsjdkTest;
import htsjdk.samtools.seekablestream.SeekableFileStream;
import htsjdk.samtools.util.CloseableIterator;
import org.testng.Assert;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.nio.file.FileSystem;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
import java.util.function.Function;

/**
 * Test index query calls against a *SamReader* on a CRAM File, which will use
 * whatever index format (.bai or .crai converted to .bai) is available for the
 * target file.
 */

public class CRAMIndexQueryTest extends HtsjdkTest {

    private static final File TEST_DATA_DIR = new File("src/test/resources/htsjdk/samtools/cram");

    private static final File cramQueryWithBAI = new File(TEST_DATA_DIR, "cramQueryWithBAI.cram");
    private static final File cramQueryWithCRAI = new File(TEST_DATA_DIR, "cramQueryWithCRAI.cram");
    private static File cramQueryWithLocalCRAI = null; // generated  by @BeforeClass from cramQueryWithCRAI
    private static final File cramQueryReference = new File(TEST_DATA_DIR, "human_g1k_v37.20.21.10M-10M200k.fasta");

    private static final File cramQueryReadsWithBAI = new File(TEST_DATA_DIR, "cramQueryTest.cram");
    private static File cramQueryReadsWithLocalCRAI = null; // generated  by @BeforeClass from cramQueryReadsWithBAI

    private static final File cramQueryTestEmptyWithBAI = new File(TEST_DATA_DIR, "cramQueryTestEmpty.cram");
    private static File cramQueryTestEmptyWithLocalCRAI = null; // generated  by @BeforeClass from cramQueryTestEmptyWithBAI
    private static final File cramQueryReadsReference = new File(TEST_DATA_DIR,"../hg19mini.fasta");


    // htsjdk currently generates .bai index files instead of .crai due to https://github.com/samtools/htsjdk/issues/531;
    // however we still want to test the htsjdk CRAI creation code. In order to run these index query tests against CRAI
    // files generated by htsjdk, we copy the checked-in test files into new CRAMs  and then manually generate .CRAI files
    // to run to use as additional test cases
    @BeforeClass
    public void createLocallyGeneratedCRAIFiles() throws IOException {
        cramQueryWithLocalCRAI = File.createTempFile("cramQueryWithLocalCRAI.", ".cram");
        cramQueryWithLocalCRAI.deleteOnExit();
        File tempCRAIOut = new File(cramQueryWithLocalCRAI.getAbsolutePath() + ".crai");
        tempCRAIOut.deleteOnExit();
        createLocalCRAMAndCRAI(
                cramQueryWithCRAI,
                cramQueryWithLocalCRAI,
                tempCRAIOut
        );

        cramQueryReadsWithLocalCRAI = File.createTempFile("cramQueryReadsWithLocalCRAI.", ".cram");
        tempCRAIOut = new File(cramQueryReadsWithLocalCRAI.getAbsolutePath() + ".crai");
        tempCRAIOut.deleteOnExit();
        cramQueryReadsWithLocalCRAI.deleteOnExit();
        createLocalCRAMAndCRAI(
                cramQueryReadsWithBAI,
                cramQueryReadsWithLocalCRAI,
                tempCRAIOut
        );

        cramQueryTestEmptyWithLocalCRAI = File.createTempFile("cramQueryTestEmptyWithLocalCRAI.", ".cram");
        tempCRAIOut = new File(cramQueryTestEmptyWithLocalCRAI.getAbsolutePath() +".crai");
        tempCRAIOut.deleteOnExit();
        cramQueryTestEmptyWithLocalCRAI.deleteOnExit();
        createLocalCRAMAndCRAI(
                cramQueryTestEmptyWithBAI,
                cramQueryTestEmptyWithLocalCRAI,
                tempCRAIOut
        );
    }

    private void createLocalCRAMAndCRAI(
            final File inputCRAM,
            final File outputCRAM,
            final File outputCRAI) throws IOException
    {
        Files.copy(inputCRAM.toPath(), outputCRAM.toPath(), StandardCopyOption.REPLACE_EXISTING);
        try (FileOutputStream bos = new FileOutputStream(outputCRAI)) {
            CRAMCRAIIndexer.writeIndex(new SeekableFileStream(outputCRAM), bos);
        }
    }

    @DataProvider(name = "singleIntervalOverlapping")
    public Object[][] singleIntervalOverlapping() {
        return new Object[][] {
            {cramQueryWithCRAI, cramQueryReference, new QueryInterval(0, 100009, 100009), new String[]{"a", "b", "c"}},
            {cramQueryWithLocalCRAI, cramQueryReference, new QueryInterval(0, 100009, 100009), new String[]{"a", "b", "c"}},
            {cramQueryWithBAI, cramQueryReference, new QueryInterval(0, 100009, 100009), new String[]{"a", "b", "c"}},

            {cramQueryWithCRAI, cramQueryReference, new QueryInterval(0, 100009, 100011), new String[]{"a", "b", "c", "d", "e"}},
            {cramQueryWithLocalCRAI, cramQueryReference, new QueryInterval(0, 100009, 100011), new String[]{"a", "b", "c", "d", "e"}},
            {cramQueryWithBAI, cramQueryReference, new QueryInterval(0, 100009, 100011), new String[]{"a", "b", "c", "d", "e"}},

            // interval with 1 start
            {cramQueryWithCRAI, cramQueryReference, new QueryInterval(0, 1, 100010), new String[]{"a", "b", "c", "d"}},
            {cramQueryWithLocalCRAI, cramQueryReference, new QueryInterval(0, 1, 100010), new String[]{"a", "b", "c", "d"}},
            {cramQueryWithBAI, cramQueryReference, new QueryInterval(0, 1, 100010), new String[]{"a", "b", "c", "d"}},

            // interval with 0 end
            {cramQueryWithCRAI, cramQueryReference, new QueryInterval(0, 100015, 0), new String[]{"a", "b", "c", "d", "e", "f"}},
            {cramQueryWithLocalCRAI, cramQueryReference, new QueryInterval(0, 100015, 0), new String[]{"a", "b", "c", "d", "e", "f"}},
            {cramQueryWithBAI, cramQueryReference, new QueryInterval(0, 100015, 0), new String[]{"a", "b", "c", "d", "e", "f"}},

            // interval with 1 start and 0 end
            {cramQueryWithCRAI, cramQueryReference, new QueryInterval(0, 1, 0), new String[]{"a", "b", "c", "d", "e", "f",  "f"}},
            {cramQueryWithLocalCRAI, cramQueryReference, new QueryInterval(0, 1, 0), new String[]{"a", "b", "c", "d", "e", "f",  "f"}},
            {cramQueryWithBAI, cramQueryReference, new QueryInterval(0, 1, 0), new String[]{"a", "b", "c", "d", "e", "f",  "f"}},

            //totally empty cram file
            {cramQueryTestEmptyWithBAI, cramQueryReadsReference, new QueryInterval(0, 1, 0), new String[]{}},
            {cramQueryTestEmptyWithLocalCRAI, cramQueryReadsReference, new QueryInterval(0, 1, 0), new String[]{}},

             { new File(TEST_DATA_DIR, "mitoAlignmentStartTest.cram"),
                     new File(TEST_DATA_DIR, "mitoAlignmentStartTest.fa"),
                     new QueryInterval(0, 631, 631),
                     new String[] {
                            "IL29_4505:7:51:11752:16255#2",
                             "IL29_4505:7:30:11521:4492#2",
                             "IL29_4505:7:78:5750:11942#2",
                             "IL29_4505:7:69:14718:6562#2",
                             "IL29_4505:7:88:5383:14756#2",
                             "IL29_4505:7:51:13987:8014#2",
                             "IL29_4505:7:26:6292:20619#2",
                             "IL29_4505:7:6:12278:3566#2",
                             "IL29_4505:7:7:7226:5753#2",
                             "IL29_4505:7:2:18466:8416#2",
                             "IL29_4505:7:42:15538:18597#2",
                             "IL29_4505:7:76:8383:16887#2",
                             "IL29_4505:7:114:15293:18740#2",
                             "IL29_4505:7:2:12262:7582#2",
                             "IL29_4505:7:29:7803:4644#2",
                             "IL29_4505:7:29:7816:4622#2",
                             "IL29_4505:7:40:16979:10045#2",
                             "IL29_4505:7:44:5928:2053#2",
                             "IL29_4505:7:93:5878:1091#2",
                             "IL29_4505:7:108:3637:10795#2",
                             "IL29_4505:7:13:6479:1635#2",
                             "IL29_4505:7:3:12568:11445#2",
                             "IL29_4505:7:32:8719:10919#2",
                             "IL29_4505:7:51:15171:6725#2",
                             "IL29_4505:7:112:9674:17564#2"
                     },
             }
        };
    }

    @Test(dataProvider="singleIntervalOverlapping")
    public void testQueryOverlappingSingleInterval(
        final File cramFileName,
        final File referenceFileName,
        final QueryInterval interval,
        final String[] expectedNames) throws IOException
    {
        doQueryTest(
                reader -> reader.queryOverlapping(new QueryInterval[]{interval}),
                cramFileName,
                referenceFileName,
                expectedNames
        );
    }

    @Test(dataProvider="singleIntervalOverlapping")
    public void testQueryOverlappingSequence(
        final File cramFileName,
        final File referenceFileName,
        final QueryInterval interval,
        final String[] expectedNames) throws IOException
    {
        doQueryTest(
            reader -> reader.queryOverlapping(
                reader.getFileHeader().getSequence(interval.referenceIndex).getSequenceName(),
                interval.start,
                interval.end
            ),
            cramFileName,
            referenceFileName,
            expectedNames
        );
    }

    @Test(dataProvider="singleIntervalOverlapping")
    public void testQuerySingleIntervalContainedFalse(
        final File cramFileName,
        final File referenceFileName,
        final QueryInterval interval,
        final String[] expectedNames) throws IOException
    {
        doQueryTest(
            reader -> reader.query(new QueryInterval[]{interval}, false),
            cramFileName,
            referenceFileName,
            expectedNames
        );
    }

    @Test(dataProvider="singleIntervalOverlapping")
    public void testQuerySequenceContainedFalse(
        final File cramFileName,
        final File referenceFileName,
        final QueryInterval interval,
        final String[] expectedNames) throws IOException
    {
        doQueryTest(
            reader -> reader.query(
                reader.getFileHeader().getSequence(interval.referenceIndex).getSequenceName(),
                interval.start,
                interval.end,
                false
            ),
            cramFileName,
            referenceFileName,
            expectedNames
        );
    }

    @DataProvider(name = "singleIntervalContained")
    public Object[][] singleIntervalContained() {
        return new Object[][] {
            {cramQueryWithCRAI, cramQueryReference, new QueryInterval(0, 100013, 100070), new String[]{"f", "f",}},
            {cramQueryWithLocalCRAI, cramQueryReference, new QueryInterval(0, 100013, 100070), new String[]{"f", "f",}},
            {cramQueryWithBAI, cramQueryReference, new QueryInterval(0, 100013, 100070), new String[]{"f", "f"}},

            // interval with 1 start
            {cramQueryWithCRAI, cramQueryReference, new QueryInterval(0, 1, 100100), new String[]{"e", "f", "f"}},
            {cramQueryWithLocalCRAI, cramQueryReference, new QueryInterval(0, 1, 100100), new String[]{"e", "f", "f"}},
            {cramQueryWithBAI, cramQueryReference, new QueryInterval(0, 1, 100100), new String[]{"e", "f", "f"}},

            // interval with 0 end
            {cramQueryWithCRAI, cramQueryReference, new QueryInterval(0, 100010, 0), new String[]{"d", "e", "f", "f",}},
            {cramQueryWithLocalCRAI, cramQueryReference, new QueryInterval(0, 100010, 0), new String[]{"d", "e", "f", "f",}},
            {cramQueryWithBAI, cramQueryReference, new QueryInterval(0, 100010, 0), new String[]{"d", "e", "f", "f",}},

            // interval with 1 start and 0 end
            {cramQueryWithCRAI, cramQueryReference, new QueryInterval(0, 1, 0), new String[]{"a", "b", "c", "d", "e", "f",  "f"}},
            {cramQueryWithLocalCRAI, cramQueryReference, new QueryInterval(0, 1, 0), new String[]{"a", "b", "c", "d", "e", "f",  "f"}},
            {cramQueryWithBAI, cramQueryReference, new QueryInterval(0, 1, 0), new String[]{"a", "b", "c", "d", "e", "f",  "f"}},

            //totally empty cram file
            {cramQueryTestEmptyWithBAI, cramQueryReadsReference, new QueryInterval(0, 1, 0), new String[]{}},
            {cramQueryTestEmptyWithLocalCRAI, cramQueryReadsReference, new QueryInterval(0, 1, 0), new String[]{}},

            { new File(TEST_DATA_DIR, "mitoAlignmentStartTest.cram"),
                    new File(TEST_DATA_DIR, "mitoAlignmentStartTest.fa"),
                    new QueryInterval(0, 631, 656),
                    new String[] {"IL29_4505:7:88:5383:14756#2"},
            }
        };
    }

    @Test(dataProvider="singleIntervalContained")
    public void testQueryContainedSingleInterval(
        final File cramFileName,
        final File referenceFileName,
        final QueryInterval interval,
        final String[] expectedNames) throws IOException
    {
        doQueryTest(
            reader -> reader.queryContained(new QueryInterval[]{interval}),
            cramFileName,
            referenceFileName,
            expectedNames
        );
    }

    @Test(dataProvider="singleIntervalContained")
    public void testQueryContainedSequence(
        final File cramFileName,
        final File referenceFileName,
        final QueryInterval interval,
        final String[] expectedNames) throws IOException
    {
        doQueryTest(
            reader -> reader.queryContained(
                reader.getFileHeader().getSequence(interval.referenceIndex).getSequenceName(),
                interval.start,
                interval.end
            ),
            cramFileName,
            referenceFileName,
            expectedNames
        );
    }

    @Test(dataProvider="singleIntervalContained")
    public void testQuerySingleIntervalContainedTrue(
            final File cramFileName,
            final File referenceFileName,
            final QueryInterval interval,
            final String[] expectedNames) throws IOException
    {
        doQueryTest(
            reader -> reader.query(new QueryInterval[]{interval}, true),
            cramFileName,
            referenceFileName,
            expectedNames
        );
    }

    @Test(dataProvider="singleIntervalContained")
    public void testQuerySequenceContainedTrue(
            final File cramFileName,
            final File referenceFileName,
            final QueryInterval interval,
            final String[] expectedNames) throws IOException
    {
        doQueryTest(
            reader -> reader.query(
                reader.getFileHeader().getSequence(interval.referenceIndex).getSequenceName(),
                interval.start,
                interval.end,
                true
            ),
            cramFileName,
            referenceFileName,
            expectedNames
        );
    }

    @DataProvider(name = "multipleIntervalOverlapping")
    public Object[][] multipleIntervalOverlapping() {
        return new Object[][]{
            {cramQueryWithCRAI, cramQueryReference,
                    new QueryInterval[]{new QueryInterval(0, 100009, 100009), new QueryInterval(0, 100011, 100011)},
                    new String[]{"a", "b", "c", "d", "e"}},
            {cramQueryWithLocalCRAI, cramQueryReference,
                    new QueryInterval[]{new QueryInterval(0, 100009, 100009), new QueryInterval(0, 100011, 100011)},
                    new String[]{"a", "b", "c", "d", "e"}},
            {cramQueryWithBAI, cramQueryReference,
                    new QueryInterval[]{new QueryInterval(0, 100009, 100009), new QueryInterval(0, 100011, 100011)},
                    new String[]{"a", "b", "c", "d", "e"}},
            // no matching reads
            {cramQueryReadsWithBAI, cramQueryReadsReference,
                    new QueryInterval[]{new QueryInterval(0, 300, 310), new QueryInterval(1, 300, 310)},
                    new String[]{}},
            {cramQueryReadsWithLocalCRAI, cramQueryReadsReference,
                    new QueryInterval[]{new QueryInterval(0, 300, 310), new QueryInterval(1, 300, 310)},
                    new String[]{}},
            // matching reads from first interval only
            {cramQueryReadsWithBAI, cramQueryReadsReference,
                    new QueryInterval[]{new QueryInterval(0, 205, 206), new QueryInterval(3, 300, 301)},
                    new String[]{"a", "b"}},
            {cramQueryReadsWithLocalCRAI, cramQueryReadsReference,
                    new QueryInterval[]{new QueryInterval(0, 205, 206), new QueryInterval(3, 300, 301)},
                    new String[]{"a", "b"}},
            // matching reads from last interval only
            {cramQueryReadsWithBAI, cramQueryReadsReference,
                    new QueryInterval[]{new QueryInterval(0, 100, 101), new QueryInterval(3, 700, 701)},
                    new String[]{"k"}},
            {cramQueryReadsWithLocalCRAI, cramQueryReadsReference,
                    new QueryInterval[]{new QueryInterval(0, 100, 101), new QueryInterval(3, 700, 701)},
                    new String[]{"k"}},
            //matching reads from each interval
            {cramQueryReadsWithBAI, cramQueryReadsReference,
                    new QueryInterval[]{new QueryInterval(0, 205, 206), new QueryInterval(3, 700, 701)},
                    new String[]{"a", "b", "k"}},
            {cramQueryReadsWithLocalCRAI, cramQueryReadsReference,
                    new QueryInterval[]{new QueryInterval(0, 205, 206), new QueryInterval(3, 700, 701)},
                    new String[]{"a", "b", "k"}},
            //matching reads from each interval - 4 intervals
            {cramQueryReadsWithBAI, cramQueryReadsReference,
                    new QueryInterval[]{
                            new QueryInterval(0, 200, 201), new QueryInterval(1, 500, 501),
                            new QueryInterval(2, 300, 301), new QueryInterval(3, 700, 701)},
                    new String[]{"a", "f", "i", "k"}},
            {cramQueryReadsWithLocalCRAI, cramQueryReadsReference,
                    new QueryInterval[]{
                            new QueryInterval(0, 200, 201), new QueryInterval(1, 500, 501),
                            new QueryInterval(2, 300, 301), new QueryInterval(3, 700, 701)},
                    new String[]{"a", "f", "i", "k"}},
            // first read is before the first interval
            {cramQueryReadsWithBAI, cramQueryReadsReference,
                    new QueryInterval[]{new QueryInterval(2, 300, 301), new QueryInterval(3, 700, 701)},
                    new String[]{"i", "k"}},
            {cramQueryReadsWithLocalCRAI, cramQueryReadsReference,
                    new QueryInterval[]{new QueryInterval(2, 300, 301), new QueryInterval(3, 700, 701)},
                    new String[]{"i", "k"}},
            // first interval is before the first read
            {cramQueryReadsWithBAI, cramQueryReadsReference,
                    new QueryInterval[]{new QueryInterval(0, 100, 101), new QueryInterval(0, 200, 201)},
                    new String[]{"a"}},
            {cramQueryReadsWithLocalCRAI, cramQueryReadsReference,
                    new QueryInterval[]{new QueryInterval(0, 100, 101), new QueryInterval(0, 200, 201)},
                    new String[]{"a"}},
            // intervals in reverse order
            {cramQueryReadsWithBAI, cramQueryReadsReference,
                    new QueryInterval[]{new QueryInterval(0, 100, 101), new QueryInterval(0, 200, 201)},
                    new String[]{"a"}},
            {cramQueryReadsWithLocalCRAI, cramQueryReadsReference,
                    new QueryInterval[]{new QueryInterval(0, 100, 101), new QueryInterval(0, 200, 201)},
                    new String[]{"a"}},
        };
    }

    @Test(dataProvider="multipleIntervalOverlapping")
    public void testQueryOverlappingMultipleIntervals(
            final File cramFileName,
            final File referenceFileName,
            final QueryInterval[] intervals,
            final String[] expectedNames) throws IOException
    {
        final QueryInterval[] optimizedIntervals = QueryInterval.optimizeIntervals(intervals);
        Assert.assertTrue(optimizedIntervals.length > 1);

        doQueryTest(
                reader -> reader.queryOverlapping(optimizedIntervals),
                cramFileName,
                referenceFileName,
                expectedNames
        );
    }

    @DataProvider(name = "otherMultipleIntervals")
    public Object[][] otherMultipleIntervals() {
        return new Object[][]{
            // accept an empty QueryIntervalArray
            {cramQueryWithBAI, cramQueryReference,
                    new QueryInterval[]{},
                    new String[]{}},
            // intervals overlapping - optimized to a single interval
            {cramQueryReadsWithBAI, cramQueryReadsReference,
                    new QueryInterval[]{new QueryInterval(0, 1000, 1030), new QueryInterval(0, 1020, 1076)},
                    new String[]{"d"}},
            {cramQueryReadsWithLocalCRAI, cramQueryReadsReference,
                    new QueryInterval[]{new QueryInterval(0, 1000, 1030), new QueryInterval(0, 1020, 1076)},
                    new String[]{"d"}}
        };
    }

    // these are tested separately because we want the normal multi-interval test to
    // assert that the interval list size is > 1 post-optimization to ensure we're
    // using more than one interval; these tests optimize down to 0 or 1 interval
    @Test(dataProvider="otherMultipleIntervals")
    public void testOtherMultipleIntervals(
            final File cramFileName,
            final File referenceFileName,
            final QueryInterval[] intervals,
            final String[] expectedNames) throws IOException
    {
        final QueryInterval[] optimizedIntervals = QueryInterval.optimizeIntervals(intervals);
        doQueryTest(
                reader -> reader.queryContained(optimizedIntervals),
                cramFileName,
                referenceFileName,
                expectedNames
        );
        doQueryTest(
                reader -> reader.queryOverlapping(optimizedIntervals),
                cramFileName,
                referenceFileName,
                expectedNames
        );
    }

    @DataProvider(name = "multipleIntervalContained")
    public Object[][] multipleIntervalContained() {
        return new Object[][]{
            {cramQueryWithCRAI, cramQueryReference,
                    new QueryInterval[]{new QueryInterval(0, 100008, 100008), new QueryInterval(0, 100013, 0)},
                    new String[]{"f", "f"}},
            {cramQueryWithLocalCRAI, cramQueryReference,
                    new QueryInterval[]{new QueryInterval(0, 100008, 100008), new QueryInterval(0, 100013, 0)},
                    new String[]{"f", "f"}},
            {cramQueryWithBAI, cramQueryReference,
                    new QueryInterval[]{new QueryInterval(0, 100008, 100008), new QueryInterval(0, 100013, 0)},
                    new String[]{"f", "f"}},
            // no matching reads
            {cramQueryReadsWithBAI, cramQueryReadsReference,
                    new QueryInterval[]{new QueryInterval(0, 300, 310), new QueryInterval(1, 300, 310)},
                    new String[]{}},
            {cramQueryReadsWithLocalCRAI, cramQueryReadsReference,
                    new QueryInterval[]{new QueryInterval(0, 300, 310), new QueryInterval(1, 300, 310)},
                    new String[]{}},
            // matching reads from first interval only
            {cramQueryReadsWithBAI, cramQueryReadsReference,
                    new QueryInterval[]{new QueryInterval(0, 205, 305), new QueryInterval(3, 300, 301)},
                    new String[]{"b", "c"}},
            {cramQueryReadsWithLocalCRAI, cramQueryReadsReference,
                    new QueryInterval[]{new QueryInterval(0, 205, 305), new QueryInterval(3, 300, 301)},
                    new String[]{"b", "c"}},
            // matching reads from last interval only
            {cramQueryReadsWithBAI, cramQueryReadsReference,
                    new QueryInterval[]{new QueryInterval(0, 100, 101), new QueryInterval(3, 700, 776)},
                    new String[]{"k"}},
            {cramQueryReadsWithLocalCRAI, cramQueryReadsReference,
                    new QueryInterval[]{new QueryInterval(0, 100, 101), new QueryInterval(3, 700, 776)},
                    new String[]{"k"}},
            //matching reads from each interval
            {cramQueryReadsWithBAI, cramQueryReadsReference,
                    new QueryInterval[]{new QueryInterval(0, 200, 281), new QueryInterval(3, 700, 776)},
                    new String[]{"a", "b", "k"}},
            {cramQueryReadsWithLocalCRAI, cramQueryReadsReference,
                    new QueryInterval[]{new QueryInterval(0, 200, 281), new QueryInterval(3, 700, 776)},
                    new String[]{"a", "b", "k"}},
            //matching reads from each interval - 4 intervals
            {cramQueryReadsWithBAI, cramQueryReadsReference,
                    new QueryInterval[]{
                            new QueryInterval(0, 200, 281), new QueryInterval(1, 500, 576),
                            new QueryInterval(2, 300, 376), new QueryInterval(3, 700, 776)},
                    new String[]{"a", "b", "f", "i", "k"}},
                {cramQueryReadsWithLocalCRAI, cramQueryReadsReference,
                    new QueryInterval[]{
                            new QueryInterval(0, 200, 281), new QueryInterval(1, 500, 576),
                            new QueryInterval(2, 300, 376), new QueryInterval(3, 700, 776)},
                    new String[]{"a", "b", "f", "i", "k"}},
            // first read is before the first interval
            {cramQueryReadsWithBAI, cramQueryReadsReference,
                    new QueryInterval[]{new QueryInterval(2, 300, 301), new QueryInterval(3, 700, 776)},
                    new String[]{"k"}},
            {cramQueryReadsWithLocalCRAI, cramQueryReadsReference,
                    new QueryInterval[]{new QueryInterval(2, 300, 301), new QueryInterval(3, 700, 776)},
                    new String[]{"k"}},
            // first interval is before the first read
            {cramQueryReadsWithBAI, cramQueryReadsReference,
                    new QueryInterval[]{new QueryInterval(0, 100, 101), new QueryInterval(0, 200, 276)},
                    new String[]{"a"}},
            {cramQueryReadsWithLocalCRAI, cramQueryReadsReference,
                    new QueryInterval[]{new QueryInterval(0, 100, 101), new QueryInterval(0, 200, 276)},
                    new String[]{"a"}},
            // intervals in reverse order
            {cramQueryReadsWithBAI, cramQueryReadsReference,
                    new QueryInterval[]{new QueryInterval(0, 200, 276), new QueryInterval(0, 100, 101)},
                    new String[]{"a"}},
            {cramQueryReadsWithLocalCRAI, cramQueryReadsReference,
                    new QueryInterval[]{new QueryInterval(0, 200, 276), new QueryInterval(0, 100, 101)},
                    new String[]{"a"}},
        };
    }

    @Test(dataProvider="multipleIntervalContained")
    public void testQueryContainedMultipleIntervals(
            final File cramFileName,
            final File referenceFileName,
            final QueryInterval[] intervals,
            final String[] expectedNames) throws IOException
    {
        final QueryInterval[] optimizedIntervals = QueryInterval.optimizeIntervals(intervals);
        Assert.assertTrue(optimizedIntervals.length > 1);
        doQueryTest(
                reader -> reader.queryContained(optimizedIntervals),
                cramFileName,
                referenceFileName,
                expectedNames
        );
    }

    @DataProvider(name = "unmappedQueries")
    public Object[][] unmappedQueries() {
        return new Object[][] {
            {cramQueryWithCRAI, cramQueryReference, new String[]{"g", "h", "h", "i", "i"}},
            {cramQueryWithLocalCRAI, cramQueryReference, new String[]{"g", "h", "h", "i", "i"}},
            {cramQueryWithBAI, cramQueryReference, new String[]{"g", "h", "h", "i", "i"}},
            //no unmapped reads
            {cramQueryReadsWithBAI, cramQueryReadsReference, new String[]{}},
            {cramQueryReadsWithLocalCRAI, cramQueryReadsReference, new String[]{}}
        };
    }

    @Test(dataProvider="unmappedQueries")
    public void testQueryUnmapped(
            final File cramFileName,
            final File referenceFileName,
            final String[] expectedNames) throws IOException
    {
        doQueryTest(
                SamReader::queryUnmapped,
                cramFileName,
                referenceFileName,
                expectedNames
        );
    }

    @DataProvider(name = "alignmentStartQueries")
    public Object[][] alignmentStartQueries() {
        return new Object[][] {
                // cram file, reference, query contig, alignment start query, number of reads with matching alignment start
                {cramQueryWithCRAI, cramQueryReference, "20", 100013, 2},
                {cramQueryWithLocalCRAI, cramQueryReference, "20", 100013, 2},
                {cramQueryWithBAI, cramQueryReference, "20", 100013, 2},
                // tests to ensure that query results on inputs that have matching reads that are followed by reads
                // in the same container that don't match the alignment start are properly constrained to only the
                // matching reads
                {new File(TEST_DATA_DIR, "mitoAlignmentStartTest.cram"),
                        new File(TEST_DATA_DIR, "mitoAlignmentStartTest.fa"),
                        "Mito", 631, 24},
                {new File(TEST_DATA_DIR, "mitoAlignmentStartTestGATKGen.cram"),
                        new File(TEST_DATA_DIR, "mitoAlignmentStartTest.fa"),
                        "Mito", 631, 24},
        };
    }

    @Test(dataProvider="alignmentStartQueries")
    public void testQueryAlignmentStart(
            final File cramFileName,
            final File referenceFileName,
            final String queryContig,
            final int alignmentStart,
            final int expectedReadCount) throws IOException {
        SamReaderFactory factory = SamReaderFactory.makeDefault().validationStringency(ValidationStringency.LENIENT);
        if (referenceFileName != null) {
            factory = factory.referenceSequence(referenceFileName);
        }
        try (final SamReader reader = factory.open(cramFileName);
             final CloseableIterator<SAMRecord> it = reader.queryAlignmentStart(queryContig, alignmentStart)) {
            int count = 0;
            while (it.hasNext()) {
                it.next();
                count++;
            }
            Assert.assertEquals(count, expectedReadCount);
        }
    }

    @DataProvider(name = "mateQueries")
    public Object[][] mateQueries() {
        return new Object[][] {
                // cram, reference, query contig, query start, name of read mates at alignment start
                {cramQueryWithCRAI, cramQueryReference, "20", 100013, "f"},
                {cramQueryWithLocalCRAI, cramQueryReference, "20", 100013, "f"},
                {cramQueryWithBAI, cramQueryReference, "20", 100013, "f"},
                {new File(TEST_DATA_DIR, "mitoAlignmentStartTest.cram"),
                        new File(TEST_DATA_DIR, "mitoAlignmentStartTest.fa"),
                        "Mito", 631, "IL29_4505:7:30:11521:4492#2"},
                {new File(TEST_DATA_DIR, "mitoAlignmentStartTestGATKGen.cram"),
                        new File(TEST_DATA_DIR, "mitoAlignmentStartTest.fa"),
                        "Mito", 631, "IL29_4505:7:30:11521:4492#2"},
        };
    }

    @Test(dataProvider="mateQueries")
    public void testQueryMate(
        final File cramFileName,
        final File referenceFileName,
        final String queryContig,
        final int alignmentStart,
        final String expectedReadName) throws IOException
    {
        // the test file ("mitoAlignmentStartTest.cram") used here, which was provided by a user who reported
        // a queryMate bug, contains reads that have a mateReferenceName == "*", but with mateAlignmentStart != 0;
        // so we have to use ValidationStringency.LENIENT to suppress validation from throwing when it sees these..
        SamReaderFactory factory = SamReaderFactory.makeDefault().validationStringency(ValidationStringency.LENIENT);
        if (referenceFileName != null) {
            factory = factory.referenceSequence(referenceFileName);
        }
        SAMRecord firstRecord;
        SAMRecord firstRecordMate;
        try (final SamReader reader = factory.open(cramFileName)) {
            final CloseableIterator<SAMRecord> it = reader.queryAlignmentStart(queryContig, alignmentStart);
            Assert.assertTrue(it.hasNext());
            firstRecord = it.next();
            Assert.assertEquals(firstRecord.getReadName(), expectedReadName);
        }

        // get the mate for the first record
        try (final SamReader reader = factory.open(cramFileName)) {
            firstRecordMate = reader.queryMate(firstRecord);
            Assert.assertEquals(firstRecordMate.getReadName(), firstRecord.getReadName());
        }

        // now query the mate's mate to ensure we get symmetric results
        try (final SamReader reader = factory.open(cramFileName)) {
            final SAMRecord matesMate = reader.queryMate(firstRecordMate);
            Assert.assertEquals(matesMate, firstRecord);
        }
    }

    private void doQueryTest(
        final Function<SamReader, CloseableIterator <SAMRecord>> getIterator,
        final File cramFileName,
        final File referenceFileName,
        final String[] expectedNames) throws IOException
    {
        SamReaderFactory factory = SamReaderFactory.makeDefault().validationStringency(ValidationStringency.LENIENT);
        if (referenceFileName != null) {
            factory = factory.referenceSequence(referenceFileName);
        }
        try (final SamReader reader = factory.open(cramFileName);
             final CloseableIterator<SAMRecord> it = getIterator.apply(reader)) {
            int count = 0;
            while (it.hasNext()) {
                final SAMRecord samRec = it.next();
                Assert.assertTrue(count < expectedNames.length);
                Assert.assertEquals(samRec.getReadName(), expectedNames[count]);
                count++;
            }
            Assert.assertEquals(count, expectedNames.length);
        }
    }


    @DataProvider(name = "iteratorStateTests")
    public Object[][] iteratorStateQueries() {
        return new Object[][] {
                {cramQueryWithCRAI, cramQueryReference},
                {cramQueryWithLocalCRAI, cramQueryReference},
                {cramQueryWithBAI, cramQueryReference}
        };
    }

    // The current CRAMFileReader implementation allows multiple iterators to exist on a
    // CRAM reader at the same time, but they're not properly isolated from each other. When
    // CRAMFileReader is changed to support the SamReader contract of one-iterator-at-a-time
    // (https://github.com/samtools/htsjdk/issues/563), these can be re-enabled.
    //
    @Test(dataProvider="iteratorStateTests", expectedExceptions=SAMException.class, enabled=false)
    public void testIteratorState(
            final File cramFileName,
            final File referenceFileName) throws IOException
    {
        SamReaderFactory factory = SamReaderFactory.makeDefault();
        if (referenceFileName != null) {
            factory = factory.referenceSequence(referenceFileName);
        }

        try (final SamReader reader = factory.open(cramFileName)) {
            final CloseableIterator<SAMRecord> origIt = reader.iterator();

            // opening the second iterator should throw
            final CloseableIterator<SAMRecord> overlapIt = reader.queryOverlapping("20", 100013, 100070);
        }
    }

    @DataProvider(name = "unmappedSliceTest")
    public Object[][] unmappedMultiSliceTest() {
        return new Object[][] {
            // the main test feature of these files is that they have several mapped reads followed by
            // some number of unmapped reads, each created with seqs_per_slice = 100 to force the unmapped
            // reads to be distributed over multiple slices (at least for large numbers of unmapped reads)
            // tests the fix to https://github.com/samtools/htsjdk/issues/562
            {new File(TEST_DATA_DIR, "NA12878.20.21.1-100.100-SeqsPerSlice.0-unMapped.cram"),
                    new File(TEST_DATA_DIR, "human_g1k_v37.20.21.1-100.fasta"), 0},
            {new File(TEST_DATA_DIR, "NA12878.20.21.1-100.100-SeqsPerSlice.1-unMapped.cram"),
                    new File(TEST_DATA_DIR, "human_g1k_v37.20.21.1-100.fasta"), 1},
            {new File(TEST_DATA_DIR, "NA12878.20.21.1-100.100-SeqsPerSlice.500-unMapped.cram"),
                    new File(TEST_DATA_DIR, "human_g1k_v37.20.21.1-100.fasta"), 500},
        };
    }

    @Test(dataProvider = "unmappedSliceTest")
    public void testUnmappedMultiSlice(
            final File cramFileName,
            final File referenceFileName,
            final int expectedCount) throws IOException
    {
        SamReaderFactory factory = SamReaderFactory.makeDefault().validationStringency(ValidationStringency.SILENT);
        factory = factory.referenceSequence(referenceFileName);

        int count = 0;
        try (final SamReader reader = factory.open(cramFileName);
             final CloseableIterator<SAMRecord> it = reader.queryUnmapped())
        {
            while (it.hasNext()) {
                it.next();
                count++;
            }
        }
        Assert.assertEquals(count, expectedCount);
    }

    @DataProvider(name = "serialQueries")
    public Object[][] serialQueries() {
        return new Object[][]{
                {cramQueryWithCRAI, cramQueryReference,
                        new QueryInterval(0, 100009, 100009), new String[]{"a", "b", "c"}, 5},
                {cramQueryWithLocalCRAI, cramQueryReference,
                        new QueryInterval(0, 100009, 100009), new String[]{"a", "b", "c"}, 5},
                {cramQueryWithBAI, cramQueryReference,
                        new QueryInterval(0, 100009, 100009), new String[]{"a", "b", "c"}, 5},
        };
    }

    @Test(dataProvider = "serialQueries")
    public void testSerialQueriesOnRemoteFile(
            final File cramFile,
            final File referenceFile,
            final QueryInterval interval,
            final String[] expectedNames,
            final int nUnmapped) throws IOException {
        final Path cramIndex = SamFiles.findIndex(cramFile.toPath());

        try (final FileSystem jimfs = Jimfs.newFileSystem(Configuration.unix())) {
            final Path jimfsCRAM = jimfs.getPath("remotecram.cram");
            final Path jimfsCRAI = jimfs.getPath("remotecram.crai");

            Files.copy(cramFile.toPath(), jimfsCRAM);
            Files.copy(cramIndex, jimfsCRAI);

            SamReaderFactory factory = SamReaderFactory.makeDefault().validationStringency(ValidationStringency.LENIENT);
            if (referenceFile != null) {
                factory = factory.referenceSequence(referenceFile);
            }
            try (final SamReader reader = factory.open(jimfsCRAM)) {
                try (final CloseableIterator<SAMRecord> it = reader.queryOverlapping(new QueryInterval[]{interval})) {
                    int count = 0;
                    while (it.hasNext()) {
                        final SAMRecord samRec = it.next();
                        Assert.assertTrue(count < expectedNames.length);
                        Assert.assertEquals(samRec.getReadName(), expectedNames[count]);
                        count++;
                    }
                    Assert.assertEquals(count, expectedNames.length);
                }

                // now execute a second query on the same SamReader, this time for unmapped reads, after having
                // closed the previous iterator
                try (final CloseableIterator<SAMRecord> it = reader.queryUnmapped()) {
                    int count = 0;
                    while (it.hasNext()) {
                        final SAMRecord samRec = it.next();
                        Assert.assertTrue(samRec.getReadUnmappedFlag());
                        count++;
                    }
                    Assert.assertEquals(count, nUnmapped);
                }
            }
        }
    }

}
