diff --git a/processing/src/test/java/org/apache/druid/segment/SchemalessIndexTest.java b/processing/src/test/java/org/apache/druid/segment/SchemalessIndexTest.java index 3b6b8ec7bced..b4c0e9c9bb72 100644 --- a/processing/src/test/java/org/apache/druid/segment/SchemalessIndexTest.java +++ b/processing/src/test/java/org/apache/druid/segment/SchemalessIndexTest.java @@ -112,6 +112,11 @@ public static IncrementalIndex getIncrementalIndex() } } + IncrementalIndex createIncrementalIndex() + { + return makeIncrementalIndex(TEST_FILE, METRIC_AGGS); + } + public static QueryableIndex getIncrementalIndex(int index1, int index2) { synchronized (log) { @@ -178,45 +183,61 @@ public QueryableIndex getMergedIncrementalIndex() return mergedIndex; } - try { - IncrementalIndex top = makeIncrementalIndex("druid.sample.json.top", METRIC_AGGS); - IncrementalIndex bottom = makeIncrementalIndex("druid.sample.json.bottom", METRIC_AGGS); + mergedIndex = createMergedIncrementalIndex(); + return mergedIndex; + } + } - File tmpFile = File.createTempFile("yay", "who"); - tmpFile.delete(); + QueryableIndex createMergedIncrementalIndex() + { + final IncrementalIndex top = makeIncrementalIndex("druid.sample.json.top", METRIC_AGGS); + final IncrementalIndex bottom = makeIncrementalIndex("druid.sample.json.bottom", METRIC_AGGS); + QueryableIndex topIndex = null; + QueryableIndex bottomIndex = null; - File topFile = new File(tmpFile, "top"); - File bottomFile = new File(tmpFile, "bottom"); - File mergedFile = new File(tmpFile, "merged"); + try { + File tmpFile = FileUtils.createTempDir("yay"); - FileUtils.mkdirp(topFile); - FileUtils.mkdirp(bottomFile); - FileUtils.mkdirp(mergedFile); - topFile.deleteOnExit(); - bottomFile.deleteOnExit(); - mergedFile.deleteOnExit(); + File topFile = new File(tmpFile, "top"); + File bottomFile = new File(tmpFile, "bottom"); + File mergedFile = new File(tmpFile, "merged"); - indexMerger.persist(top, topFile, INDEX_SPEC, null); - indexMerger.persist(bottom, bottomFile, INDEX_SPEC, null); + FileUtils.mkdirp(topFile); + FileUtils.mkdirp(bottomFile); + FileUtils.mkdirp(mergedFile); + topFile.deleteOnExit(); + bottomFile.deleteOnExit(); + mergedFile.deleteOnExit(); - mergedIndex = indexIO.loadIndex( - indexMerger.mergeQueryableIndex( - Arrays.asList(indexIO.loadIndex(topFile), indexIO.loadIndex(bottomFile)), - true, - METRIC_AGGS, - mergedFile, - INDEX_SPEC, - null, - -1 - ) - ); + indexMerger.persist(top, topFile, INDEX_SPEC, null); + indexMerger.persist(bottom, bottomFile, INDEX_SPEC, null); + topIndex = indexIO.loadIndex(topFile); + bottomIndex = indexIO.loadIndex(bottomFile); - return mergedIndex; + return indexIO.loadIndex( + indexMerger.mergeQueryableIndex( + Arrays.asList(topIndex, bottomIndex), + true, + METRIC_AGGS, + mergedFile, + INDEX_SPEC, + null, + -1 + ) + ); + } + catch (IOException e) { + throw new RuntimeException(e); + } + finally { + if (topIndex != null) { + topIndex.close(); } - catch (IOException e) { - mergedIndex = null; - throw new RuntimeException(e); + if (bottomIndex != null) { + bottomIndex.close(); } + top.close(); + bottom.close(); } } @@ -239,8 +260,7 @@ public QueryableIndex getMergedIncrementalIndex(int index1, int index2) } try { - File tmpFile = File.createTempFile("yay", "who"); - tmpFile.delete(); + File tmpFile = FileUtils.createTempDir("yay"); File mergedFile = new File(tmpFile, "merged"); @@ -277,8 +297,7 @@ public QueryableIndex getMergedIncrementalIndex(int[] indexes) } try { - File tmpFile = File.createTempFile("yay", "who"); - tmpFile.delete(); + File tmpFile = FileUtils.createTempDir("yay"); File mergedFile = new File(tmpFile, "merged"); @@ -364,9 +383,7 @@ private void makeRowPersistedIndexes() new MapBasedInputRow(timestamp, dims, event) ); - File tmpFile = File.createTempFile("billy", "yay"); - tmpFile.delete(); - FileUtils.mkdirp(tmpFile); + File tmpFile = FileUtils.createTempDir("billy"); tmpFile.deleteOnExit(); indexMerger.persist(rowIndex, tmpFile, INDEX_SPEC, null); @@ -444,8 +461,7 @@ private List makeFilesToMap(File tmpFile, Iterable> files) { try { - File tmpFile = File.createTempFile("yay", "who"); - tmpFile.delete(); + File tmpFile = FileUtils.createTempDir("yay"); File mergedFile = new File(tmpFile, "merged"); FileUtils.mkdirp(mergedFile); mergedFile.deleteOnExit(); diff --git a/processing/src/test/java/org/apache/druid/segment/SchemalessTestSimpleTest.java b/processing/src/test/java/org/apache/druid/segment/SchemalessTestSimpleTest.java index e12c6c099aa9..198c59155aa0 100644 --- a/processing/src/test/java/org/apache/druid/segment/SchemalessTestSimpleTest.java +++ b/processing/src/test/java/org/apache/druid/segment/SchemalessTestSimpleTest.java @@ -58,11 +58,11 @@ import org.apache.druid.segment.writeout.SegmentWriteOutMediumFactory; import org.apache.druid.testing.InitializedNullHandlingTest; import org.apache.druid.timeline.SegmentId; -import org.junit.Ignore; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; +import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; @@ -72,19 +72,17 @@ /** */ -@RunWith(Parameterized.class) public class SchemalessTestSimpleTest extends InitializedNullHandlingTest { - @Parameterized.Parameters public static Collection constructorFeeder() { List argumentArrays = new ArrayList<>(); for (SegmentWriteOutMediumFactory segmentWriteOutMediumFactory : SegmentWriteOutMediumFactory.builtInFactories()) { SchemalessIndexTest schemalessIndexTest = new SchemalessIndexTest(segmentWriteOutMediumFactory); - final IncrementalIndex incrementalIndex = SchemalessIndexTest.getIncrementalIndex(); + final IncrementalIndex incrementalIndex = schemalessIndexTest.createIncrementalIndex(); final QueryableIndex persistedIncrementalIndex = TestIndex.persistAndMemoryMap(incrementalIndex); - final QueryableIndex mergedIncrementalIndex = schemalessIndexTest.getMergedIncrementalIndex(); + final QueryableIndex mergedIncrementalIndex = schemalessIndexTest.createMergedIncrementalIndex(); argumentArrays.add(new Object[] {new IncrementalIndexSegment(incrementalIndex, SegmentId.dummy("test"))}); argumentArrays.add(new Object[] {new QueryableIndexSegment(persistedIncrementalIndex, SegmentId.dummy("test"))}); argumentArrays.add(new Object[] {new QueryableIndexSegment(mergedIncrementalIndex, SegmentId.dummy("test"))}); @@ -115,180 +113,186 @@ public static Collection constructorFeeder() Collections.singletonList(Intervals.of("1970-01-01T00:00:00.000Z/2020-01-01T00:00:00.000Z")) ); - private final Segment segment; - - public SchemalessTestSimpleTest(Segment segment) + @ParameterizedTest + @MethodSource("constructorFeeder") + public void testFullOnTimeseries(final Segment segment) throws IOException { - this.segment = segment; - } - - @Test - public void testFullOnTimeseries() - { - TimeseriesQuery query = Druids.newTimeseriesQueryBuilder() - .dataSource(dataSource) - .granularity(ALL_GRAN) - .intervals(fullOnInterval) - .aggregators( - Lists.newArrayList( - Iterables.concat( - commonAggregators, - Lists.newArrayList( - new DoubleMaxAggregatorFactory("maxIndex", "index"), - new DoubleMinAggregatorFactory("minIndex", "index") - ) - ) - ) - ) - .postAggregators(addRowsIndexConstant) - .build(); + try (segment) { + TimeseriesQuery query = Druids.newTimeseriesQueryBuilder() + .dataSource(dataSource) + .granularity(ALL_GRAN) + .intervals(fullOnInterval) + .aggregators( + Lists.newArrayList( + Iterables.concat( + commonAggregators, + Lists.newArrayList( + new DoubleMaxAggregatorFactory("maxIndex", "index"), + new DoubleMinAggregatorFactory("minIndex", "index") + ) + ) + ) + ) + .postAggregators(addRowsIndexConstant) + .build(); - List> expectedResults = Collections.singletonList( - new Result( - DateTimes.of("2011-01-12T00:00:00.000Z"), - new TimeseriesResultValue( - ImmutableMap.builder() - .put("rows", 11L) - .put("index", 900.0) - .put("addRowsIndexConstant", 912.0) - .put("uniques", 2.000977198748901D) - .put("maxIndex", 100.0) - .put("minIndex", 100.0) - .build() - ) - ) - ); - QueryRunner runner = TestQueryRunners.makeTimeSeriesQueryRunner(segment); - TestHelper.assertExpectedResults(expectedResults, runner.run(QueryPlus.wrap(query))); + List> expectedResults = Collections.singletonList( + new Result( + DateTimes.of("2011-01-12T00:00:00.000Z"), + new TimeseriesResultValue( + ImmutableMap.builder() + .put("rows", 11L) + .put("index", 900.0) + .put("addRowsIndexConstant", 912.0) + .put("uniques", 2.000977198748901D) + .put("maxIndex", 100.0) + .put("minIndex", 100.0) + .build() + ) + ) + ); + QueryRunner runner = TestQueryRunners.makeTimeSeriesQueryRunner(segment); + TestHelper.assertExpectedResults(expectedResults, runner.run(QueryPlus.wrap(query))); + } } // @Test TODO: Handling of null values is inconsistent right now, need to make it all consistent and re-enable test // TODO: Complain to Eric when you see this. It shouldn't be like this... - @Ignore + @Disabled + @ParameterizedTest + @MethodSource("constructorFeeder") @SuppressWarnings("unused") - public void testFullOnTopN() + public void testFullOnTopN(final Segment segment) throws IOException { - TopNQuery query = new TopNQueryBuilder() - .dataSource(dataSource) - .granularity(ALL_GRAN) - .dimension(marketDimension) - .metric(indexMetric) - .threshold(3) - .intervals(fullOnInterval) - .aggregators( - Lists.newArrayList( - Iterables.concat( - commonAggregators, - Lists.newArrayList( - new DoubleMaxAggregatorFactory("maxIndex", "index"), - new DoubleMinAggregatorFactory("minIndex", "index") - ) - ) - ) - ) - .postAggregators(addRowsIndexConstant) - .build(); + try (segment) { + TopNQuery query = new TopNQueryBuilder() + .dataSource(dataSource) + .granularity(ALL_GRAN) + .dimension(marketDimension) + .metric(indexMetric) + .threshold(3) + .intervals(fullOnInterval) + .aggregators( + Lists.newArrayList( + Iterables.concat( + commonAggregators, + Lists.newArrayList( + new DoubleMaxAggregatorFactory("maxIndex", "index"), + new DoubleMinAggregatorFactory("minIndex", "index") + ) + ) + ) + ) + .postAggregators(addRowsIndexConstant) + .build(); - List> expectedResults = Collections.singletonList( - new Result<>( - DateTimes.of("2011-01-12T00:00:00.000Z"), - TopNResultValue.create( - Arrays.asList( - new DimensionAndMetricValueExtractor( - ImmutableMap.builder() - .put("market", "spot") - .put("rows", 4L) - .put("index", 400.0D) - .put("addRowsIndexConstant", 405.0D) - .put("uniques", 1.0002442201269182D) - .put("maxIndex", 100.0) - .put("minIndex", 100.0) - .build() - ), - new DimensionAndMetricValueExtractor( - ImmutableMap.builder() - .put("market", "") - .put("rows", 2L) - .put("index", 200.0D) - .put("addRowsIndexConstant", 203.0D) - .put("uniques", 0.0) - .put("maxIndex", 100.0D) - .put("minIndex", 100.0D) - .build() - ), - new DimensionAndMetricValueExtractor( - ImmutableMap.builder() - .put("market", "total_market") - .put("rows", 2L) - .put("index", 200.0D) - .put("addRowsIndexConstant", 203.0D) - .put("uniques", 1.0002442201269182D) - .put("maxIndex", 100.0D) - .put("minIndex", 100.0D) - .build() - ) - ) - ) - ) - ); + List> expectedResults = Collections.singletonList( + new Result<>( + DateTimes.of("2011-01-12T00:00:00.000Z"), + TopNResultValue.create( + Arrays.asList( + new DimensionAndMetricValueExtractor( + ImmutableMap.builder() + .put("market", "spot") + .put("rows", 4L) + .put("index", 400.0D) + .put("addRowsIndexConstant", 405.0D) + .put("uniques", 1.0002442201269182D) + .put("maxIndex", 100.0) + .put("minIndex", 100.0) + .build() + ), + new DimensionAndMetricValueExtractor( + ImmutableMap.builder() + .put("market", "") + .put("rows", 2L) + .put("index", 200.0D) + .put("addRowsIndexConstant", 203.0D) + .put("uniques", 0.0) + .put("maxIndex", 100.0D) + .put("minIndex", 100.0D) + .build() + ), + new DimensionAndMetricValueExtractor( + ImmutableMap.builder() + .put("market", "total_market") + .put("rows", 2L) + .put("index", 200.0D) + .put("addRowsIndexConstant", 203.0D) + .put("uniques", 1.0002442201269182D) + .put("maxIndex", 100.0D) + .put("minIndex", 100.0D) + .build() + ) + ) + ) + ) + ); - try (CloseableStupidPool pool = TestQueryRunners.createDefaultNonBlockingPool()) { - QueryRunner runner = TestQueryRunners.makeTopNQueryRunner(segment, pool); - TestHelper.assertExpectedResults(expectedResults, runner.run(QueryPlus.wrap(query))); + try (CloseableStupidPool pool = TestQueryRunners.createDefaultNonBlockingPool()) { + QueryRunner runner = TestQueryRunners.makeTopNQueryRunner(segment, pool); + TestHelper.assertExpectedResults(expectedResults, runner.run(QueryPlus.wrap(query))); + } } } - @Test - public void testFullOnSearch() + @ParameterizedTest + @MethodSource("constructorFeeder") + public void testFullOnSearch(final Segment segment) throws IOException { - SearchQuery query = Druids.newSearchQueryBuilder() - .dataSource(dataSource) - .granularity(ALL_GRAN) - .intervals(fullOnInterval) - .query("a") - .build(); + try (segment) { + SearchQuery query = Druids.newSearchQueryBuilder() + .dataSource(dataSource) + .granularity(ALL_GRAN) + .intervals(fullOnInterval) + .query("a") + .build(); - List> expectedResults = Collections.singletonList( - new Result<>( - DateTimes.of("2011-01-12T00:00:00.000Z"), - new SearchResultValue( - Arrays.asList( - new SearchHit(placementishDimension, "a"), - new SearchHit(qualityDimension, "automotive"), - new SearchHit(placementDimension, "mezzanine"), - new SearchHit(marketDimension, "total_market") - ) - ) - ) - ); + List> expectedResults = Collections.singletonList( + new Result<>( + DateTimes.of("2011-01-12T00:00:00.000Z"), + new SearchResultValue( + Arrays.asList( + new SearchHit(placementishDimension, "a"), + new SearchHit(qualityDimension, "automotive"), + new SearchHit(placementDimension, "mezzanine"), + new SearchHit(marketDimension, "total_market") + ) + ) + ) + ); - QueryRunner runner = TestQueryRunners.makeSearchQueryRunner(segment); - TestHelper.assertExpectedResults(expectedResults, runner.run(QueryPlus.wrap(query))); + QueryRunner runner = TestQueryRunners.makeSearchQueryRunner(segment); + TestHelper.assertExpectedResults(expectedResults, runner.run(QueryPlus.wrap(query))); + } } - @Test - public void testTimeBoundary() + @ParameterizedTest + @MethodSource("constructorFeeder") + public void testTimeBoundary(final Segment segment) throws IOException { - TimeBoundaryQuery query = Druids.newTimeBoundaryQueryBuilder() - .dataSource("testing") - .build(); + try (segment) { + TimeBoundaryQuery query = Druids.newTimeBoundaryQueryBuilder() + .dataSource("testing") + .build(); - List> expectedResults = Collections.singletonList( - new Result<>( - DateTimes.of("2011-01-12T00:00:00.000Z"), - new TimeBoundaryResultValue( - ImmutableMap.of( - TimeBoundaryQuery.MIN_TIME, - DateTimes.of("2011-01-12T00:00:00.000Z"), - TimeBoundaryQuery.MAX_TIME, - DateTimes.of("2011-01-13T00:00:00.000Z") - ) - ) - ) - ); + List> expectedResults = Collections.singletonList( + new Result<>( + DateTimes.of("2011-01-12T00:00:00.000Z"), + new TimeBoundaryResultValue( + ImmutableMap.of( + TimeBoundaryQuery.MIN_TIME, + DateTimes.of("2011-01-12T00:00:00.000Z"), + TimeBoundaryQuery.MAX_TIME, + DateTimes.of("2011-01-13T00:00:00.000Z") + ) + ) + ) + ); - QueryRunner runner = TestQueryRunners.makeTimeBoundaryQueryRunner(segment); - TestHelper.assertExpectedResults(expectedResults, runner.run(QueryPlus.wrap(query))); + QueryRunner runner = TestQueryRunners.makeTimeBoundaryQueryRunner(segment); + TestHelper.assertExpectedResults(expectedResults, runner.run(QueryPlus.wrap(query))); + } } }