Skip to content

Commit 70ccb6c

Browse files
DilumAluthgeDilumAluthge
committed
Add docker-1 through docker-4 test groups
1 parent d4679df commit 70ccb6c

File tree

6 files changed

+113
-48
lines changed

6 files changed

+113
-48
lines changed

‎TEST_GROUPS.md

+18-12
Original file line numberDiff line numberDiff line change
@@ -13,15 +13,21 @@ Pkg.test("PredictMDFull")
1313

1414
## Available test groups
1515

16-
| group | default | all | test-plots | import-only | travis-1 | travis-2 | travis-3 | travis-4 | travis-5 | travis-6 | travis-7 |
17-
| :--- | :--- | :--- | :--- | :--- | :--- | :--- | :--- | :--- | :--- | :--- | :--- |
18-
| Import package | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: |
19-
| Unit tests | :white_check_mark: | :white_check_mark: | :white_check_mark: | :x: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
20-
| Integration tests (1/7) | :white_check_mark: | :white_check_mark: | :white_check_mark: | :x: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
21-
| Integration tests (2/7) | :white_check_mark: | :white_check_mark: | :white_check_mark: | :x: | :x: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: |
22-
| Integration tests (3/7) | :white_check_mark: | :white_check_mark: | :white_check_mark: | :x: | :x: | :x: | :white_check_mark: | :x: | :x: | :x: | :x: |
23-
| Integration tests (4/7) | :white_check_mark: | :white_check_mark: | :white_check_mark: | :x: | :x: | :x: | :x: | :white_check_mark: | :x: | :x: | :x: |
24-
| Integration tests (5/7) | :white_check_mark: | :white_check_mark: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :white_check_mark: | :x: | :x: |
25-
| Integration tests (6/7) | :white_check_mark: | :white_check_mark: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :white_check_mark: | :x: |
26-
| Integration tests (7/7) | :white_check_mark: | :white_check_mark: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :white_check_mark: |
27-
| Test plots | :x: | :white_check_mark: | :white_check_mark: | :x: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: |
16+
| group | `default` | `all` | `test-plots` | `import-only` | `travis-1` | `travis-2` | `travis-3` | `travis-4` | `travis-5` | `travis-6` | `travis-7` | `docker-1` | `docker-2` | `docker-3` | `docker-4` |
17+
| :--- | :--- | :--- | :--- | :--- | :--- | :--- | :--- | :--- | :--- | :--- | :--- | :--- | :--- | :--- | :--- |
18+
| Import package | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: |
19+
| Unit tests | :white_check_mark: | :white_check_mark: | :white_check_mark: | :x: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :white_check_mark: | :x: | :x: | :x: |
20+
| Integration tests 1/7 | :white_check_mark: | :white_check_mark: | :white_check_mark: | :x: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :white_check_mark: | :x: | :x: |
21+
| Integration tests 2/7 | :white_check_mark: | :white_check_mark: | :white_check_mark: | :x: | :x: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :white_check_mark: | :x: | :x: |
22+
| Integration tests 3/7 | :white_check_mark: | :white_check_mark: | :white_check_mark: | :x: | :x: | :x: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :white_check_mark: | :x: | :x: |
23+
| Integration tests 4/7 | :white_check_mark: | :white_check_mark: | :white_check_mark: | :x: | :x: | :x: | :x: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :white_check_mark: | :x: |
24+
| Integration tests 5/7 | :white_check_mark: | :white_check_mark: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :white_check_mark: | :x: | :x: | :x: | :x: | :white_check_mark: | :x: |
25+
| Integration tests 6/7 | :white_check_mark: | :white_check_mark: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :white_check_mark: | :x: | :x: | :x: | :white_check_mark: | :x: |
26+
| Integration tests 7/7 | :white_check_mark: | :white_check_mark: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :white_check_mark: | :x: | :x: | :white_check_mark: | :x: |
27+
| Plot tests 1/7 | :x: | :white_check_mark: | :white_check_mark: | :x: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :white_check_mark: | :x: | :x: |
28+
| Plot tests 2/7 | :x: | :white_check_mark: | :white_check_mark: | :x: | :x: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :white_check_mark: | :x: | :x: |
29+
| Plot tests 3/7 | :x: | :white_check_mark: | :white_check_mark: | :x: | :x: | :x: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :white_check_mark: | :x: | :x: |
30+
| Plot tests 4/7 | :x: | :white_check_mark: | :white_check_mark: | :x: | :x: | :x: | :x: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :white_check_mark: | :x: |
31+
| Plot tests 5/7 | :x: | :white_check_mark: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :white_check_mark: | :x: | :x: | :x: | :x: | :white_check_mark: | :x: |
32+
| Plot tests 6/7 | :x: | :white_check_mark: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :white_check_mark: | :x: | :x: | :x: | :white_check_mark: | :x: |
33+
| Plot tests 7/7 | :x: | :white_check_mark: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :white_check_mark: | :x: | :x: | :white_check_mark: | :x: |

‎src/toplevel/plotting/probability_calibration_plots.jl

+9-9
Original file line numberDiff line numberDiff line change
@@ -36,10 +36,11 @@ end
3636
"""
3737
"""
3838
function probability_calibration_scores_and_fractions(
39-
ytrue::AbstractVector{<:Integer},
40-
yscore::AbstractVector{<:AbstractFloat};
39+
ytrue::AbstractVector{<: Integer},
40+
yscore::AbstractVector{F};
4141
window::Real = 0.1,
42-
)
42+
)::Tuple{Vector{F}, Vector{F}} where
43+
F <: AbstractFloat
4344
scores = sort(
4445
unique(
4546
vcat(
@@ -49,25 +50,25 @@ function probability_calibration_scores_and_fractions(
4950
)
5051
)
5152
)
52-
fractions = fill(Cfloat(0), size(scores))
53+
fractions = fill(F(0), size(scores))
5354
num_rows = fill(Int(0), length(scores))
5455
for k = 1:length(scores)
5556
rows_that_have_approximately_the_kth_score = findall(
5657
( scores[k] - window ) .<= ( yscore ) .<= ( scores[k] + window )
5758
)
5859
num_rows[k] = length(rows_that_have_approximately_the_kth_score)
5960
if length(rows_that_have_approximately_the_kth_score) == 0
60-
fractions[k] = -999
61+
fractions[k] = NaN
6162
else
6263
fractions[k] = Statistics.mean(
6364
ytrue[rows_that_have_approximately_the_kth_score]
6465
)
6566
end
6667
end
6768
nonzero_indices = findall(num_rows .!= 0)
68-
scores = scores[nonzero_indices]
69-
fractions = fractions[nonzero_indices]
70-
return scores, fractions
69+
scores_nonzero_indices::Vector{F} = scores[nonzero_indices]
70+
fraction_nonzero_indicess::Vector{F} = fractions[nonzero_indices]
71+
return scores_nonzero_indices, fraction_nonzero_indices
7172
end
7273

7374
"""
@@ -265,4 +266,3 @@ function probability_calibration_metrics(
265266
end
266267
return result
267268
end
268-

‎test/define-test-blocks.jl

+35
Original file line numberDiff line numberDiff line change
@@ -89,3 +89,38 @@ group_includes_block(::TestGroupTravis7, ::TestBlockIntegration5) = false
8989
group_includes_block(::TestGroupTravis7, ::TestBlockIntegration6) = false
9090
group_includes_block(::TestGroupTravis7, ::TestBlockIntegration7) = true
9191

92+
group_includes_block(::TestGroupDocker1, ::TestBlockUnitTests) = true
93+
group_includes_block(::TestGroupDocker1, ::TestBlockIntegration1) = false
94+
group_includes_block(::TestGroupDocker1, ::TestBlockIntegration2) = false
95+
group_includes_block(::TestGroupDocker1, ::TestBlockIntegration3) = false
96+
group_includes_block(::TestGroupDocker1, ::TestBlockIntegration4) = false
97+
group_includes_block(::TestGroupDocker1, ::TestBlockIntegration5) = false
98+
group_includes_block(::TestGroupDocker1, ::TestBlockIntegration6) = false
99+
group_includes_block(::TestGroupDocker1, ::TestBlockIntegration7) = false
100+
101+
group_includes_block(::TestGroupDocker2, ::TestBlockUnitTests) = false
102+
group_includes_block(::TestGroupDocker2, ::TestBlockIntegration1) = true
103+
group_includes_block(::TestGroupDocker2, ::TestBlockIntegration2) = true
104+
group_includes_block(::TestGroupDocker2, ::TestBlockIntegration3) = true
105+
group_includes_block(::TestGroupDocker2, ::TestBlockIntegration4) = false
106+
group_includes_block(::TestGroupDocker2, ::TestBlockIntegration5) = false
107+
group_includes_block(::TestGroupDocker2, ::TestBlockIntegration6) = false
108+
group_includes_block(::TestGroupDocker2, ::TestBlockIntegration7) = false
109+
110+
group_includes_block(::TestGroupDocker3, ::TestBlockUnitTests) = false
111+
group_includes_block(::TestGroupDocker3, ::TestBlockIntegration1) = false
112+
group_includes_block(::TestGroupDocker3, ::TestBlockIntegration2) = false
113+
group_includes_block(::TestGroupDocker3, ::TestBlockIntegration3) = false
114+
group_includes_block(::TestGroupDocker3, ::TestBlockIntegration4) = true
115+
group_includes_block(::TestGroupDocker3, ::TestBlockIntegration5) = true
116+
group_includes_block(::TestGroupDocker3, ::TestBlockIntegration6) = true
117+
group_includes_block(::TestGroupDocker3, ::TestBlockIntegration7) = true
118+
119+
group_includes_block(::TestGroupDocker4, ::TestBlockUnitTests) = false
120+
group_includes_block(::TestGroupDocker4, ::TestBlockIntegration1) = false
121+
group_includes_block(::TestGroupDocker4, ::TestBlockIntegration2) = false
122+
group_includes_block(::TestGroupDocker4, ::TestBlockIntegration3) = false
123+
group_includes_block(::TestGroupDocker4, ::TestBlockIntegration4) = false
124+
group_includes_block(::TestGroupDocker4, ::TestBlockIntegration5) = false
125+
group_includes_block(::TestGroupDocker4, ::TestBlockIntegration6) = false
126+
group_includes_block(::TestGroupDocker4, ::TestBlockIntegration7) = false

‎test/define-test-groups.jl

+32-26
Original file line numberDiff line numberDiff line change
@@ -1,46 +1,52 @@
11
import PredictMD
22

3-
abstract type AbstractTestGroup
4-
end
5-
6-
struct TestGroupDefault <: AbstractTestGroup
7-
end
8-
struct TestGroupAll <: AbstractTestGroup
9-
end
10-
struct TestGroupTestPlots <: AbstractTestGroup
11-
end
12-
struct TestGroupImportOnly <: AbstractTestGroup
13-
end
14-
struct TestGroupTravis1 <: AbstractTestGroup
15-
end
16-
struct TestGroupTravis2 <: AbstractTestGroup
17-
end
18-
struct TestGroupTravis3 <: AbstractTestGroup
19-
end
20-
struct TestGroupTravis4 <: AbstractTestGroup
21-
end
22-
struct TestGroupTravis5 <: AbstractTestGroup
23-
end
24-
struct TestGroupTravis6 <: AbstractTestGroup
25-
end
26-
struct TestGroupTravis7 <: AbstractTestGroup
27-
end
3+
abstract type AbstractTestGroup end
4+
5+
struct TestGroupDefault <: AbstractTestGroup end
6+
7+
struct TestGroupAll <: AbstractTestGroup end
8+
9+
struct TestGroupTestPlots <: AbstractTestGroup end
10+
11+
struct TestGroupImportOnly <: AbstractTestGroup end
12+
13+
struct TestGroupTravis1 <: AbstractTestGroup end
14+
struct TestGroupTravis2 <: AbstractTestGroup end
15+
struct TestGroupTravis3 <: AbstractTestGroup end
16+
struct TestGroupTravis4 <: AbstractTestGroup end
17+
struct TestGroupTravis5 <: AbstractTestGroup end
18+
struct TestGroupTravis6 <: AbstractTestGroup end
19+
struct TestGroupTravis7 <: AbstractTestGroup end
20+
21+
struct TestGroupDocker1 <: AbstractTestGroup end
22+
struct TestGroupDocker2 <: AbstractTestGroup end
23+
struct TestGroupDocker3 <: AbstractTestGroup end
24+
struct TestGroupDocker4 <: AbstractTestGroup end
2825

2926
const TEST_GROUP_STRING_TO_INSTANCE = Dict{String, AbstractTestGroup}(
27+
#
3028
"default" => TestGroupDefault(),
29+
#
3130
"all" => TestGroupAll(),
31+
#
3232
"test-plots" => TestGroupTestPlots(),
33+
#
3334
"import-only" => TestGroupImportOnly(),
35+
#
3436
"travis-1" => TestGroupTravis1(),
3537
"travis-2" => TestGroupTravis2(),
3638
"travis-3" => TestGroupTravis3(),
3739
"travis-4" => TestGroupTravis4(),
3840
"travis-5" => TestGroupTravis5(),
3941
"travis-6" => TestGroupTravis6(),
4042
"travis-7" => TestGroupTravis7(),
43+
#
44+
"docker-1" => TestGroupDocker1(),
45+
"docker-2" => TestGroupDocker2(),
46+
"docker-3" => TestGroupDocker3(),
47+
"docker-4" => TestGroupDocker4(),
4148
)
4249

4350
const TEST_GROUP_INSTANCE_TO_STRING = PredictMD.inverse(
4451
TEST_GROUP_STRING_TO_INSTANCE
4552
)
46-

‎test/set-predictmd-test-plots.jl

+13-1
Original file line numberDiff line numberDiff line change
@@ -29,11 +29,23 @@ if isa(TEST_GROUP, TestGroupTravis6)
2929
ENV["PREDICTMD_TEST_PLOTS"] = "true"
3030
end
3131

32+
if isa(TEST_GROUP, TestGroupDocker1)
33+
ENV["PREDICTMD_TEST_PLOTS"] = "true"
34+
end
35+
if isa(TEST_GROUP, TestGroupDocker2)
36+
ENV["PREDICTMD_TEST_PLOTS"] = "true"
37+
end
38+
if isa(TEST_GROUP, TestGroupDocker3)
39+
ENV["PREDICTMD_TEST_PLOTS"] = "true"
40+
end
41+
if isa(TEST_GROUP, TestGroupDocker4)
42+
ENV["PREDICTMD_TEST_PLOTS"] = "true"
43+
end
44+
3245
@info(
3346
string(
3447
"PREDICTMD_TEST_PLOTS: \"",
3548
ENV["PREDICTMD_TEST_PLOTS"],
3649
"\"",
3750
)
3851
)
39-

‎test/unit/toplevel/plotting/probability_calibration_plots.jl

+6
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
import PredictMD
2+
3+
probability_calibration_scores_and_fractions(
4+
[0, 0, 0], [0., 0., 0.];
5+
window = -1,
6+
)

0 commit comments

Comments
 (0)