View Javadoc
1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one or more
3    * contributor license agreements.  See the NOTICE file distributed with
4    * this work for additional information regarding copyright ownership.
5    * The ASF licenses this file to You under the Apache License, Version 2.0
6    * (the "License"); you may not use this file except in compliance with
7    * the License.  You may obtain a copy of the License at
8    *
9    *      https://www.apache.org/licenses/LICENSE-2.0
10   *
11   * Unless required by applicable law or agreed to in writing, software
12   * distributed under the License is distributed on an "AS IS" BASIS,
13   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14   * See the License for the specific language governing permissions and
15   * limitations under the License.
16   */
17  
18  package org.apache.commons.statistics.distribution;
19  
20  import java.io.BufferedReader;
21  import java.io.IOException;
22  import java.io.InputStream;
23  import java.io.InputStreamReader;
24  import java.util.stream.Stream;
25  import org.apache.commons.math3.stat.descriptive.SummaryStatistics;
26  import org.apache.commons.numbers.gamma.LanczosApproximation;
27  import org.junit.jupiter.api.Assertions;
28  import org.junit.jupiter.params.ParameterizedTest;
29  import org.junit.jupiter.params.provider.Arguments;
30  import org.junit.jupiter.params.provider.CsvSource;
31  import org.junit.jupiter.params.provider.MethodSource;
32  
33  /**
34   * Test cases for {@link GammaDistribution}.
35   * Extends {@link BaseContinuousDistributionTest}. See javadoc of that class for details.
36   */
37  class GammaDistributionTest extends BaseContinuousDistributionTest {
38      private static final double HALF_LOG_2_PI = 0.5 * Math.log(2.0 * Math.PI);
39  
40      @Override
41      ContinuousDistribution makeDistribution(Object... parameters) {
42          final double shape = (Double) parameters[0];
43          final double scale = (Double) parameters[1];
44          return GammaDistribution.of(shape, scale);
45      }
46  
47      @Override
48      Object[][] makeInvalidParameters() {
49          return new Object[][] {
50              {0.0, 1.0},
51              {-0.1, 1.0},
52              {1.0, 0.0},
53              {1.0, -0.1},
54          };
55      }
56  
57      @Override
58      String[] getParameterNames() {
59          return new String[] {"Shape", "Scale"};
60      }
61  
62      @Override
63      protected double getRelativeTolerance() {
64          // Tolerance is 8.881784197001252E-16
65          return 4 * RELATIVE_EPS;
66      }
67  
68      //-------------------- Additional test cases -------------------------------
69  
70      @ParameterizedTest
71      @MethodSource
72      void testAdditionalMoments(double shape, double scale, double mean, double variance) {
73          final GammaDistribution dist = GammaDistribution.of(shape, scale);
74          testMoments(dist, mean, variance, DoubleTolerances.equals());
75      }
76  
77      static Stream<Arguments> testAdditionalMoments() {
78          return Stream.of(
79              Arguments.of(1, 2, 2, 4),
80              Arguments.of(1.1, 4.2, 1.1 * 4.2, 1.1 * 4.2 * 4.2),
81              // scipy.stats.gamma(shape, scale=scale).stats()
82              Arguments.of(0.5, 10, 5, 50),
83              Arguments.of(0.5, 7.5, 3.75, 28.125),
84              Arguments.of(0.25, 10, 2.5, 25)
85          );
86      }
87  
88      @ParameterizedTest
89      @CsvSource({
90          "4.0, 2.0, -1.0, 0.0",
91          "4.0, 2.0, 15.501, 0.94989465156755404",
92          "4.0, 1.0, 0.504, 0.0018026739713985257",
93          "1.0, 2.0, 10.011, 0.99329900998454213",
94          "2.0, 2.0, 5.000, 0.71270250481635422",
95      })
96      void testAdditionalCumulativeProbability(double a, double b, double x, double expected) {
97          final GammaDistribution dist = GammaDistribution.of(a, b);
98          final double actual = dist.cumulativeProbability(x);
99          Assertions.assertEquals(expected, actual, expected * 1e-15, () -> "probability for " + x);
100     }
101 
102     @ParameterizedTest
103     @CsvSource({
104         "4.0, 2.0, 0.94989465156755404, 15.501",
105         "4.0, 1.0, 0.0018026739713985257, 0.504",
106         "1.0, 2.0, 0.99329900998454213, 10.011",
107         "2.0, 2.0, 0.71270250481635422, 5.0",
108     })
109     void testAdditionalInverseCumulativeProbability(double a, double b, double p, double expected) {
110         final GammaDistribution dist = GammaDistribution.of(a, b);
111         final double actual = dist.inverseCumulativeProbability(p);
112         Assertions.assertEquals(expected, actual, expected * 5e-15, () -> "critical value for " + p);
113     }
114 
115     @ParameterizedTest
116     @MethodSource
117     void testAdditionalDensity(double alpha, double rate, double[] x, double[] expected) {
118         final GammaDistribution dist = GammaDistribution.of(alpha, 1 / rate);
119         testDensity(dist, x, expected, createRelTolerance(1e-9));
120     }
121 
122     static Stream<Arguments> testAdditionalDensity() {
123         final double[] x = {-0.1, 1e-6, 0.5, 1, 2, 5};
124         final double[] x1 = new double[]{1e-100, 1e-10, 1e-5, 0.1};
125         return Stream.of(
126             // R2.5: print(dgamma((x, shape=1, rate=1), digits=10)
127             Arguments.of(1, 1, x, new double[]{0.000000000000, 0.999999000001, 0.606530659713, 0.367879441171, 0.135335283237, 0.006737946999}),
128             // R2.5: print(dgamma((x, shape=2, rate=1), digits=10)
129             Arguments.of(2, 1, x, new double[]{0.000000000000, 0.000000999999, 0.303265329856, 0.367879441171, 0.270670566473, 0.033689734995}),
130             // R2.5: print(dgamma((x, shape=4, rate=1), digits=10)
131             Arguments.of(4, 1, x, new double[]{0.000000000e+00, 1.666665000e-19, 1.263605541e-02, 6.131324020e-02, 1.804470443e-01, 1.403738958e-01}),
132             // R2.5: print(dgamma((x, shape=4, rate=10), digits=10)
133             Arguments.of(4, 10, x, new double[]{0.000000000e+00, 1.666650000e-15, 1.403738958e+00, 7.566654960e-02, 2.748204830e-05, 4.018228850e-17}),
134             // R2.5: print(dgamma((x, shape=.1, rate=10), digits=10)
135             Arguments.of(0.1, 10, x, new double[]{0.000000000e+00, 3.323953832e+04, 1.663849010e-03, 6.007786726e-06, 1.461647647e-10, 5.996008322e-24}),
136             // R2.5: print(dgamma((x, shape=.1, rate=20), digits=10)
137             Arguments.of(0.1, 20, x, new double[]{0.000000000e+00, 3.562489883e+04, 1.201557345e-05, 2.923295295e-10, 3.228910843e-19, 1.239484589e-45}),
138             // R2.5: print(dgamma((x, shape=.1, rate=4), digits=10)
139             Arguments.of(0.1, 4, x, new double[]{0.000000000e+00, 3.032938388e+04, 3.049322494e-02, 2.211502311e-03, 2.170613371e-05, 5.846590589e-11}),
140             // R2.5: print(dgamma((x, shape=.1, rate=1), digits=10)
141             Arguments.of(0.1, 1, x, new double[]{0.000000000e+00, 2.640334143e+04, 1.189704437e-01, 3.866916944e-02, 7.623306235e-03, 1.663849010e-04}),
142             // To force overflow condition
143             // R2.5: print(dgamma((x, shape=1000, rate=100), digits=10)
144             Arguments.of(1000, 100, x, new double[]{0.000000000e+00, 0.000000000e+00, 0.000000000e+00, 0.000000000e+00, 0.000000000e+00, 3.304830256e-84}),
145 
146             // Test a shape far below 1.
147             // Support for very small shape parameters was fixed in STATISTICS-39.
148             // R2.5: print(dgamma((x1, shape=0.05, rate=1), digits=20)
149             Arguments.of(0.05, 1, x1,
150                 new double[] {
151                     5.1360843263583843333e+93, 1.6241724724359893799e+08,
152                     2.8882035841935007738e+03, 4.1419294512123655538e-01
153                 })
154         );
155     }
156 
157     @ParameterizedTest
158     @MethodSource
159     void testAdditionalLogDensity(double alpha, double rate, double[] x, double[] expected) {
160         final GammaDistribution dist = GammaDistribution.of(alpha, 1 / rate);
161         testLogDensity(dist, x, expected, createRelTolerance(1e-9));
162     }
163 
164     static Stream<Arguments> testAdditionalLogDensity() {
165         final double[] x = {-0.1, 1e-6, 0.5, 1, 2, 5};
166         final double[] x1 = {1e-315, 1e-320, 1e-323};
167         final double inf = Double.POSITIVE_INFINITY;
168         return Stream.of(
169             // R2.5: print(dgamma((x, shape=1, rate=1, log=TRUE), digits=10)
170             Arguments.of(1, 1, x, new double[]{-inf, -1e-06, -5e-01, -1e+00, -2e+00, -5e+00}),
171             // R2.5: print(dgamma((x, shape=2, rate=1, log=TRUE), digits=10)
172             Arguments.of(2, 1, x, new double[]{-inf, -13.815511558, -1.193147181, -1.000000000, -1.306852819, -3.390562088}),
173             // R2.5: print(dgamma((x, shape=4, rate=1, log=TRUE), digits=10)
174             Arguments.of(4, 1, x, new double[]{-inf, -43.238292143, -4.371201011, -2.791759469, -1.712317928, -1.963445732}),
175             // R2.5: print(dgamma((x, shape=4, rate=10, log=TRUE), digits=10)
176             Arguments.of(4, 10, x, new double[]{-inf, -34.0279607711, 0.3391393611, -2.5814190973, -10.5019775556, -37.7531053599}),
177             // R2.5: print(dgamma((x, shape=.1, rate=10, log=TRUE), digits=10)
178             Arguments.of(0.1, 10, x, new double[]{-inf, 10.41149536, -6.39862168, -12.02245414, -22.64628660, -53.47094826}),
179             // R2.5: print(dgamma((x, shape=.1, rate=20, log=TRUE), digits=10)
180             Arguments.of(0.1, 20, x, new double[]{-inf, 10.48080008, -11.32930696, -21.95313942, -42.57697189, -103.40163355}),
181             // R2.5: print(dgamma((x, shape=.1, rate=4, log=TRUE), digits=10)
182             Arguments.of(0.1, 4, x, new double[]{-inf, 10.319872287, -3.490250753, -6.114083216, -10.737915678, -23.562577337}),
183             // R2.5: print(dgamma((x, shape=.1, rate=1, log=TRUE), digits=10)
184             Arguments.of(0.1, 1, x, new double[]{-inf, 10.181245850, -2.128880189, -3.252712652, -4.876545114, -8.701206773}),
185             // To force overflow condition to pdf=zero
186             // R2.5: print(dgamma((x, shape=1000, rate=100, log=TRUE), digits=10)
187             Arguments.of(1000, 100, x, new double[]{-inf, -15101.7453846, -2042.5042706, -1400.0502372, -807.5962038, -192.2217627}),
188             // To force overflow condition to pdf=infinity
189             // scipy.stats gamma(1e-2).logpdf(x1)
190             Arguments.of(0.01, 1, x1,
191                 new double[]{713.46168137365419, 724.85948860402209, 731.70997561537104})
192         );
193     }
194 
195     private static double logGamma(double x) {
196         /*
197          * This is a copy of
198          * double Gamma.logGamma(double)
199          * prior to MATH-849
200          */
201         if (Double.isNaN(x) || x <= 0.0) {
202             return Double.NaN;
203         }
204         final double sum = LanczosApproximation.value(x);
205         final double tmp = x + LanczosApproximation.g() + .5;
206         return ((x + .5) * Math.log(tmp)) - tmp +
207             HALF_LOG_2_PI + Math.log(sum / x);
208     }
209 
210     private static double density(double x,
211                                   double shape,
212                                   double scale) {
213         /*
214          * This is a copy of
215          * double GammaDistribution.density(double)
216          * prior to MATH-753.
217          */
218         if (x < 0) {
219             return 0;
220         }
221         return Math.pow(x / scale, shape - 1) / scale *
222                Math.exp(-x / scale) / Math.exp(logGamma(shape));
223     }
224 
225     /**
226      * MATH-753: large values of x or shape parameter cause density(double) to
227      * overflow. Reference data is generated with the Maxima script
228      * gamma-distribution.mac, which can be found in
229      * src/test/resources/org/apache/commons/statistics/distribution.
230      *
231      * @param shape Shape of gamma distribution (scale is assumed to be 1)
232      * @param meanNoOF Allowed mean ULP error when the computed value does not overflow using the old method
233      * @param sdNoOF Allowed SD ULP error when the computed value does not overflow using the old method
234      * @param meanOF Allowed mean ULP error when the computed value overflows using the old method
235      * @param sdOF Allowed SD ULP error when the computed value overflows using the old method
236      * @param resourceName Resource name containing a pair of comma separated values for the
237      * random variable x and the expected value of the gamma distribution: x, gamma(x; shape, scale=1)
238      */
239     @ParameterizedTest
240     @MethodSource
241     void testMath753(double shape,
242                      double meanNoOF, double sdNoOF,
243                      double meanOF, double sdOF,
244                      String resourceName) {
245         final GammaDistribution dist = GammaDistribution.of(shape, 1.0);
246         final SummaryStatistics statOld = new SummaryStatistics();
247         // statNewNoOF = No overflow of old function
248         // statNewOF   = Overflow of old function
249         final SummaryStatistics statNewNoOF = new SummaryStatistics();
250         final SummaryStatistics statNewOF = new SummaryStatistics();
251 
252         final InputStream resourceAsStream = this.getClass().getResourceAsStream(resourceName);
253         Assertions.assertNotNull(resourceAsStream, () -> "Could not find resource " + resourceName);
254 
255         try (BufferedReader in = new BufferedReader(new InputStreamReader(resourceAsStream))) {
256             for (String line = in.readLine(); line != null; line = in.readLine()) {
257                 if (line.startsWith("#")) {
258                     continue;
259                 }
260                 final String[] tokens = line.split(", ");
261                 Assertions.assertEquals(2, tokens.length, "expected two floating-point values");
262                 final double x = Double.parseDouble(tokens[0]);
263                 final String msg = "x = " + x + ", shape = " + shape +
264                                    ", scale = 1.0";
265                 final double expected = Double.parseDouble(tokens[1]);
266                 final double ulp = Math.ulp(expected);
267                 final double actualOld = density(x, shape, 1.0);
268                 final double actualNew = dist.density(x);
269                 final double errOld = Math.abs((actualOld - expected) / ulp);
270                 final double errNew = Math.abs((actualNew - expected) / ulp);
271 
272                 if (!Double.isFinite(actualOld)) {
273                     Assertions.assertTrue(Double.isFinite(actualNew), msg);
274                     statNewOF.addValue(errNew);
275                 } else {
276                     statOld.addValue(errOld);
277                     statNewNoOF.addValue(errNew);
278                 }
279             }
280             if (statOld.getN() != 0) {
281                 /*
282                  * If no overflow occurs, check that new implementation is
283                  * better than old one.
284                  */
285                 final StringBuilder sb = new StringBuilder("shape = ");
286                 sb.append(shape);
287                 sb.append(", scale = 1.0\n");
288                 sb.append("Old implementation\n");
289                 sb.append("------------------\n");
290                 sb.append(statOld.toString());
291                 sb.append("New implementation\n");
292                 sb.append("------------------\n");
293                 sb.append(statNewNoOF.toString());
294                 final String msg = sb.toString();
295 
296                 final double oldMin = statOld.getMin();
297                 final double newMin = statNewNoOF.getMin();
298                 Assertions.assertTrue(newMin <= oldMin, msg);
299 
300                 final double oldMax = statOld.getMax();
301                 final double newMax = statNewNoOF.getMax();
302                 Assertions.assertTrue(newMax <= oldMax, msg);
303 
304                 final double oldMean = statOld.getMean();
305                 final double newMean = statNewNoOF.getMean();
306                 Assertions.assertTrue(newMean <= oldMean, msg);
307 
308                 final double oldSd = statOld.getStandardDeviation();
309                 final double newSd = statNewNoOF.getStandardDeviation();
310                 Assertions.assertTrue(newSd <= oldSd, msg);
311 
312                 Assertions.assertTrue(newMean <= meanNoOF, msg);
313                 Assertions.assertTrue(newSd <= sdNoOF, msg);
314             }
315             if (statNewOF.getN() != 0) {
316                 final double newMean = statNewOF.getMean();
317                 final double newSd = statNewOF.getStandardDeviation();
318 
319                 final StringBuilder sb = new StringBuilder("shape = ");
320                 sb.append(shape);
321                 sb.append(", scale = 1.0");
322                 sb.append(", max. mean error (ulps) = ");
323                 sb.append(meanOF);
324                 sb.append(", actual mean error (ulps) = ");
325                 sb.append(newMean);
326                 sb.append(", max. sd of error (ulps) = ");
327                 sb.append(sdOF);
328                 sb.append(", actual sd of error (ulps) = ");
329                 sb.append(newSd);
330                 final String msg = sb.toString();
331 
332                 Assertions.assertTrue(newMean <= meanOF, msg);
333                 Assertions.assertTrue(newSd <= sdOF, msg);
334             }
335         } catch (final IOException e) {
336             Assertions.fail(e);
337         }
338     }
339 
340     static Stream<Arguments> testMath753() {
341         return Stream.of(
342             Arguments.of(0.25, 1.0, 1.0, 0.0, 0.0, "gamma-distribution-shape-0.25.csv"),
343             Arguments.of(0.5, 1.0, 1.0, 0.0, 0.0, "gamma-distribution-shape-0.5.csv"),
344             Arguments.of(0.75, 1.0, 1.0, 0.0, 0.0, "gamma-distribution-shape-0.75.csv"),
345             Arguments.of(1.0, 1.0, 0.5, 0.0, 0.0, "gamma-distribution-shape-1.csv"),
346             Arguments.of(8.0, 1.0, 1.0, 0.0, 0.0, "gamma-distribution-shape-8.csv"),
347             Arguments.of(10.0, 1.0, 1.0, 0.0, 0.0, "gamma-distribution-shape-10.csv"),
348             Arguments.of(100.0, 2.0, 1.0, 0.0, 0.0, "gamma-distribution-shape-100.csv"),
349             Arguments.of(142.0, 1.5, 1.0, 40.0, 40.0, "gamma-distribution-shape-142.csv"),
350             Arguments.of(1000.0, 1.0, 1.0, 160.0, 200.0, "gamma-distribution-shape-1000.csv")
351         );
352     }
353 }