View Javadoc
1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one or more
3    * contributor license agreements.  See the NOTICE file distributed with
4    * this work for additional information regarding copyright ownership.
5    * The ASF licenses this file to You under the Apache License, Version 2.0
6    * (the "License"); you may not use this file except in compliance with
7    * the License.  You may obtain a copy of the License at
8    *
9    *      http://www.apache.org/licenses/LICENSE-2.0
10   *
11   * Unless required by applicable law or agreed to in writing, software
12   * distributed under the License is distributed on an "AS IS" BASIS,
13   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14   * See the License for the specific language governing permissions and
15   * limitations under the License.
16   */
17  package org.apache.commons.statistics.descriptive;
18  
19  /**
20   * Computes the standard deviation of the available values. The default implementation uses the
21   * following definition of the <em>sample standard deviation</em>:
22   *
23   * <p>\[ \sqrt{ \tfrac{1}{n-1} \sum_{i=1}^n (x_i-\overline{x})^2 } \]
24   *
25   * <p>where \( \overline{x} \) is the sample mean, and \( n \) is the number of samples.
26   *
27   * <ul>
28   *   <li>The result is {@code NaN} if no values are added.
29   *   <li>The result is zero if there is one value in the data set.
30   * </ul>
31   *
32   * <p>The use of the term \( n − 1 \) is called Bessel's correction. Omitting the square root,
33   * this provides an unbiased estimator of the variance of a hypothetical infinite population. If the
34   * {@link #setBiased(boolean) biased} option is enabled the normalisation factor is
35   * changed to \( \frac{1}{n} \) for a biased estimator of the <em>sample variance</em>.
36   * Note however that square root is a concave function and thus introduces negative bias
37   * (by Jensen's inequality), which depends on the distribution, and thus the corrected sample
38   * standard deviation (using Bessel's correction) is less biased, but still biased.
39   *
40   * <p>The implementation uses an exact integer sum to compute the scaled (by \( n \))
41   * sum of squared deviations from the mean; this is normalised by the scaled correction factor.
42   *
43   * <p>\[ \frac {n \times \sum_{i=1}^n x_i^2 - (\sum_{i=1}^n x_i)^2}{n \times (n - 1)} \]
44   *
45   * <p>Supports up to 2<sup>63</sup> (exclusive) observations.
46   * This implementation does not check for overflow of the count.
47   *
48   * <p>This class is designed to work with (though does not require)
49   * {@linkplain java.util.stream streams}.
50   *
51   * <p><strong>This implementation is not thread safe.</strong>
52   * If multiple threads access an instance of this class concurrently,
53   * and at least one of the threads invokes the {@link java.util.function.IntConsumer#accept(int) accept} or
54   * {@link StatisticAccumulator#combine(StatisticResult) combine} method, it must be synchronized externally.
55   *
56   * <p>However, it is safe to use {@link java.util.function.IntConsumer#accept(int) accept}
57   * and {@link StatisticAccumulator#combine(StatisticResult) combine}
58   * as {@code accumulator} and {@code combiner} functions of
59   * {@link java.util.stream.Collector Collector} on a parallel stream,
60   * because the parallel implementation of {@link java.util.stream.Stream#collect Stream.collect()}
61   * provides the necessary partitioning, isolation, and merging of results for
62   * safe and efficient parallel execution.
63   *
64   * @see <a href="https://en.wikipedia.org/wiki/Standard_deviation">Standard deviation (Wikipedia)</a>
65   * @see <a href="https://en.wikipedia.org/wiki/Bessel%27s_correction">Bessel&#39;s correction</a>
66   * @see <a href="https://en.wikipedia.org/wiki/Jensen%27s_inequality">Jensen&#39;s inequality</a>
67   * @see IntVariance
68   * @since 1.1
69   */
70  public final class IntStandardDeviation implements IntStatistic, StatisticAccumulator<IntStandardDeviation> {
71  
72      /** Sum of the squared values. */
73      private final UInt128 sumSq;
74      /** Sum of the values. */
75      private final Int128 sum;
76      /** Count of values that have been added. */
77      private long n;
78  
79      /** Flag to control if the statistic is biased, or should use a bias correction. */
80      private boolean biased;
81  
82      /**
83       * Create an instance.
84       */
85      private IntStandardDeviation() {
86          this(UInt128.create(), Int128.create(), 0);
87      }
88  
89      /**
90       * Create an instance.
91       *
92       * @param sumSq Sum of the squared values.
93       * @param sum Sum of the values.
94       * @param n Count of values that have been added.
95       */
96      private IntStandardDeviation(UInt128 sumSq, Int128 sum, int n) {
97          this.sumSq = sumSq;
98          this.sum = sum;
99          this.n = n;
100     }
101 
102     /**
103      * Creates an instance.
104      *
105      * <p>The initial result is {@code NaN}.
106      *
107      * @return {@code IntStandardDeviation} instance.
108      */
109     public static IntStandardDeviation create() {
110         return new IntStandardDeviation();
111     }
112 
113     /**
114      * Returns an instance populated using the input {@code values}.
115      *
116      * @param values Values.
117      * @return {@code IntStandardDeviation} instance.
118      */
119     public static IntStandardDeviation of(int... values) {
120         return createFromRange(values, 0, values.length);
121     }
122 
123     /**
124      * Returns an instance populated using the specified range of {@code values}.
125      *
126      * @param values Values.
127      * @param from Inclusive start of the range.
128      * @param to Exclusive end of the range.
129      * @return {@code IntStandardDeviation} instance.
130      * @throws IndexOutOfBoundsException if the sub-range is out of bounds
131      * @since 1.2
132      */
133     public static IntStandardDeviation ofRange(int[] values, int from, int to) {
134         Statistics.checkFromToIndex(from, to, values.length);
135         return createFromRange(values, from, to);
136     }
137 
138     /**
139      * Create an instance using the specified range of {@code values}.
140      *
141      * <p>Warning: No range checks are performed.
142      *
143      * @param values Values.
144      * @param from Inclusive start of the range.
145      * @param to Exclusive end of the range.
146      * @return {@code IntStandardDeviation} instance.
147      */
148     static IntStandardDeviation createFromRange(int[] values, int from, int to) {
149         // Small arrays can be processed using the object
150         final int length = to - from;
151         if (length < IntVariance.SMALL_SAMPLE) {
152             final IntStandardDeviation stat = new IntStandardDeviation();
153             for (int i = from; i < to; i++) {
154                 stat.accept(values[i]);
155             }
156             return stat;
157         }
158 
159         // Arrays can be processed using specialised counts knowing the maximum limit
160         // for an array is 2^31 values.
161         long s = 0;
162         final UInt96 ss = UInt96.create();
163         // Process pairs as we know two maximum value int^2 will not overflow
164         // an unsigned long.
165         final int end = from + (length & ~0x1);
166         for (int i = from; i < end; i += 2) {
167             final long x = values[i];
168             final long y = values[i + 1];
169             s += x + y;
170             ss.addPositive(x * x + y * y);
171         }
172         if (end < to) {
173             final long x = values[end];
174             s += x;
175             ss.addPositive(x * x);
176         }
177 
178         // Convert
179         return new IntStandardDeviation(UInt128.of(ss), Int128.of(s), length);
180     }
181 
182     /**
183      * Updates the state of the statistic to reflect the addition of {@code value}.
184      *
185      * @param value Value.
186      */
187     @Override
188     public void accept(int value) {
189         sumSq.addPositive((long) value * value);
190         sum.add(value);
191         n++;
192     }
193 
194     /**
195      * Gets the standard deviation of all input values.
196      *
197      * <p>When no values have been added, the result is {@code NaN}.
198      *
199      * @return standard deviation of all values.
200      */
201     @Override
202     public double getAsDouble() {
203         return IntVariance.computeVarianceOrStd(sumSq, sum, n, biased, true);
204     }
205 
206     @Override
207     public IntStandardDeviation combine(IntStandardDeviation other) {
208         sumSq.add(other.sumSq);
209         sum.add(other.sum);
210         n += other.n;
211         return this;
212     }
213 
214     /**
215      * Sets the value of the biased flag. The default value is {@code false}. The bias
216      * term refers to the computation of the variance; the standard deviation is returned
217      * as the square root of the biased or unbiased <em>sample variance</em>. For further
218      * details see {@link IntVariance#setBiased(boolean) IntVariance.setBiased}.
219      *
220      * <p>This flag only controls the final computation of the statistic. The value of
221      * this flag will not affect compatibility between instances during a
222      * {@link #combine(IntStandardDeviation) combine} operation.
223      *
224      * @param v Value.
225      * @return {@code this} instance
226      * @see IntVariance#setBiased(boolean)
227      */
228     public IntStandardDeviation setBiased(boolean v) {
229         biased = v;
230         return this;
231     }
232 }