001/*
002 * Licensed to the Apache Software Foundation (ASF) under one or more
003 * contributor license agreements.  See the NOTICE file distributed with
004 * this work for additional information regarding copyright ownership.
005 * The ASF licenses this file to You under the Apache License, Version 2.0
006 * (the "License"); you may not use this file except in compliance with
007 * the License.  You may obtain a copy of the License at
008 *
009 *      http://www.apache.org/licenses/LICENSE-2.0
010 *
011 * Unless required by applicable law or agreed to in writing, software
012 * distributed under the License is distributed on an "AS IS" BASIS,
013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
014 * See the License for the specific language governing permissions and
015 * limitations under the License.
016 */
017package org.apache.commons.statistics.descriptive;
018
019import java.math.BigInteger;
020
021/**
022 * Computes the variance of the available values. The default implementation uses the
023 * following definition of the <em>sample variance</em>:
024 *
025 * <p>\[ \tfrac{1}{n-1} \sum_{i=1}^n (x_i-\overline{x})^2 \]
026 *
027 * <p>where \( \overline{x} \) is the sample mean, and \( n \) is the number of samples.
028 *
029 * <ul>
030 *   <li>The result is {@code NaN} if no values are added.
031 *   <li>The result is zero if there is one value in the data set.
032 * </ul>
033 *
034 * <p>The use of the term \( n − 1 \) is called Bessel's correction. This is an unbiased
035 * estimator of the variance of a hypothetical infinite population. If the
036 * {@link #setBiased(boolean) biased} option is enabled the normalisation factor is
037 * changed to \( \frac{1}{n} \) for a biased estimator of the <em>sample variance</em>.
038 *
039 * <p>The implementation uses an exact integer sum to compute the scaled (by \( n \))
040 * sum of squared deviations from the mean; this is normalised by the scaled correction factor.
041 *
042 * <p>\[ \frac {n \times \sum_{i=1}^n x_i^2 - (\sum_{i=1}^n x_i)^2}{n \times (n - 1)} \]
043 *
044 * <p>Supports up to 2<sup>63</sup> (exclusive) observations.
045 * This implementation does not check for overflow of the count.
046 *
047 * <p>This class is designed to work with (though does not require)
048 * {@linkplain java.util.stream streams}.
049 *
050 * <p><strong>This implementation is not thread safe.</strong>
051 * If multiple threads access an instance of this class concurrently,
052 * and at least one of the threads invokes the {@link java.util.function.LongConsumer#accept(long) accept} or
053 * {@link StatisticAccumulator#combine(StatisticResult) combine} method, it must be synchronized externally.
054 *
055 * <p>However, it is safe to use {@link java.util.function.LongConsumer#accept(long) accept}
056 * and {@link StatisticAccumulator#combine(StatisticResult) combine}
057 * as {@code accumulator} and {@code combiner} functions of
058 * {@link java.util.stream.Collector Collector} on a parallel stream,
059 * because the parallel implementation of {@link java.util.stream.Stream#collect Stream.collect()}
060 * provides the necessary partitioning, isolation, and merging of results for
061 * safe and efficient parallel execution.
062 *
063 * @see <a href="https://en.wikipedia.org/wiki/variance">variance (Wikipedia)</a>
064 * @see <a href="https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance">
065 *   Algorithms for computing the variance (Wikipedia)</a>
066 * @see <a href="https://en.wikipedia.org/wiki/Bessel%27s_correction">Bessel&#39;s correction</a>
067 * @since 1.1
068 */
069public final class LongVariance implements LongStatistic, StatisticAccumulator<LongVariance> {
070
071    /** Sum of the squared values. */
072    private final UInt192 sumSq;
073    /** Sum of the values. */
074    private final Int128 sum;
075    /** Count of values that have been added. */
076    private long n;
077
078    /** Flag to control if the statistic is biased, or should use a bias correction. */
079    private boolean biased;
080
081    /**
082     * Create an instance.
083     */
084    private LongVariance() {
085        this(UInt192.create(), Int128.create(), 0);
086    }
087
088    /**
089     * Create an instance.
090     *
091     * @param sumSq Sum of the squared values.
092     * @param sum Sum of the values.
093     * @param n Count of values that have been added.
094     */
095    private LongVariance(UInt192 sumSq, Int128 sum, int n) {
096        this.sumSq = sumSq;
097        this.sum = sum;
098        this.n = n;
099    }
100
101    /**
102     * Creates an instance.
103     *
104     * <p>The initial result is {@code NaN}.
105     *
106     * @return {@code LongVariance} instance.
107     */
108    public static LongVariance create() {
109        return new LongVariance();
110    }
111
112    /**
113     * Returns an instance populated using the input {@code values}.
114     *
115     * @param values Values.
116     * @return {@code LongVariance} instance.
117     */
118    public static LongVariance of(long... values) {
119        return createFromRange(values, 0, values.length);
120    }
121
122    /**
123     * Returns an instance populated using the specified range of {@code values}.
124     *
125     * @param values Values.
126     * @param from Inclusive start of the range.
127     * @param to Exclusive end of the range.
128     * @return {@code LongVariance} instance.
129     * @throws IndexOutOfBoundsException if the sub-range is out of bounds
130     * @since 1.2
131     */
132    public static LongVariance ofRange(long[] values, int from, int to) {
133        Statistics.checkFromToIndex(from, to, values.length);
134        return createFromRange(values, from, to);
135    }
136
137    /**
138     * Create an instance using the specified range of {@code values}.
139     *
140     * <p>Warning: No range checks are performed.
141     *
142     * @param values Values.
143     * @param from Inclusive start of the range.
144     * @param to Exclusive end of the range.
145     * @return {@code LongVariance} instance.
146     */
147    static LongVariance createFromRange(long[] values, int from, int to) {
148        // Note: Arrays could be processed using specialised counts knowing the maximum limit
149        // for an array is 2^31 values. Requires a UInt160.
150
151        final Int128 s = Int128.create();
152        final UInt192 ss = UInt192.create();
153        for (int i = from; i < to; i++) {
154            final long x = values[i];
155            s.add(x);
156            ss.addSquare(x);
157        }
158        return new LongVariance(ss, s, to - from);
159    }
160
161    /**
162     * Updates the state of the statistic to reflect the addition of {@code value}.
163     *
164     * @param value Value.
165     */
166    @Override
167    public void accept(long value) {
168        sumSq.addSquare(value);
169        sum.add(value);
170        n++;
171    }
172
173    /**
174     * Gets the variance of all input values.
175     *
176     * <p>When no values have been added, the result is {@code NaN}.
177     *
178     * @return variance of all values.
179     */
180    @Override
181    public double getAsDouble() {
182        return computeVarianceOrStd(sumSq, sum, n, biased, false);
183    }
184
185    /**
186     * Compute the variance (or standard deviation).
187     *
188     * <p>The {@code std} flag controls if the result is returned as the standard deviation
189     * using the {@link Math#sqrt(double) square root} function.
190     *
191     * @param sumSq Sum of the squared values.
192     * @param sum Sum of the values.
193     * @param n Count of values that have been added.
194     * @param biased Flag to control if the statistic is biased, or should use a bias correction.
195     * @param std Flag to control if the statistic is the standard deviation.
196     * @return the variance (or standard deviation)
197     */
198    static double computeVarianceOrStd(UInt192 sumSq, Int128 sum, long n, boolean biased, boolean std) {
199        if (n == 0) {
200            return Double.NaN;
201        }
202        // Avoid a divide by zero
203        if (n == 1) {
204            return 0;
205        }
206        // Sum-of-squared deviations: sum(x^2) - sum(x)^2 / n
207        // Sum-of-squared deviations precursor: n * sum(x^2) - sum(x)^2
208        // The precursor is computed in integer precision.
209        // The divide uses double precision.
210        // This ensures we avoid cancellation in the difference and use a fast divide.
211        // The result is limited to by the rounding in the double computation.
212        final double diff = computeSSDevN(sumSq, sum, n);
213        final long n0 = biased ? n : n - 1;
214        final double v = diff / IntMath.unsignedMultiplyToDouble(n, n0);
215        if (std) {
216            return Math.sqrt(v);
217        }
218        return v;
219    }
220
221    /**
222     * Compute the sum-of-squared deviations multiplied by the count of values:
223     * {@code n * sum(x^2) - sum(x)^2}.
224     *
225     * @param sumSq Sum of the squared values.
226     * @param sum Sum of the values.
227     * @param n Count of values that have been added.
228     * @return the sum-of-squared deviations precursor
229     */
230    private static double computeSSDevN(UInt192 sumSq, Int128 sum, long n) {
231        // Compute the term if possible using fast integer arithmetic.
232        // 192-bit sum(x^2) * n will be OK when the upper 32-bits are zero.
233        // 128-bit sum(x)^2 will be OK when the upper 64-bits are zero.
234        // The first is safe when n < 2^32 but we must check the sum high bits.
235        if (((n >>> Integer.SIZE) | sum.hi64()) == 0) {
236            return sumSq.unsignedMultiply((int) n).subtract(sum.squareLow()).toDouble();
237        } else {
238            return sumSq.toBigInteger().multiply(BigInteger.valueOf(n))
239                .subtract(square(sum.toBigInteger())).doubleValue();
240        }
241    }
242
243    /**
244     * Compute the sum of the squared deviations from the mean.
245     *
246     * <p>This is a helper method used in higher order moments.
247     *
248     * @return the sum of the squared deviations
249     */
250    double computeSumOfSquaredDeviations() {
251        return computeSSDevN(sumSq, sum, n) / n;
252    }
253
254    /**
255     * Compute the mean.
256     *
257     * <p>This is a helper method used in higher order moments.
258     *
259     * @return the mean
260     */
261    double computeMean() {
262        return LongMean.computeMean(sum, n);
263    }
264
265    /**
266     * Convenience method to square a BigInteger.
267     *
268     * @param x Value
269     * @return x^2
270     */
271    private static BigInteger square(BigInteger x) {
272        return x.multiply(x);
273    }
274
275    @Override
276    public LongVariance combine(LongVariance other) {
277        sumSq.add(other.sumSq);
278        sum.add(other.sum);
279        n += other.n;
280        return this;
281    }
282
283    /**
284     * Sets the value of the biased flag. The default value is {@code false}.
285     *
286     * <p>If {@code false} the sum of squared deviations from the sample mean is normalised by
287     * {@code n - 1} where {@code n} is the number of samples. This is Bessel's correction
288     * for an unbiased estimator of the variance of a hypothetical infinite population.
289     *
290     * <p>If {@code true} the sum of squared deviations is normalised by the number of samples
291     * {@code n}.
292     *
293     * <p>Note: This option only applies when {@code n > 1}. The variance of {@code n = 1} is
294     * always 0.
295     *
296     * <p>This flag only controls the final computation of the statistic. The value of this flag
297     * will not affect compatibility between instances during a {@link #combine(LongVariance) combine}
298     * operation.
299     *
300     * @param v Value.
301     * @return {@code this} instance
302     */
303    public LongVariance setBiased(boolean v) {
304        biased = v;
305        return this;
306    }
307}