HypergeometricDistribution.java
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.commons.statistics.distribution;
import java.util.function.DoublePredicate;
/**
* Implementation of the hypergeometric distribution.
*
* <p>The probability mass function of \( X \) is:
*
* <p>\[ f(k; N, K, n) = \frac{\binom{K}{k} \binom{N - K}{n-k}}{\binom{N}{n}} \]
*
* <p>for \( N \in \{0, 1, 2, \dots\} \) the population size,
* \( K \in \{0, 1, \dots, N\} \) the number of success states,
* \( n \in \{0, 1, \dots, N\} \) the number of samples,
* \( k \in \{\max(0, n+K-N), \dots, \min(n, K)\} \) the number of successes, and
*
* <p>\[ \binom{a}{b} = \frac{a!}{b! \, (a-b)!} \]
*
* <p>is the binomial coefficient.
*
* @see <a href="https://en.wikipedia.org/wiki/Hypergeometric_distribution">Hypergeometric distribution (Wikipedia)</a>
* @see <a href="https://mathworld.wolfram.com/HypergeometricDistribution.html">Hypergeometric distribution (MathWorld)</a>
*/
public final class HypergeometricDistribution extends AbstractDiscreteDistribution {
/** 1/2. */
private static final double HALF = 0.5;
/** The number of successes in the population. */
private final int numberOfSuccesses;
/** The population size. */
private final int populationSize;
/** The sample size. */
private final int sampleSize;
/** The lower bound of the support (inclusive). */
private final int lowerBound;
/** The upper bound of the support (inclusive). */
private final int upperBound;
/** Binomial probability of success (sampleSize / populationSize). */
private final double bp;
/** Binomial probability of failure ((populationSize - sampleSize) / populationSize). */
private final double bq;
/** Cached midpoint of the CDF/SF. The array holds [x, cdf(x)] for the midpoint x.
* Used for the cumulative probability functions. */
private double[] midpoint;
/**
* @param populationSize Population size.
* @param numberOfSuccesses Number of successes in the population.
* @param sampleSize Sample size.
*/
private HypergeometricDistribution(int populationSize,
int numberOfSuccesses,
int sampleSize) {
this.numberOfSuccesses = numberOfSuccesses;
this.populationSize = populationSize;
this.sampleSize = sampleSize;
lowerBound = getLowerDomain(populationSize, numberOfSuccesses, sampleSize);
upperBound = getUpperDomain(numberOfSuccesses, sampleSize);
bp = (double) sampleSize / populationSize;
bq = (double) (populationSize - sampleSize) / populationSize;
}
/**
* Creates a hypergeometric distribution.
*
* @param populationSize Population size.
* @param numberOfSuccesses Number of successes in the population.
* @param sampleSize Sample size.
* @return the distribution
* @throws IllegalArgumentException if {@code numberOfSuccesses < 0}, or
* {@code populationSize <= 0} or {@code numberOfSuccesses > populationSize}, or
* {@code sampleSize > populationSize}.
*/
public static HypergeometricDistribution of(int populationSize,
int numberOfSuccesses,
int sampleSize) {
if (populationSize <= 0) {
throw new DistributionException(DistributionException.NOT_STRICTLY_POSITIVE,
populationSize);
}
if (numberOfSuccesses < 0) {
throw new DistributionException(DistributionException.NEGATIVE,
numberOfSuccesses);
}
if (sampleSize < 0) {
throw new DistributionException(DistributionException.NEGATIVE,
sampleSize);
}
if (numberOfSuccesses > populationSize) {
throw new DistributionException(DistributionException.TOO_LARGE,
numberOfSuccesses, populationSize);
}
if (sampleSize > populationSize) {
throw new DistributionException(DistributionException.TOO_LARGE,
sampleSize, populationSize);
}
return new HypergeometricDistribution(populationSize, numberOfSuccesses, sampleSize);
}
/**
* Return the lowest domain value for the given hypergeometric distribution
* parameters.
*
* @param nn Population size.
* @param k Number of successes in the population.
* @param n Sample size.
* @return the lowest domain value of the hypergeometric distribution.
*/
private static int getLowerDomain(int nn, int k, int n) {
// Avoid overflow given N > n:
// n + K - N == K - (N - n)
return Math.max(0, k - (nn - n));
}
/**
* Return the highest domain value for the given hypergeometric distribution
* parameters.
*
* @param k Number of successes in the population.
* @param n Sample size.
* @return the highest domain value of the hypergeometric distribution.
*/
private static int getUpperDomain(int k, int n) {
return Math.min(n, k);
}
/**
* Gets the population size parameter of this distribution.
*
* @return the population size.
*/
public int getPopulationSize() {
return populationSize;
}
/**
* Gets the number of successes parameter of this distribution.
*
* @return the number of successes.
*/
public int getNumberOfSuccesses() {
return numberOfSuccesses;
}
/**
* Gets the sample size parameter of this distribution.
*
* @return the sample size.
*/
public int getSampleSize() {
return sampleSize;
}
/** {@inheritDoc} */
@Override
public double probability(int x) {
return Math.exp(logProbability(x));
}
/** {@inheritDoc} */
@Override
public double probability(int x0, int x1) {
if (x0 > x1) {
throw new DistributionException(DistributionException.INVALID_RANGE_LOW_GT_HIGH, x0, x1);
}
if (x0 == x1 || x1 < lowerBound) {
return 0;
}
// If the range is outside the bounds use the appropriate cumulative probability
if (x0 < lowerBound) {
return cumulativeProbability(x1);
}
if (x1 >= upperBound) {
// 1 - cdf(x0)
return survivalProbability(x0);
}
// Here: lower <= x0 < x1 < upper:
// sum(pdf(x)) for x in (x0, x1]
final int lo = x0 + 1;
// Sum small values first by starting at the point the greatest distance from the mode.
final int mode = (int) Math.floor((sampleSize + 1.0) * (numberOfSuccesses + 1.0) / (populationSize + 2.0));
return Math.abs(mode - lo) > Math.abs(mode - x1) ?
innerCumulativeProbability(lo, x1) :
innerCumulativeProbability(x1, lo);
}
/** {@inheritDoc} */
@Override
public double logProbability(int x) {
if (x < lowerBound || x > upperBound) {
return Double.NEGATIVE_INFINITY;
}
return computeLogProbability(x);
}
/**
* Compute the log probability.
*
* @param x Value.
* @return log(P(X = x))
*/
private double computeLogProbability(int x) {
final double p1 =
SaddlePointExpansionUtils.logBinomialProbability(x, numberOfSuccesses, bp, bq);
final double p2 =
SaddlePointExpansionUtils.logBinomialProbability(sampleSize - x,
populationSize - numberOfSuccesses, bp, bq);
final double p3 =
SaddlePointExpansionUtils.logBinomialProbability(sampleSize, populationSize, bp, bq);
return p1 + p2 - p3;
}
/** {@inheritDoc} */
@Override
public double cumulativeProbability(int x) {
if (x < lowerBound) {
return 0.0;
} else if (x >= upperBound) {
return 1.0;
}
final double[] mid = getMidPoint();
final int m = (int) mid[0];
if (x < m) {
return innerCumulativeProbability(lowerBound, x);
} else if (x > m) {
return 1 - innerCumulativeProbability(upperBound, x + 1);
}
// cdf(x)
return mid[1];
}
/** {@inheritDoc} */
@Override
public double survivalProbability(int x) {
if (x < lowerBound) {
return 1.0;
} else if (x >= upperBound) {
return 0.0;
}
final double[] mid = getMidPoint();
final int m = (int) mid[0];
if (x < m) {
return 1 - innerCumulativeProbability(lowerBound, x);
} else if (x > m) {
return innerCumulativeProbability(upperBound, x + 1);
}
// 1 - cdf(x)
return 1 - mid[1];
}
/**
* For this distribution, {@code X}, this method returns
* {@code P(x0 <= X <= x1)}.
* This probability is computed by summing the point probabilities for the
* values {@code x0, x0 + dx, x0 + 2 * dx, ..., x1}; the direction {@code dx} is determined
* using a comparison of the input bounds.
* This should be called by using {@code x0} as the domain limit and {@code x1}
* as the internal value. This will result in an initial sum of increasing larger magnitudes.
*
* @param x0 Inclusive domain bound.
* @param x1 Inclusive internal bound.
* @return {@code P(x0 <= X <= x1)}.
*/
private double innerCumulativeProbability(int x0, int x1) {
// Assume the range is within the domain.
// Reuse the computation for probability(x) but avoid checking the domain for each call.
int x = x0;
double ret = Math.exp(computeLogProbability(x));
if (x0 < x1) {
while (x != x1) {
x++;
ret += Math.exp(computeLogProbability(x));
}
} else {
while (x != x1) {
x--;
ret += Math.exp(computeLogProbability(x));
}
}
return ret;
}
@Override
public int inverseCumulativeProbability(double p) {
ArgumentUtils.checkProbability(p);
return computeInverseProbability(p, 1 - p, false);
}
@Override
public int inverseSurvivalProbability(double p) {
ArgumentUtils.checkProbability(p);
return computeInverseProbability(1 - p, p, true);
}
/**
* Implementation for the inverse cumulative or survival probability.
*
* @param p Cumulative probability.
* @param q Survival probability.
* @param complement Set to true to compute the inverse survival probability.
* @return the value
*/
private int computeInverseProbability(double p, double q, boolean complement) {
if (p == 0) {
return lowerBound;
}
if (q == 0) {
return upperBound;
}
// Sum the PDF(x) until the appropriate p-value is obtained
// CDF: require smallest x where P(X<=x) >= p
// SF: require smallest x where P(X>x) <= q
// The choice of summation uses the mid-point.
// The test on the CDF or SF is based on the appropriate input p-value.
final double[] mid = getMidPoint();
final int m = (int) mid[0];
final double mp = mid[1];
final int midPointComparison = complement ?
Double.compare(1 - mp, q) :
Double.compare(p, mp);
if (midPointComparison < 0) {
return inverseLower(p, q, complement);
} else if (midPointComparison > 0) {
// Avoid floating-point summation error when the mid-point computed using the
// lower sum is different to the midpoint computed using the upper sum.
// Here we know the result must be above the midpoint so we can clip the result.
return Math.max(m + 1, inverseUpper(p, q, complement));
}
// Exact mid-point
return m;
}
/**
* Compute the inverse cumulative or survival probability using the lower sum.
*
* @param p Cumulative probability.
* @param q Survival probability.
* @param complement Set to true to compute the inverse survival probability.
* @return the value
*/
private int inverseLower(double p, double q, boolean complement) {
// Sum from the lower bound (computing the cdf)
int x = lowerBound;
final DoublePredicate test = complement ?
i -> 1 - i > q :
i -> i < p;
double cdf = Math.exp(computeLogProbability(x));
while (test.test(cdf)) {
x++;
cdf += Math.exp(computeLogProbability(x));
}
return x;
}
/**
* Compute the inverse cumulative or survival probability using the upper sum.
*
* @param p Cumulative probability.
* @param q Survival probability.
* @param complement Set to true to compute the inverse survival probability.
* @return the value
*/
private int inverseUpper(double p, double q, boolean complement) {
// Sum from the upper bound (computing the sf)
int x = upperBound;
final DoublePredicate test = complement ?
i -> i < q :
i -> 1 - i > p;
double sf = 0;
while (test.test(sf)) {
sf += Math.exp(computeLogProbability(x));
x--;
}
// Here either sf(x) >= q, or cdf(x) <= p
// Ensure sf(x) <= q, or cdf(x) >= p
if (complement && sf > q ||
!complement && 1 - sf < p) {
x++;
}
return x;
}
/**
* {@inheritDoc}
*
* <p>For population size \( N \), number of successes \( K \), and sample
* size \( n \), the mean is:
*
* <p>\[ n \frac{K}{N} \]
*/
@Override
public double getMean() {
return getSampleSize() * (getNumberOfSuccesses() / (double) getPopulationSize());
}
/**
* {@inheritDoc}
*
* <p>For population size \( N \), number of successes \( K \), and sample
* size \( n \), the variance is:
*
* <p>\[ n \frac{K}{N} \frac{N-K}{N} \frac{N-n}{N-1} \]
*/
@Override
public double getVariance() {
final double N = getPopulationSize();
final double K = getNumberOfSuccesses();
final double n = getSampleSize();
return (n * K * (N - K) * (N - n)) / (N * N * (N - 1));
}
/**
* {@inheritDoc}
*
* <p>For population size \( N \), number of successes \( K \), and sample
* size \( n \), the lower bound of the support is \( \max \{ 0, n + K - N \} \).
*
* @return lower bound of the support
*/
@Override
public int getSupportLowerBound() {
return lowerBound;
}
/**
* {@inheritDoc}
*
* <p>For number of successes \( K \), and sample
* size \( n \), the upper bound of the support is \( \min \{ n, K \} \).
*
* @return upper bound of the support
*/
@Override
public int getSupportUpperBound() {
return upperBound;
}
/**
* Return the mid-point {@code x} of the distribution, and the cdf(x).
*
* <p>This is not the true median. It is the value where the CDF(x) is closest to 0.5;
* as such the CDF may be below 0.5 if the next value of x is further from 0.5.
*
* @return the mid-point ([x, cdf(x)])
*/
private double[] getMidPoint() {
double[] v = midpoint;
if (v == null) {
// Find the closest sum(PDF) to 0.5
int x = lowerBound;
double p0 = 0;
double p1 = Math.exp(computeLogProbability(x));
// No check of the upper bound required here as the CDF should sum to 1 and 0.5
// is exceeded before a bounds error.
while (p1 < HALF) {
x++;
p0 = p1;
p1 += Math.exp(computeLogProbability(x));
}
// p1 >= 0.5 > p0
// Pick closet
if (p1 - HALF >= HALF - p0) {
x--;
p1 = p0;
}
midpoint = v = new double[] {x, p1};
}
return v;
}
}