Gaussian Mixture Model

posted in: Cool Stuff | 0

Gaussian Mixture Model algorithm and example.

Algorithm

#
# Parameters
#
nclusters = 3
maxiterations = 100

#
# Initialization
#
# initial mean vectors
U = matrix
    size(nclusters, dimensionality)
    with randomly selected samples
# initial covariance matrices
S = 3D matrix
    size(nclusters, dimensionality, dimensionality)
    with each S[cluster] equal to identity matrix
# cluster probabilities
Z = matrix
    size(nsamples, nclusters)
# mixture weights
alpha = vector
    size(nclusters)
    with each alpha[cluster] equal to 1/nclusters

#
# Loop
#
for it = 1:maxiterations

    #
    # E-Step
    #
    for i = 1:nsamples
        for k = 1:nclusters
            x = vector subtraction (sample[i,] - U[k,])
            s = invert matrix S[k]
            v = x . s . transpose(x)
            d = determinant of S[k]
            Z[i,k] = alpha[k] * exp(-v) / d
        endfor
        divide all elements in the row Z[i,] by sum(Z[i,])
    endfor

    #
    # Convergence check
    #
    verify whether Z changed from previous iteration

    #
    # M-Step
    #
    # update mixture weights
    for k = 1:nclusters
        alpha[k] = sum(Z[,k]) / nsamples
    endfor
    # update mean vectors U
    for k = 1:nclusters
        for d = 1:dimensionality
            v = multiply column Z[,k] and column sample[,d]
                element by element
            U[k,d] = sum(v) / sum(Z[,k])
        endfor
    endfor
    # update covariance matrices S
    for k = 1:nclusters
        for di = 1:dimensionality
            for dj = 1:dimensionality
                x1 = subtract columns of sample[, di] by U[k, di]
                x2 = subtract columns of sample[, dj] by U[k, dj]
                s = multiply Z[,k] and x1 and x2 element by element
                S[k, di, dj] = sum(s) / (sum(Z[,k]) - 1)
            endfor
        endfor
    endfor
endfor

# Z contains probabilities of each sample
# E-Step can be used to determine probabilities of test samples

Example

[code]