001/*
002 * Licensed to the Apache Software Foundation (ASF) under one or more
003 * contributor license agreements.  See the NOTICE file distributed with
004 * this work for additional information regarding copyright ownership.
005 * The ASF licenses this file to You under the Apache License, Version 2.0
006 * (the "License"); you may not use this file except in compliance with
007 * the License.  You may obtain a copy of the License at
008 *
009 *      http://www.apache.org/licenses/LICENSE-2.0
010 *
011 * Unless required by applicable law or agreed to in writing, software
012 * distributed under the License is distributed on an "AS IS" BASIS,
013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
014 * See the License for the specific language governing permissions and
015 * limitations under the License.
016 */
017package org.apache.commons.text.similarity;
018
019import java.util.Arrays;
020
021/**
022 * An algorithm for measuring the difference between two character sequences.
023 *
024 * <p>
025 * This is the number of changes needed to change one sequence into another, where each change is a single character modification (deletion, insertion or
026 * substitution).
027 * </p>
028 * <p>
029 * This code has been adapted from Apache Commons Lang 3.3.
030 * </p>
031 *
032 * @since 1.0
033 */
034public class LevenshteinDistance implements EditDistance<Integer> {
035
036    /**
037     * Singleton instance.
038     */
039    private static final LevenshteinDistance INSTANCE = new LevenshteinDistance();
040
041    /**
042     * Gets the default instance.
043     *
044     * @return The default instance
045     */
046    public static LevenshteinDistance getDefaultInstance() {
047        return INSTANCE;
048    }
049
050    /**
051     * Find the Levenshtein distance between two CharSequences if it's less than or equal to a given threshold.
052     *
053     * <p>
054     * This implementation follows from Algorithms on Strings, Trees and Sequences by Dan Gusfield and Chas Emerick's implementation of the Levenshtein distance
055     * algorithm from <a href="http://www.merriampark.com/ld.htm">http://www.merriampark.com/ld.htm</a>
056     * </p>
057     *
058     * <pre>
059     * limitedCompare(null, *, *)             = IllegalArgumentException
060     * limitedCompare(*, null, *)             = IllegalArgumentException
061     * limitedCompare(*, *, -1)               = IllegalArgumentException
062     * limitedCompare("","", 0)               = 0
063     * limitedCompare("aaapppp", "", 8)       = 7
064     * limitedCompare("aaapppp", "", 7)       = 7
065     * limitedCompare("aaapppp", "", 6))      = -1
066     * limitedCompare("elephant", "hippo", 7) = 7
067     * limitedCompare("elephant", "hippo", 6) = -1
068     * limitedCompare("hippo", "elephant", 7) = 7
069     * limitedCompare("hippo", "elephant", 6) = -1
070     * </pre>
071     *
072     * @param left      the first SimilarityInput, must not be null
073     * @param right     the second SimilarityInput, must not be null
074     * @param threshold the target threshold, must not be negative
075     * @return result distance, or -1
076     */
077    private static <E> int limitedCompare(SimilarityInput<E> left, SimilarityInput<E> right, final int threshold) { // NOPMD
078        if (left == null || right == null) {
079            throw new IllegalArgumentException("CharSequences must not be null");
080        }
081        if (threshold < 0) {
082            throw new IllegalArgumentException("Threshold must not be negative");
083        }
084
085        /*
086         * This implementation only computes the distance if it's less than or equal to the threshold value, returning -1 if it's greater. The advantage is
087         * performance: unbounded distance is O(nm), but a bound of k allows us to reduce it to O(km) time by only computing a diagonal stripe of width 2k + 1
088         * of the cost table. It is also possible to use this to compute the unbounded Levenshtein distance by starting the threshold at 1 and doubling each
089         * time until the distance is found; this is O(dm), where d is the distance.
090         *
091         * One subtlety comes from needing to ignore entries on the border of our stripe eg. p[] = |#|#|#|* d[] = *|#|#|#| We must ignore the entry to the left
092         * of the leftmost member We must ignore the entry above the rightmost member
093         *
094         * Another subtlety comes from our stripe running off the matrix if the strings aren't of the same size. Since string s is always swapped to be the
095         * shorter of the two, the stripe will always run off to the upper right instead of the lower left of the matrix.
096         *
097         * As a concrete example, suppose s is of length 5, t is of length 7, and our threshold is 1. In this case we're going to walk a stripe of length 3. The
098         * matrix would look like so:
099         *
100         * <pre> 1 2 3 4 5 1 |#|#| | | | 2 |#|#|#| | | 3 | |#|#|#| | 4 | | |#|#|#| 5 | | | |#|#| 6 | | | | |#| 7 | | | | | | </pre>
101         *
102         * Note how the stripe leads off the table as there is no possible way to turn a string of length 5 into one of length 7 in edit distance of 1.
103         *
104         * Additionally, this implementation decreases memory usage by using two single-dimensional arrays and swapping them back and forth instead of
105         * allocating an entire n by m matrix. This requires a few minor changes, such as immediately returning when it's detected that the stripe has run off
106         * the matrix and initially filling the arrays with large values so that entries we don't compute are ignored.
107         *
108         * See Algorithms on Strings, Trees and Sequences by Dan Gusfield for some discussion.
109         */
110
111        int n = left.length(); // length of left
112        int m = right.length(); // length of right
113
114        // if one string is empty, the edit distance is necessarily the length
115        // of the other
116        if (n == 0) {
117            return m <= threshold ? m : -1;
118        }
119        if (m == 0) {
120            return n <= threshold ? n : -1;
121        }
122
123        if (n > m) {
124            // swap the two strings to consume less memory
125            final SimilarityInput<E> tmp = left;
126            left = right;
127            right = tmp;
128            n = m;
129            m = right.length();
130        }
131
132        // the edit distance cannot be less than the length difference
133        if (m - n > threshold) {
134            return -1;
135        }
136
137        int[] p = new int[n + 1]; // 'previous' cost array, horizontally
138        int[] d = new int[n + 1]; // cost array, horizontally
139        int[] tempD; // placeholder to assist in swapping p and d
140
141        // fill in starting table values
142        final int boundary = Math.min(n, threshold) + 1;
143        for (int i = 0; i < boundary; i++) {
144            p[i] = i;
145        }
146        // these fills ensure that the value above the rightmost entry of our
147        // stripe will be ignored in following loop iterations
148        Arrays.fill(p, boundary, p.length, Integer.MAX_VALUE);
149        Arrays.fill(d, Integer.MAX_VALUE);
150
151        // iterates through t
152        for (int j = 1; j <= m; j++) {
153            final E rightJ = right.at(j - 1); // jth character of right
154            d[0] = j;
155
156            // compute stripe indices, constrain to array size
157            final int min = Math.max(1, j - threshold);
158            final int max = j > Integer.MAX_VALUE - threshold ? n : Math.min(n, j + threshold);
159
160            // ignore entry left of leftmost
161            if (min > 1) {
162                d[min - 1] = Integer.MAX_VALUE;
163            }
164
165            int lowerBound = Integer.MAX_VALUE;
166            // iterates through [min, max] in s
167            for (int i = min; i <= max; i++) {
168                if (left.at(i - 1).equals(rightJ)) {
169                    // diagonally left and up
170                    d[i] = p[i - 1];
171                } else {
172                    // 1 + minimum of cell to the left, to the top, diagonally
173                    // left and up
174                    d[i] = 1 + Math.min(Math.min(d[i - 1], p[i]), p[i - 1]);
175                }
176                lowerBound = Math.min(lowerBound, d[i]);
177            }
178            // if the lower bound is greater than the threshold, then exit early
179            if (lowerBound > threshold) {
180                return -1;
181            }
182
183            // copy current distance counts to 'previous row' distance counts
184            tempD = p;
185            p = d;
186            d = tempD;
187        }
188
189        // if p[n] is greater than the threshold, there's no guarantee on it
190        // being the correct
191        // distance
192        if (p[n] <= threshold) {
193            return p[n];
194        }
195        return -1;
196    }
197
198    /**
199     * Finds the Levenshtein distance between two Strings.
200     *
201     * <p>
202     * A higher score indicates a greater distance.
203     * </p>
204     *
205     * <p>
206     * The previous implementation of the Levenshtein distance algorithm was from
207     * <a href="https://web.archive.org/web/20120526085419/http://www.merriampark.com/ldjava.htm">
208     * https://web.archive.org/web/20120526085419/http://www.merriampark.com/ldjava.htm</a>
209     * </p>
210     *
211     * <p>
212     * This implementation only need one single-dimensional arrays of length s.length() + 1
213     * </p>
214     *
215     * <pre>
216     * unlimitedCompare(null, *)             = IllegalArgumentException
217     * unlimitedCompare(*, null)             = IllegalArgumentException
218     * unlimitedCompare("","")               = 0
219     * unlimitedCompare("","a")              = 1
220     * unlimitedCompare("aaapppp", "")       = 7
221     * unlimitedCompare("frog", "fog")       = 1
222     * unlimitedCompare("fly", "ant")        = 3
223     * unlimitedCompare("elephant", "hippo") = 7
224     * unlimitedCompare("hippo", "elephant") = 7
225     * unlimitedCompare("hippo", "zzzzzzzz") = 8
226     * unlimitedCompare("hello", "hallo")    = 1
227     * </pre>
228     *
229     * @param left  the first CharSequence, must not be null
230     * @param right the second CharSequence, must not be null
231     * @return result distance, or -1
232     * @throws IllegalArgumentException if either CharSequence input is {@code null}
233     */
234    private static <E> int unlimitedCompare(SimilarityInput<E> left, SimilarityInput<E> right) {
235        if (left == null || right == null) {
236            throw new IllegalArgumentException("CharSequences must not be null");
237        }
238        /*
239         * This implementation use two variable to record the previous cost counts, So this implementation use less memory than previous impl.
240         */
241        int n = left.length(); // length of left
242        int m = right.length(); // length of right
243
244        if (n == 0) {
245            return m;
246        }
247        if (m == 0) {
248            return n;
249        }
250        if (n > m) {
251            // swap the input strings to consume less memory
252            final SimilarityInput<E> tmp = left;
253            left = right;
254            right = tmp;
255            n = m;
256            m = right.length();
257        }
258        final int[] p = new int[n + 1];
259        // indexes into strings left and right
260        int i; // iterates through left
261        int j; // iterates through right
262        int upperLeft;
263        int upper;
264        E rightJ; // jth character of right
265        int cost; // cost
266        for (i = 0; i <= n; i++) {
267            p[i] = i;
268        }
269        for (j = 1; j <= m; j++) {
270            upperLeft = p[0];
271            rightJ = right.at(j - 1);
272            p[0] = j;
273
274            for (i = 1; i <= n; i++) {
275                upper = p[i];
276                cost = left.at(i - 1).equals(rightJ) ? 0 : 1;
277                // minimum of cell to the left+1, to the top+1, diagonally left and up +cost
278                p[i] = Math.min(Math.min(p[i - 1] + 1, p[i] + 1), upperLeft + cost);
279                upperLeft = upper;
280            }
281        }
282        return p[n];
283    }
284
285    /**
286     * Threshold.
287     */
288    private final Integer threshold;
289
290    /**
291     * This returns the default instance that uses a version of the algorithm that does not use a threshold parameter.
292     *
293     * @see LevenshteinDistance#getDefaultInstance()
294     * @deprecated Use {@link #getDefaultInstance()}.
295     */
296    @Deprecated
297    public LevenshteinDistance() {
298        this(null);
299    }
300
301    /**
302     * If the threshold is not null, distance calculations will be limited to a maximum length. If the threshold is null, the unlimited version of the algorithm
303     * will be used.
304     *
305     * @param threshold If this is null then distances calculations will not be limited. This may not be negative.
306     */
307    public LevenshteinDistance(final Integer threshold) {
308        if (threshold != null && threshold < 0) {
309            throw new IllegalArgumentException("Threshold must not be negative");
310        }
311        this.threshold = threshold;
312    }
313
314    /**
315     * Computes the Levenshtein distance between two Strings.
316     *
317     * <p>
318     * A higher score indicates a greater distance.
319     * </p>
320     *
321     * <p>
322     * The previous implementation of the Levenshtein distance algorithm was from
323     * <a href="http://www.merriampark.com/ld.htm">http://www.merriampark.com/ld.htm</a>
324     * </p>
325     *
326     * <p>
327     * Chas Emerick has written an implementation in Java, which avoids an OutOfMemoryError which can occur when my Java implementation is used with very large
328     * strings.<br>
329     * This implementation of the Levenshtein distance algorithm is from
330     * <a href="http://www.merriampark.com/ldjava.htm">http://www.merriampark.com/ldjava.htm</a>
331     * </p>
332     *
333     * <pre>
334     * distance.apply(null, *)             = IllegalArgumentException
335     * distance.apply(*, null)             = IllegalArgumentException
336     * distance.apply("","")               = 0
337     * distance.apply("","a")              = 1
338     * distance.apply("aaapppp", "")       = 7
339     * distance.apply("frog", "fog")       = 1
340     * distance.apply("fly", "ant")        = 3
341     * distance.apply("elephant", "hippo") = 7
342     * distance.apply("hippo", "elephant") = 7
343     * distance.apply("hippo", "zzzzzzzz") = 8
344     * distance.apply("hello", "hallo")    = 1
345     * </pre>
346     *
347     * @param left  the first input, must not be null
348     * @param right the second input, must not be null
349     * @return result distance, or -1
350     * @throws IllegalArgumentException if either String input {@code null}
351     */
352    @Override
353    public Integer apply(final CharSequence left, final CharSequence right) {
354        return apply(SimilarityInput.input(left), SimilarityInput.input(right));
355    }
356
357    /**
358     * Computes the Levenshtein distance between two inputs.
359     *
360     * <p>
361     * A higher score indicates a greater distance.
362     * </p>
363     *
364     * <pre>
365     * distance.apply(null, *)             = IllegalArgumentException
366     * distance.apply(*, null)             = IllegalArgumentException
367     * distance.apply("","")               = 0
368     * distance.apply("","a")              = 1
369     * distance.apply("aaapppp", "")       = 7
370     * distance.apply("frog", "fog")       = 1
371     * distance.apply("fly", "ant")        = 3
372     * distance.apply("elephant", "hippo") = 7
373     * distance.apply("hippo", "elephant") = 7
374     * distance.apply("hippo", "zzzzzzzz") = 8
375     * distance.apply("hello", "hallo")    = 1
376     * </pre>
377     *
378     * @param <E>   The type of similarity score unit.
379     * @param left  the first input, must not be null.
380     * @param right the second input, must not be null.
381     * @return result distance, or -1.
382     * @throws IllegalArgumentException if either String input {@code null}.
383     * @since 1.13.0
384     */
385    public <E> Integer apply(final SimilarityInput<E> left, final SimilarityInput<E> right) {
386        if (threshold != null) {
387            return limitedCompare(left, right, threshold);
388        }
389        return unlimitedCompare(left, right);
390    }
391
392    /**
393     * Gets the distance threshold.
394     *
395     * @return The distance threshold
396     */
397    public Integer getThreshold() {
398        return threshold;
399    }
400
401}