001/* 002 * Licensed to the Apache Software Foundation (ASF) under one or more 003 * contributor license agreements. See the NOTICE file distributed with 004 * this work for additional information regarding copyright ownership. 005 * The ASF licenses this file to You under the Apache License, Version 2.0 006 * (the "License"); you may not use this file except in compliance with 007 * the License. You may obtain a copy of the License at 008 * 009 * http://www.apache.org/licenses/LICENSE-2.0 010 * 011 * Unless required by applicable law or agreed to in writing, software 012 * distributed under the License is distributed on an "AS IS" BASIS, 013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 014 * See the License for the specific language governing permissions and 015 * limitations under the License. 016 */ 017package org.apache.commons.text.similarity; 018 019import java.util.Arrays; 020 021/** 022 * An algorithm for measuring the difference between two character sequences. 023 * 024 * <p> 025 * This is the number of changes needed to change one sequence into another, where each change is a single character modification (deletion, insertion or 026 * substitution). 027 * </p> 028 * <p> 029 * This code has been adapted from Apache Commons Lang 3.3. 030 * </p> 031 * 032 * @since 1.0 033 */ 034public class LevenshteinDistance implements EditDistance<Integer> { 035 036 /** 037 * Singleton instance. 038 */ 039 private static final LevenshteinDistance INSTANCE = new LevenshteinDistance(); 040 041 /** 042 * Gets the default instance. 043 * 044 * @return The default instance 045 */ 046 public static LevenshteinDistance getDefaultInstance() { 047 return INSTANCE; 048 } 049 050 /** 051 * Find the Levenshtein distance between two CharSequences if it's less than or equal to a given threshold. 052 * 053 * <p> 054 * This implementation follows from Algorithms on Strings, Trees and Sequences by Dan Gusfield and Chas Emerick's implementation of the Levenshtein distance 055 * algorithm from <a href="http://www.merriampark.com/ld.htm">http://www.merriampark.com/ld.htm</a> 056 * </p> 057 * 058 * <pre> 059 * limitedCompare(null, *, *) = IllegalArgumentException 060 * limitedCompare(*, null, *) = IllegalArgumentException 061 * limitedCompare(*, *, -1) = IllegalArgumentException 062 * limitedCompare("","", 0) = 0 063 * limitedCompare("aaapppp", "", 8) = 7 064 * limitedCompare("aaapppp", "", 7) = 7 065 * limitedCompare("aaapppp", "", 6)) = -1 066 * limitedCompare("elephant", "hippo", 7) = 7 067 * limitedCompare("elephant", "hippo", 6) = -1 068 * limitedCompare("hippo", "elephant", 7) = 7 069 * limitedCompare("hippo", "elephant", 6) = -1 070 * </pre> 071 * 072 * @param left the first SimilarityInput, must not be null 073 * @param right the second SimilarityInput, must not be null 074 * @param threshold the target threshold, must not be negative 075 * @return result distance, or -1 076 */ 077 private static <E> int limitedCompare(SimilarityInput<E> left, SimilarityInput<E> right, final int threshold) { // NOPMD 078 if (left == null || right == null) { 079 throw new IllegalArgumentException("CharSequences must not be null"); 080 } 081 if (threshold < 0) { 082 throw new IllegalArgumentException("Threshold must not be negative"); 083 } 084 085 /* 086 * This implementation only computes the distance if it's less than or equal to the threshold value, returning -1 if it's greater. The advantage is 087 * performance: unbounded distance is O(nm), but a bound of k allows us to reduce it to O(km) time by only computing a diagonal stripe of width 2k + 1 088 * of the cost table. It is also possible to use this to compute the unbounded Levenshtein distance by starting the threshold at 1 and doubling each 089 * time until the distance is found; this is O(dm), where d is the distance. 090 * 091 * One subtlety comes from needing to ignore entries on the border of our stripe eg. p[] = |#|#|#|* d[] = *|#|#|#| We must ignore the entry to the left 092 * of the leftmost member We must ignore the entry above the rightmost member 093 * 094 * Another subtlety comes from our stripe running off the matrix if the strings aren't of the same size. Since string s is always swapped to be the 095 * shorter of the two, the stripe will always run off to the upper right instead of the lower left of the matrix. 096 * 097 * As a concrete example, suppose s is of length 5, t is of length 7, and our threshold is 1. In this case we're going to walk a stripe of length 3. The 098 * matrix would look like so: 099 * 100 * <pre> 1 2 3 4 5 1 |#|#| | | | 2 |#|#|#| | | 3 | |#|#|#| | 4 | | |#|#|#| 5 | | | |#|#| 6 | | | | |#| 7 | | | | | | </pre> 101 * 102 * Note how the stripe leads off the table as there is no possible way to turn a string of length 5 into one of length 7 in edit distance of 1. 103 * 104 * Additionally, this implementation decreases memory usage by using two single-dimensional arrays and swapping them back and forth instead of 105 * allocating an entire n by m matrix. This requires a few minor changes, such as immediately returning when it's detected that the stripe has run off 106 * the matrix and initially filling the arrays with large values so that entries we don't compute are ignored. 107 * 108 * See Algorithms on Strings, Trees and Sequences by Dan Gusfield for some discussion. 109 */ 110 111 int n = left.length(); // length of left 112 int m = right.length(); // length of right 113 114 // if one string is empty, the edit distance is necessarily the length 115 // of the other 116 if (n == 0) { 117 return m <= threshold ? m : -1; 118 } 119 if (m == 0) { 120 return n <= threshold ? n : -1; 121 } 122 123 if (n > m) { 124 // swap the two strings to consume less memory 125 final SimilarityInput<E> tmp = left; 126 left = right; 127 right = tmp; 128 n = m; 129 m = right.length(); 130 } 131 132 // the edit distance cannot be less than the length difference 133 if (m - n > threshold) { 134 return -1; 135 } 136 137 int[] p = new int[n + 1]; // 'previous' cost array, horizontally 138 int[] d = new int[n + 1]; // cost array, horizontally 139 int[] tempD; // placeholder to assist in swapping p and d 140 141 // fill in starting table values 142 final int boundary = Math.min(n, threshold) + 1; 143 for (int i = 0; i < boundary; i++) { 144 p[i] = i; 145 } 146 // these fills ensure that the value above the rightmost entry of our 147 // stripe will be ignored in following loop iterations 148 Arrays.fill(p, boundary, p.length, Integer.MAX_VALUE); 149 Arrays.fill(d, Integer.MAX_VALUE); 150 151 // iterates through t 152 for (int j = 1; j <= m; j++) { 153 final E rightJ = right.at(j - 1); // jth character of right 154 d[0] = j; 155 156 // compute stripe indices, constrain to array size 157 final int min = Math.max(1, j - threshold); 158 final int max = j > Integer.MAX_VALUE - threshold ? n : Math.min(n, j + threshold); 159 160 // ignore entry left of leftmost 161 if (min > 1) { 162 d[min - 1] = Integer.MAX_VALUE; 163 } 164 165 int lowerBound = Integer.MAX_VALUE; 166 // iterates through [min, max] in s 167 for (int i = min; i <= max; i++) { 168 if (left.at(i - 1).equals(rightJ)) { 169 // diagonally left and up 170 d[i] = p[i - 1]; 171 } else { 172 // 1 + minimum of cell to the left, to the top, diagonally 173 // left and up 174 d[i] = 1 + Math.min(Math.min(d[i - 1], p[i]), p[i - 1]); 175 } 176 lowerBound = Math.min(lowerBound, d[i]); 177 } 178 // if the lower bound is greater than the threshold, then exit early 179 if (lowerBound > threshold) { 180 return -1; 181 } 182 183 // copy current distance counts to 'previous row' distance counts 184 tempD = p; 185 p = d; 186 d = tempD; 187 } 188 189 // if p[n] is greater than the threshold, there's no guarantee on it 190 // being the correct 191 // distance 192 if (p[n] <= threshold) { 193 return p[n]; 194 } 195 return -1; 196 } 197 198 /** 199 * Finds the Levenshtein distance between two Strings. 200 * 201 * <p> 202 * A higher score indicates a greater distance. 203 * </p> 204 * 205 * <p> 206 * The previous implementation of the Levenshtein distance algorithm was from 207 * <a href="https://web.archive.org/web/20120526085419/http://www.merriampark.com/ldjava.htm"> 208 * https://web.archive.org/web/20120526085419/http://www.merriampark.com/ldjava.htm</a> 209 * </p> 210 * 211 * <p> 212 * This implementation only need one single-dimensional arrays of length s.length() + 1 213 * </p> 214 * 215 * <pre> 216 * unlimitedCompare(null, *) = IllegalArgumentException 217 * unlimitedCompare(*, null) = IllegalArgumentException 218 * unlimitedCompare("","") = 0 219 * unlimitedCompare("","a") = 1 220 * unlimitedCompare("aaapppp", "") = 7 221 * unlimitedCompare("frog", "fog") = 1 222 * unlimitedCompare("fly", "ant") = 3 223 * unlimitedCompare("elephant", "hippo") = 7 224 * unlimitedCompare("hippo", "elephant") = 7 225 * unlimitedCompare("hippo", "zzzzzzzz") = 8 226 * unlimitedCompare("hello", "hallo") = 1 227 * </pre> 228 * 229 * @param left the first CharSequence, must not be null 230 * @param right the second CharSequence, must not be null 231 * @return result distance, or -1 232 * @throws IllegalArgumentException if either CharSequence input is {@code null} 233 */ 234 private static <E> int unlimitedCompare(SimilarityInput<E> left, SimilarityInput<E> right) { 235 if (left == null || right == null) { 236 throw new IllegalArgumentException("CharSequences must not be null"); 237 } 238 /* 239 * This implementation use two variable to record the previous cost counts, So this implementation use less memory than previous impl. 240 */ 241 int n = left.length(); // length of left 242 int m = right.length(); // length of right 243 244 if (n == 0) { 245 return m; 246 } 247 if (m == 0) { 248 return n; 249 } 250 if (n > m) { 251 // swap the input strings to consume less memory 252 final SimilarityInput<E> tmp = left; 253 left = right; 254 right = tmp; 255 n = m; 256 m = right.length(); 257 } 258 final int[] p = new int[n + 1]; 259 // indexes into strings left and right 260 int i; // iterates through left 261 int j; // iterates through right 262 int upperLeft; 263 int upper; 264 E rightJ; // jth character of right 265 int cost; // cost 266 for (i = 0; i <= n; i++) { 267 p[i] = i; 268 } 269 for (j = 1; j <= m; j++) { 270 upperLeft = p[0]; 271 rightJ = right.at(j - 1); 272 p[0] = j; 273 274 for (i = 1; i <= n; i++) { 275 upper = p[i]; 276 cost = left.at(i - 1).equals(rightJ) ? 0 : 1; 277 // minimum of cell to the left+1, to the top+1, diagonally left and up +cost 278 p[i] = Math.min(Math.min(p[i - 1] + 1, p[i] + 1), upperLeft + cost); 279 upperLeft = upper; 280 } 281 } 282 return p[n]; 283 } 284 285 /** 286 * Threshold. 287 */ 288 private final Integer threshold; 289 290 /** 291 * This returns the default instance that uses a version of the algorithm that does not use a threshold parameter. 292 * 293 * @see LevenshteinDistance#getDefaultInstance() 294 * @deprecated Use {@link #getDefaultInstance()}. 295 */ 296 @Deprecated 297 public LevenshteinDistance() { 298 this(null); 299 } 300 301 /** 302 * If the threshold is not null, distance calculations will be limited to a maximum length. If the threshold is null, the unlimited version of the algorithm 303 * will be used. 304 * 305 * @param threshold If this is null then distances calculations will not be limited. This may not be negative. 306 */ 307 public LevenshteinDistance(final Integer threshold) { 308 if (threshold != null && threshold < 0) { 309 throw new IllegalArgumentException("Threshold must not be negative"); 310 } 311 this.threshold = threshold; 312 } 313 314 /** 315 * Computes the Levenshtein distance between two Strings. 316 * 317 * <p> 318 * A higher score indicates a greater distance. 319 * </p> 320 * 321 * <p> 322 * The previous implementation of the Levenshtein distance algorithm was from 323 * <a href="http://www.merriampark.com/ld.htm">http://www.merriampark.com/ld.htm</a> 324 * </p> 325 * 326 * <p> 327 * Chas Emerick has written an implementation in Java, which avoids an OutOfMemoryError which can occur when my Java implementation is used with very large 328 * strings.<br> 329 * This implementation of the Levenshtein distance algorithm is from 330 * <a href="http://www.merriampark.com/ldjava.htm">http://www.merriampark.com/ldjava.htm</a> 331 * </p> 332 * 333 * <pre> 334 * distance.apply(null, *) = IllegalArgumentException 335 * distance.apply(*, null) = IllegalArgumentException 336 * distance.apply("","") = 0 337 * distance.apply("","a") = 1 338 * distance.apply("aaapppp", "") = 7 339 * distance.apply("frog", "fog") = 1 340 * distance.apply("fly", "ant") = 3 341 * distance.apply("elephant", "hippo") = 7 342 * distance.apply("hippo", "elephant") = 7 343 * distance.apply("hippo", "zzzzzzzz") = 8 344 * distance.apply("hello", "hallo") = 1 345 * </pre> 346 * 347 * @param left the first input, must not be null 348 * @param right the second input, must not be null 349 * @return result distance, or -1 350 * @throws IllegalArgumentException if either String input {@code null} 351 */ 352 @Override 353 public Integer apply(final CharSequence left, final CharSequence right) { 354 return apply(SimilarityInput.input(left), SimilarityInput.input(right)); 355 } 356 357 /** 358 * Computes the Levenshtein distance between two inputs. 359 * 360 * <p> 361 * A higher score indicates a greater distance. 362 * </p> 363 * 364 * <pre> 365 * distance.apply(null, *) = IllegalArgumentException 366 * distance.apply(*, null) = IllegalArgumentException 367 * distance.apply("","") = 0 368 * distance.apply("","a") = 1 369 * distance.apply("aaapppp", "") = 7 370 * distance.apply("frog", "fog") = 1 371 * distance.apply("fly", "ant") = 3 372 * distance.apply("elephant", "hippo") = 7 373 * distance.apply("hippo", "elephant") = 7 374 * distance.apply("hippo", "zzzzzzzz") = 8 375 * distance.apply("hello", "hallo") = 1 376 * </pre> 377 * 378 * @param <E> The type of similarity score unit. 379 * @param left the first input, must not be null. 380 * @param right the second input, must not be null. 381 * @return result distance, or -1. 382 * @throws IllegalArgumentException if either String input {@code null}. 383 * @since 1.13.0 384 */ 385 public <E> Integer apply(final SimilarityInput<E> left, final SimilarityInput<E> right) { 386 if (threshold != null) { 387 return limitedCompare(left, right, threshold); 388 } 389 return unlimitedCompare(left, right); 390 } 391 392 /** 393 * Gets the distance threshold. 394 * 395 * @return The distance threshold 396 */ 397 public Integer getThreshold() { 398 return threshold; 399 } 400 401}