001/* 002 * Licensed to the Apache Software Foundation (ASF) under one or more 003 * contributor license agreements. See the NOTICE file distributed with 004 * this work for additional information regarding copyright ownership. 005 * The ASF licenses this file to You under the Apache License, Version 2.0 006 * (the "License"); you may not use this file except in compliance with 007 * the License. You may obtain a copy of the License at 008 * 009 * http://www.apache.org/licenses/LICENSE-2.0 010 * 011 * Unless required by applicable law or agreed to in writing, software 012 * distributed under the License is distributed on an "AS IS" BASIS, 013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 014 * See the License for the specific language governing permissions and 015 * limitations under the License. 016 */ 017package org.apache.commons.text.similarity; 018 019import java.util.Arrays; 020 021/** 022 * An algorithm for measuring the difference between two character sequences. 023 * 024 * <p> 025 * This is the number of changes needed to change one sequence into another, 026 * where each change is a single character modification (deletion, insertion 027 * or substitution). 028 * </p> 029 * 030 * @since 1.0 031 */ 032public class LevenshteinDetailedDistance implements EditDistance<LevenshteinResults> { 033 034 /** 035 * Singleton instance. 036 */ 037 private static final LevenshteinDetailedDistance INSTANCE = new LevenshteinDetailedDistance(); 038 039 /** 040 * Finds count for each of the three [insert, delete, substitute] operations 041 * needed. This is based on the matrix formed based on the two character 042 * sequence. 043 * 044 * @param left character sequence which need to be converted from 045 * @param right character sequence which need to be converted to 046 * @param matrix two dimensional array containing 047 * @param swapped tells whether the value for left character sequence and right 048 * character sequence were swapped to save memory 049 * @return result object containing the count of insert, delete and substitute and total count needed 050 */ 051 private static LevenshteinResults findDetailedResults(final CharSequence left, 052 final CharSequence right, 053 final int[][] matrix, 054 final boolean swapped) { 055 056 int delCount = 0; 057 int addCount = 0; 058 int subCount = 0; 059 060 int rowIndex = right.length(); 061 int columnIndex = left.length(); 062 063 int dataAtLeft = 0; 064 int dataAtTop = 0; 065 int dataAtDiagonal = 0; 066 int data = 0; 067 boolean deleted = false; 068 boolean added = false; 069 070 while (rowIndex >= 0 && columnIndex >= 0) { 071 072 if (columnIndex == 0) { 073 dataAtLeft = -1; 074 } else { 075 dataAtLeft = matrix[rowIndex][columnIndex - 1]; 076 } 077 if (rowIndex == 0) { 078 dataAtTop = -1; 079 } else { 080 dataAtTop = matrix[rowIndex - 1][columnIndex]; 081 } 082 if (rowIndex > 0 && columnIndex > 0) { 083 dataAtDiagonal = matrix[rowIndex - 1][columnIndex - 1]; 084 } else { 085 dataAtDiagonal = -1; 086 } 087 if (dataAtLeft == -1 && dataAtTop == -1 && dataAtDiagonal == -1) { 088 break; 089 } 090 data = matrix[rowIndex][columnIndex]; 091 092 // case in which the character at left and right are the same, 093 // in this case none of the counters will be incremented. 094 if (columnIndex > 0 && rowIndex > 0 && left.charAt(columnIndex - 1) == right.charAt(rowIndex - 1)) { 095 columnIndex--; 096 rowIndex--; 097 continue; 098 } 099 100 // handling insert and delete cases. 101 deleted = false; 102 added = false; 103 if (data - 1 == dataAtLeft && data <= dataAtDiagonal && data <= dataAtTop 104 || dataAtDiagonal == -1 && dataAtTop == -1) { // NOPMD 105 columnIndex--; 106 if (swapped) { 107 addCount++; 108 added = true; 109 } else { 110 delCount++; 111 deleted = true; 112 } 113 } else if (data - 1 == dataAtTop && data <= dataAtDiagonal && data <= dataAtLeft 114 || dataAtDiagonal == -1 && dataAtLeft == -1) { // NOPMD 115 rowIndex--; 116 if (swapped) { 117 delCount++; 118 deleted = true; 119 } else { 120 addCount++; 121 added = true; 122 } 123 } 124 125 // substituted case 126 if (!added && !deleted) { 127 subCount++; 128 columnIndex--; 129 rowIndex--; 130 } 131 } 132 return new LevenshteinResults(addCount + delCount + subCount, addCount, delCount, subCount); 133 } 134 135 /** 136 * Gets the default instance. 137 * 138 * @return The default instace 139 */ 140 public static LevenshteinDetailedDistance getDefaultInstance() { 141 return INSTANCE; 142 } 143 144 /** 145 * Finds the Levenshtein distance between two CharSequences if it's less than or 146 * equal to a given threshold. 147 * 148 * <p> 149 * This implementation follows from Algorithms on Strings, Trees and 150 * Sequences by Dan Gusfield and Chas Emerick's implementation of the 151 * Levenshtein distance algorithm from <a 152 * href="http://www.merriampark.com/ld.htm" 153 * >http://www.merriampark.com/ld.htm</a> 154 * </p> 155 * 156 * <pre> 157 * limitedCompare(null, *, *) = IllegalArgumentException 158 * limitedCompare(*, null, *) = IllegalArgumentException 159 * limitedCompare(*, *, -1) = IllegalArgumentException 160 * limitedCompare("","", 0) = 0 161 * limitedCompare("aaapppp", "", 8) = 7 162 * limitedCompare("aaapppp", "", 7) = 7 163 * limitedCompare("aaapppp", "", 6)) = -1 164 * limitedCompare("elephant", "hippo", 7) = 7 165 * limitedCompare("elephant", "hippo", 6) = -1 166 * limitedCompare("hippo", "elephant", 7) = 7 167 * limitedCompare("hippo", "elephant", 6) = -1 168 * </pre> 169 * 170 * @param left the first CharSequence, must not be null 171 * @param right the second CharSequence, must not be null 172 * @param threshold the target threshold, must not be negative 173 * @return result distance, or -1 174 */ 175 private static LevenshteinResults limitedCompare(CharSequence left, 176 CharSequence right, 177 final int threshold) { //NOPMD 178 if (left == null || right == null) { 179 throw new IllegalArgumentException("CharSequences must not be null"); 180 } 181 if (threshold < 0) { 182 throw new IllegalArgumentException("Threshold must not be negative"); 183 } 184 185 /* 186 * This implementation only computes the distance if it's less than or 187 * equal to the threshold value, returning -1 if it's greater. The 188 * advantage is performance: unbounded distance is O(nm), but a bound of 189 * k allows us to reduce it to O(km) time by only computing a diagonal 190 * stripe of width 2k + 1 of the cost table. It is also possible to use 191 * this to compute the unbounded Levenshtein distance by starting the 192 * threshold at 1 and doubling each time until the distance is found; 193 * this is O(dm), where d is the distance. 194 * 195 * One subtlety comes from needing to ignore entries on the border of 196 * our stripe eg. p[] = |#|#|#|* d[] = *|#|#|#| We must ignore the entry 197 * to the left of the leftmost member We must ignore the entry above the 198 * rightmost member 199 * 200 * Another subtlety comes from our stripe running off the matrix if the 201 * strings aren't of the same size. Since string s is always swapped to 202 * be the shorter of the two, the stripe will always run off to the 203 * upper right instead of the lower left of the matrix. 204 * 205 * As a concrete example, suppose s is of length 5, t is of length 7, 206 * and our threshold is 1. In this case we're going to walk a stripe of 207 * length 3. The matrix would look like so: 208 * 209 * <pre> 210 * 1 2 3 4 5 211 * 1 |#|#| | | | 212 * 2 |#|#|#| | | 213 * 3 | |#|#|#| | 214 * 4 | | |#|#|#| 215 * 5 | | | |#|#| 216 * 6 | | | | |#| 217 * 7 | | | | | | 218 * </pre> 219 * 220 * Note how the stripe leads off the table as there is no possible way 221 * to turn a string of length 5 into one of length 7 in edit distance of 222 * 1. 223 * 224 * Additionally, this implementation decreases memory usage by using two 225 * single-dimensional arrays and swapping them back and forth instead of 226 * allocating an entire n by m matrix. This requires a few minor 227 * changes, such as immediately returning when it's detected that the 228 * stripe has run off the matrix and initially filling the arrays with 229 * large values so that entries we don't compute are ignored. 230 * 231 * See Algorithms on Strings, Trees and Sequences by Dan Gusfield for 232 * some discussion. 233 */ 234 235 int n = left.length(); // length of left 236 int m = right.length(); // length of right 237 238 // if one string is empty, the edit distance is necessarily the length of the other 239 if (n == 0) { 240 return m <= threshold ? new LevenshteinResults(m, m, 0, 0) : new LevenshteinResults(-1, 0, 0, 0); 241 } 242 if (m == 0) { 243 return n <= threshold ? new LevenshteinResults(n, 0, n, 0) : new LevenshteinResults(-1, 0, 0, 0); 244 } 245 246 boolean swapped = false; 247 if (n > m) { 248 // swap the two strings to consume less memory 249 final CharSequence tmp = left; 250 left = right; 251 right = tmp; 252 n = m; 253 m = right.length(); 254 swapped = true; 255 } 256 257 int[] p = new int[n + 1]; // 'previous' cost array, horizontally 258 int[] d = new int[n + 1]; // cost array, horizontally 259 int[] tempD; // placeholder to assist in swapping p and d 260 final int[][] matrix = new int[m + 1][n + 1]; 261 262 //filling the first row and first column values in the matrix 263 for (int index = 0; index <= n; index++) { 264 matrix[0][index] = index; 265 } 266 for (int index = 0; index <= m; index++) { 267 matrix[index][0] = index; 268 } 269 270 // fill in starting table values 271 final int boundary = Math.min(n, threshold) + 1; 272 for (int i = 0; i < boundary; i++) { 273 p[i] = i; 274 } 275 // these fills ensure that the value above the rightmost entry of our 276 // stripe will be ignored in following loop iterations 277 Arrays.fill(p, boundary, p.length, Integer.MAX_VALUE); 278 Arrays.fill(d, Integer.MAX_VALUE); 279 280 // iterates through t 281 for (int j = 1; j <= m; j++) { 282 final char rightJ = right.charAt(j - 1); // jth character of right 283 d[0] = j; 284 285 // compute stripe indices, constrain to array size 286 final int min = Math.max(1, j - threshold); 287 final int max = j > Integer.MAX_VALUE - threshold ? n : Math.min( 288 n, j + threshold); 289 290 // the stripe may lead off of the table if s and t are of different sizes 291 if (min > max) { 292 return new LevenshteinResults(-1, 0, 0, 0); 293 } 294 295 // ignore entry left of leftmost 296 if (min > 1) { 297 d[min - 1] = Integer.MAX_VALUE; 298 } 299 300 // iterates through [min, max] in s 301 for (int i = min; i <= max; i++) { 302 if (left.charAt(i - 1) == rightJ) { 303 // diagonally left and up 304 d[i] = p[i - 1]; 305 } else { 306 // 1 + minimum of cell to the left, to the top, diagonally left and up 307 d[i] = 1 + Math.min(Math.min(d[i - 1], p[i]), p[i - 1]); 308 } 309 matrix[j][i] = d[i]; 310 } 311 312 // copy current distance counts to 'previous row' distance counts 313 tempD = p; 314 p = d; 315 d = tempD; 316 } 317 318 // if p[n] is greater than the threshold, there's no guarantee on it being the correct distance 319 if (p[n] <= threshold) { 320 return findDetailedResults(left, right, matrix, swapped); 321 } 322 return new LevenshteinResults(-1, 0, 0, 0); 323 } 324 325 /** 326 * Finds the Levenshtein distance between two Strings. 327 * 328 * <p>A higher score indicates a greater distance.</p> 329 * 330 * <p>The previous implementation of the Levenshtein distance algorithm 331 * was from <a href="http://www.merriampark.com/ld.htm">http://www.merriampark.com/ld.htm</a></p> 332 * 333 * <p>Chas Emerick has written an implementation in Java, which avoids an OutOfMemoryError 334 * which can occur when my Java implementation is used with very large strings.<br> 335 * This implementation of the Levenshtein distance algorithm 336 * is from <a href="http://www.merriampark.com/ldjava.htm">http://www.merriampark.com/ldjava.htm</a></p> 337 * 338 * <pre> 339 * unlimitedCompare(null, *) = IllegalArgumentException 340 * unlimitedCompare(*, null) = IllegalArgumentException 341 * unlimitedCompare("","") = 0 342 * unlimitedCompare("","a") = 1 343 * unlimitedCompare("aaapppp", "") = 7 344 * unlimitedCompare("frog", "fog") = 1 345 * unlimitedCompare("fly", "ant") = 3 346 * unlimitedCompare("elephant", "hippo") = 7 347 * unlimitedCompare("hippo", "elephant") = 7 348 * unlimitedCompare("hippo", "zzzzzzzz") = 8 349 * unlimitedCompare("hello", "hallo") = 1 350 * </pre> 351 * 352 * @param left the first CharSequence, must not be null 353 * @param right the second CharSequence, must not be null 354 * @return result distance, or -1 355 * @throws IllegalArgumentException if either CharSequence input is {@code null} 356 */ 357 private static LevenshteinResults unlimitedCompare(CharSequence left, CharSequence right) { 358 if (left == null || right == null) { 359 throw new IllegalArgumentException("CharSequences must not be null"); 360 } 361 362 /* 363 The difference between this impl. and the previous is that, rather 364 than creating and retaining a matrix of size s.length() + 1 by t.length() + 1, 365 we maintain two single-dimensional arrays of length s.length() + 1. The first, d, 366 is the 'current working' distance array that maintains the newest distance cost 367 counts as we iterate through the characters of String s. Each time we increment 368 the index of String t we are comparing, d is copied to p, the second int[]. Doing so 369 allows us to retain the previous cost counts as required by the algorithm (taking 370 the minimum of the cost count to the left, up one, and diagonally up and to the left 371 of the current cost count being calculated). (Note that the arrays aren't really 372 copied anymore, just switched...this is clearly much better than cloning an array 373 or doing a System.arraycopy() each time through the outer loop.) 374 375 Effectively, the difference between the two implementations is this one does not 376 cause an out of memory condition when calculating the LD over two very large strings. 377 */ 378 379 int n = left.length(); // length of left 380 int m = right.length(); // length of right 381 382 if (n == 0) { 383 return new LevenshteinResults(m, m, 0, 0); 384 } 385 if (m == 0) { 386 return new LevenshteinResults(n, 0, n, 0); 387 } 388 boolean swapped = false; 389 if (n > m) { 390 // swap the input strings to consume less memory 391 final CharSequence tmp = left; 392 left = right; 393 right = tmp; 394 n = m; 395 m = right.length(); 396 swapped = true; 397 } 398 399 int[] p = new int[n + 1]; // 'previous' cost array, horizontally 400 int[] d = new int[n + 1]; // cost array, horizontally 401 int[] tempD; //placeholder to assist in swapping p and d 402 final int[][] matrix = new int[m + 1][n + 1]; 403 404 // filling the first row and first column values in the matrix 405 for (int index = 0; index <= n; index++) { 406 matrix[0][index] = index; 407 } 408 for (int index = 0; index <= m; index++) { 409 matrix[index][0] = index; 410 } 411 412 // indexes into strings left and right 413 int i; // iterates through left 414 int j; // iterates through right 415 416 char rightJ; // jth character of right 417 418 int cost; // cost 419 for (i = 0; i <= n; i++) { 420 p[i] = i; 421 } 422 423 for (j = 1; j <= m; j++) { 424 rightJ = right.charAt(j - 1); 425 d[0] = j; 426 427 for (i = 1; i <= n; i++) { 428 cost = left.charAt(i - 1) == rightJ ? 0 : 1; 429 // minimum of cell to the left+1, to the top+1, diagonally left and up +cost 430 d[i] = Math.min(Math.min(d[i - 1] + 1, p[i] + 1), p[i - 1] + cost); 431 //filling the matrix 432 matrix[j][i] = d[i]; 433 } 434 435 // copy current distance counts to 'previous row' distance counts 436 tempD = p; 437 p = d; 438 d = tempD; 439 } 440 return findDetailedResults(left, right, matrix, swapped); 441 } 442 443 /** 444 * Threshold. 445 */ 446 private final Integer threshold; 447 448 /** 449 * <p> 450 * This returns the default instance that uses a version 451 * of the algorithm that does not use a threshold parameter. 452 * </p> 453 * 454 * @see LevenshteinDetailedDistance#getDefaultInstance() 455 */ 456 public LevenshteinDetailedDistance() { 457 this(null); 458 } 459 460 /** 461 * If the threshold is not null, distance calculations will be limited to a maximum length. 462 * 463 * <p>If the threshold is null, the unlimited version of the algorithm will be used.</p> 464 * 465 * @param threshold If this is null then distances calculations will not be limited. This may not be negative. 466 */ 467 public LevenshteinDetailedDistance(final Integer threshold) { 468 if (threshold != null && threshold < 0) { 469 throw new IllegalArgumentException("Threshold must not be negative"); 470 } 471 this.threshold = threshold; 472 } 473 474 /** 475 * Finds the Levenshtein distance between two Strings. 476 * 477 * <p>A higher score indicates a greater distance.</p> 478 * 479 * <p>The previous implementation of the Levenshtein distance algorithm 480 * was from <a href="http://www.merriampark.com/ld.htm">http://www.merriampark.com/ld.htm</a></p> 481 * 482 * <p>Chas Emerick has written an implementation in Java, which avoids an OutOfMemoryError 483 * which can occur when my Java implementation is used with very large strings.<br> 484 * This implementation of the Levenshtein distance algorithm 485 * is from <a href="http://www.merriampark.com/ldjava.htm">http://www.merriampark.com/ldjava.htm</a></p> 486 * 487 * <pre> 488 * distance.apply(null, *) = IllegalArgumentException 489 * distance.apply(*, null) = IllegalArgumentException 490 * distance.apply("","") = 0 491 * distance.apply("","a") = 1 492 * distance.apply("aaapppp", "") = 7 493 * distance.apply("frog", "fog") = 1 494 * distance.apply("fly", "ant") = 3 495 * distance.apply("elephant", "hippo") = 7 496 * distance.apply("hippo", "elephant") = 7 497 * distance.apply("hippo", "zzzzzzzz") = 8 498 * distance.apply("hello", "hallo") = 1 499 * </pre> 500 * 501 * @param left the first string, must not be null 502 * @param right the second string, must not be null 503 * @return result distance, or -1 504 * @throws IllegalArgumentException if either String input {@code null} 505 */ 506 @Override 507 public LevenshteinResults apply(final CharSequence left, final CharSequence right) { 508 if (threshold != null) { 509 return limitedCompare(left, right, threshold); 510 } 511 return unlimitedCompare(left, right); 512 } 513 514 /** 515 * Gets the distance threshold. 516 * 517 * @return The distance threshold 518 */ 519 public Integer getThreshold() { 520 return threshold; 521 } 522}