001/* 002 * Licensed to the Apache Software Foundation (ASF) under one or more 003 * contributor license agreements. See the NOTICE file distributed with 004 * this work for additional information regarding copyright ownership. 005 * The ASF licenses this file to You under the Apache License, Version 2.0 006 * (the "License"); you may not use this file except in compliance with 007 * the License. You may obtain a copy of the License at 008 * 009 * http://www.apache.org/licenses/LICENSE-2.0 010 * 011 * Unless required by applicable law or agreed to in writing, software 012 * distributed under the License is distributed on an "AS IS" BASIS, 013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 014 * See the License for the specific language governing permissions and 015 * limitations under the License. 016 */ 017package org.apache.commons.text.similarity; 018 019import java.util.Arrays; 020 021/** 022 * An algorithm for measuring the difference between two character sequences. 023 * 024 * <p> 025 * This is the number of changes needed to change one sequence into another, 026 * where each change is a single character modification (deletion, insertion 027 * or substitution). 028 * </p> 029 * 030 * @since 1.0 031 */ 032public class LevenshteinDetailedDistance implements EditDistance<LevenshteinResults> { 033 034 /** 035 * Default instance. 036 */ 037 private static final LevenshteinDetailedDistance DEFAULT_INSTANCE = new LevenshteinDetailedDistance(); 038 /** 039 * Threshold. 040 */ 041 private final Integer threshold; 042 043 /** 044 * <p> 045 * This returns the default instance that uses a version 046 * of the algorithm that does not use a threshold parameter. 047 * </p> 048 * 049 * @see LevenshteinDetailedDistance#getDefaultInstance() 050 */ 051 public LevenshteinDetailedDistance() { 052 this(null); 053 } 054 055 /** 056 * If the threshold is not null, distance calculations will be limited to a maximum length. 057 * 058 * <p>If the threshold is null, the unlimited version of the algorithm will be used.</p> 059 * 060 * @param threshold If this is null then distances calculations will not be limited. This may not be negative. 061 */ 062 public LevenshteinDetailedDistance(final Integer threshold) { 063 if (threshold != null && threshold < 0) { 064 throw new IllegalArgumentException("Threshold must not be negative"); 065 } 066 this.threshold = threshold; 067 } 068 069 /** 070 * <p>Find the Levenshtein distance between two Strings.</p> 071 * 072 * <p>A higher score indicates a greater distance.</p> 073 * 074 * <p>The previous implementation of the Levenshtein distance algorithm 075 * was from <a href="http://www.merriampark.com/ld.htm">http://www.merriampark.com/ld.htm</a></p> 076 * 077 * <p>Chas Emerick has written an implementation in Java, which avoids an OutOfMemoryError 078 * which can occur when my Java implementation is used with very large strings.<br> 079 * This implementation of the Levenshtein distance algorithm 080 * is from <a href="http://www.merriampark.com/ldjava.htm">http://www.merriampark.com/ldjava.htm</a></p> 081 * 082 * <pre> 083 * distance.apply(null, *) = IllegalArgumentException 084 * distance.apply(*, null) = IllegalArgumentException 085 * distance.apply("","") = 0 086 * distance.apply("","a") = 1 087 * distance.apply("aaapppp", "") = 7 088 * distance.apply("frog", "fog") = 1 089 * distance.apply("fly", "ant") = 3 090 * distance.apply("elephant", "hippo") = 7 091 * distance.apply("hippo", "elephant") = 7 092 * distance.apply("hippo", "zzzzzzzz") = 8 093 * distance.apply("hello", "hallo") = 1 094 * </pre> 095 * 096 * @param left the first string, must not be null 097 * @param right the second string, must not be null 098 * @return result distance, or -1 099 * @throws IllegalArgumentException if either String input {@code null} 100 */ 101 @Override 102 public LevenshteinResults apply(final CharSequence left, final CharSequence right) { 103 if (threshold != null) { 104 return limitedCompare(left, right, threshold); 105 } 106 return unlimitedCompare(left, right); 107 } 108 109 /** 110 * Gets the default instance. 111 * 112 * @return the default instace 113 */ 114 public static LevenshteinDetailedDistance getDefaultInstance() { 115 return DEFAULT_INSTANCE; 116 } 117 118 /** 119 * Gets the distance threshold. 120 * 121 * @return the distance threshold 122 */ 123 public Integer getThreshold() { 124 return threshold; 125 } 126 127 /** 128 * Find the Levenshtein distance between two CharSequences if it's less than or 129 * equal to a given threshold. 130 * 131 * <p> 132 * This implementation follows from Algorithms on Strings, Trees and 133 * Sequences by Dan Gusfield and Chas Emerick's implementation of the 134 * Levenshtein distance algorithm from <a 135 * href="http://www.merriampark.com/ld.htm" 136 * >http://www.merriampark.com/ld.htm</a> 137 * </p> 138 * 139 * <pre> 140 * limitedCompare(null, *, *) = IllegalArgumentException 141 * limitedCompare(*, null, *) = IllegalArgumentException 142 * limitedCompare(*, *, -1) = IllegalArgumentException 143 * limitedCompare("","", 0) = 0 144 * limitedCompare("aaapppp", "", 8) = 7 145 * limitedCompare("aaapppp", "", 7) = 7 146 * limitedCompare("aaapppp", "", 6)) = -1 147 * limitedCompare("elephant", "hippo", 7) = 7 148 * limitedCompare("elephant", "hippo", 6) = -1 149 * limitedCompare("hippo", "elephant", 7) = 7 150 * limitedCompare("hippo", "elephant", 6) = -1 151 * </pre> 152 * 153 * @param left the first string, must not be null 154 * @param right the second string, must not be null 155 * @param threshold the target threshold, must not be negative 156 * @return result distance, or -1 157 */ 158 private static LevenshteinResults limitedCompare(CharSequence left, CharSequence right, final int threshold) { //NOPMD 159 if (left == null || right == null) { 160 throw new IllegalArgumentException("Strings must not be null"); 161 } 162 if (threshold < 0) { 163 throw new IllegalArgumentException("Threshold must not be negative"); 164 } 165 166 /* 167 * This implementation only computes the distance if it's less than or 168 * equal to the threshold value, returning -1 if it's greater. The 169 * advantage is performance: unbounded distance is O(nm), but a bound of 170 * k allows us to reduce it to O(km) time by only computing a diagonal 171 * stripe of width 2k + 1 of the cost table. It is also possible to use 172 * this to compute the unbounded Levenshtein distance by starting the 173 * threshold at 1 and doubling each time until the distance is found; 174 * this is O(dm), where d is the distance. 175 * 176 * One subtlety comes from needing to ignore entries on the border of 177 * our stripe eg. p[] = |#|#|#|* d[] = *|#|#|#| We must ignore the entry 178 * to the left of the leftmost member We must ignore the entry above the 179 * rightmost member 180 * 181 * Another subtlety comes from our stripe running off the matrix if the 182 * strings aren't of the same size. Since string s is always swapped to 183 * be the shorter of the two, the stripe will always run off to the 184 * upper right instead of the lower left of the matrix. 185 * 186 * As a concrete example, suppose s is of length 5, t is of length 7, 187 * and our threshold is 1. In this case we're going to walk a stripe of 188 * length 3. The matrix would look like so: 189 * 190 * <pre> 191 * 1 2 3 4 5 192 * 1 |#|#| | | | 193 * 2 |#|#|#| | | 194 * 3 | |#|#|#| | 195 * 4 | | |#|#|#| 196 * 5 | | | |#|#| 197 * 6 | | | | |#| 198 * 7 | | | | | | 199 * </pre> 200 * 201 * Note how the stripe leads off the table as there is no possible way 202 * to turn a string of length 5 into one of length 7 in edit distance of 203 * 1. 204 * 205 * Additionally, this implementation decreases memory usage by using two 206 * single-dimensional arrays and swapping them back and forth instead of 207 * allocating an entire n by m matrix. This requires a few minor 208 * changes, such as immediately returning when it's detected that the 209 * stripe has run off the matrix and initially filling the arrays with 210 * large values so that entries we don't compute are ignored. 211 * 212 * See Algorithms on Strings, Trees and Sequences by Dan Gusfield for 213 * some discussion. 214 */ 215 216 int n = left.length(); // length of left 217 int m = right.length(); // length of right 218 219 // if one string is empty, the edit distance is necessarily the length of the other 220 if (n == 0) { 221 return m <= threshold ? new LevenshteinResults(m, m, 0, 0) : new LevenshteinResults(-1, 0, 0, 0); 222 } else if (m == 0) { 223 return n <= threshold ? new LevenshteinResults(n, 0, n, 0) : new LevenshteinResults(-1, 0, 0, 0); 224 } 225 226 boolean swapped = false; 227 if (n > m) { 228 // swap the two strings to consume less memory 229 final CharSequence tmp = left; 230 left = right; 231 right = tmp; 232 n = m; 233 m = right.length(); 234 swapped = true; 235 } 236 237 int[] p = new int[n + 1]; // 'previous' cost array, horizontally 238 int[] d = new int[n + 1]; // cost array, horizontally 239 int[] tempD; // placeholder to assist in swapping p and d 240 final int[][] matrix = new int[m + 1][n + 1]; 241 242 //filling the first row and first column values in the matrix 243 for (int index = 0; index <= n; index++) { 244 matrix[0][index] = index; 245 } 246 for (int index = 0; index <= m; index++) { 247 matrix[index][0] = index; 248 } 249 250 // fill in starting table values 251 final int boundary = Math.min(n, threshold) + 1; 252 for (int i = 0; i < boundary; i++) { 253 p[i] = i; 254 } 255 // these fills ensure that the value above the rightmost entry of our 256 // stripe will be ignored in following loop iterations 257 Arrays.fill(p, boundary, p.length, Integer.MAX_VALUE); 258 Arrays.fill(d, Integer.MAX_VALUE); 259 260 // iterates through t 261 for (int j = 1; j <= m; j++) { 262 final char rightJ = right.charAt(j - 1); // jth character of right 263 d[0] = j; 264 265 // compute stripe indices, constrain to array size 266 final int min = Math.max(1, j - threshold); 267 final int max = j > Integer.MAX_VALUE - threshold ? n : Math.min( 268 n, j + threshold); 269 270 // the stripe may lead off of the table if s and t are of different sizes 271 if (min > max) { 272 return new LevenshteinResults(-1, 0, 0, 0); 273 } 274 275 // ignore entry left of leftmost 276 if (min > 1) { 277 d[min - 1] = Integer.MAX_VALUE; 278 } 279 280 // iterates through [min, max] in s 281 for (int i = min; i <= max; i++) { 282 if (left.charAt(i - 1) == rightJ) { 283 // diagonally left and up 284 d[i] = p[i - 1]; 285 } else { 286 // 1 + minimum of cell to the left, to the top, diagonally left and up 287 d[i] = 1 + Math.min(Math.min(d[i - 1], p[i]), p[i - 1]); 288 } 289 matrix[j][i] = d[i]; 290 } 291 292 // copy current distance counts to 'previous row' distance counts 293 tempD = p; 294 p = d; 295 d = tempD; 296 } 297 298 // if p[n] is greater than the threshold, there's no guarantee on it being the correct distance 299 if (p[n] <= threshold) { 300 return findDetailedResults(left, right, matrix, swapped); 301 } 302 return new LevenshteinResults(-1, 0, 0, 0); 303 } 304 305 /** 306 * <p>Find the Levenshtein distance between two Strings.</p> 307 * 308 * <p>A higher score indicates a greater distance.</p> 309 * 310 * <p>The previous implementation of the Levenshtein distance algorithm 311 * was from <a href="http://www.merriampark.com/ld.htm">http://www.merriampark.com/ld.htm</a></p> 312 * 313 * <p>Chas Emerick has written an implementation in Java, which avoids an OutOfMemoryError 314 * which can occur when my Java implementation is used with very large strings.<br> 315 * This implementation of the Levenshtein distance algorithm 316 * is from <a href="http://www.merriampark.com/ldjava.htm">http://www.merriampark.com/ldjava.htm</a></p> 317 * 318 * <pre> 319 * unlimitedCompare(null, *) = IllegalArgumentException 320 * unlimitedCompare(*, null) = IllegalArgumentException 321 * unlimitedCompare("","") = 0 322 * unlimitedCompare("","a") = 1 323 * unlimitedCompare("aaapppp", "") = 7 324 * unlimitedCompare("frog", "fog") = 1 325 * unlimitedCompare("fly", "ant") = 3 326 * unlimitedCompare("elephant", "hippo") = 7 327 * unlimitedCompare("hippo", "elephant") = 7 328 * unlimitedCompare("hippo", "zzzzzzzz") = 8 329 * unlimitedCompare("hello", "hallo") = 1 330 * </pre> 331 * 332 * @param left the first String, must not be null 333 * @param right the second String, must not be null 334 * @return result distance, or -1 335 * @throws IllegalArgumentException if either String input {@code null} 336 */ 337 private static LevenshteinResults unlimitedCompare(CharSequence left, CharSequence right) { 338 if (left == null || right == null) { 339 throw new IllegalArgumentException("Strings must not be null"); 340 } 341 342 /* 343 The difference between this impl. and the previous is that, rather 344 than creating and retaining a matrix of size s.length() + 1 by t.length() + 1, 345 we maintain two single-dimensional arrays of length s.length() + 1. The first, d, 346 is the 'current working' distance array that maintains the newest distance cost 347 counts as we iterate through the characters of String s. Each time we increment 348 the index of String t we are comparing, d is copied to p, the second int[]. Doing so 349 allows us to retain the previous cost counts as required by the algorithm (taking 350 the minimum of the cost count to the left, up one, and diagonally up and to the left 351 of the current cost count being calculated). (Note that the arrays aren't really 352 copied anymore, just switched...this is clearly much better than cloning an array 353 or doing a System.arraycopy() each time through the outer loop.) 354 355 Effectively, the difference between the two implementations is this one does not 356 cause an out of memory condition when calculating the LD over two very large strings. 357 */ 358 359 int n = left.length(); // length of left 360 int m = right.length(); // length of right 361 362 if (n == 0) { 363 return new LevenshteinResults(m, m, 0, 0); 364 } else if (m == 0) { 365 return new LevenshteinResults(n, 0, n, 0); 366 } 367 boolean swapped = false; 368 if (n > m) { 369 // swap the input strings to consume less memory 370 final CharSequence tmp = left; 371 left = right; 372 right = tmp; 373 n = m; 374 m = right.length(); 375 swapped = true; 376 } 377 378 int[] p = new int[n + 1]; // 'previous' cost array, horizontally 379 int[] d = new int[n + 1]; // cost array, horizontally 380 int[] tempD; //placeholder to assist in swapping p and d 381 final int[][] matrix = new int[m + 1][n + 1]; 382 383 // filling the first row and first column values in the matrix 384 for (int index = 0; index <= n; index++) { 385 matrix[0][index] = index; 386 } 387 for (int index = 0; index <= m; index++) { 388 matrix[index][0] = index; 389 } 390 391 // indexes into strings left and right 392 int i; // iterates through left 393 int j; // iterates through right 394 395 char rightJ; // jth character of right 396 397 int cost; // cost 398 for (i = 0; i <= n; i++) { 399 p[i] = i; 400 } 401 402 for (j = 1; j <= m; j++) { 403 rightJ = right.charAt(j - 1); 404 d[0] = j; 405 406 for (i = 1; i <= n; i++) { 407 cost = left.charAt(i - 1) == rightJ ? 0 : 1; 408 // minimum of cell to the left+1, to the top+1, diagonally left and up +cost 409 d[i] = Math.min(Math.min(d[i - 1] + 1, p[i] + 1), p[i - 1] + cost); 410 //filling the matrix 411 matrix[j][i] = d[i]; 412 } 413 414 // copy current distance counts to 'previous row' distance counts 415 tempD = p; 416 p = d; 417 d = tempD; 418 } 419 return findDetailedResults(left, right, matrix, swapped); 420 } 421 422 /** 423 * Finds count for each of the three [insert, delete, substitute] operations 424 * needed. This is based on the matrix formed based on the two character 425 * sequence. 426 * 427 * @param left character sequence which need to be converted from 428 * @param right character sequence which need to be converted to 429 * @param matrix two dimensional array containing 430 * @param swapped tells whether the value for left character sequence and right 431 * character sequence were swapped to save memory 432 * @return result object containing the count of insert, delete and substitute and total count needed 433 */ 434 private static LevenshteinResults findDetailedResults(final CharSequence left, final CharSequence right, final int[][] matrix, 435 final boolean swapped) { 436 437 int delCount = 0; 438 int addCount = 0; 439 int subCount = 0; 440 441 int rowIndex = right.length(); 442 int columnIndex = left.length(); 443 444 int dataAtLeft = 0; 445 int dataAtTop = 0; 446 int dataAtDiagonal = 0; 447 int data = 0; 448 boolean deleted = false; 449 boolean added = false; 450 451 while (rowIndex >= 0 && columnIndex >= 0) { 452 453 if (columnIndex == 0) { 454 dataAtLeft = -1; 455 } else { 456 dataAtLeft = matrix[rowIndex][columnIndex - 1]; 457 } 458 if (rowIndex == 0) { 459 dataAtTop = -1; 460 } else { 461 dataAtTop = matrix[rowIndex - 1][columnIndex]; 462 } 463 if (rowIndex > 0 && columnIndex > 0) { 464 dataAtDiagonal = matrix[rowIndex - 1][columnIndex - 1]; 465 } else { 466 dataAtDiagonal = -1; 467 } 468 if (dataAtLeft == -1 && dataAtTop == -1 && dataAtDiagonal == -1) { 469 break; 470 } 471 data = matrix[rowIndex][columnIndex]; 472 473 // case in which the character at left and right are the same, 474 // in this case none of the counters will be incremented. 475 if (columnIndex > 0 && rowIndex > 0 && left.charAt(columnIndex - 1) == right.charAt(rowIndex - 1)) { 476 columnIndex--; 477 rowIndex--; 478 continue; 479 } 480 481 // handling insert and delete cases. 482 deleted = false; 483 added = false; 484 if (data - 1 == dataAtLeft && (data <= dataAtDiagonal && data <= dataAtTop) 485 || (dataAtDiagonal == -1 && dataAtTop == -1)) { // NOPMD 486 columnIndex--; 487 if (swapped) { 488 addCount++; 489 added = true; 490 } else { 491 delCount++; 492 deleted = true; 493 } 494 } else if (data - 1 == dataAtTop && (data <= dataAtDiagonal && data <= dataAtLeft) 495 || (dataAtDiagonal == -1 && dataAtLeft == -1)) { // NOPMD 496 rowIndex--; 497 if (swapped) { 498 delCount++; 499 deleted = true; 500 } else { 501 addCount++; 502 added = true; 503 } 504 } 505 506 // substituted case 507 if (!added && !deleted) { 508 subCount++; 509 columnIndex--; 510 rowIndex--; 511 } 512 } 513 return new LevenshteinResults(addCount + delCount + subCount, addCount, delCount, subCount); 514 } 515}