001/* 002 * Licensed to the Apache Software Foundation (ASF) under one or more 003 * contributor license agreements. See the NOTICE file distributed with 004 * this work for additional information regarding copyright ownership. 005 * The ASF licenses this file to You under the Apache License, Version 2.0 006 * (the "License"); you may not use this file except in compliance with 007 * the License. You may obtain a copy of the License at 008 * 009 * http://www.apache.org/licenses/LICENSE-2.0 010 * 011 * Unless required by applicable law or agreed to in writing, software 012 * distributed under the License is distributed on an "AS IS" BASIS, 013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 014 * See the License for the specific language governing permissions and 015 * limitations under the License. 016 */ 017package org.apache.commons.text.similarity; 018 019import java.util.Arrays; 020 021/** 022 * An algorithm for measuring the difference between two character sequences. 023 * 024 * <p> 025 * This is the number of changes needed to change one sequence into another, 026 * where each change is a single character modification (deletion, insertion 027 * or substitution). 028 * </p> 029 * 030 * <p> 031 * This code has been adapted from Apache Commons Lang 3.3. 032 * </p> 033 * 034 * @since 1.0 035 */ 036public class LevenshteinDistance implements EditDistance<Integer> { 037 038 /** 039 * Singleton instance. 040 */ 041 private static final LevenshteinDistance INSTANCE = new LevenshteinDistance(); 042 043 /** 044 * Gets the default instance. 045 * 046 * @return The default instance 047 */ 048 public static LevenshteinDistance getDefaultInstance() { 049 return INSTANCE; 050 } 051 052 /** 053 * Find the Levenshtein distance between two CharSequences if it's less than or 054 * equal to a given threshold. 055 * 056 * <p> 057 * This implementation follows from Algorithms on Strings, Trees and 058 * Sequences by Dan Gusfield and Chas Emerick's implementation of the 059 * Levenshtein distance algorithm from <a 060 * href="http://www.merriampark.com/ld.htm" 061 * >http://www.merriampark.com/ld.htm</a> 062 * </p> 063 * 064 * <pre> 065 * limitedCompare(null, *, *) = IllegalArgumentException 066 * limitedCompare(*, null, *) = IllegalArgumentException 067 * limitedCompare(*, *, -1) = IllegalArgumentException 068 * limitedCompare("","", 0) = 0 069 * limitedCompare("aaapppp", "", 8) = 7 070 * limitedCompare("aaapppp", "", 7) = 7 071 * limitedCompare("aaapppp", "", 6)) = -1 072 * limitedCompare("elephant", "hippo", 7) = 7 073 * limitedCompare("elephant", "hippo", 6) = -1 074 * limitedCompare("hippo", "elephant", 7) = 7 075 * limitedCompare("hippo", "elephant", 6) = -1 076 * </pre> 077 * 078 * @param left the first CharSequence, must not be null 079 * @param right the second CharSequence, must not be null 080 * @param threshold the target threshold, must not be negative 081 * @return result distance, or -1 082 */ 083 private static int limitedCompare(CharSequence left, CharSequence right, final int threshold) { // NOPMD 084 if (left == null || right == null) { 085 throw new IllegalArgumentException("CharSequences must not be null"); 086 } 087 if (threshold < 0) { 088 throw new IllegalArgumentException("Threshold must not be negative"); 089 } 090 091 /* 092 * This implementation only computes the distance if it's less than or 093 * equal to the threshold value, returning -1 if it's greater. The 094 * advantage is performance: unbounded distance is O(nm), but a bound of 095 * k allows us to reduce it to O(km) time by only computing a diagonal 096 * stripe of width 2k + 1 of the cost table. It is also possible to use 097 * this to compute the unbounded Levenshtein distance by starting the 098 * threshold at 1 and doubling each time until the distance is found; 099 * this is O(dm), where d is the distance. 100 * 101 * One subtlety comes from needing to ignore entries on the border of 102 * our stripe eg. p[] = |#|#|#|* d[] = *|#|#|#| We must ignore the entry 103 * to the left of the leftmost member We must ignore the entry above the 104 * rightmost member 105 * 106 * Another subtlety comes from our stripe running off the matrix if the 107 * strings aren't of the same size. Since string s is always swapped to 108 * be the shorter of the two, the stripe will always run off to the 109 * upper right instead of the lower left of the matrix. 110 * 111 * As a concrete example, suppose s is of length 5, t is of length 7, 112 * and our threshold is 1. In this case we're going to walk a stripe of 113 * length 3. The matrix would look like so: 114 * 115 * <pre> 116 * 1 2 3 4 5 117 * 1 |#|#| | | | 118 * 2 |#|#|#| | | 119 * 3 | |#|#|#| | 120 * 4 | | |#|#|#| 121 * 5 | | | |#|#| 122 * 6 | | | | |#| 123 * 7 | | | | | | 124 * </pre> 125 * 126 * Note how the stripe leads off the table as there is no possible way 127 * to turn a string of length 5 into one of length 7 in edit distance of 128 * 1. 129 * 130 * Additionally, this implementation decreases memory usage by using two 131 * single-dimensional arrays and swapping them back and forth instead of 132 * allocating an entire n by m matrix. This requires a few minor 133 * changes, such as immediately returning when it's detected that the 134 * stripe has run off the matrix and initially filling the arrays with 135 * large values so that entries we don't compute are ignored. 136 * 137 * See Algorithms on Strings, Trees and Sequences by Dan Gusfield for 138 * some discussion. 139 */ 140 141 int n = left.length(); // length of left 142 int m = right.length(); // length of right 143 144 // if one string is empty, the edit distance is necessarily the length 145 // of the other 146 if (n == 0) { 147 return m <= threshold ? m : -1; 148 } 149 if (m == 0) { 150 return n <= threshold ? n : -1; 151 } 152 153 if (n > m) { 154 // swap the two strings to consume less memory 155 final CharSequence tmp = left; 156 left = right; 157 right = tmp; 158 n = m; 159 m = right.length(); 160 } 161 162 // the edit distance cannot be less than the length difference 163 if (m - n > threshold) { 164 return -1; 165 } 166 167 int[] p = new int[n + 1]; // 'previous' cost array, horizontally 168 int[] d = new int[n + 1]; // cost array, horizontally 169 int[] tempD; // placeholder to assist in swapping p and d 170 171 // fill in starting table values 172 final int boundary = Math.min(n, threshold) + 1; 173 for (int i = 0; i < boundary; i++) { 174 p[i] = i; 175 } 176 // these fills ensure that the value above the rightmost entry of our 177 // stripe will be ignored in following loop iterations 178 Arrays.fill(p, boundary, p.length, Integer.MAX_VALUE); 179 Arrays.fill(d, Integer.MAX_VALUE); 180 181 // iterates through t 182 for (int j = 1; j <= m; j++) { 183 final char rightJ = right.charAt(j - 1); // jth character of right 184 d[0] = j; 185 186 // compute stripe indices, constrain to array size 187 final int min = Math.max(1, j - threshold); 188 final int max = j > Integer.MAX_VALUE - threshold ? n : Math.min( 189 n, j + threshold); 190 191 // ignore entry left of leftmost 192 if (min > 1) { 193 d[min - 1] = Integer.MAX_VALUE; 194 } 195 196 int lowerBound = Integer.MAX_VALUE; 197 // iterates through [min, max] in s 198 for (int i = min; i <= max; i++) { 199 if (left.charAt(i - 1) == rightJ) { 200 // diagonally left and up 201 d[i] = p[i - 1]; 202 } else { 203 // 1 + minimum of cell to the left, to the top, diagonally 204 // left and up 205 d[i] = 1 + Math.min(Math.min(d[i - 1], p[i]), p[i - 1]); 206 } 207 lowerBound = Math.min(lowerBound, d[i]); 208 } 209 // if the lower bound is greater than the threshold, then exit early 210 if (lowerBound > threshold) { 211 return -1; 212 } 213 214 // copy current distance counts to 'previous row' distance counts 215 tempD = p; 216 p = d; 217 d = tempD; 218 } 219 220 // if p[n] is greater than the threshold, there's no guarantee on it 221 // being the correct 222 // distance 223 if (p[n] <= threshold) { 224 return p[n]; 225 } 226 return -1; 227 } 228 229 /** 230 * Finds the Levenshtein distance between two Strings. 231 * 232 * <p>A higher score indicates a greater distance.</p> 233 * 234 * <p>The previous implementation of the Levenshtein distance algorithm 235 * was from <a href="https://web.archive.org/web/20120526085419/http://www.merriampark.com/ldjava.htm"> 236 * https://web.archive.org/web/20120526085419/http://www.merriampark.com/ldjava.htm</a></p> 237 * 238 * <p>This implementation only need one single-dimensional arrays of length s.length() + 1</p> 239 * 240 * <pre> 241 * unlimitedCompare(null, *) = IllegalArgumentException 242 * unlimitedCompare(*, null) = IllegalArgumentException 243 * unlimitedCompare("","") = 0 244 * unlimitedCompare("","a") = 1 245 * unlimitedCompare("aaapppp", "") = 7 246 * unlimitedCompare("frog", "fog") = 1 247 * unlimitedCompare("fly", "ant") = 3 248 * unlimitedCompare("elephant", "hippo") = 7 249 * unlimitedCompare("hippo", "elephant") = 7 250 * unlimitedCompare("hippo", "zzzzzzzz") = 8 251 * unlimitedCompare("hello", "hallo") = 1 252 * </pre> 253 * 254 * @param left the first CharSequence, must not be null 255 * @param right the second CharSequence, must not be null 256 * @return result distance, or -1 257 * @throws IllegalArgumentException if either CharSequence input is {@code null} 258 */ 259 private static int unlimitedCompare(CharSequence left, CharSequence right) { 260 if (left == null || right == null) { 261 throw new IllegalArgumentException("CharSequences must not be null"); 262 } 263 264 /* 265 This implementation use two variable to record the previous cost counts, 266 So this implementation use less memory than previous impl. 267 */ 268 269 int n = left.length(); // length of left 270 int m = right.length(); // length of right 271 272 if (n == 0) { 273 return m; 274 } 275 if (m == 0) { 276 return n; 277 } 278 279 if (n > m) { 280 // swap the input strings to consume less memory 281 final CharSequence tmp = left; 282 left = right; 283 right = tmp; 284 n = m; 285 m = right.length(); 286 } 287 288 final int[] p = new int[n + 1]; 289 290 // indexes into strings left and right 291 int i; // iterates through left 292 int j; // iterates through right 293 int upperLeft; 294 int upper; 295 296 char rightJ; // jth character of right 297 int cost; // cost 298 299 for (i = 0; i <= n; i++) { 300 p[i] = i; 301 } 302 303 for (j = 1; j <= m; j++) { 304 upperLeft = p[0]; 305 rightJ = right.charAt(j - 1); 306 p[0] = j; 307 308 for (i = 1; i <= n; i++) { 309 upper = p[i]; 310 cost = left.charAt(i - 1) == rightJ ? 0 : 1; 311 // minimum of cell to the left+1, to the top+1, diagonally left and up +cost 312 p[i] = Math.min(Math.min(p[i - 1] + 1, p[i] + 1), upperLeft + cost); 313 upperLeft = upper; 314 } 315 } 316 317 return p[n]; 318 } 319 320 /** 321 * Threshold. 322 */ 323 private final Integer threshold; 324 325 /** 326 * This returns the default instance that uses a version 327 * of the algorithm that does not use a threshold parameter. 328 * 329 * @see LevenshteinDistance#getDefaultInstance() 330 */ 331 public LevenshteinDistance() { 332 this(null); 333 } 334 335 /** 336 * If the threshold is not null, distance calculations will be limited to a maximum length. 337 * If the threshold is null, the unlimited version of the algorithm will be used. 338 * 339 * @param threshold 340 * If this is null then distances calculations will not be limited. 341 * This may not be negative. 342 */ 343 public LevenshteinDistance(final Integer threshold) { 344 if (threshold != null && threshold < 0) { 345 throw new IllegalArgumentException("Threshold must not be negative"); 346 } 347 this.threshold = threshold; 348 } 349 350 /** 351 * Finds the Levenshtein distance between two Strings. 352 * 353 * <p>A higher score indicates a greater distance.</p> 354 * 355 * <p>The previous implementation of the Levenshtein distance algorithm 356 * was from <a href="http://www.merriampark.com/ld.htm">http://www.merriampark.com/ld.htm</a></p> 357 * 358 * <p>Chas Emerick has written an implementation in Java, which avoids an OutOfMemoryError 359 * which can occur when my Java implementation is used with very large strings.<br> 360 * This implementation of the Levenshtein distance algorithm 361 * is from <a href="http://www.merriampark.com/ldjava.htm">http://www.merriampark.com/ldjava.htm</a></p> 362 * 363 * <pre> 364 * distance.apply(null, *) = IllegalArgumentException 365 * distance.apply(*, null) = IllegalArgumentException 366 * distance.apply("","") = 0 367 * distance.apply("","a") = 1 368 * distance.apply("aaapppp", "") = 7 369 * distance.apply("frog", "fog") = 1 370 * distance.apply("fly", "ant") = 3 371 * distance.apply("elephant", "hippo") = 7 372 * distance.apply("hippo", "elephant") = 7 373 * distance.apply("hippo", "zzzzzzzz") = 8 374 * distance.apply("hello", "hallo") = 1 375 * </pre> 376 * 377 * @param left the first string, must not be null 378 * @param right the second string, must not be null 379 * @return result distance, or -1 380 * @throws IllegalArgumentException if either String input {@code null} 381 */ 382 @Override 383 public Integer apply(final CharSequence left, final CharSequence right) { 384 if (threshold != null) { 385 return limitedCompare(left, right, threshold); 386 } 387 return unlimitedCompare(left, right); 388 } 389 390 /** 391 * Gets the distance threshold. 392 * 393 * @return The distance threshold 394 */ 395 public Integer getThreshold() { 396 return threshold; 397 } 398 399}