001/*
002 * Licensed to the Apache Software Foundation (ASF) under one or more
003 * contributor license agreements.  See the NOTICE file distributed with
004 * this work for additional information regarding copyright ownership.
005 * The ASF licenses this file to You under the Apache License, Version 2.0
006 * (the "License"); you may not use this file except in compliance with
007 * the License.  You may obtain a copy of the License at
008 *
009 *      http://www.apache.org/licenses/LICENSE-2.0
010 *
011 * Unless required by applicable law or agreed to in writing, software
012 * distributed under the License is distributed on an "AS IS" BASIS,
013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
014 * See the License for the specific language governing permissions and
015 * limitations under the License.
016 */
017package org.apache.commons.text.similarity;
018
019import java.util.Arrays;
020
021/**
022 * An algorithm for measuring the difference between two character sequences.
023 *
024 * <p>
025 * This is the number of changes needed to change one sequence into another,
026 * where each change is a single character modification (deletion, insertion
027 * or substitution).
028 * </p>
029 *
030 * @since 1.0
031 */
032public class LevenshteinDetailedDistance implements EditDistance<LevenshteinResults> {
033
034    /**
035     * Singleton instance.
036     */
037    private static final LevenshteinDetailedDistance INSTANCE = new LevenshteinDetailedDistance();
038
039    /**
040     * Finds count for each of the three [insert, delete, substitute] operations
041     * needed. This is based on the matrix formed based on the two character
042     * sequence.
043     *
044     * @param <E> The type of similarity score unit.
045     * @param left character sequence which need to be converted from
046     * @param right character sequence which need to be converted to
047     * @param matrix two dimensional array containing
048     * @param swapped tells whether the value for left character sequence and right
049     *            character sequence were swapped to save memory
050     * @return result object containing the count of insert, delete and substitute and total count needed
051     */
052    private static <E> LevenshteinResults findDetailedResults(final SimilarityInput<E> left,
053                                                          final SimilarityInput<E> right,
054                                                          final int[][] matrix,
055                                                          final boolean swapped) {
056
057        int delCount = 0;
058        int addCount = 0;
059        int subCount = 0;
060
061        int rowIndex = right.length();
062        int columnIndex = left.length();
063
064        int dataAtLeft = 0;
065        int dataAtTop = 0;
066        int dataAtDiagonal = 0;
067        int data = 0;
068        boolean deleted = false;
069        boolean added = false;
070
071        while (rowIndex >= 0 && columnIndex >= 0) {
072
073            if (columnIndex == 0) {
074                dataAtLeft = -1;
075            } else {
076                dataAtLeft = matrix[rowIndex][columnIndex - 1];
077            }
078            if (rowIndex == 0) {
079                dataAtTop = -1;
080            } else {
081                dataAtTop = matrix[rowIndex - 1][columnIndex];
082            }
083            if (rowIndex > 0 && columnIndex > 0) {
084                dataAtDiagonal = matrix[rowIndex - 1][columnIndex - 1];
085            } else {
086                dataAtDiagonal = -1;
087            }
088            if (dataAtLeft == -1 && dataAtTop == -1 && dataAtDiagonal == -1) {
089                break;
090            }
091            data = matrix[rowIndex][columnIndex];
092
093            // case in which the character at left and right are the same,
094            // in this case none of the counters will be incremented.
095            if (columnIndex > 0 && rowIndex > 0 && left.at(columnIndex - 1).equals(right.at(rowIndex - 1))) {
096                columnIndex--;
097                rowIndex--;
098                continue;
099            }
100
101            // handling insert and delete cases.
102            deleted = false;
103            added = false;
104            if (data - 1 == dataAtLeft && data <= dataAtDiagonal && data <= dataAtTop
105                    || dataAtDiagonal == -1 && dataAtTop == -1) { // NOPMD
106                columnIndex--;
107                if (swapped) {
108                    addCount++;
109                    added = true;
110                } else {
111                    delCount++;
112                    deleted = true;
113                }
114            } else if (data - 1 == dataAtTop && data <= dataAtDiagonal && data <= dataAtLeft
115                    || dataAtDiagonal == -1 && dataAtLeft == -1) { // NOPMD
116                rowIndex--;
117                if (swapped) {
118                    delCount++;
119                    deleted = true;
120                } else {
121                    addCount++;
122                    added = true;
123                }
124            }
125
126            // substituted case
127            if (!added && !deleted) {
128                subCount++;
129                columnIndex--;
130                rowIndex--;
131            }
132        }
133        return new LevenshteinResults(addCount + delCount + subCount, addCount, delCount, subCount);
134    }
135
136    /**
137     * Gets the default instance.
138     *
139     * @return The default instace
140     */
141    public static LevenshteinDetailedDistance getDefaultInstance() {
142        return INSTANCE;
143    }
144
145    /**
146     * Finds the Levenshtein distance between two CharSequences if it's less than or
147     * equal to a given threshold.
148     *
149     * <p>
150     * This implementation follows from Algorithms on Strings, Trees and
151     * Sequences by Dan Gusfield and Chas Emerick's implementation of the
152     * Levenshtein distance algorithm from <a
153     * href="http://www.merriampark.com/ld.htm"
154     * >http://www.merriampark.com/ld.htm</a>
155     * </p>
156     *
157     * <pre>
158     * limitedCompare(null, *, *)             = IllegalArgumentException
159     * limitedCompare(*, null, *)             = IllegalArgumentException
160     * limitedCompare(*, *, -1)               = IllegalArgumentException
161     * limitedCompare("","", 0)               = 0
162     * limitedCompare("aaapppp", "", 8)       = 7
163     * limitedCompare("aaapppp", "", 7)       = 7
164     * limitedCompare("aaapppp", "", 6))      = -1
165     * limitedCompare("elephant", "hippo", 7) = 7
166     * limitedCompare("elephant", "hippo", 6) = -1
167     * limitedCompare("hippo", "elephant", 7) = 7
168     * limitedCompare("hippo", "elephant", 6) = -1
169     * </pre>
170     *
171     * @param <E> The type of similarity score unit.
172     * @param left the first CharSequence, must not be null
173     * @param right the second CharSequence, must not be null
174     * @param threshold the target threshold, must not be negative
175     * @return result distance, or -1
176     */
177    private static <E> LevenshteinResults limitedCompare(SimilarityInput<E> left, SimilarityInput<E> right, final int threshold) { //NOPMD
178        if (left == null || right == null) {
179            throw new IllegalArgumentException("CharSequences must not be null");
180        }
181        if (threshold < 0) {
182            throw new IllegalArgumentException("Threshold must not be negative");
183        }
184
185        /*
186         * This implementation only computes the distance if it's less than or
187         * equal to the threshold value, returning -1 if it's greater. The
188         * advantage is performance: unbounded distance is O(nm), but a bound of
189         * k allows us to reduce it to O(km) time by only computing a diagonal
190         * stripe of width 2k + 1 of the cost table. It is also possible to use
191         * this to compute the unbounded Levenshtein distance by starting the
192         * threshold at 1 and doubling each time until the distance is found;
193         * this is O(dm), where d is the distance.
194         *
195         * One subtlety comes from needing to ignore entries on the border of
196         * our stripe eg. p[] = |#|#|#|* d[] = *|#|#|#| We must ignore the entry
197         * to the left of the leftmost member We must ignore the entry above the
198         * rightmost member
199         *
200         * Another subtlety comes from our stripe running off the matrix if the
201         * strings aren't of the same size. Since string s is always swapped to
202         * be the shorter of the two, the stripe will always run off to the
203         * upper right instead of the lower left of the matrix.
204         *
205         * As a concrete example, suppose s is of length 5, t is of length 7,
206         * and our threshold is 1. In this case we're going to walk a stripe of
207         * length 3. The matrix would look like so:
208         *
209         * <pre>
210         *    1 2 3 4 5
211         * 1 |#|#| | | |
212         * 2 |#|#|#| | |
213         * 3 | |#|#|#| |
214         * 4 | | |#|#|#|
215         * 5 | | | |#|#|
216         * 6 | | | | |#|
217         * 7 | | | | | |
218         * </pre>
219         *
220         * Note how the stripe leads off the table as there is no possible way
221         * to turn a string of length 5 into one of length 7 in edit distance of
222         * 1.
223         *
224         * Additionally, this implementation decreases memory usage by using two
225         * single-dimensional arrays and swapping them back and forth instead of
226         * allocating an entire n by m matrix. This requires a few minor
227         * changes, such as immediately returning when it's detected that the
228         * stripe has run off the matrix and initially filling the arrays with
229         * large values so that entries we don't compute are ignored.
230         *
231         * See Algorithms on Strings, Trees and Sequences by Dan Gusfield for
232         * some discussion.
233         */
234
235        int n = left.length(); // length of left
236        int m = right.length(); // length of right
237
238        // if one string is empty, the edit distance is necessarily the length of the other
239        if (n == 0) {
240            return m <= threshold ? new LevenshteinResults(m, m, 0, 0) : new LevenshteinResults(-1, 0, 0, 0);
241        }
242        if (m == 0) {
243            return n <= threshold ? new LevenshteinResults(n, 0, n, 0) : new LevenshteinResults(-1, 0, 0, 0);
244        }
245
246        boolean swapped = false;
247        if (n > m) {
248            // swap the two strings to consume less memory
249            final SimilarityInput<E> tmp = left;
250            left = right;
251            right = tmp;
252            n = m;
253            m = right.length();
254            swapped = true;
255        }
256
257        int[] p = new int[n + 1]; // 'previous' cost array, horizontally
258        int[] d = new int[n + 1]; // cost array, horizontally
259        int[] tempD; // placeholder to assist in swapping p and d
260        final int[][] matrix = new int[m + 1][n + 1];
261
262        //filling the first row and first column values in the matrix
263        for (int index = 0; index <= n; index++) {
264            matrix[0][index] = index;
265        }
266        for (int index = 0; index <= m; index++) {
267            matrix[index][0] = index;
268        }
269
270        // fill in starting table values
271        final int boundary = Math.min(n, threshold) + 1;
272        for (int i = 0; i < boundary; i++) {
273            p[i] = i;
274        }
275        // these fills ensure that the value above the rightmost entry of our
276        // stripe will be ignored in following loop iterations
277        Arrays.fill(p, boundary, p.length, Integer.MAX_VALUE);
278        Arrays.fill(d, Integer.MAX_VALUE);
279
280        // iterates through t
281        for (int j = 1; j <= m; j++) {
282            final E rightJ = right.at(j - 1); // jth character of right
283            d[0] = j;
284
285            // compute stripe indices, constrain to array size
286            final int min = Math.max(1, j - threshold);
287            final int max = j > Integer.MAX_VALUE - threshold ? n : Math.min(
288                    n, j + threshold);
289
290            // the stripe may lead off of the table if s and t are of different sizes
291            if (min > max) {
292                return new LevenshteinResults(-1, 0, 0, 0);
293            }
294
295            // ignore entry left of leftmost
296            if (min > 1) {
297                d[min - 1] = Integer.MAX_VALUE;
298            }
299
300            // iterates through [min, max] in s
301            for (int i = min; i <= max; i++) {
302                if (left.at(i - 1).equals(rightJ)) {
303                    // diagonally left and up
304                    d[i] = p[i - 1];
305                } else {
306                    // 1 + minimum of cell to the left, to the top, diagonally left and up
307                    d[i] = 1 + Math.min(Math.min(d[i - 1], p[i]), p[i - 1]);
308                }
309                matrix[j][i] = d[i];
310            }
311
312            // copy current distance counts to 'previous row' distance counts
313            tempD = p;
314            p = d;
315            d = tempD;
316        }
317
318        // if p[n] is greater than the threshold, there's no guarantee on it being the correct distance
319        if (p[n] <= threshold) {
320            return findDetailedResults(left, right, matrix, swapped);
321        }
322        return new LevenshteinResults(-1, 0, 0, 0);
323    }
324
325    /**
326     * Finds the Levenshtein distance between two Strings.
327     *
328     * <p>A higher score indicates a greater distance.</p>
329     *
330     * <p>The previous implementation of the Levenshtein distance algorithm
331     * was from <a href="http://www.merriampark.com/ld.htm">http://www.merriampark.com/ld.htm</a></p>
332     *
333     * <p>Chas Emerick has written an implementation in Java, which avoids an OutOfMemoryError
334     * which can occur when my Java implementation is used with very large strings.<br>
335     * This implementation of the Levenshtein distance algorithm
336     * is from <a href="http://www.merriampark.com/ldjava.htm">http://www.merriampark.com/ldjava.htm</a></p>
337     *
338     * <pre>
339     * unlimitedCompare(null, *)             = IllegalArgumentException
340     * unlimitedCompare(*, null)             = IllegalArgumentException
341     * unlimitedCompare("","")               = 0
342     * unlimitedCompare("","a")              = 1
343     * unlimitedCompare("aaapppp", "")       = 7
344     * unlimitedCompare("frog", "fog")       = 1
345     * unlimitedCompare("fly", "ant")        = 3
346     * unlimitedCompare("elephant", "hippo") = 7
347     * unlimitedCompare("hippo", "elephant") = 7
348     * unlimitedCompare("hippo", "zzzzzzzz") = 8
349     * unlimitedCompare("hello", "hallo")    = 1
350     * </pre>
351     *
352     * @param <E> The type of similarity score unit.
353     * @param left the first CharSequence, must not be null
354     * @param right the second CharSequence, must not be null
355     * @return result distance, or -1
356     * @throws IllegalArgumentException if either CharSequence input is {@code null}
357     */
358    private static <E> LevenshteinResults unlimitedCompare(SimilarityInput<E> left, SimilarityInput<E> right) {
359        if (left == null || right == null) {
360            throw new IllegalArgumentException("CharSequences must not be null");
361        }
362
363        /*
364           The difference between this impl. and the previous is that, rather
365           than creating and retaining a matrix of size s.length() + 1 by t.length() + 1,
366           we maintain two single-dimensional arrays of length s.length() + 1.  The first, d,
367           is the 'current working' distance array that maintains the newest distance cost
368           counts as we iterate through the characters of String s.  Each time we increment
369           the index of String t we are comparing, d is copied to p, the second int[].  Doing so
370           allows us to retain the previous cost counts as required by the algorithm (taking
371           the minimum of the cost count to the left, up one, and diagonally up and to the left
372           of the current cost count being calculated).  (Note that the arrays aren't really
373           copied anymore, just switched...this is clearly much better than cloning an array
374           or doing a System.arraycopy() each time  through the outer loop.)
375
376           Effectively, the difference between the two implementations is this one does not
377           cause an out of memory condition when calculating the LD over two very large strings.
378         */
379
380        int n = left.length(); // length of left
381        int m = right.length(); // length of right
382
383        if (n == 0) {
384            return new LevenshteinResults(m, m, 0, 0);
385        }
386        if (m == 0) {
387            return new LevenshteinResults(n, 0, n, 0);
388        }
389        boolean swapped = false;
390        if (n > m) {
391            // swap the input strings to consume less memory
392            final SimilarityInput<E> tmp = left;
393            left = right;
394            right = tmp;
395            n = m;
396            m = right.length();
397            swapped = true;
398        }
399
400        int[] p = new int[n + 1]; // 'previous' cost array, horizontally
401        int[] d = new int[n + 1]; // cost array, horizontally
402        int[] tempD; //placeholder to assist in swapping p and d
403        final int[][] matrix = new int[m + 1][n + 1];
404
405        // filling the first row and first column values in the matrix
406        for (int index = 0; index <= n; index++) {
407            matrix[0][index] = index;
408        }
409        for (int index = 0; index <= m; index++) {
410            matrix[index][0] = index;
411        }
412
413        // indexes into strings left and right
414        int i; // iterates through left
415        int j; // iterates through right
416
417        E rightJ; // jth character of right
418
419        int cost; // cost
420        for (i = 0; i <= n; i++) {
421            p[i] = i;
422        }
423
424        for (j = 1; j <= m; j++) {
425            rightJ = right.at(j - 1);
426            d[0] = j;
427
428            for (i = 1; i <= n; i++) {
429                cost = left.at(i - 1).equals(rightJ) ? 0 : 1;
430                // minimum of cell to the left+1, to the top+1, diagonally left and up +cost
431                d[i] = Math.min(Math.min(d[i - 1] + 1, p[i] + 1), p[i - 1] + cost);
432                //filling the matrix
433                matrix[j][i] = d[i];
434            }
435
436            // copy current distance counts to 'previous row' distance counts
437            tempD = p;
438            p = d;
439            d = tempD;
440        }
441        return findDetailedResults(left, right, matrix, swapped);
442    }
443
444    /**
445     * Threshold.
446     */
447    private final Integer threshold;
448
449    /**
450     * <p>
451     * This returns the default instance that uses a version
452     * of the algorithm that does not use a threshold parameter.
453     * </p>
454     *
455     * @see LevenshteinDetailedDistance#getDefaultInstance()
456     * @deprecated Use {@link #getDefaultInstance()}.
457     */
458    @Deprecated
459    public LevenshteinDetailedDistance() {
460        this(null);
461    }
462
463    /**
464     * If the threshold is not null, distance calculations will be limited to a maximum length.
465     *
466     * <p>If the threshold is null, the unlimited version of the algorithm will be used.</p>
467     *
468     * @param threshold If this is null then distances calculations will not be limited. This may not be negative.
469     */
470    public LevenshteinDetailedDistance(final Integer threshold) {
471        if (threshold != null && threshold < 0) {
472            throw new IllegalArgumentException("Threshold must not be negative");
473        }
474        this.threshold = threshold;
475    }
476
477    /**
478     * Computes the Levenshtein distance between two Strings.
479     *
480     * <p>A higher score indicates a greater distance.</p>
481     *
482     * <p>The previous implementation of the Levenshtein distance algorithm
483     * was from <a href="http://www.merriampark.com/ld.htm">http://www.merriampark.com/ld.htm</a></p>
484     *
485     * <p>Chas Emerick has written an implementation in Java, which avoids an OutOfMemoryError
486     * which can occur when my Java implementation is used with very large strings.<br>
487     * This implementation of the Levenshtein distance algorithm
488     * is from <a href="http://www.merriampark.com/ldjava.htm">http://www.merriampark.com/ldjava.htm</a></p>
489     *
490     * <pre>
491     * distance.apply(null, *)             = IllegalArgumentException
492     * distance.apply(*, null)             = IllegalArgumentException
493     * distance.apply("","")               = 0
494     * distance.apply("","a")              = 1
495     * distance.apply("aaapppp", "")       = 7
496     * distance.apply("frog", "fog")       = 1
497     * distance.apply("fly", "ant")        = 3
498     * distance.apply("elephant", "hippo") = 7
499     * distance.apply("hippo", "elephant") = 7
500     * distance.apply("hippo", "zzzzzzzz") = 8
501     * distance.apply("hello", "hallo")    = 1
502     * </pre>
503     *
504     * @param left the first input, must not be null
505     * @param right the second input, must not be null
506     * @return result distance, or -1
507     * @throws IllegalArgumentException if either String input {@code null}
508     */
509    @Override
510    public LevenshteinResults apply(final CharSequence left, final CharSequence right) {
511        return apply(SimilarityInput.input(left), SimilarityInput.input(right));
512    }
513
514    /**
515     * Computes the Levenshtein distance between two Strings.
516     *
517     * <p>A higher score indicates a greater distance.</p>
518     *
519     * <p>The previous implementation of the Levenshtein distance algorithm
520     * was from <a href="http://www.merriampark.com/ld.htm">http://www.merriampark.com/ld.htm</a></p>
521     *
522     * <p>Chas Emerick has written an implementation in Java, which avoids an OutOfMemoryError
523     * which can occur when my Java implementation is used with very large strings.<br>
524     * This implementation of the Levenshtein distance algorithm
525     * is from <a href="http://www.merriampark.com/ldjava.htm">http://www.merriampark.com/ldjava.htm</a></p>
526     *
527     * <pre>
528     * distance.apply(null, *)             = IllegalArgumentException
529     * distance.apply(*, null)             = IllegalArgumentException
530     * distance.apply("","")               = 0
531     * distance.apply("","a")              = 1
532     * distance.apply("aaapppp", "")       = 7
533     * distance.apply("frog", "fog")       = 1
534     * distance.apply("fly", "ant")        = 3
535     * distance.apply("elephant", "hippo") = 7
536     * distance.apply("hippo", "elephant") = 7
537     * distance.apply("hippo", "zzzzzzzz") = 8
538     * distance.apply("hello", "hallo")    = 1
539     * </pre>
540     *
541     * @param <E> The type of similarity score unit.
542     * @param left the first input, must not be null
543     * @param right the second input, must not be null
544     * @return result distance, or -1
545     * @throws IllegalArgumentException if either String input {@code null}
546     * @since 1.13.0
547     */
548    public <E> LevenshteinResults apply(final SimilarityInput<E> left, final SimilarityInput<E> right) {
549        if (threshold != null) {
550            return limitedCompare(left, right, threshold);
551        }
552        return unlimitedCompare(left, right);
553    }
554
555    /**
556     * Gets the distance threshold.
557     *
558     * @return The distance threshold
559     */
560    public Integer getThreshold() {
561        return threshold;
562    }
563}