001/*
002 * Licensed to the Apache Software Foundation (ASF) under one or more
003 * contributor license agreements.  See the NOTICE file distributed with
004 * this work for additional information regarding copyright ownership.
005 * The ASF licenses this file to You under the Apache License, Version 2.0
006 * (the "License"); you may not use this file except in compliance with
007 * the License.  You may obtain a copy of the License at
008 *
009 *      https://www.apache.org/licenses/LICENSE-2.0
010 *
011 * Unless required by applicable law or agreed to in writing, software
012 * distributed under the License is distributed on an "AS IS" BASIS,
013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
014 * See the License for the specific language governing permissions and
015 * limitations under the License.
016 */
017package org.apache.commons.text.similarity;
018
019import java.util.Arrays;
020
021/**
022 * An algorithm for measuring the difference between two character sequences using the <a href="https://en.wikipedia.org/wiki/Levenshtein_distance">Levenshtein
023 * Distance</a>.
024 *
025 * <p>
026 * This is the number of changes needed to change one sequence into another, where each change is a single character modification (deletion, insertion or
027 * substitution).
028 * </p>
029 * <p>
030 * This code has been adapted from Apache Commons Lang 3.3.
031 * </p>
032 *
033 * @since 1.0
034 * @see <a href="https://en.wikipedia.org/wiki/Levenshtein_distance">Levenshtein Distance on Wikipedia</a>
035 * @see <a href="https://xlinux.nist.gov/dads/HTML/Levenshtein.html">Levenshtein Distance on NIST</a>
036 */
037public class LevenshteinDistance implements EditDistance<Integer> {
038
039    /**
040     * The singleton instance.
041     */
042    private static final LevenshteinDistance INSTANCE = new LevenshteinDistance();
043
044    /**
045     * Gets the default instance.
046     *
047     * @return The default instance.
048     */
049    public static LevenshteinDistance getDefaultInstance() {
050        return INSTANCE;
051    }
052
053    /**
054     * Finds the Levenshtein distance between two CharSequences if it's less than or equal to a given threshold.
055     *
056     * <p>
057     * This implementation follows from Algorithms on Strings, Trees and Sequences by Dan Gusfield and Chas Emerick's implementation of the Levenshtein distance
058     * algorithm.
059     * </p>
060     *
061     * <pre>
062     * limitedCompare(null, *, *)             = Throws {@link IllegalArgumentException}
063     * limitedCompare(*, null, *)             = Throws {@link IllegalArgumentException}
064     * limitedCompare(*, *, -1)               = Throws {@link IllegalArgumentException}
065     * limitedCompare("","", 0)               = 0
066     * limitedCompare("aaapppp", "", 8)       = 7
067     * limitedCompare("aaapppp", "", 7)       = 7
068     * limitedCompare("aaapppp", "", 6))      = -1
069     * limitedCompare("elephant", "hippo", 7) = 7
070     * limitedCompare("elephant", "hippo", 6) = -1
071     * limitedCompare("hippo", "elephant", 7) = 7
072     * limitedCompare("hippo", "elephant", 6) = -1
073     * </pre>
074     *
075     * @param left      the first SimilarityInput, must not be null.
076     * @param right     the second SimilarityInput, must not be null.
077     * @param threshold the target threshold, must not be negative.
078     * @return result distance, or -1
079     */
080    private static <E> int limitedCompare(SimilarityInput<E> left, SimilarityInput<E> right, final int threshold) { // NOPMD
081        if (left == null || right == null) {
082            throw new IllegalArgumentException("CharSequences must not be null");
083        }
084
085        /*
086         * This implementation only computes the distance if it's less than or equal to the threshold value, returning -1 if it's greater. The advantage is
087         * performance: unbounded distance is O(nm), but a bound of k allows us to reduce it to O(km) time by only computing a diagonal stripe of width 2k + 1
088         * of the cost table. It is also possible to use this to compute the unbounded Levenshtein distance by starting the threshold at 1 and doubling each
089         * time until the distance is found; this is O(dm), where d is the distance.
090         *
091         * One subtlety comes from needing to ignore entries on the border of our stripe, for example,
092         * p[] = |#|#|#|* d[] = *|#|#|#| We must ignore the entry to the left
093         * of the leftmost member We must ignore the entry above the rightmost member
094         *
095         * Another subtlety comes from our stripe running off the matrix if the strings aren't of the same size. Since string s is always swapped to be the
096         * shorter of the two, the stripe will always run off to the upper right instead of the lower left of the matrix.
097         *
098         * As a concrete example, suppose s is of length 5, t is of length 7, and our threshold is 1. In this case we're going to walk a stripe of length 3. The
099         * matrix would look like so:
100         *
101         * <pre> 1 2 3 4 5 1 |#|#| | | | 2 |#|#|#| | | 3 | |#|#|#| | 4 | | |#|#|#| 5 | | | |#|#| 6 | | | | |#| 7 | | | | | | </pre>
102         *
103         * Note how the stripe leads off the table as there is no possible way to turn a string of length 5 into one of length 7 in edit distance of 1.
104         *
105         * Additionally, this implementation decreases memory usage by using two single-dimensional arrays and swapping them back and forth instead of
106         * allocating an entire n by m matrix. This requires a few minor changes, such as immediately returning when it's detected that the stripe has run off
107         * the matrix and initially filling the arrays with large values so that entries we don't compute are ignored.
108         *
109         * See Algorithms on Strings, Trees and Sequences by Dan Gusfield for some discussion.
110         */
111
112        int n = left.length(); // length of left
113        int m = right.length(); // length of right
114
115        // if one string is empty, the edit distance is necessarily the length
116        // of the other
117        if (n == 0) {
118            return m <= threshold ? m : -1;
119        }
120        if (m == 0) {
121            return n <= threshold ? n : -1;
122        }
123
124        if (n > m) {
125            // swap the two strings to consume less memory
126            final SimilarityInput<E> tmp = left;
127            left = right;
128            right = tmp;
129            n = m;
130            m = right.length();
131        }
132
133        // the edit distance cannot be less than the length difference
134        if (m - n > threshold) {
135            return -1;
136        }
137
138        int[] p = new int[n + 1]; // 'previous' cost array, horizontally
139        int[] d = new int[n + 1]; // cost array, horizontally
140        int[] tempD; // placeholder to assist in swapping p and d
141
142        // fill in starting table values
143        final int boundary = Math.min(n, threshold) + 1;
144        for (int i = 0; i < boundary; i++) {
145            p[i] = i;
146        }
147        // these fills ensure that the value above the rightmost entry of our
148        // stripe will be ignored in following loop iterations
149        Arrays.fill(p, boundary, p.length, Integer.MAX_VALUE);
150        Arrays.fill(d, Integer.MAX_VALUE);
151
152        // iterates through t
153        for (int j = 1; j <= m; j++) {
154            final E rightJ = right.at(j - 1); // jth character of right
155            d[0] = j;
156
157            // compute stripe indices, constrain to array size
158            final int min = Math.max(1, j - threshold);
159            final int max = j > Integer.MAX_VALUE - threshold ? n : Math.min(n, j + threshold);
160
161            // ignore entry left of leftmost
162            if (min > 1) {
163                d[min - 1] = Integer.MAX_VALUE;
164            }
165
166            int lowerBound = Integer.MAX_VALUE;
167            // iterates through [min, max] in s
168            for (int i = min; i <= max; i++) {
169                if (left.at(i - 1).equals(rightJ)) {
170                    // diagonally left and up
171                    d[i] = p[i - 1];
172                } else {
173                    // 1 + minimum of cell to the left, to the top, diagonally
174                    // left and up
175                    d[i] = 1 + Math.min(Math.min(d[i - 1], p[i]), p[i - 1]);
176                }
177                lowerBound = Math.min(lowerBound, d[i]);
178            }
179            // if the lower bound is greater than the threshold, then exit early
180            if (lowerBound > threshold) {
181                return -1;
182            }
183
184            // copy current distance counts to 'previous row' distance counts
185            tempD = p;
186            p = d;
187            d = tempD;
188        }
189
190        // if p[n] is greater than the threshold, there's no guarantee on it
191        // being the correct
192        // distance
193        if (p[n] <= threshold) {
194            return p[n];
195        }
196        return -1;
197    }
198
199    /**
200     * Finds the Levenshtein distance between two Strings.
201     *
202     * <p>
203     * A higher score indicates a greater distance.
204     * </p>
205     *
206     * <p>
207     * This implementation only need one single-dimensional arrays of length s.length() + 1
208     * </p>
209     *
210     * <pre>
211     * unlimitedCompare(null, *)             = Throws {@link IllegalArgumentException}
212     * unlimitedCompare(*, null)             = Throws {@link IllegalArgumentException}
213     * unlimitedCompare("","")               = 0
214     * unlimitedCompare("","a")              = 1
215     * unlimitedCompare("aaapppp", "")       = 7
216     * unlimitedCompare("frog", "fog")       = 1
217     * unlimitedCompare("fly", "ant")        = 3
218     * unlimitedCompare("elephant", "hippo") = 7
219     * unlimitedCompare("hippo", "elephant") = 7
220     * unlimitedCompare("hippo", "zzzzzzzz") = 8
221     * unlimitedCompare("hello", "hallo")    = 1
222     * </pre>
223     *
224     * @param left  the first CharSequence, must not be null.
225     * @param right the second CharSequence, must not be null.
226     * @return result distance, or -1.
227     * @throws IllegalArgumentException if either CharSequence input is {@code null}.
228     */
229    private static <E> int unlimitedCompare(SimilarityInput<E> left, SimilarityInput<E> right) {
230        if (left == null || right == null) {
231            throw new IllegalArgumentException("CharSequences must not be null");
232        }
233        /*
234         * This implementation use two variable to record the previous cost counts, So this implementation use less memory than previous impl.
235         */
236        int n = left.length(); // length of left
237        int m = right.length(); // length of right
238
239        if (n == 0) {
240            return m;
241        }
242        if (m == 0) {
243            return n;
244        }
245        if (n > m) {
246            // swap the input strings to consume less memory
247            final SimilarityInput<E> tmp = left;
248            left = right;
249            right = tmp;
250            n = m;
251            m = right.length();
252        }
253        final int[] p = new int[n + 1];
254        // indexes into strings left and right
255        int i; // iterates through left
256        int j; // iterates through right
257        int upperLeft;
258        int upper;
259        E rightJ; // jth character of right
260        int cost; // cost
261        for (i = 0; i <= n; i++) {
262            p[i] = i;
263        }
264        for (j = 1; j <= m; j++) {
265            upperLeft = p[0];
266            rightJ = right.at(j - 1);
267            p[0] = j;
268
269            for (i = 1; i <= n; i++) {
270                upper = p[i];
271                cost = left.at(i - 1).equals(rightJ) ? 0 : 1;
272                // minimum of cell to the left+1, to the top+1, diagonally left and up +cost
273                p[i] = Math.min(Math.min(p[i - 1] + 1, p[i] + 1), upperLeft + cost);
274                upperLeft = upper;
275            }
276        }
277        return p[n];
278    }
279
280    /**
281     * Threshold.
282     */
283    private final Integer threshold;
284
285    /**
286     * Constructs a default instance that uses a version of the algorithm that does not use a threshold parameter.
287     *
288     * @see LevenshteinDistance#getDefaultInstance()
289     * @deprecated Use {@link #getDefaultInstance()}.
290     */
291    @Deprecated
292    public LevenshteinDistance() {
293        this(null);
294    }
295
296    /**
297     * Constructs a new instance. If the threshold is not null, distance calculations will be limited to a maximum length. If the threshold is null, the
298     * unlimited version of the algorithm will be used.
299     *
300     * @param threshold If this is null then distances calculations will not be limited. This may not be negative.
301     */
302    public LevenshteinDistance(final Integer threshold) {
303        if (threshold != null && threshold < 0) {
304            throw new IllegalArgumentException("Threshold must not be negative");
305        }
306        this.threshold = threshold;
307    }
308
309    /**
310     * Computes the Levenshtein distance between two Strings.
311     *
312     * <p>
313     * A higher score indicates a greater distance.
314     * </p>
315     *
316     * <p>
317     * Chas Emerick has written an implementation in Java, which avoids an OutOfMemoryError which can occur when my Java implementation is used with very large
318     * strings.
319     * </p>
320     *
321     * <pre>
322     * distance.apply(null, *)             = Throws {@link IllegalArgumentException}
323     * distance.apply(*, null)             = Throws {@link IllegalArgumentException}
324     * distance.apply("","")               = 0
325     * distance.apply("","a")              = 1
326     * distance.apply("aaapppp", "")       = 7
327     * distance.apply("frog", "fog")       = 1
328     * distance.apply("fly", "ant")        = 3
329     * distance.apply("elephant", "hippo") = 7
330     * distance.apply("hippo", "elephant") = 7
331     * distance.apply("hippo", "zzzzzzzz") = 8
332     * distance.apply("hello", "hallo")    = 1
333     * </pre>
334     *
335     * @param left  the first input, must not be null.
336     * @param right the second input, must not be null.
337     * @return result distance, or -1.
338     * @throws IllegalArgumentException if either String input {@code null}.
339     */
340    @Override
341    public Integer apply(final CharSequence left, final CharSequence right) {
342        return apply(SimilarityInput.input(left), SimilarityInput.input(right));
343    }
344
345    /**
346     * Computes the Levenshtein distance between two inputs.
347     *
348     * <p>
349     * A higher score indicates a greater distance.
350     * </p>
351     *
352     * <pre>
353     * distance.apply(null, *)             = Throws {@link IllegalArgumentException}
354     * distance.apply(*, null)             = Throws {@link IllegalArgumentException}
355     * distance.apply("","")               = 0
356     * distance.apply("","a")              = 1
357     * distance.apply("aaapppp", "")       = 7
358     * distance.apply("frog", "fog")       = 1
359     * distance.apply("fly", "ant")        = 3
360     * distance.apply("elephant", "hippo") = 7
361     * distance.apply("hippo", "elephant") = 7
362     * distance.apply("hippo", "zzzzzzzz") = 8
363     * distance.apply("hello", "hallo")    = 1
364     * </pre>
365     *
366     * @param <E>   The type of similarity score unit.
367     * @param left  the first input, must not be null.
368     * @param right the second input, must not be null.
369     * @return result distance, or -1.
370     * @throws IllegalArgumentException if either String input {@code null}.
371     * @since 1.13.0
372     */
373    public <E> Integer apply(final SimilarityInput<E> left, final SimilarityInput<E> right) {
374        if (threshold != null) {
375            return limitedCompare(left, right, threshold);
376        }
377        return unlimitedCompare(left, right);
378    }
379
380    /**
381     * Gets the distance threshold.
382     *
383     * @return The distance threshold.
384     */
385    public Integer getThreshold() {
386        return threshold;
387    }
388
389}