001/*
002 * Licensed to the Apache Software Foundation (ASF) under one or more
003 * contributor license agreements.  See the NOTICE file distributed with
004 * this work for additional information regarding copyright ownership.
005 * The ASF licenses this file to You under the Apache License, Version 2.0
006 * (the "License"); you may not use this file except in compliance with
007 * the License.  You may obtain a copy of the License at
008 *
009 *      http://www.apache.org/licenses/LICENSE-2.0
010 *
011 * Unless required by applicable law or agreed to in writing, software
012 * distributed under the License is distributed on an "AS IS" BASIS,
013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
014 * See the License for the specific language governing permissions and
015 * limitations under the License.
016 */
017package org.apache.commons.text.similarity;
018
019import java.util.Arrays;
020
021/**
022 * An algorithm for measuring the difference between two character sequences.
023 *
024 * <p>
025 * This is the number of changes needed to change one sequence into another,
026 * where each change is a single character modification (deletion, insertion
027 * or substitution).
028 * </p>
029 *
030 * <p>
031 * This code has been adapted from Apache Commons Lang 3.3.
032 * </p>
033 *
034 * @since 1.0
035 */
036public class LevenshteinDistance implements EditDistance<Integer> {
037
038    /**
039     * Singleton instance.
040     */
041    private static final LevenshteinDistance INSTANCE = new LevenshteinDistance();
042
043    /**
044     * Gets the default instance.
045     *
046     * @return The default instance
047     */
048    public static LevenshteinDistance getDefaultInstance() {
049        return INSTANCE;
050    }
051
052    /**
053     * Find the Levenshtein distance between two CharSequences if it's less than or
054     * equal to a given threshold.
055     *
056     * <p>
057     * This implementation follows from Algorithms on Strings, Trees and
058     * Sequences by Dan Gusfield and Chas Emerick's implementation of the
059     * Levenshtein distance algorithm from <a
060     * href="http://www.merriampark.com/ld.htm"
061     * >http://www.merriampark.com/ld.htm</a>
062     * </p>
063     *
064     * <pre>
065     * limitedCompare(null, *, *)             = IllegalArgumentException
066     * limitedCompare(*, null, *)             = IllegalArgumentException
067     * limitedCompare(*, *, -1)               = IllegalArgumentException
068     * limitedCompare("","", 0)               = 0
069     * limitedCompare("aaapppp", "", 8)       = 7
070     * limitedCompare("aaapppp", "", 7)       = 7
071     * limitedCompare("aaapppp", "", 6))      = -1
072     * limitedCompare("elephant", "hippo", 7) = 7
073     * limitedCompare("elephant", "hippo", 6) = -1
074     * limitedCompare("hippo", "elephant", 7) = 7
075     * limitedCompare("hippo", "elephant", 6) = -1
076     * </pre>
077     *
078     * @param left the first CharSequence, must not be null
079     * @param right the second CharSequence, must not be null
080     * @param threshold the target threshold, must not be negative
081     * @return result distance, or -1
082     */
083    private static int limitedCompare(CharSequence left, CharSequence right, final int threshold) { // NOPMD
084        if (left == null || right == null) {
085            throw new IllegalArgumentException("CharSequences must not be null");
086        }
087        if (threshold < 0) {
088            throw new IllegalArgumentException("Threshold must not be negative");
089        }
090
091        /*
092         * This implementation only computes the distance if it's less than or
093         * equal to the threshold value, returning -1 if it's greater. The
094         * advantage is performance: unbounded distance is O(nm), but a bound of
095         * k allows us to reduce it to O(km) time by only computing a diagonal
096         * stripe of width 2k + 1 of the cost table. It is also possible to use
097         * this to compute the unbounded Levenshtein distance by starting the
098         * threshold at 1 and doubling each time until the distance is found;
099         * this is O(dm), where d is the distance.
100         *
101         * One subtlety comes from needing to ignore entries on the border of
102         * our stripe eg. p[] = |#|#|#|* d[] = *|#|#|#| We must ignore the entry
103         * to the left of the leftmost member We must ignore the entry above the
104         * rightmost member
105         *
106         * Another subtlety comes from our stripe running off the matrix if the
107         * strings aren't of the same size. Since string s is always swapped to
108         * be the shorter of the two, the stripe will always run off to the
109         * upper right instead of the lower left of the matrix.
110         *
111         * As a concrete example, suppose s is of length 5, t is of length 7,
112         * and our threshold is 1. In this case we're going to walk a stripe of
113         * length 3. The matrix would look like so:
114         *
115         * <pre>
116         *    1 2 3 4 5
117         * 1 |#|#| | | |
118         * 2 |#|#|#| | |
119         * 3 | |#|#|#| |
120         * 4 | | |#|#|#|
121         * 5 | | | |#|#|
122         * 6 | | | | |#|
123         * 7 | | | | | |
124         * </pre>
125         *
126         * Note how the stripe leads off the table as there is no possible way
127         * to turn a string of length 5 into one of length 7 in edit distance of
128         * 1.
129         *
130         * Additionally, this implementation decreases memory usage by using two
131         * single-dimensional arrays and swapping them back and forth instead of
132         * allocating an entire n by m matrix. This requires a few minor
133         * changes, such as immediately returning when it's detected that the
134         * stripe has run off the matrix and initially filling the arrays with
135         * large values so that entries we don't compute are ignored.
136         *
137         * See Algorithms on Strings, Trees and Sequences by Dan Gusfield for
138         * some discussion.
139         */
140
141        int n = left.length(); // length of left
142        int m = right.length(); // length of right
143
144        // if one string is empty, the edit distance is necessarily the length
145        // of the other
146        if (n == 0) {
147            return m <= threshold ? m : -1;
148        }
149        if (m == 0) {
150            return n <= threshold ? n : -1;
151        }
152
153        if (n > m) {
154            // swap the two strings to consume less memory
155            final CharSequence tmp = left;
156            left = right;
157            right = tmp;
158            n = m;
159            m = right.length();
160        }
161
162        // the edit distance cannot be less than the length difference
163        if (m - n > threshold) {
164            return -1;
165        }
166
167        int[] p = new int[n + 1]; // 'previous' cost array, horizontally
168        int[] d = new int[n + 1]; // cost array, horizontally
169        int[] tempD; // placeholder to assist in swapping p and d
170
171        // fill in starting table values
172        final int boundary = Math.min(n, threshold) + 1;
173        for (int i = 0; i < boundary; i++) {
174            p[i] = i;
175        }
176        // these fills ensure that the value above the rightmost entry of our
177        // stripe will be ignored in following loop iterations
178        Arrays.fill(p, boundary, p.length, Integer.MAX_VALUE);
179        Arrays.fill(d, Integer.MAX_VALUE);
180
181        // iterates through t
182        for (int j = 1; j <= m; j++) {
183            final char rightJ = right.charAt(j - 1); // jth character of right
184            d[0] = j;
185
186            // compute stripe indices, constrain to array size
187            final int min = Math.max(1, j - threshold);
188            final int max = j > Integer.MAX_VALUE - threshold ? n : Math.min(
189                    n, j + threshold);
190
191            // ignore entry left of leftmost
192            if (min > 1) {
193                d[min - 1] = Integer.MAX_VALUE;
194            }
195
196            int lowerBound = Integer.MAX_VALUE;
197            // iterates through [min, max] in s
198            for (int i = min; i <= max; i++) {
199                if (left.charAt(i - 1) == rightJ) {
200                    // diagonally left and up
201                    d[i] = p[i - 1];
202                } else {
203                    // 1 + minimum of cell to the left, to the top, diagonally
204                    // left and up
205                    d[i] = 1 + Math.min(Math.min(d[i - 1], p[i]), p[i - 1]);
206                }
207                lowerBound = Math.min(lowerBound, d[i]);
208            }
209            // if the lower bound is greater than the threshold, then exit early
210            if (lowerBound > threshold) {
211                return -1;
212            }
213
214            // copy current distance counts to 'previous row' distance counts
215            tempD = p;
216            p = d;
217            d = tempD;
218        }
219
220        // if p[n] is greater than the threshold, there's no guarantee on it
221        // being the correct
222        // distance
223        if (p[n] <= threshold) {
224            return p[n];
225        }
226        return -1;
227    }
228
229    /**
230     * Finds the Levenshtein distance between two Strings.
231     *
232     * <p>A higher score indicates a greater distance.</p>
233     *
234     * <p>The previous implementation of the Levenshtein distance algorithm
235     * was from <a href="https://web.archive.org/web/20120526085419/http://www.merriampark.com/ldjava.htm">
236     * https://web.archive.org/web/20120526085419/http://www.merriampark.com/ldjava.htm</a></p>
237     *
238     * <p>This implementation only need one single-dimensional arrays of length s.length() + 1</p>
239     *
240     * <pre>
241     * unlimitedCompare(null, *)             = IllegalArgumentException
242     * unlimitedCompare(*, null)             = IllegalArgumentException
243     * unlimitedCompare("","")               = 0
244     * unlimitedCompare("","a")              = 1
245     * unlimitedCompare("aaapppp", "")       = 7
246     * unlimitedCompare("frog", "fog")       = 1
247     * unlimitedCompare("fly", "ant")        = 3
248     * unlimitedCompare("elephant", "hippo") = 7
249     * unlimitedCompare("hippo", "elephant") = 7
250     * unlimitedCompare("hippo", "zzzzzzzz") = 8
251     * unlimitedCompare("hello", "hallo")    = 1
252     * </pre>
253     *
254     * @param left the first CharSequence, must not be null
255     * @param right the second CharSequence, must not be null
256     * @return result distance, or -1
257     * @throws IllegalArgumentException if either CharSequence input is {@code null}
258     */
259    private static int unlimitedCompare(CharSequence left, CharSequence right) {
260        if (left == null || right == null) {
261            throw new IllegalArgumentException("CharSequences must not be null");
262        }
263
264        /*
265           This implementation use two variable to record the previous cost counts,
266           So this implementation use less memory than previous impl.
267         */
268
269        int n = left.length(); // length of left
270        int m = right.length(); // length of right
271
272        if (n == 0) {
273            return m;
274        }
275        if (m == 0) {
276            return n;
277        }
278
279        if (n > m) {
280            // swap the input strings to consume less memory
281            final CharSequence tmp = left;
282            left = right;
283            right = tmp;
284            n = m;
285            m = right.length();
286        }
287
288        final int[] p = new int[n + 1];
289
290        // indexes into strings left and right
291        int i; // iterates through left
292        int j; // iterates through right
293        int upperLeft;
294        int upper;
295
296        char rightJ; // jth character of right
297        int cost; // cost
298
299        for (i = 0; i <= n; i++) {
300            p[i] = i;
301        }
302
303        for (j = 1; j <= m; j++) {
304            upperLeft = p[0];
305            rightJ = right.charAt(j - 1);
306            p[0] = j;
307
308            for (i = 1; i <= n; i++) {
309                upper = p[i];
310                cost = left.charAt(i - 1) == rightJ ? 0 : 1;
311                // minimum of cell to the left+1, to the top+1, diagonally left and up +cost
312                p[i] = Math.min(Math.min(p[i - 1] + 1, p[i] + 1), upperLeft + cost);
313                upperLeft = upper;
314            }
315        }
316
317        return p[n];
318    }
319
320    /**
321     * Threshold.
322     */
323    private final Integer threshold;
324
325    /**
326     * This returns the default instance that uses a version
327     * of the algorithm that does not use a threshold parameter.
328     *
329     * @see LevenshteinDistance#getDefaultInstance()
330     */
331    public LevenshteinDistance() {
332        this(null);
333    }
334
335    /**
336     * If the threshold is not null, distance calculations will be limited to a maximum length.
337     * If the threshold is null, the unlimited version of the algorithm will be used.
338     *
339     * @param threshold
340     *        If this is null then distances calculations will not be limited.
341     *        This may not be negative.
342     */
343    public LevenshteinDistance(final Integer threshold) {
344        if (threshold != null && threshold < 0) {
345            throw new IllegalArgumentException("Threshold must not be negative");
346        }
347        this.threshold = threshold;
348    }
349
350    /**
351     * Finds the Levenshtein distance between two Strings.
352     *
353     * <p>A higher score indicates a greater distance.</p>
354     *
355     * <p>The previous implementation of the Levenshtein distance algorithm
356     * was from <a href="http://www.merriampark.com/ld.htm">http://www.merriampark.com/ld.htm</a></p>
357     *
358     * <p>Chas Emerick has written an implementation in Java, which avoids an OutOfMemoryError
359     * which can occur when my Java implementation is used with very large strings.<br>
360     * This implementation of the Levenshtein distance algorithm
361     * is from <a href="http://www.merriampark.com/ldjava.htm">http://www.merriampark.com/ldjava.htm</a></p>
362     *
363     * <pre>
364     * distance.apply(null, *)             = IllegalArgumentException
365     * distance.apply(*, null)             = IllegalArgumentException
366     * distance.apply("","")               = 0
367     * distance.apply("","a")              = 1
368     * distance.apply("aaapppp", "")       = 7
369     * distance.apply("frog", "fog")       = 1
370     * distance.apply("fly", "ant")        = 3
371     * distance.apply("elephant", "hippo") = 7
372     * distance.apply("hippo", "elephant") = 7
373     * distance.apply("hippo", "zzzzzzzz") = 8
374     * distance.apply("hello", "hallo")    = 1
375     * </pre>
376     *
377     * @param left the first string, must not be null
378     * @param right the second string, must not be null
379     * @return result distance, or -1
380     * @throws IllegalArgumentException if either String input {@code null}
381     */
382    @Override
383    public Integer apply(final CharSequence left, final CharSequence right) {
384        if (threshold != null) {
385            return limitedCompare(left, right, threshold);
386        }
387        return unlimitedCompare(left, right);
388    }
389
390    /**
391     * Gets the distance threshold.
392     *
393     * @return The distance threshold
394     */
395    public Integer getThreshold() {
396        return threshold;
397    }
398
399}