View Javadoc
1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one or more
3    * contributor license agreements.  See the NOTICE file distributed with
4    * this work for additional information regarding copyright ownership.
5    * The ASF licenses this file to You under the Apache License, Version 2.0
6    * (the "License"); you may not use this file except in compliance with
7    * the License.  You may obtain a copy of the License at
8    *
9    *      https://www.apache.org/licenses/LICENSE-2.0
10   *
11   * Unless required by applicable law or agreed to in writing, software
12   * distributed under the License is distributed on an "AS IS" BASIS,
13   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14   * See the License for the specific language governing permissions and
15   * limitations under the License.
16   */
17  
18  package org.apache.commons.codec.language.bm;
19  
20  import static org.junit.jupiter.api.Assertions.assertEquals;
21  
22  import java.util.Arrays;
23  import java.util.HashSet;
24  import java.util.Map;
25  import java.util.TreeMap;
26  import java.util.regex.Pattern;
27  
28  import org.junit.jupiter.api.Test;
29  
30  /**
31   * Tests PhoneticEngine and Languages.LanguageSet in ways very similar to code found in solr-3.6.0.
32   */
33  class PhoneticEngineRegressionTest {
34  
35      private static final Pattern COMMA_PATTERN = Pattern.compile(",");
36  
37      /**
38       * This code is similar in style to code found in Solr:
39       * solr/core/src/java/org/apache/solr/analysis/BeiderMorseFilterFactory.java
40       *
41       * Making a JUnit test out of it to protect Solr from possible future
42       * regressions in Commons-Codec.
43       */
44      private static String encode(final Map<String, String> args, final boolean concat, final String input) {
45          // PhoneticEngine = NameType + RuleType + concat
46          // we use common-codec's defaults: GENERIC + APPROX + true
47          final String nameTypeArg = args.get("nameType");
48          final NameType nameType = nameTypeArg == null ? NameType.GENERIC : NameType.valueOf(nameTypeArg);
49  
50          final String ruleTypeArg = args.get("ruleType");
51          final RuleType ruleType = ruleTypeArg == null ? RuleType.APPROX : RuleType.valueOf(ruleTypeArg);
52  
53          // @formatter:off
54          final PhoneticEngine engine = PhoneticEngine.builder()
55              .setNameType(nameType)
56              .setRuleType(ruleType)
57              .setConcat(concat)
58              .get();
59          // @formatter:on
60  
61          // LanguageSet: defaults to automagic, otherwise a comma-separated list.
62          final Languages.LanguageSet languageSet;
63          final String languageSetArg = args.get("languageSet");
64          if (languageSetArg == null || languageSetArg.equals("auto")) {
65              languageSet = null;
66          } else {
67              languageSet = Languages.LanguageSet.from(new HashSet<>(Arrays.asList(COMMA_PATTERN.split(languageSetArg))));
68          }
69  
70          /*
71              org/apache/lucene/analysis/phonetic/BeiderMorseFilter.java (lines 96-98) does this:
72  
73              encoded = (languages == null)
74                  ? engine.encode(termAtt.toString())
75                  : engine.encode(termAtt.toString(), languages);
76  
77              Hence our approach, below:
78          */
79          if (languageSet == null) {
80              return engine.encode(input);
81          }
82          return engine.encode(input, languageSet);
83      }
84  
85      @Test
86      void testCompatibilityWithOriginalVersion() {
87          // see CODEC-187
88          // comparison: https://stevemorse.org/census/soundex.html
89  
90          final Map<String, String> args = new TreeMap<>();
91          args.put("nameType", "GENERIC");
92          args.put("ruleType", "APPROX");
93  
94          assertEquals("Ybram|Ybrom|abram|abran|abrom|abron|avram|avrom|obram|obran|obrom|obron|ovram|ovrom",
95              encode(args, true, "abram"));
96          assertEquals("bndzn|bntsn|bnzn|vndzn|vntsn",
97              encode(args, true, "Bendzin"));
98  
99          args.put("nameType", "ASHKENAZI");
100         args.put("ruleType", "APPROX");
101 
102         assertEquals("Ybram|Ybrom|abram|abrom|avram|avrom|imbram|imbrom|obram|obrom|ombram|ombrom|ovram|ovrom",
103             encode(args, true, "abram"));
104         assertEquals("YlpYrn|Ylpirn|alpYrn|alpirn|olpYrn|olpirn|xalpirn|xolpirn",
105             encode(args, true, "Halpern"));
106 
107     }
108 
109     @Test
110     void testSolrASHKENAZI() {
111         Map<String, String> args;
112 
113         // concat is true, ruleType is EXACT
114         args = new TreeMap<>();
115         args.put("nameType", "ASHKENAZI");
116         assertEquals("YngYlo|Yngilo|angYlo|angilo|anilo|anxilo|anzilo|ongYlo|ongilo|onilo|onxilo|onzilo",
117             encode(args, true, "Angelo"));
118         args.put("ruleType", "EXACT");
119         assertEquals("andZelo|angelo|anhelo|anxelo",
120             encode(args, true, "Angelo"));
121         assertEquals("dandZelo|dangelo|danhelo|danxelo",
122             encode(args, true, "D'Angelo"));
123         args.put("languageSet", "italian,greek,spanish");
124         assertEquals("angelo|anxelo",
125             encode(args, true, "Angelo"));
126         assertEquals(encode(args, true, "1234"), "");
127 
128         // concat is false, ruleType is EXACT
129         args = new TreeMap<>();
130         args.put("nameType", "ASHKENAZI");
131         assertEquals("YngYlo|Yngilo|angYlo|angilo|anilo|anxilo|anzilo|ongYlo|ongilo|onilo|onxilo|onzilo",
132             encode(args, false, "Angelo"));
133         args.put("ruleType", "EXACT");
134         assertEquals("andZelo|angelo|anhelo|anxelo",
135             encode(args, false, "Angelo"));
136         assertEquals("dandZelo|dangelo|danhelo|danxelo",
137             encode(args, false, "D'Angelo"));
138         args.put("languageSet", "italian,greek,spanish");
139         assertEquals("angelo|anxelo",
140             encode(args, false, "Angelo"));
141         assertEquals(encode(args, false, "1234"), "");
142 
143         // concat is true, ruleType is APPROX
144         args = new TreeMap<>();
145         args.put("nameType", "ASHKENAZI");
146         assertEquals("YngYlo|Yngilo|angYlo|angilo|anilo|anxilo|anzilo|ongYlo|ongilo|onilo|onxilo|onzilo",
147             encode(args, true, "Angelo"));
148         args.put("ruleType", "APPROX");
149         assertEquals("YngYlo|Yngilo|angYlo|angilo|anilo|anxilo|anzilo|ongYlo|ongilo|onilo|onxilo|onzilo",
150             encode(args, true, "Angelo"));
151         assertEquals("dYngYlo|dYngilo|dangYlo|dangilo|danilo|danxilo|danzilo|dongYlo|dongilo|donilo|donxilo|donzilo",
152             encode(args, true, "D'Angelo"));
153         args.put("languageSet", "italian,greek,spanish");
154         assertEquals("angilo|anxilo|ongilo|onxilo",
155             encode(args, true, "Angelo"));
156         assertEquals(encode(args, true, "1234"), "");
157 
158         // concat is false, ruleType is APPROX
159         args = new TreeMap<>();
160         args.put("nameType", "ASHKENAZI");
161         assertEquals("YngYlo|Yngilo|angYlo|angilo|anilo|anxilo|anzilo|ongYlo|ongilo|onilo|onxilo|onzilo",
162             encode(args, false, "Angelo"));
163         args.put("ruleType", "APPROX");
164         assertEquals("YngYlo|Yngilo|angYlo|angilo|anilo|anxilo|anzilo|ongYlo|ongilo|onilo|onxilo|onzilo",
165             encode(args, false, "Angelo"));
166         assertEquals("dYngYlo|dYngilo|dangYlo|dangilo|danilo|danxilo|danzilo|dongYlo|dongilo|donilo|donxilo|donzilo",
167             encode(args, false, "D'Angelo"));
168         args.put("languageSet", "italian,greek,spanish");
169         assertEquals("angilo|anxilo|ongilo|onxilo",
170             encode(args, false, "Angelo"));
171         assertEquals(encode(args, false, "1234"), "");
172     }
173 
174     @Test
175     void testSolrGENERIC() {
176         Map<String, String> args;
177 
178         // concat is true, ruleType is EXACT
179         args = new TreeMap<>();
180         args.put("nameType", "GENERIC");
181         assertEquals("YngYlo|Yngilo|agilo|angYlo|angilo|aniilo|anilo|anxilo|anzilo|ogilo|ongYlo|ongilo|oniilo|onilo|onxilo|onzilo",
182             encode(args, true, "Angelo"));
183         args.put("ruleType", "EXACT");
184         assertEquals("anZelo|andZelo|angelo|anhelo|anjelo|anxelo",
185             encode(args, true, "Angelo"));
186         assertEquals("(anZelo|andZelo|angelo|anhelo|anjelo|anxelo)-(danZelo|dandZelo|dangelo|danhelo|danjelo|danxelo)",
187             encode(args, true, "D'Angelo"));
188         args.put("languageSet", "italian,greek,spanish");
189         assertEquals("andZelo|angelo|anxelo",
190             encode(args, true, "Angelo"));
191         assertEquals(encode(args, true, "1234"), "");
192 
193         // concat is false, ruleType is EXACT
194         args = new TreeMap<>();
195         assertEquals("YngYlo|Yngilo|agilo|angYlo|angilo|aniilo|anilo|anxilo|anzilo|ogilo|ongYlo|ongilo|oniilo|onilo|onxilo|onzilo",
196             encode(args, false, "Angelo"));
197         args.put("ruleType", "EXACT");
198         assertEquals("anZelo|andZelo|angelo|anhelo|anjelo|anxelo",
199             encode(args, false, "Angelo"));
200         assertEquals("(anZelo|andZelo|angelo|anhelo|anjelo|anxelo)-(danZelo|dandZelo|dangelo|danhelo|danjelo|danxelo)",
201             encode(args, false, "D'Angelo"));
202         args.put("languageSet", "italian,greek,spanish");
203         assertEquals("andZelo|angelo|anxelo",
204             encode(args, false, "Angelo"));
205         assertEquals(encode(args, false, "1234"), "");
206 
207         // concat is true, ruleType is APPROX
208         args = new TreeMap<>();
209         assertEquals("YngYlo|Yngilo|agilo|angYlo|angilo|aniilo|anilo|anxilo|anzilo|ogilo|ongYlo|ongilo|oniilo|onilo|onxilo|onzilo",
210             encode(args, true, "Angelo"));
211         args.put("ruleType", "APPROX");
212         assertEquals("YngYlo|Yngilo|agilo|angYlo|angilo|aniilo|anilo|anxilo|anzilo|ogilo|ongYlo|ongilo|oniilo|onilo|onxilo|onzilo",
213             encode(args, true, "Angelo"));
214         assertEquals("(YngYlo|Yngilo|agilo|angYlo|angilo|aniilo|anilo|anxilo|anzilo|ogilo|ongYlo|ongilo|oniilo|onilo|onxilo|onzilo)-(dYngYlo|dYngilo|dagilo|dangYlo|dangilo|daniilo|danilo|danxilo|danzilo|dogilo|dongYlo|dongilo|doniilo|donilo|donxilo|donzilo)",
215             encode(args, true, "D'Angelo"));
216         args.put("languageSet", "italian,greek,spanish");
217         assertEquals("angilo|anxilo|anzilo|ongilo|onxilo|onzilo",
218             encode(args, true, "Angelo"));
219         assertEquals(encode(args, true, "1234"), "");
220 
221         // concat is false, ruleType is APPROX
222         args = new TreeMap<>();
223         assertEquals("YngYlo|Yngilo|agilo|angYlo|angilo|aniilo|anilo|anxilo|anzilo|ogilo|ongYlo|ongilo|oniilo|onilo|onxilo|onzilo",
224             encode(args, false, "Angelo"));
225         args.put("ruleType", "APPROX");
226         assertEquals("YngYlo|Yngilo|agilo|angYlo|angilo|aniilo|anilo|anxilo|anzilo|ogilo|ongYlo|ongilo|oniilo|onilo|onxilo|onzilo",
227             encode(args, false, "Angelo"));
228         assertEquals("(YngYlo|Yngilo|agilo|angYlo|angilo|aniilo|anilo|anxilo|anzilo|ogilo|ongYlo|ongilo|oniilo|onilo|onxilo|onzilo)-(dYngYlo|dYngilo|dagilo|dangYlo|dangilo|daniilo|danilo|danxilo|danzilo|dogilo|dongYlo|dongilo|doniilo|donilo|donxilo|donzilo)",
229             encode(args, false, "D'Angelo"));
230         args.put("languageSet", "italian,greek,spanish");
231         assertEquals("angilo|anxilo|anzilo|ongilo|onxilo|onzilo",
232             encode(args, false, "Angelo"));
233         assertEquals(encode(args, false, "1234"), "");
234     }
235 
236     @Test
237     void testSolrSEPHARDIC() {
238         Map<String, String> args;
239 
240         // concat is true, ruleType is EXACT
241         args = new TreeMap<>();
242         args.put("nameType", "SEPHARDIC");
243         assertEquals("anhila|anhilu|anzila|anzilu|nhila|nhilu|nzila|nzilu",
244             encode(args, true, "Angelo"));
245         args.put("ruleType", "EXACT");
246         assertEquals("anZelo|andZelo|anxelo",
247             encode(args, true, "Angelo"));
248         assertEquals("anZelo|andZelo|anxelo",
249             encode(args, true, "D'Angelo"));
250         args.put("languageSet", "italian,greek,spanish");
251         assertEquals("andZelo|anxelo",
252             encode(args, true, "Angelo"));
253         assertEquals(encode(args, true, "1234"), "");
254 
255         // concat is false, ruleType is EXACT
256         args = new TreeMap<>();
257         args.put("nameType", "SEPHARDIC");
258         assertEquals("anhila|anhilu|anzila|anzilu|nhila|nhilu|nzila|nzilu",
259             encode(args, false, "Angelo"));
260         args.put("ruleType", "EXACT");
261         assertEquals("anZelo|andZelo|anxelo",
262             encode(args, false, "Angelo"));
263         assertEquals("danZelo|dandZelo|danxelo",
264             encode(args, false, "D'Angelo"));
265         args.put("languageSet", "italian,greek,spanish");
266         assertEquals("andZelo|anxelo",
267             encode(args, false, "Angelo"));
268         assertEquals(encode(args, false, "1234"), "");
269 
270         // concat is true, ruleType is APPROX
271         args = new TreeMap<>();
272         args.put("nameType", "SEPHARDIC");
273         assertEquals("anhila|anhilu|anzila|anzilu|nhila|nhilu|nzila|nzilu",
274             encode(args, true, "Angelo"));
275         args.put("ruleType", "APPROX");
276         assertEquals("anhila|anhilu|anzila|anzilu|nhila|nhilu|nzila|nzilu",
277             encode(args, true, "Angelo"));
278         assertEquals("anhila|anhilu|anzila|anzilu|nhila|nhilu|nzila|nzilu",
279             encode(args, true, "D'Angelo"));
280         args.put("languageSet", "italian,greek,spanish");
281         assertEquals("anhila|anhilu|anzila|anzilu|nhila|nhilu|nzila|nzilu",
282             encode(args, true, "Angelo"));
283         assertEquals(encode(args, true, "1234"), "");
284 
285         // concat is false, ruleType is APPROX
286         args = new TreeMap<>();
287         args.put("nameType", "SEPHARDIC");
288         assertEquals("anhila|anhilu|anzila|anzilu|nhila|nhilu|nzila|nzilu",
289             encode(args, false, "Angelo"));
290         args.put("ruleType", "APPROX");
291         assertEquals("anhila|anhilu|anzila|anzilu|nhila|nhilu|nzila|nzilu",
292             encode(args, false, "Angelo"));
293         assertEquals("danhila|danhilu|danzila|danzilu|nhila|nhilu|nzila|nzilu",
294             encode(args, false, "D'Angelo"));
295         args.put("languageSet", "italian,greek,spanish");
296         assertEquals("anhila|anhilu|anzila|anzilu|nhila|nhilu|nzila|nzilu",
297             encode(args, false, "Angelo"));
298         assertEquals(encode(args, false, "1234"), "");
299     }
300 }