001/* 002 * Licensed to the Apache Software Foundation (ASF) under one or more 003 * contributor license agreements. See the NOTICE file distributed with 004 * this work for additional information regarding copyright ownership. 005 * The ASF licenses this file to You under the Apache License, Version 2.0 006 * (the "License"); you may not use this file except in compliance with 007 * the License. You may obtain a copy of the License at 008 * 009 * https://www.apache.org/licenses/LICENSE-2.0 010 * 011 * Unless required by applicable law or agreed to in writing, software 012 * distributed under the License is distributed on an "AS IS" BASIS, 013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 014 * See the License for the specific language governing permissions and 015 * limitations under the License. 016 */ 017 018package org.apache.commons.codec.language.bm; 019 020import java.util.ArrayList; 021import java.util.Arrays; 022import java.util.Collections; 023import java.util.EnumMap; 024import java.util.HashSet; 025import java.util.LinkedHashSet; 026import java.util.List; 027import java.util.Locale; 028import java.util.Map; 029import java.util.Objects; 030import java.util.Set; 031import java.util.TreeMap; 032import java.util.regex.Pattern; 033import java.util.stream.Collectors; 034 035import org.apache.commons.codec.language.bm.Languages.LanguageSet; 036import org.apache.commons.codec.language.bm.Rule.Phoneme; 037 038/** 039 * Converts words into potential phonetic representations. 040 * <p> 041 * This is a two-stage process. Firstly, the word is converted into a phonetic representation that takes 042 * into account the likely source language. Next, this phonetic representation is converted into a 043 * pan-European 'average' representation, allowing comparison between different versions of essentially 044 * the same word from different languages. 045 * </p> 046 * <p> 047 * This class is intentionally immutable and thread-safe. 048 * If you wish to alter the settings for a PhoneticEngine, you 049 * must make a new one with the updated settings. 050 * </p> 051 * <p> 052 * Ported from phoneticengine.php 053 * </p> 054 * 055 * @since 1.6 056 */ 057public class PhoneticEngine { 058 059 /** 060 * Utility for manipulating a set of phonemes as they are being built up. Not intended for use outside 061 * this package, and probably not outside the {@link PhoneticEngine} class. 062 * 063 * @since 1.6 064 */ 065 static final class PhonemeBuilder { 066 067 /** 068 * An empty builder where all phonemes must come from some set of languages. This will contain a single 069 * phoneme of zero characters. This can then be appended to. This should be the only way to create a new 070 * phoneme from scratch. 071 * 072 * @param languages the set of languages 073 * @return a new, empty phoneme builder 074 */ 075 public static PhonemeBuilder empty(final Languages.LanguageSet languages) { 076 return new PhonemeBuilder(new Rule.Phoneme("", languages)); 077 } 078 079 private final Set<Rule.Phoneme> phonemes; 080 081 private PhonemeBuilder(final Rule.Phoneme phoneme) { 082 this.phonemes = new LinkedHashSet<>(); 083 this.phonemes.add(phoneme); 084 } 085 086 private PhonemeBuilder(final Set<Rule.Phoneme> phonemes) { 087 this.phonemes = phonemes; 088 } 089 090 /** 091 * Creates a new phoneme builder containing all phonemes in this one extended by {@code str}. 092 * 093 * @param str the characters to append to the phonemes 094 */ 095 public void append(final CharSequence str) { 096 phonemes.forEach(ph -> ph.append(str)); 097 } 098 099 /** 100 * Applies the given phoneme expression to all phonemes in this phoneme builder. 101 * <p> 102 * This will lengthen phonemes that have compatible language sets to the expression, and drop those that are 103 * incompatible. 104 * </p> 105 * 106 * @param phonemeExpr the expression to apply 107 * @param maxPhonemes the maximum number of phonemes to build up 108 */ 109 public void apply(final Rule.PhonemeExpr phonemeExpr, final int maxPhonemes) { 110 final Set<Rule.Phoneme> newPhonemes = new LinkedHashSet<>(Math.min(phonemes.size() * phonemeExpr.size(), maxPhonemes)); 111 EXPR: for (final Rule.Phoneme left : phonemes) { 112 for (final Rule.Phoneme right : phonemeExpr.getPhonemes()) { 113 final LanguageSet languages = left.getLanguages().restrictTo(right.getLanguages()); 114 if (!languages.isEmpty()) { 115 final Rule.Phoneme join = new Phoneme(left, right, languages); 116 if (newPhonemes.size() < maxPhonemes) { 117 newPhonemes.add(join); 118 if (newPhonemes.size() >= maxPhonemes) { 119 break EXPR; 120 } 121 } 122 } 123 } 124 } 125 phonemes.clear(); 126 phonemes.addAll(newPhonemes); 127 } 128 129 /** 130 * Gets underlying phoneme set. Please don't mutate. 131 * 132 * @return the phoneme set 133 */ 134 public Set<Rule.Phoneme> getPhonemes() { 135 return phonemes; 136 } 137 138 /** 139 * Stringifies the phoneme set. This produces a single string of the strings of each phoneme, 140 * joined with a pipe. This is explicitly provided in place of toString as it is a potentially 141 * expensive operation, which should be avoided when debugging. 142 * 143 * @return the stringified phoneme set 144 */ 145 public String makeString() { 146 return phonemes.stream().map(Rule.Phoneme::getPhonemeText).collect(Collectors.joining("|")); 147 } 148 } 149 150 /** 151 * A function closure capturing the application of a list of rules to an input sequence at a particular offset. 152 * After invocation, the values {@code i} and {@code found} are updated. {@code i} points to the 153 * index of the next char in {@code input} that must be processed next (the input up to that index having been 154 * processed already), and {@code found} indicates if a matching rule was found or not. In the case where a 155 * matching rule was found, {@code phonemeBuilder} is replaced with a new builder containing the phonemes 156 * updated by the matching rule. 157 * <p> 158 * Although this class is not thread-safe (it has mutable unprotected fields), it is not shared between threads 159 * as it is constructed as needed by the calling methods. 160 * </p> 161 * 162 * @since 1.6 163 */ 164 private static final class RulesApplication { 165 166 private final Map<String, List<Rule>> finalRules; 167 private final CharSequence input; 168 private final PhonemeBuilder phonemeBuilder; 169 private int i; 170 private final int maxPhonemes; 171 private boolean found; 172 173 RulesApplication(final Map<String, List<Rule>> finalRules, final CharSequence input, final PhonemeBuilder phonemeBuilder, final int i, 174 final int maxPhonemes) { 175 Objects.requireNonNull(finalRules, "finalRules"); 176 this.finalRules = finalRules; 177 this.phonemeBuilder = phonemeBuilder; 178 this.input = input; 179 this.i = i; 180 this.maxPhonemes = maxPhonemes; 181 } 182 183 public int getI() { 184 return i; 185 } 186 187 public PhonemeBuilder getPhonemeBuilder() { 188 return phonemeBuilder; 189 } 190 191 /** 192 * Invokes the rules. Loops over the rules list, stopping at the first one that has a matching context 193 * and pattern. Then applies this rule to the phoneme builder to produce updated phonemes. If there was no 194 * match, {@code i} is advanced one and the character is silently dropped from the phonetic spelling. 195 * 196 * @return {@code this} 197 */ 198 public RulesApplication invoke() { 199 found = false; 200 int patternLength = 1; 201 final List<Rule> rules = finalRules.get(input.subSequence(i, i + patternLength)); 202 if (rules != null) { 203 for (final Rule rule : rules) { 204 final String pattern = rule.getPattern(); 205 patternLength = pattern.length(); 206 if (rule.patternAndContextMatches(input, i)) { 207 phonemeBuilder.apply(rule.getPhoneme(), maxPhonemes); 208 found = true; 209 break; 210 } 211 } 212 } 213 214 if (!found) { 215 patternLength = 1; 216 } 217 218 i += patternLength; 219 return this; 220 } 221 222 public boolean isFound() { 223 return found; 224 } 225 } 226 227 private static final int DEFAULT_MAX_PHONEMES = 20; 228 229 private static final Map<NameType, Set<String>> NAME_PREFIXES = new EnumMap<>(NameType.class); 230 231 private static final Pattern QUOTE = Pattern.compile("'"); 232 233 static { 234 NAME_PREFIXES.put(NameType.ASHKENAZI, 235 Collections.unmodifiableSet( 236 new HashSet<>(Arrays.asList("bar", "ben", "da", "de", "van", "von")))); 237 NAME_PREFIXES.put(NameType.SEPHARDIC, 238 Collections.unmodifiableSet( 239 new HashSet<>(Arrays.asList("al", "el", "da", "dal", "de", "del", "dela", "de la", 240 "della", "des", "di", "do", "dos", "du", "van", "von")))); 241 NAME_PREFIXES.put(NameType.GENERIC, 242 Collections.unmodifiableSet( 243 new HashSet<>(Arrays.asList("da", "dal", "de", "del", "dela", "de la", "della", 244 "des", "di", "do", "dos", "du", "van", "von")))); 245 } 246 247 /** 248 * Joins some strings with an internal separator. 249 * 250 * @param strings Strings to join 251 * @param sep String to separate them with 252 * @return a single String consisting of each element of {@code strings} interleaved by {@code sep} 253 */ 254 private static String join(final List<String> strings, final String sep) { 255 return strings.stream().collect(Collectors.joining(sep)); 256 } 257 258 private final Lang lang; 259 260 private final NameType nameType; 261 262 private final RuleType ruleType; 263 264 private final boolean concat; 265 266 private final int maxPhonemes; 267 268 /** 269 * Generates a new, fully-configured phonetic engine. 270 * 271 * @param nameType 272 * the type of names it will use 273 * @param ruleType 274 * the type of rules it will apply 275 * @param concatenate 276 * if it will concatenate multiple encodings 277 */ 278 public PhoneticEngine(final NameType nameType, final RuleType ruleType, final boolean concatenate) { 279 this(nameType, ruleType, concatenate, DEFAULT_MAX_PHONEMES); 280 } 281 282 /** 283 * Generates a new, fully-configured phonetic engine. 284 * 285 * @param nameType 286 * the type of names it will use 287 * @param ruleType 288 * the type of rules it will apply 289 * @param concatenate 290 * if it will concatenate multiple encodings 291 * @param maxPhonemes 292 * the maximum number of phonemes that will be handled 293 * @since 1.7 294 */ 295 public PhoneticEngine(final NameType nameType, final RuleType ruleType, final boolean concatenate, final int maxPhonemes) { 296 if (ruleType == RuleType.RULES) { 297 throw new IllegalArgumentException("ruleType must not be " + RuleType.RULES); 298 } 299 this.nameType = nameType; 300 this.ruleType = ruleType; 301 this.concat = concatenate; 302 this.lang = Lang.instance(nameType); 303 this.maxPhonemes = maxPhonemes; 304 } 305 306 /** 307 * Applies the final rules to convert from a language-specific phonetic representation to a 308 * language-independent representation. 309 * 310 * @param phonemeBuilder the current phonemes 311 * @param finalRules the final rules to apply 312 * @return the resulting phonemes 313 */ 314 private PhonemeBuilder applyFinalRules(final PhonemeBuilder phonemeBuilder, 315 final Map<String, List<Rule>> finalRules) { 316 Objects.requireNonNull(finalRules, "finalRules"); 317 if (finalRules.isEmpty()) { 318 return phonemeBuilder; 319 } 320 321 final Map<Rule.Phoneme, Rule.Phoneme> phonemes = new TreeMap<>(Rule.Phoneme.COMPARATOR); 322 323 phonemeBuilder.getPhonemes().forEach(phoneme -> { 324 PhonemeBuilder subBuilder = PhonemeBuilder.empty(phoneme.getLanguages()); 325 final CharSequence phonemeText = phoneme.getPhonemeText(); 326 327 for (int i = 0; i < phonemeText.length();) { 328 final RulesApplication rulesApplication = new RulesApplication(finalRules, phonemeText, subBuilder, i, maxPhonemes).invoke(); 329 final boolean found = rulesApplication.isFound(); 330 subBuilder = rulesApplication.getPhonemeBuilder(); 331 332 if (!found) { 333 // not found, appending as-is 334 subBuilder.append(phonemeText.subSequence(i, i + 1)); 335 } 336 337 i = rulesApplication.getI(); 338 } 339 340 // the phonemes map orders the phonemes only based on their text, but ignores the language set 341 // when adding new phonemes, check for equal phonemes and merge their language set, otherwise 342 // phonemes with the same text but different language set get lost 343 subBuilder.getPhonemes().forEach(newPhoneme -> { 344 if (phonemes.containsKey(newPhoneme)) { 345 final Rule.Phoneme oldPhoneme = phonemes.remove(newPhoneme); 346 final Rule.Phoneme mergedPhoneme = oldPhoneme.mergeWithLanguage(newPhoneme.getLanguages()); 347 phonemes.put(mergedPhoneme, mergedPhoneme); 348 } else { 349 phonemes.put(newPhoneme, newPhoneme); 350 } 351 }); 352 }); 353 354 return new PhonemeBuilder(phonemes.keySet()); 355 } 356 357 /** 358 * Encodes a string to its phonetic representation. 359 * 360 * @param input 361 * the String to encode 362 * @return the encoding of the input 363 */ 364 public String encode(final String input) { 365 final Languages.LanguageSet languageSet = this.lang.guessLanguages(input); 366 return encode(input, languageSet); 367 } 368 369 /** 370 * Encodes an input string into an output phonetic representation, given a set of possible origin languages. 371 * 372 * @param input 373 * String to phoneticise; a String with dashes or spaces separating each word 374 * @param languageSet 375 * set of possible origin languages 376 * @return a phonetic representation of the input; a String containing '-'-separated phonetic representations of the 377 * input 378 */ 379 public String encode(String input, final Languages.LanguageSet languageSet) { 380 final Map<String, List<Rule>> rules = Rule.getInstanceMap(this.nameType, RuleType.RULES, languageSet); 381 // rules common across many (all) languages 382 final Map<String, List<Rule>> finalRules1 = Rule.getInstanceMap(this.nameType, this.ruleType, "common"); 383 // rules that apply to a specific language that may be ambiguous or wrong if applied to other languages 384 final Map<String, List<Rule>> finalRules2 = Rule.getInstanceMap(this.nameType, this.ruleType, languageSet); 385 386 // tidy the input 387 // lower case is a locale-dependent operation 388 input = input.toLowerCase(Locale.ENGLISH).replace('-', ' ').trim(); 389 390 if (this.nameType == NameType.GENERIC) { 391 if (input.startsWith("d'")) { // check for d' 392 final String remainder = input.substring(2); 393 final String combined = "d" + remainder; 394 return "(" + encode(remainder) + ")-(" + encode(combined) + ")"; 395 } 396 for (final String l : NAME_PREFIXES.get(this.nameType)) { 397 // handle generic prefixes 398 if (input.startsWith(l + " ")) { 399 // check for any prefix in the words list 400 final String remainder = input.substring(l.length() + 1); // input without the prefix 401 final String combined = l + remainder; // input with prefix without space 402 return "(" + encode(remainder) + ")-(" + encode(combined) + ")"; 403 } 404 } 405 } 406 407 final List<String> words = Arrays.asList(ResourceConstants.SPACES.split(input)); 408 final List<String> words2 = new ArrayList<>(); 409 410 // special-case handling of word prefixes based upon the name type 411 switch (this.nameType) { 412 case SEPHARDIC: 413 words.forEach(aWord -> { 414 final String[] parts = QUOTE.split(aWord, -1); 415 words2.add(parts[parts.length - 1]); 416 }); 417 words2.removeAll(NAME_PREFIXES.get(this.nameType)); 418 break; 419 case ASHKENAZI: 420 words2.addAll(words); 421 words2.removeAll(NAME_PREFIXES.get(this.nameType)); 422 break; 423 case GENERIC: 424 words2.addAll(words); 425 break; 426 default: 427 throw new IllegalStateException("Unreachable case: " + this.nameType); 428 } 429 430 if (this.concat) { 431 // concat mode enabled 432 input = join(words2, " "); 433 } else if (words2.size() == 1) { 434 // not a multi-word name 435 input = words.iterator().next(); 436 } else if (!words2.isEmpty()) { 437 // encode each word in a multi-word name separately (normally used for approx matches) 438 final StringBuilder result = new StringBuilder(); 439 words2.forEach(word -> result.append("-").append(encode(word))); 440 // return the result without the leading "-" 441 return result.substring(1); 442 } 443 444 PhonemeBuilder phonemeBuilder = PhonemeBuilder.empty(languageSet); 445 446 // loop over each char in the input - we will handle the increment manually 447 for (int i = 0; i < input.length();) { 448 final RulesApplication rulesApplication = 449 new RulesApplication(rules, input, phonemeBuilder, i, maxPhonemes).invoke(); 450 i = rulesApplication.getI(); 451 phonemeBuilder = rulesApplication.getPhonemeBuilder(); 452 } 453 454 // Apply the general rules 455 phonemeBuilder = applyFinalRules(phonemeBuilder, finalRules1); 456 // Apply the language-specific rules 457 phonemeBuilder = applyFinalRules(phonemeBuilder, finalRules2); 458 459 return phonemeBuilder.makeString(); 460 } 461 462 /** 463 * Gets the Lang language guessing rules being used. 464 * 465 * @return the Lang in use 466 */ 467 public Lang getLang() { 468 return this.lang; 469 } 470 471 /** 472 * Gets the maximum number of phonemes the engine will calculate for a given input. 473 * 474 * @return the maximum number of phonemes 475 * @since 1.7 476 */ 477 public int getMaxPhonemes() { 478 return this.maxPhonemes; 479 } 480 481 /** 482 * Gets the NameType being used. 483 * 484 * @return the NameType in use 485 */ 486 public NameType getNameType() { 487 return this.nameType; 488 } 489 490 /** 491 * Gets the RuleType being used. 492 * 493 * @return the RuleType in use 494 */ 495 public RuleType getRuleType() { 496 return this.ruleType; 497 } 498 499 /** 500 * Gets if multiple phonetic encodings are concatenated or if just the first one is kept. 501 * 502 * @return true if multiple phonetic encodings are returned, false if just the first is 503 */ 504 public boolean isConcat() { 505 return this.concat; 506 } 507}