001/*
002 * Licensed to the Apache Software Foundation (ASF) under one or more
003 * contributor license agreements.  See the NOTICE file distributed with
004 * this work for additional information regarding copyright ownership.
005 * The ASF licenses this file to You under the Apache License, Version 2.0
006 * (the "License"); you may not use this file except in compliance with
007 * the License.  You may obtain a copy of the License at
008 *
009 *      https://www.apache.org/licenses/LICENSE-2.0
010 *
011 * Unless required by applicable law or agreed to in writing, software
012 * distributed under the License is distributed on an "AS IS" BASIS,
013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
014 * See the License for the specific language governing permissions and
015 * limitations under the License.
016 */
017package org.apache.commons.validator.routines;
018
019import java.io.Serializable;
020import java.net.URI;
021import java.net.URISyntaxException;
022import java.util.Collections;
023import java.util.HashSet;
024import java.util.Locale;
025import java.util.Set;
026import java.util.regex.Matcher;
027import java.util.regex.Pattern;
028
029import org.apache.commons.validator.GenericValidator;
030
031/**
032 * <p><strong>URL Validation</strong> routines.</p>
033 * Behavior of validation is modified by passing in options:
034 * <ul>
035 * <li>ALLOW_2_SLASHES - [FALSE]  Allows double '/' characters in the path
036 * component.</li>
037 * <li>NO_FRAGMENT- [FALSE]  By default fragments are allowed, if this option is
038 * included then fragments are flagged as illegal.</li>
039 * <li>ALLOW_ALL_SCHEMES - [FALSE] By default only http, https, and ftp are
040 * considered valid schemes.  Enabling this option will let any scheme pass validation.</li>
041 * </ul>
042 *
043 * <p>Originally based in on php script by Debbie Dyer, validation.php v1.2b, Date: 03/07/02,
044 * https://javascript.internet.com. However, this validation now bears little resemblance
045 * to the php original.</p>
046 * <pre>
047 *   Example of usage:
048 *   Construct a UrlValidator with valid schemes of "http", and "https".
049 *
050 *    String[] schemes = {"http","https"}.
051 *    UrlValidator urlValidator = new UrlValidator(schemes);
052 *    if (urlValidator.isValid("ftp://foo.bar.com/")) {
053 *       System.out.println("URL is valid");
054 *    } else {
055 *       System.out.println("URL is invalid");
056 *    }
057 *
058 *    prints "URL is invalid"
059 *   If instead the default constructor is used.
060 *
061 *    UrlValidator urlValidator = new UrlValidator();
062 *    if (urlValidator.isValid("ftp://foo.bar.com/")) {
063 *       System.out.println("URL is valid");
064 *    } else {
065 *       System.out.println("URL is invalid");
066 *    }
067 *
068 *   prints out "URL is valid"
069 *  </pre>
070 *
071 * @see
072 * <a href="http://www.ietf.org/rfc/rfc2396.txt">
073 *  Uniform Resource Identifiers (URI): Generic Syntax
074 * </a>
075 *
076 * @since 1.4
077 */
078public class UrlValidator implements Serializable {
079
080    private static final long serialVersionUID = 7557161713937335013L;
081
082    private static final int MAX_UNSIGNED_16_BIT_INT = 0xFFFF; // port max
083
084    /**
085     * Allows all validly formatted schemes to pass validation instead of
086     * supplying a set of valid schemes.
087     */
088    public static final long ALLOW_ALL_SCHEMES = 1 << 0;
089
090    /**
091     * Allow two slashes in the path component of the URL.
092     */
093    public static final long ALLOW_2_SLASHES = 1 << 1;
094
095    /**
096     * Enabling this options disallows any URL fragments.
097     */
098    public static final long NO_FRAGMENTS = 1 << 2;
099
100    /**
101     * Allow local URLs, such as https://localhost/ or https://machine/ .
102     * This enables a broad-brush check, for complex local machine name
103     *  validation requirements you should create your validator with
104     *  a {@link RegexValidator} instead ({@link #UrlValidator(RegexValidator, long)})
105     */
106    public static final long ALLOW_LOCAL_URLS = 1 << 3; // CHECKSTYLE IGNORE MagicNumber
107
108    /**
109     * Protocol scheme (for example, http, ftp, https).
110     */
111    private static final String SCHEME_REGEX = "^\\p{Alpha}[\\p{Alnum}\\+\\-\\.]*";
112    private static final Pattern SCHEME_PATTERN = Pattern.compile(SCHEME_REGEX);
113
114    // Drop numeric, and  "+-." for now
115    // TODO does not allow for optional userinfo.
116    // Validation of character set is done by isValidAuthority
117    private static final String AUTHORITY_CHARS_REGEX = "\\p{Alnum}\\-\\."; // allows for IPV4 but not IPV6
118    // Allow for IPv4 mapped addresses: ::FFF:123.123.123.123
119    private static final String IPV6_REGEX = "::FFFF:(?:\\d{1,3}\\.){3}\\d{1,3}|[0-9a-fA-F:]+"; // do this as separate match because : could cause ambiguity with port prefix
120
121    // userinfo    = *( unreserved / pct-encoded / sub-delims / ":" )
122    // unreserved    = ALPHA / DIGIT / "-" / "." / "_" / "~"
123    // sub-delims    = "!" / "$" / "&" / "'" / "(" / ")" / "*" / "+" / "," / ";" / "="
124    // We assume that password has the same valid chars as user info
125    private static final String USERINFO_CHARS_REGEX = "[a-zA-Z0-9%-._~!$&'()*+,;=]";
126
127    // since neither ':' nor '@' are allowed chars, we don't need to use non-greedy matching
128    private static final String USERINFO_FIELD_REGEX =
129            USERINFO_CHARS_REGEX + "+" + // At least one character for the name
130            "(?::" + USERINFO_CHARS_REGEX + "*)?@"; // colon and password may be absent
131
132    private static final String AUTHORITY_REGEX =
133            "(?:\\[(" + IPV6_REGEX + ")\\]|(?:(?:" + USERINFO_FIELD_REGEX + ")?([" + AUTHORITY_CHARS_REGEX + "]*)))(?::(\\d*))?(.*)?";
134    //             1                         for example, user:pass@           2                                       3       4
135    private static final Pattern AUTHORITY_PATTERN = Pattern.compile(AUTHORITY_REGEX);
136
137    private static final int PARSE_AUTHORITY_IPV6 = 1;
138
139    private static final int PARSE_AUTHORITY_HOST_IP = 2; // excludes userinfo, if present
140
141    private static final int PARSE_AUTHORITY_PORT = 3; // excludes leading colon
142
143    /**
144     * Should always be empty. The code currently allows spaces.
145     */
146    private static final int PARSE_AUTHORITY_EXTRA = 4;
147
148    private static final String PATH_REGEX = "^(/[-\\w:@&?=+,.!/~*'%$_;\\(\\)]*)?$";
149    private static final Pattern PATH_PATTERN = Pattern.compile(PATH_REGEX);
150
151    private static final String QUERY_REGEX = "^(\\S*)$";
152    private static final Pattern QUERY_PATTERN = Pattern.compile(QUERY_REGEX);
153
154    /**
155     * If no schemes are provided, default to this set.
156     */
157    private static final String[] DEFAULT_SCHEMES = {"http", "https", "ftp"}; // Must be lower-case
158
159    /**
160     * Singleton instance of this class with default schemes and options.
161     */
162    private static final UrlValidator DEFAULT_URL_VALIDATOR = new UrlValidator();
163
164    /**
165     * Returns the singleton instance of this class with default schemes and options.
166     * @return singleton instance with default schemes and options
167     */
168    public static UrlValidator getInstance() {
169        return DEFAULT_URL_VALIDATOR;
170    }
171
172    /**
173     * Tests whether the given flag is on. If the flag is not a power of 2
174     * (for example, 3) this tests whether the combination of flags is on.
175     *
176     * @param flag Flag value to check.
177     * @param options what to check
178     * @return whether the specified flag value is on.
179     */
180    private static boolean isOn(final long flag, final long options) {
181        return (options & flag) > 0;
182    }
183
184    /**
185     * Holds the set of current validation options.
186     */
187    private final long options;
188
189    /**
190     * The set of schemes that are allowed to be in a URL.
191     */
192    private final Set<String> allowedSchemes; // Must be lower-case
193
194    /**
195     * Regular expressions used to manually validate authorities if IANA
196     * domain name validation isn't desired.
197     */
198    private final RegexValidator authorityValidator;
199
200    /**
201     * The domain validator.
202     */
203    private final DomainValidator domainValidator;
204
205    /**
206     * Constructs a new instance with default properties.
207     */
208    public UrlValidator() {
209        this(null);
210    }
211
212    /**
213     * Constructs a new instance with the given validation options.
214     * @param options The options should be set using the public constants declared in
215     * this class.  To set multiple options you simply add them together.  For example,
216     * ALLOW_2_SLASHES + NO_FRAGMENTS enables both of those options.
217     */
218    public UrlValidator(final long options) {
219        this(null, null, options);
220    }
221
222    /**
223     * Constructs a new instance with the given validation options.
224     * @param authorityValidator Regular expression validator used to validate the authority part
225     * This allows the user to override the standard set of domains.
226     * @param options Validation options. Set using the public constants of this class.
227     * To set multiple options, simply add them together:
228     * <p>{@code ALLOW_2_SLASHES + NO_FRAGMENTS}</p>
229     * enables both of those options.
230     */
231    public UrlValidator(final RegexValidator authorityValidator, final long options) {
232        this(null, authorityValidator, options);
233    }
234
235    /**
236     * Behavior of validation is modified by passing in several strings options:
237     * @param schemes Pass in one or more URL schemes to consider valid, passing in
238     *        a null will default to "http,https,ftp" being valid.
239     *        If a non-null schemes is specified then all valid schemes must
240     *        be specified. Setting the ALLOW_ALL_SCHEMES option will
241     *        ignore the contents of schemes.
242     */
243    public UrlValidator(final String[] schemes) {
244        this(schemes, 0L);
245    }
246
247    /**
248     * Behavior of validation is modified by passing in options:
249     * @param schemes The set of valid schemes. Ignored if the ALLOW_ALL_SCHEMES option is set.
250     * @param options The options should be set using the public constants declared in
251     * this class.  To set multiple options you simply add them together.  For example,
252     * ALLOW_2_SLASHES + NO_FRAGMENTS enables both of those options.
253     */
254    public UrlValidator(final String[] schemes, final long options) {
255        this(schemes, null, options);
256    }
257
258    /**
259     * Customizable constructor. Validation behavior is modified by passing in options.
260     * @param schemes the set of valid schemes. Ignored if the ALLOW_ALL_SCHEMES option is set.
261     * @param authorityValidator Regular expression validator used to validate the authority part
262     * @param options Validation options. Set using the public constants of this class.
263     * To set multiple options, simply add them together:
264     * <p>{@code ALLOW_2_SLASHES + NO_FRAGMENTS}</p>
265     * enables both of those options.
266     */
267    public UrlValidator(final String[] schemes, final RegexValidator authorityValidator, final long options) {
268        this(schemes, authorityValidator, options, DomainValidator.getInstance(isOn(ALLOW_LOCAL_URLS, options)));
269    }
270
271    /**
272     * Customizable constructor. Validation behavior is modified by passing in options.
273     * @param schemes the set of valid schemes. Ignored if the ALLOW_ALL_SCHEMES option is set.
274     * @param authorityValidator Regular expression validator used to validate the authority part
275     * @param options Validation options. Set using the public constants of this class.
276     * To set multiple options, simply add them together:
277     * <p>{@code ALLOW_2_SLASHES + NO_FRAGMENTS}</p>
278     * enables both of those options.
279     * @param domainValidator the DomainValidator to use; must agree with ALLOW_LOCAL_URLS setting
280     * @since 1.7
281     */
282    public UrlValidator(String[] schemes, final RegexValidator authorityValidator, final long options, final DomainValidator domainValidator) {
283        this.options = options;
284        if (domainValidator == null) {
285            throw new IllegalArgumentException("DomainValidator must not be null");
286        }
287        if (domainValidator.isAllowLocal() != (options & ALLOW_LOCAL_URLS) > 0) {
288            throw new IllegalArgumentException("DomainValidator disagrees with ALLOW_LOCAL_URLS setting");
289        }
290        this.domainValidator = domainValidator;
291
292        if (isOn(ALLOW_ALL_SCHEMES)) {
293            allowedSchemes = Collections.emptySet();
294        } else {
295            if (schemes == null) {
296                schemes = DEFAULT_SCHEMES;
297            }
298            allowedSchemes = new HashSet<>(schemes.length);
299            for (final String scheme : schemes) {
300                allowedSchemes.add(scheme.toLowerCase(Locale.ENGLISH));
301            }
302        }
303
304        this.authorityValidator = authorityValidator;
305    }
306
307    /**
308     * Returns the number of times the token appears in the target.
309     * @param token Token value to be counted.
310     * @param target Target value to count tokens in.
311     * @return the number of tokens.
312     */
313    protected int countToken(final String token, final String target) {
314        int tokenIndex = 0;
315        int count = 0;
316        while (tokenIndex != -1) {
317            tokenIndex = target.indexOf(token, tokenIndex);
318            if (tokenIndex > -1) {
319                tokenIndex++;
320                count++;
321            }
322        }
323        return count;
324    }
325
326    /**
327     * Tests whether the given flag is off.  If the flag is not a power of 2
328     * (for example, 3) this tests whether the combination of flags is off.
329     *
330     * @param flag Flag value to check.
331     * @return whether the specified flag value is off.
332     */
333    private boolean isOff(final long flag) {
334        return (options & flag) == 0;
335    }
336
337    /**
338     * Tests whether the given flag is on.  If the flag is not a power of 2
339     * (for example, 3) this tests whether the combination of flags is on.
340     *
341     * @param flag Flag value to check.
342     * @return whether the specified flag value is on.
343     */
344    private boolean isOn(final long flag) {
345        return (options & flag) > 0;
346    }
347
348    /**
349     * <p>Checks if a field has a valid URL address.</p>
350     *
351     * Note that the method calls #isValidAuthority()
352     * which checks that the domain is valid.
353     *
354     * @param value The value validation is being performed on.  A {@code null}
355     * value is considered invalid.
356     * @return true if the URL is valid.
357     */
358    public boolean isValid(final String value) {
359        if (value == null) {
360            return false;
361        }
362        final URI uri; // ensure value is a valid URI
363        try {
364            uri = new URI(value);
365        } catch (final URISyntaxException e) {
366            return false;
367        }
368        // OK, perform additional validation
369        final String scheme = uri.getScheme();
370        if (!isValidScheme(scheme)) {
371            return false;
372        }
373        final String authority = uri.getRawAuthority();
374        if ("file".equals(scheme) && GenericValidator.isBlankOrNull(authority)) { // Special case - file: allows an empty authority
375            return true; // this is a local file - nothing more to do here
376        }
377        // Validate the authority
378        if ("file".equals(scheme) && authority != null && authority.contains(":") || !isValidAuthority(authority)) {
379            return false;
380        }
381        if (!isValidPath(uri.getRawPath()) || !isValidQuery(uri.getRawQuery()) || !isValidFragment(uri.getRawFragment())) {
382            return false;
383        }
384        return true;
385    }
386
387    /**
388     * Returns true if the authority is properly formatted.  An authority is the combination
389     * of hostname and port.  A {@code null} authority value is considered invalid.
390     * Note: this implementation validates the domain unless a RegexValidator was provided.
391     * If a RegexValidator was supplied, and it matches, then the authority is regarded
392     * as valid with no further checks, otherwise the method checks against the
393     * AUTHORITY_PATTERN and the DomainValidator (ALLOW_LOCAL_URLS)
394     * @param authority Authority value to validate, allows IDN
395     * @return true if authority (hostname and port) is valid.
396     */
397    protected boolean isValidAuthority(final String authority) {
398        if (authority == null) {
399            return false;
400        }
401
402        // check manual authority validation if specified
403        if (authorityValidator != null && authorityValidator.isValid(authority)) {
404            return true;
405        }
406        // convert to ASCII if possible
407        final String authorityASCII = DomainValidator.unicodeToASCII(authority);
408
409        final Matcher authorityMatcher = AUTHORITY_PATTERN.matcher(authorityASCII);
410        if (!authorityMatcher.matches()) {
411            return false;
412        }
413
414        // We have to process IPV6 separately because that is parsed in a different group
415        final String ipv6 = authorityMatcher.group(PARSE_AUTHORITY_IPV6);
416        if (ipv6 != null) {
417            final InetAddressValidator inetAddressValidator = InetAddressValidator.getInstance();
418            if (!inetAddressValidator.isValidInet6Address(ipv6)) {
419                return false;
420            }
421        } else {
422            final String hostLocation = authorityMatcher.group(PARSE_AUTHORITY_HOST_IP);
423            // check if authority is hostname or IP address:
424            // try a hostname first since that's much more likely
425            if (!domainValidator.isValid(hostLocation)) {
426                // try an IPv4 address
427                final InetAddressValidator inetAddressValidator = InetAddressValidator.getInstance();
428                if (!inetAddressValidator.isValidInet4Address(hostLocation)) {
429                    // isn't IPv4, so the URL is invalid
430                    return false;
431                }
432            }
433            final String port = authorityMatcher.group(PARSE_AUTHORITY_PORT);
434            if (!GenericValidator.isBlankOrNull(port)) {
435                try {
436                    final int iPort = Integer.parseInt(port);
437                    if (iPort < 0 || iPort > MAX_UNSIGNED_16_BIT_INT) {
438                        return false;
439                    }
440                } catch (final NumberFormatException nfe) {
441                    return false; // this can happen for big numbers
442                }
443            }
444        }
445
446        final String extra = authorityMatcher.group(PARSE_AUTHORITY_EXTRA);
447        if (extra != null && !extra.trim().isEmpty()) {
448            return false;
449        }
450
451        return true;
452    }
453
454    /**
455     * Returns true if the given fragment is null or fragments are allowed.
456     * @param fragment Fragment value to validate.
457     * @return true if fragment is valid.
458     */
459    protected boolean isValidFragment(final String fragment) {
460        if (fragment == null) {
461            return true;
462        }
463
464        return isOff(NO_FRAGMENTS);
465    }
466
467    /**
468     * Returns true if the path is valid.  A {@code null} value is considered invalid.
469     * @param path Path value to validate.
470     * @return true if path is valid.
471     */
472    protected boolean isValidPath(final String path) {
473        if (path == null || !PATH_PATTERN.matcher(path).matches()) {
474            return false;
475        }
476
477        try {
478            // Don't omit host otherwise leading path may be taken as host if it starts with //
479            final URI uri = new URI(null, "localhost", path, null);
480            final String norm = uri.normalize().getPath();
481            if (norm.startsWith("/../") // Trying to go via the parent dir
482                    || norm.equals("/..")) { // Trying to go to the parent dir
483                return false;
484            }
485        } catch (final URISyntaxException e) {
486            return false;
487        }
488
489        final int slash2Count = countToken("//", path);
490        if (isOff(ALLOW_2_SLASHES) && slash2Count > 0) {
491            return false;
492        }
493
494        return true;
495    }
496
497    /**
498     * Returns true if the query is null, or it's a properly formatted query string.
499     * @param query Query value to validate.
500     * @return true if query is valid.
501     */
502    protected boolean isValidQuery(final String query) {
503        if (query == null) {
504            return true;
505        }
506        return QUERY_PATTERN.matcher(query).matches();
507    }
508
509    /**
510     * Validate scheme. If schemes[] was initialized to a non-null,
511     * then only those schemes are allowed.
512     * Otherwise, the default schemes are "http", "https", "ftp".
513     * Matching is case-blind.
514     * @param scheme The scheme to validate.  A {@code null} value is considered
515     * invalid.
516     * @return true if valid.
517     */
518    protected boolean isValidScheme(final String scheme) {
519        if (scheme == null || !SCHEME_PATTERN.matcher(scheme).matches()
520                || isOff(ALLOW_ALL_SCHEMES) && !allowedSchemes.contains(scheme.toLowerCase(Locale.ENGLISH))) {
521            return false;
522        }
523
524        return true;
525    }
526
527}