/* LaxURI
   *
   * $Id: LaxURI.java 5919 2008-07-30 22:46:02Z gojomo $
   *
   * Created on Aug 3, 2005
   *
   * Copyright (C) 2005 Internet Archive.
   *
   * This file is part of the Heritrix web crawler (crawler.archive.org).
  *
  * Heritrix is free software; you can redistribute it and/or modify
  * it under the terms of the GNU Lesser Public License as published by
  * the Free Software Foundation; either version 2.1 of the License, or
  * any later version.
  *
  * Heritrix is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  * GNU Lesser Public License for more details.
  *
  * You should have received a copy of the GNU Lesser Public License
  * along with Heritrix; if not, write to the Free Software
  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  */ 
  package org.archive.net;
  
  import java.util.Arrays;
  import java.util.BitSet;
  
  import org.apache.commons.httpclient.URI;
  import org.apache.commons.httpclient.URIException;
  import org.apache.commons.httpclient.util.EncodingUtil;
  
  /***
   * URI subclass which allows partial/inconsistent encoding, matching
   * the URIs which will be relayed in requests from popular web
   * browsers (esp. Mozilla Firefox and MS IE).
   * 
   * @author gojomo
   */
  public class LaxURI extends URI {
  
      private static final long serialVersionUID = 5273922211722239537L;
      
      final protected static char[] HTTP_SCHEME = {'h','t','t','p'};
      final protected static char[] HTTPS_SCHEME = {'h','t','t','p','s'};
      
      protected static final BitSet lax_rel_segment = new BitSet(256);
      // Static initializer for lax_rel_segment
      static {
          lax_rel_segment.or(rel_segment);
          lax_rel_segment.set(':'); // allow ':'
          // TODO: add additional allowances as need is demonstrated
      }
  
      protected static final BitSet lax_abs_path = new BitSet(256);
      static {
          lax_abs_path.or(abs_path);
          lax_abs_path.set('|'); // tests indicate Firefox (1.0.6) doesn't escape.
      }
      
      protected static final BitSet lax_query = new BitSet(256);
      static {
          lax_query.or(query);
          lax_query.set('{'); // tests indicate FF doesn't escape { in query
          lax_query.set('}'); // tests indicate FF doesn't escape } in query
          lax_query.set('|'); // tests indicate FF doesn't escape | in query
          lax_query.set('['); // tests indicate FF doesn't escape [ in query
          lax_query.set(']'); // tests indicate FF doesn't escape ] in query
          lax_query.set('^'); // tests indicate FF doesn't escape ^ in query
      }
      
      // passthrough initializers
      public LaxURI(String uri, boolean escaped, String charset)
      throws URIException {
          super(uri,escaped,charset);
      }
      public LaxURI(URI base, URI relative) throws URIException {
          super(base,relative);
      }
      public LaxURI(String uri, boolean escaped) throws URIException {
          super(uri,escaped);
      }
      public LaxURI() {
          super();
      }
  
      // overridden to use this class's static decode()
      public String getURI() throws URIException {
          return (_uri == null) ? null : decode(_uri, getProtocolCharset());
      }
      
      // overridden to use this class's static decode()
      public String getPath() throws URIException {
          char[] p = getRawPath();
          return (p == null) ? null : decode(p, getProtocolCharset());
      }
  
      // overridden to use this class's static decode()
     public String getPathQuery() throws URIException {
         char[] rawPathQuery = getRawPathQuery();
         return (rawPathQuery == null) ? null : decode(rawPathQuery,
                 getProtocolCharset());
     }
     // overridden to use this class's static decode()
     protected static String decode(char[] component, String charset)
             throws URIException {
         if (component == null) {
             throw new IllegalArgumentException(
                     "Component array of chars may not be null");
         }
         return decode(new String(component), charset);
     }
 
     // overridden to use IA's LaxURLCodec, which never throws DecoderException
     protected static String decode(String component, String charset)
             throws URIException {
         if (component == null) {
             throw new IllegalArgumentException(
                     "Component array of chars may not be null");
         }
         byte[] rawdata = null;
         //     try {
         rawdata = LaxURLCodec.decodeUrlLoose(EncodingUtil
                 .getAsciiBytes(component));
         //     } catch (DecoderException e) {
         //         throw new URIException(e.getMessage());
         //     }
         return EncodingUtil.getString(rawdata, charset);
     }
     
     // overidden to lax() the acceptable-char BitSet passed in
     protected boolean validate(char[] component, BitSet generous) {
         return super.validate(component, lax(generous));
     }
 
     // overidden to lax() the acceptable-char BitSet passed in
     protected boolean validate(char[] component, int soffset, int eoffset,
             BitSet generous) {
         return super.validate(component, soffset, eoffset, lax(generous));
     }
     
     /***
      * Given a BitSet -- typically one of the URI superclass's
      * predefined static variables -- possibly replace it with
      * a more-lax version to better match the character sets
      * actually left unencoded in web browser requests
      * 
      * @param generous original BitSet
      * @return (possibly more lax) BitSet to use
      */
     protected BitSet lax(BitSet generous) {
         if (generous == rel_segment) {
             // Swap in more lax allowable set
             return lax_rel_segment;
         }
         if (generous == abs_path) {
             return lax_abs_path;
         }
         if (generous == query) {
             return lax_query;
         }
         // otherwise, leave as is
         return generous;
     }
     
     /*** 
      * Coalesce the _host and _authority fields where 
      * possible.
      * 
      * In the web crawl/http domain, most URIs have an 
      * identical _host and _authority. (There is no port
      * or user info.) However, the superclass always 
      * creates two separate char[] instances. 
      * 
      * Notably, the lengths of these char[] fields are 
      * equal if and only if their values are identical.
      * This method makes use of this fact to reduce the
      * two instances to one where possible, slimming 
      * instances.  
      * 
      * @see org.apache.commons.httpclient.URI#parseAuthority(java.lang.String, boolean)
      */
     protected void parseAuthority(String original, boolean escaped)
             throws URIException {
         super.parseAuthority(original, escaped);
         if (_host != null && _authority != null
                 && _host.length == _authority.length) {
             _host = _authority;
         }
     }
     
     
     /*** 
      * Coalesce _scheme to existing instances, where appropriate.
      * 
      * In the web-crawl domain, most _schemes are 'http' or 'https',
      * but the superclass always creates a new char[] instance. For
      * these two cases, we replace the created instance with a 
      * long-lived instance from a static field, saving 12-14 bytes
      * per instance. 
      * 
      * @see org.apache.commons.httpclient.URI#setURI()
      */
     protected void setURI() {
         if (_scheme != null) {
             if (_scheme.length == 4 && Arrays.equals(_scheme, HTTP_SCHEME)) {
                 _scheme = HTTP_SCHEME;
             } else if (_scheme.length == 5
                     && Arrays.equals(_scheme, HTTP_SCHEME)) {
                 _scheme = HTTPS_SCHEME;
             }
         }
         super.setURI();
     }
     
     /***
      * IA OVERRIDDEN IN LaxURI TO INCLUDE FIX FOR 
      * http://issues.apache.org/jira/browse/HTTPCLIENT-588
      * AND
      * http://webteam.archive.org/jira/browse/HER-1268
      * 
      * In order to avoid any possilbity of conflict with non-ASCII characters,
      * Parse a URI reference as a <code>String</code> with the character
      * encoding of the local system or the document.
      * <p>
      * The following line is the regular expression for breaking-down a URI
      * reference into its components.
      * <p><blockquote><pre>
      *   ^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?
      *    12            3  4          5       6  7        8 9
      * </pre></blockquote><p>
      * For example, matching the above expression to
      *   http://jakarta.apache.org/ietf/uri/#Related
      * results in the following subexpression matches:
      * <p><blockquote><pre>
      *               $1 = http:
      *  scheme    =  $2 = http
      *               $3 = //jakarta.apache.org
      *  authority =  $4 = jakarta.apache.org
      *  path      =  $5 = /ietf/uri/
      *               $6 = <undefined>
      *  query     =  $7 = <undefined>
      *               $8 = #Related
      *  fragment  =  $9 = Related
      * </pre></blockquote><p>
      *
      * @param original the original character sequence
      * @param escaped <code>true</code> if <code>original</code> is escaped
      * @throws URIException If an error occurs.
      */
     protected void parseUriReference(String original, boolean escaped)
         throws URIException {
 
         // validate and contruct the URI character sequence
         if (original == null) {
             throw new URIException("URI-Reference required");
         }
 
         /* @
          *  ^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?
          */
         String tmp = original.trim();
         
         /*
          * The length of the string sequence of characters.
          * It may not be equal to the length of the byte array.
          */
         int length = tmp.length();
 
         /*
          * Remove the delimiters like angle brackets around an URI.
          */
         if (length > 0) {
             char[] firstDelimiter = { tmp.charAt(0) };
             if (validate(firstDelimiter, delims)) {
                 if (length >= 2) {
                     char[] lastDelimiter = { tmp.charAt(length - 1) };
                     if (validate(lastDelimiter, delims)) {
                         tmp = tmp.substring(1, length - 1);
                         length = length - 2;
                     }
                 }
             }
         }
 
         /*
          * The starting index
          */
         int from = 0;
 
         /*
          * The test flag whether the URI is started from the path component.
          */
         boolean isStartedFromPath = false;
         int atColon = tmp.indexOf(':');
         int atSlash = tmp.indexOf('/');
         if ((atColon <= 0 && !tmp.startsWith("//"))
             || (atSlash >= 0 && atSlash < atColon)) {
             isStartedFromPath = true;
         }
 
         /*
          * <p><blockquote><pre>
          *     @@@@@@@@
          *  ^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?
          * </pre></blockquote><p>
          */
         int at = indexFirstOf(tmp, isStartedFromPath ? "/?#" : ":/?#", from);
         if (at == -1) { 
             at = 0;
         }
 
         /*
          * Parse the scheme.
          * <p><blockquote><pre>
          *  scheme    =  $2 = http
          *              @
          *  ^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?
          * </pre></blockquote><p>
          */
         if (at > 0 && at < length && tmp.charAt(at) == ':') {
             char[] target = tmp.substring(0, at).toLowerCase().toCharArray();
             if (validate(target, scheme)) {
                 _scheme = target;
                 from = ++at;
             } else {
                 // IA CHANGE:
                 // do nothing; allow interpretation as URI with 
                 // later colon in other syntactical component
             }
             
         }
 
         /*
          * Parse the authority component.
          * <p><blockquote><pre>
          *  authority =  $4 = jakarta.apache.org
          *                  @@
          *  ^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?
          * </pre></blockquote><p>
          */
         // Reset flags
         _is_net_path = _is_abs_path = _is_rel_path = _is_hier_part = false;
         if (0 <= at && at < length && tmp.charAt(at) == '/') {
             // Set flag
             _is_hier_part = true;
             if (at + 2 < length && tmp.charAt(at + 1) == '/' 
                 && !isStartedFromPath) {
                 // the temporary index to start the search from
                 int next = indexFirstOf(tmp, "/?#", at + 2);
                 if (next == -1) {
                     next = (tmp.substring(at + 2).length() == 0) ? at + 2 
                         : tmp.length();
                 }
                 parseAuthority(tmp.substring(at + 2, next), escaped);
                 from = at = next;
                 // Set flag
                 _is_net_path = true;
             }
             if (from == at) {
                 // Set flag
                 _is_abs_path = true;
             }
         }
 
         /*
          * Parse the path component.
          * <p><blockquote><pre>
          *  path      =  $5 = /ietf/uri/
          *                                @@@@@@
          *  ^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?
          * </pre></blockquote><p>
          */
         if (from < length) {
             // rel_path = rel_segment [ abs_path ]
             int next = indexFirstOf(tmp, "?#", from);
             if (next == -1) {
                 next = tmp.length();
             }
             if (!_is_abs_path) {
                 if (!escaped 
                     && prevalidate(tmp.substring(from, next), disallowed_rel_path) 
                     || escaped 
                     && validate(tmp.substring(from, next).toCharArray(), rel_path)) {
                     // Set flag
                     _is_rel_path = true;
                 } else if (!escaped 
                     && prevalidate(tmp.substring(from, next), disallowed_opaque_part) 
                     || escaped 
                     && validate(tmp.substring(from, next).toCharArray(), opaque_part)) {
                     // Set flag
                     _is_opaque_part = true;
                 } else {
                     // the path component may be empty
                     _path = null;
                 }
             }
             String s = tmp.substring(from, next);
             if (escaped) {
                 setRawPath(s.toCharArray());
             } else {
                 setPath(s);
             }
             at = next;
         }
 
         // set the charset to do escape encoding
         String charset = getProtocolCharset();
 
         /*
          * Parse the query component.
          * <p><blockquote><pre>
          *  query     =  $7 = <undefined>
          *                                        @@@@@@@@@
          *  ^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?
          * </pre></blockquote><p>
          */
         if (0 <= at && at + 1 < length && tmp.charAt(at) == '?') {
             int next = tmp.indexOf('#', at + 1);
             if (next == -1) {
                 next = tmp.length();
             }
             if (escaped) {
                 _query = tmp.substring(at + 1, next).toCharArray();
                 if (!validate(_query, query)) {
                     throw new URIException("Invalid query");
                 }
             } else {
                 _query = encode(tmp.substring(at + 1, next), allowed_query, charset);
             }
             at = next;
         }
 
         /*
          * Parse the fragment component.
          * <p><blockquote><pre>
          *  fragment  =  $9 = Related
          *                                                   @@@@@@@@
          *  ^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?
          * </pre></blockquote><p>
          */
         if (0 <= at && at + 1 <= length && tmp.charAt(at) == '#') {
             if (at + 1 == length) { // empty fragment
                 _fragment = "".toCharArray();
             } else {
                 _fragment = (escaped) ? tmp.substring(at + 1).toCharArray() 
                     : encode(tmp.substring(at + 1), allowed_fragment, charset);
             }
         }
 
         // set this URI.
         setURI();
     }
     
 }