View Javadoc

1   /* PathologicalFilter
2    *
3    * $Id: PathologicalPathFilter.java 4652 2006-09-25 18:41:10Z paul_jack $
4    *
5    * Created on Feb 20, 2004
6    *
7    * Copyright (C) 2004 Internet Archive.
8    *
9    * This file is part of the Heritrix web crawler (crawler.archive.org).
10   *
11   * Heritrix is free software; you can redistribute it and/or modify
12   * it under the terms of the GNU Lesser Public License as published by
13   * the Free Software Foundation; either version 2.1 of the License, or
14   * any later version.
15   *
16   * Heritrix is distributed in the hope that it will be useful,
17   * but WITHOUT ANY WARRANTY; without even the implied warranty of
18   * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
19   * GNU Lesser Public License for more details.
20   *
21   * You should have received a copy of the GNU Lesser Public License
22   * along with Heritrix; if not, write to the Free Software
23   * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
24   */
25  package org.archive.crawler.filter;
26  
27  import java.util.logging.Logger;
28  
29  import javax.management.AttributeNotFoundException;
30  
31  import org.archive.crawler.datamodel.CrawlURI;
32  import org.archive.crawler.deciderules.DecideRule;
33  import org.archive.crawler.deciderules.DecidingFilter;
34  import org.archive.crawler.settings.SimpleType;
35  import org.archive.crawler.settings.Type;
36  
37  /*** 
38   * Checks if a URI contains a repeated pattern.
39   *
40   * This filter is checking if a pattern is repeated a specific number of times.
41   * The use is to avoid crawler traps where the server adds the same pattern to
42   * the requested URI like: <code>http://host/img/img/img/img....</code>. This
43   * filter returns TRUE if the path is pathological.  FALSE otherwise.
44   *
45   * @author John Erik Halse
46   * @deprecated As of release 1.10.0.  Replaced by {@link DecidingFilter} and
47   * equivalent {@link DecideRule}.
48   */
49  public class PathologicalPathFilter extends URIRegExpFilter {
50  
51      private static final long serialVersionUID = 2797805167250054353L;
52  
53      private static final Logger logger =
54          Logger.getLogger(PathologicalPathFilter.class.getName());
55  
56      public static final String ATTR_REPETITIONS = "repetitions";
57  
58      public static final Integer DEFAULT_REPETITIONS = new Integer(3);
59      
60      private final String REGEX_PREFIX = ".*?/(.*?/)//1{";
61      private final String REGEX_SUFFIX = ",}.*";
62  
63      /*** Constructs a new PathologicalPathFilter.
64       *
65       * @param name the name of the filter.
66       */
67      public PathologicalPathFilter(String name) {
68          super(name);
69          setDescription("Pathological path filter *Deprecated* Use" +
70          		"DecidingFilter and equivalent DecideRule instead. " +
71          		"The Pathologicalpath filter" +
72                  " is used to avoid crawler traps by adding a constraint on" +
73                  " how many times a pattern in the URI could be repeated." +
74                  " Returns false if the path is NOT pathological (There" +
75                  " are no subpath reptitions or reptitions are less than" +
76                  " the '" + ATTR_REPETITIONS + "' limit).");
77  
78          Type type = getElementFromDefinition(ATTR_MATCH_RETURN_VALUE);
79          type.setTransient(true);
80  
81          type = getElementFromDefinition(ATTR_REGEXP);
82          type.setTransient(true);
83  
84          addElementToDefinition(new SimpleType(ATTR_REPETITIONS,
85                  "Number of times the pattern should be allowed to occur. \n" +
86                  "This filter returns true if number of repetitions of a" +
87                  " pattern exceeds this value",
88                  DEFAULT_REPETITIONS));
89      }
90  
91      /*** 
92       * Construct the regexp string to be matched aginst the URI.
93       * @param o an object to extract a URI from.
94       * @return the regexp pattern.
95       */
96      protected String getRegexp(Object o) {
97          int rep = 0;
98          try {
99              rep = ((Integer)getAttribute(o, ATTR_REPETITIONS)).intValue();
100         } catch (AttributeNotFoundException e) {
101             logger.severe(e.getMessage());
102         }
103         return rep == 0? null: REGEX_PREFIX + (rep - 1) + REGEX_SUFFIX;
104     }
105     
106     protected boolean getFilterOffPosition(CrawlURI curi) {
107         return false;
108     }
109 }