用java简单实现爬虫抓取网页代码案例
|
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 |
package com.gaodaima; import java.io.IOException; import java.util.LinkedList; import java.util.List; import java.util.Queue; import java.util.regex.Matcher; import java.util.regex.Pattern; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; public class Robot { // robot url private List<String> urlList; // cache url private Queue<String> urlQueue; // define Host public final static String HOST = "gaodaima.com"; // constructor public Robot() { super(); // initialization robot's member setUrlList(new LinkedList<String>()); setUrlQueue(new LinkedList<String>()); } // url public List<String> getUrlList() { return urlList; } public void setUrlList(List<String> urlList) { this.urlList = urlList; } // cache public Queue<String> getUrlQueue() { return urlQueue; } public void setUrlQueue(Queue<String> urlQueue) { this.urlQueue = urlQueue; } // Legal link private boolean isURL(String url) { try { // judge url Pattern pattern = Pattern.compile("^[a-zA-z]+://[^\\s]*"); Matcher matcher = pattern.matcher(url); if (matcher.matches()) { return true; } else { return false; } } catch (Exception e) { e.printStackTrace(); return false; } } // whether the url is belong to host public static boolean isHost(String url) { return url.contains(HOST); } // travel all url public void traverse(String seed) { for (this.getUrlQueue().add(seed); !this.getUrlQueue().isEmpty();) { boolean flag = true; Document document = null; try { document = Jsoup.connect(seed).timeout(5000).get(); } catch (IOException e) { e.printStackTrace(); // whether connect success flag = false; } // whether connect success,then select a tag // add these aTag into queue if (flag) { // get url Elements elements = document.select("a[href]"); for (Element e : elements) { String s = e.attr("abs:href"); // Legal link and belong host // and url not in list // then add it if (isURL(s) && s.contains(HOST) && (!getUrlQueue().contains(s)) && (!getUrlList().contains(s))) { this.getUrlQueue().add(s); } } } // get head of queue // and set it seed // travel seed it again seed = this.getUrlQueue().poll(); this.getUrlList().add(seed); // show information // System.out.println("SIZE:" // + this.getUrlQueue().size() + "---" // + seed + " connect!"); } } public static void main(String[] args) { Robot robot = new Robot(); robot.traverse("http://www.gaodaima.com"); List<String> list = robot.getUrlList(); for (String s : list) { System.out.println(s); } } } |
原创文章，转载请注明： 转载自搞代码
本文链接地址: 用java简单实现爬虫抓取网页代码案例
微信 赏一包辣条吧~
支付宝 赏一听可乐吧~
发表评论