多线程网页爬虫

/**
 * Leetcode 1242: Web Crawler Multithreaded
 * Problem statement: Given a start URL and an interface HtmlParser, implement a multithreaded web crawler to visit all URLs from the same domain.
 * 
 * Approach:
 * 1. Use a ConcurrentHashMap to store visited URLs for thread safety.
 * 2. Use a ForkJoinPool to parallelize the crawling process.
 * 3. For each URL, we create a task that fetches the URLs from the page.
 * 4. Filter and add new URLs from the same domain to the pool for further processing.
 */

import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ForkJoinPool;
import java.util.concurrent.RecursiveAction;
import java.util.stream.Collectors;

class WebCrawlerStream {
    private Set<String> visited = ConcurrentHashMap.newKeySet();
    private HtmlParser parser;
    private String startHost;

    public WebCrawlerStream(HtmlParser parser) {
        this.parser = parser;
    }

    public void startCrawling(String startUrl) {
        startHost = getHostName(startUrl);
        visited.add(startUrl);
        ForkJoinPool.commonPool().invoke(new CrawlTask(startUrl));
    }

    private class CrawlTask extends RecursiveAction {
        private String url;

        CrawlTask(String url) {
            this.url = url;
        }

        @Override
        protected void compute() {
            if (getHostName(url).equals(startHost)) {
                parser.getUrls(url).stream()
                    .filter(nextUrl -> getHostName(nextUrl).equals(startHost))
                    .filter(visited::add)
                    .map(CrawlTask::new)
                    .collect(Collectors.toList())
                    .forEach(ForkJoinPool.commonPool()::invoke);
            }
        }
    }

    private String getHostName(String url) {
        int idx = url.indexOf('/', 7); // Skip the 'http://' part
        return idx == -1 ? url : url.substring(0, idx);
    }
}