网络爬虫

/*
 * LeetCode 1236: Network Crawler
 * Using Java Streams to perform the crawling operation. The approach is similar to the BFS method but leverages Streams to process the URLs.
 */

import java.util.*;
import java.util.stream.*;

interface HtmlParser {
    List<String> getUrls(String url);
}

public class Solution {
    public List<String> crawl(String startUrl, HtmlParser htmlParser) {
        String hostName = getHostName(startUrl);
        Set<String> visited = Collections.synchronizedSet(new HashSet<>(Collections.singleton(startUrl)));
        Queue<String> queue = new LinkedList<>(visited);
        while (!queue.isEmpty()) {
            String currentUrl = queue.poll();
            List<String> newUrls = htmlParser.getUrls(currentUrl).stream()
                .filter(url -> url.contains(hostName) && visited.add(url))
                .collect(Collectors.toList());
            queue.addAll(newUrls);
        }
        return new ArrayList<>(visited);
    }

    private String getHostName(String url) {
        return url.split("/")[2]; // Splitting by '/' and taking the 3rd element which is the host name
    }
}