JAVA爬虫系列

准备工作

导入依赖

XML 复制代码
 <dependency>
            <groupId>org.apache.httpcomponents</groupId>
            <artifactId>httpclient</artifactId>
            <version>4.5.2</version>
        </dependency>

yml

XML 复制代码
logging:
  level:
    root: info
    com.lrm: debug

1.入门程序(获取到静态页面)

java 复制代码
package com.itheima.reggie.utils;


import org.apache.http.HttpEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;

import java.io.IOException;

/**
 * @Author lpc
 **/
public class CrawlerFirst {
    public static void main(String[] args) throws Exception {

        //1.打开浏览器,创建Httpclient对象
        CloseableHttpClient httpClient = HttpClients.createDefault();

       //2.输入网址,发起get请求创建HttpGet对象
        HttpGet httpGet = new HttpGet("http://www.itcast.cn");

        //3.按回车,发起请求,返回响应,使用Httpclient对象发起请求
        CloseableHttpResponse response = httpClient.execute(httpGet);
        //4.解析响应,获取数据
        //判斯状态码是否是200
        if (response.getStatusLine().getStatusCode()==200){
            HttpEntity httpEntity = response.getEntity();
            //获取前端静态页面
            String content = EntityUtils.toString(httpEntity,"utf8");
            System.out.println(content);
        }


    }
}

2.HttpClient---Get

java 复制代码
package com.itheima.reggie.utils;


import org.apache.http.HttpEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;

import java.io.IOException;

/**
 * @Author lpc
 * @Date 2024 03 12 00 23
 **/
public class CrawlerFirst {
    public static void main(String[] args){

        //1.打开浏览器,创建Httpclient对象
        CloseableHttpClient httpClient = HttpClients.createDefault();

       //2.输入网址,发起get请求创建HttpGet对象
        HttpGet httpGet = new HttpGet("http://www.itcast.cn");

        //3.按回车,发起请求,返回响应,使用Httpclient对象发起请求
        CloseableHttpResponse response = null;

        try {
            response = httpClient.execute(httpGet);
            //4.解析响应,获取数据
            //判斯状态码是否是200
            if (response.getStatusLine().getStatusCode()==200){
                HttpEntity httpEntity = response.getEntity();
                //获取前端静态页面
                String content = EntityUtils.toString(httpEntity,"utf8");
                System.out.println(content.length());
            }
        } catch (IOException e) {
            throw new RuntimeException(e);
        }finally {
            try {
                //关闭response
                response.close();
            } catch (IOException e) {
                throw new RuntimeException(e);
            }
            try {
                //关闭浏览器
                httpClient.close();
            } catch (IOException e) {
                throw new RuntimeException(e);
            }
        }


    }
}
相关推荐
张槊哲13 分钟前
函数的定义与使用(python)
开发语言·python
iuyou️20 分钟前
Spring Boot知识点详解
java·spring boot·后端
北辰浮光22 分钟前
[Mybatis-plus]
java·开发语言·mybatis
一弓虽32 分钟前
SpringBoot 学习
java·spring boot·后端·学习
南客先生36 分钟前
互联网大厂Java面试:RocketMQ、RabbitMQ与Kafka的深度解析
java·面试·kafka·rabbitmq·rocketmq·消息中间件
ai大佬39 分钟前
Java 开发玩转 MCP:从 Claude 自动化到 Spring AI Alibaba 生态整合
java·spring·自动化·api中转·apikey
光而不耀@lgy1 小时前
C++初登门槛
linux·开发语言·网络·c++·后端
lkbhua莱克瓦241 小时前
用C语言实现——一个中缀表达式的计算器。支持用户输入和动画演示过程。
c语言·开发语言·数据结构·链表·学习方法·交友·计算器
Mr__Miss1 小时前
面试踩过的坑
java·开发语言
爱喝一杯白开水1 小时前
POI从入门到上手(一)-轻松完成Apache POI使用,完成Excel导入导出.
java·poi