一、异步处理单任务
1、cargo.toml
[dependencies]
tokio = { version = "1.0.0", features = ["full", "tracing"] }
tokio-util = { version = "0.7.0", features = ["full"] }
tokio-stream = { version = "0.1" }
tracing = "0.1"
tracing-subscriber = { version = "0.3.1", default-features = false, features = ["fmt", "ansi", "env-filter", "tracing-log"] }
bytes = "1.0.0"
futures = { version = "0.3.0", features = ["thread-pool"]}
http = "1.0.0"
serde = "1.0.55"
serde_derive = "1.0"
serde_json = "1.0"
httparse = "1.0"
httpdate = "1.0"
once_cell = "1.5.2"
rand = "0.8.3"
csv = "1.3.0"
encoding= "0.2.33"
reqwest = { version = "0.11", features = ["json"] }
主要看tokio和reqwest设置就可以了。其它是个人其它的库,没有删除。
2、代码
use reqwest::{self};
#[tokio::main]
async fn main() -> Result<(), reqwest::Error> {
let url = "https://vip.stock.finance.sina.com.cn/quotes_service/view/vMS_tradedetail.php?symbol=sz000001";
let content = reqwest::get(url).await?.text_with_charset("utf-8").await?;
println!("{content}");
Ok(())
}
或者
use reqwest::{self};
#[tokio::main]
async fn main() -> Result<(), reqwest::Error> {
let url = "https://vip.stock.finance.sina.com.cn/quotes_service/view/vMS_tradedetail.php?symbol=sz000001";
let _body = reqwest::get(url).await?.text().await?;
println!("{_body}");
Ok(())
}
二、异步处理多任务
下面代码参考了
https://blog.csdn.net/wsp_1138886114/article/details/111354010。谨仅参考。
新测是有效的。也可以处理中文字符的问题。
cargo.toml文件,同上。
有异步要爬多个网站的任务,可以参考如下:
相关代码:
use tokio;
use futures::future::join_all;
async fn fetch_path(path:String) -> Result<String,reqwest::Error>{
let mut back_string = String::new();
match reqwest::get(&path).await {
Ok(response) => {
match response.text().await{
Ok(text) =>{
println!("Read response text {},{}" ,text.len(),text);
back_string = format!("Read response text {} \t {}\t {}",path,text.len(),text)
}
Err(_) => {
println!("Read response text Error!")
}
};
}
Err(_) => {
println!("reqwest get Error!")
}
}
Ok(back_string)
}
#[tokio::main]
async fn main() -> Result<(),reqwest::Error>{
let paths = vec![
"https://www.qq.com.cn".to_string(),
"https://www.sina.com.cn".to_string(),
"https://www.sohu.com.cn".to_string()
];
let result_list = join_all(paths.into_iter().map(|path|{
fetch_path(path)
})).await;
let mut list_string:Vec<String> = vec![];
for ele in result_list.into_iter(){
if ele.is_ok(){
list_string.push(ele.unwrap())
}else {
return Err(ele.unwrap_err())
}
}
println!("请求输出:{:?}",list_string);
Ok(())
}