Spaces:
Runtime error
Runtime error
File size: 3,370 Bytes
15fc415 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 |
use std::collections::HashMap;
use reqwest::header::USER_AGENT;
use scraper::{Html, Selector};
use crate::search_results_handler::aggregation_models::RawSearchResult;
// This function scrapes results from the upstream engine duckduckgo and puts all the scraped
// results like title, visiting_url (href in html),engine (from which engine it was fetched from)
// and description in a RawSearchResult and then adds that to HashMap whose keys are url and
// values are RawSearchResult struct and then returns it within a Result enum.
pub async fn results(
query: &str,
page: Option<u32>,
user_agent: &str,
) -> Result<HashMap<String, RawSearchResult>, Box<dyn std::error::Error>> {
// Page number can be missing or empty string and so appropriate handling is required
// so that upstream server recieves valid page number.
let url: String = match page {
Some(page_number) => {
if page_number <= 1 {
format!("https://html.duckduckgo.com/html/?q={query}&s=&dc=&v=1&o=json&api=/d.js")
} else {
format!(
"https://duckduckgo.com/html/?q={}&s={}&dc={}&v=1&o=json&api=/d.js",
query,
page_number / 2 * 30,
page_number / 2 * 30 + 1
)
}
}
None => format!("https://html.duckduckgo.com/html/?q={query}&s=&dc=&v=1&o=json&api=/d.js"),
};
// fetch the html from upstream duckduckgo engine
// TODO: Write better error handling code to handle no results case.
let results: String = reqwest::Client::new()
.get(url)
.header(USER_AGENT, user_agent)
.send()
.await?
.text()
.await?;
let document: Html = Html::parse_document(&results);
let results: Selector = Selector::parse(".result")?;
let result_title: Selector = Selector::parse(".result__a")?;
let result_url: Selector = Selector::parse(".result__url")?;
let result_desc: Selector = Selector::parse(".result__snippet")?;
let mut search_results: HashMap<String, RawSearchResult> = HashMap::new();
// scrape all the results from the html
for result in document.select(&results) {
let search_result: RawSearchResult = RawSearchResult {
title: result
.select(&result_title)
.next()
.unwrap()
.inner_html()
.trim()
.to_string(),
visiting_url: format!(
"https://{}",
result
.select(&result_url)
.next()
.unwrap()
.inner_html()
.trim()
),
description: result
.select(&result_desc)
.next()
.unwrap()
.inner_html()
.trim()
.to_string(),
engine: vec!["duckduckgo".to_string()],
};
search_results.insert(
format!(
"https://{}",
result
.select(&result_url)
.next()
.unwrap()
.inner_html()
.trim()
),
search_result,
);
}
Ok(search_results)
}
|