fix broken featured snippets

This commit is contained in:
mat 2023-12-20 04:23:02 -06:00
parent bcf42a733a
commit 2b2771c132
3 changed files with 18 additions and 10 deletions

View File

@ -193,16 +193,17 @@ pub async fn search_with_client_and_engines(
for engine in engines { for engine in engines {
requests.push(async { requests.push(async {
let engine = *engine; let engine = *engine;
let request_response = engine.request(client, query).into();
let response = match request_response {
RequestResponse::Http(request) => {
progress_tx.send(ProgressUpdate::new( progress_tx.send(ProgressUpdate::new(
ProgressUpdateKind::Requesting, ProgressUpdateKind::Requesting,
engine, engine,
start_time, start_time,
))?; ))?;
let request_response = engine.request(client, query).into();
let response = match request_response {
RequestResponse::Http(request) => {
let res = request.send().await?; let res = request.send().await?;
progress_tx.send(ProgressUpdate::new( progress_tx.send(ProgressUpdate::new(

View File

@ -1,6 +1,7 @@
use url::Url; use url::Url;
pub fn normalize_url(url: &str) -> eyre::Result<String> { pub fn normalize_url(url: &str) -> eyre::Result<String> {
let url = url.trim_end_matches('#');
if url.is_empty() { if url.is_empty() {
return Ok(String::new()); return Ok(String::new());
} }

View File

@ -161,11 +161,17 @@ pub(super) fn parse_html_response_with_opts(
let url = normalize_url(&url)?; let url = normalize_url(&url)?;
let description = featured_snippet_description_query_method.call(&featured_snippet)?; let description = featured_snippet_description_query_method.call(&featured_snippet)?;
// this can happen on google if you search "what's my user agent"
let is_empty = description.is_empty() && title.is_empty() && url.is_empty();
if is_empty {
None
} else {
Some(EngineFeaturedSnippet { Some(EngineFeaturedSnippet {
url, url,
title, title,
description, description,
}) })
}
} else { } else {
None None
} }