Spaces:
Runtime error
Runtime error
Zsombor Gegesy
commited on
Commit
•
519ebe0
1
Parent(s):
996ff84
Instead of caching jsons, we can cache the original structure
Browse files- src/cache/cacher.rs +21 -11
- src/cache/error.rs +4 -0
- src/lib.rs +2 -1
- src/models/aggregation_models.rs +1 -1
- src/server/routes/search.rs +6 -9
src/cache/cacher.rs
CHANGED
@@ -6,6 +6,8 @@ use mini_moka::sync::Cache as MokaCache;
|
|
6 |
use std::time::Duration;
|
7 |
use tokio::sync::Mutex;
|
8 |
|
|
|
|
|
9 |
use super::{error::PoolError, redis_cacher::RedisCache};
|
10 |
|
11 |
/// Different implementations for caching, currently it is possible to cache in-memory or in Redis.
|
@@ -14,7 +16,7 @@ pub enum Cache {
|
|
14 |
/// Encapsulates the Redis based cache
|
15 |
Redis(RedisCache),
|
16 |
/// Contains the in-memory cache.
|
17 |
-
InMemory(MokaCache<String,
|
18 |
}
|
19 |
|
20 |
impl Cache {
|
@@ -37,9 +39,13 @@ impl Cache {
|
|
37 |
/// # Arguments
|
38 |
///
|
39 |
/// * `url` - It takes an url as a string.
|
40 |
-
pub async fn cached_json(&mut self, url: &str) -> Result<
|
41 |
match self {
|
42 |
-
Cache::Redis(redis_cache) =>
|
|
|
|
|
|
|
|
|
43 |
Cache::InMemory(in_memory) => match in_memory.get(&url.to_string()) {
|
44 |
Some(res) => Ok(res),
|
45 |
None => Err(Report::new(PoolError::MissingValue)),
|
@@ -56,13 +62,17 @@ impl Cache {
|
|
56 |
/// * `url` - It takes the url as a String.
|
57 |
pub async fn cache_results(
|
58 |
&mut self,
|
59 |
-
|
60 |
url: &str,
|
61 |
) -> Result<(), Report<PoolError>> {
|
62 |
match self {
|
63 |
-
Cache::Redis(redis_cache) =>
|
|
|
|
|
|
|
|
|
64 |
Cache::InMemory(cache) => {
|
65 |
-
cache.insert(url.to_string(),
|
66 |
Ok(())
|
67 |
}
|
68 |
}
|
@@ -82,20 +92,20 @@ impl SharedCache {
|
|
82 |
}
|
83 |
}
|
84 |
|
85 |
-
/// A function which
|
86 |
-
pub async fn cached_json(&self, url: &str) -> Result<
|
87 |
let mut mut_cache = self.cache.lock().await;
|
88 |
mut_cache.cached_json(url).await
|
89 |
}
|
90 |
|
91 |
/// A function which caches the results by using the `url` as the key and
|
92 |
-
/// `
|
93 |
pub async fn cache_results(
|
94 |
&self,
|
95 |
-
|
96 |
url: &str,
|
97 |
) -> Result<(), Report<PoolError>> {
|
98 |
let mut mut_cache = self.cache.lock().await;
|
99 |
-
mut_cache.cache_results(
|
100 |
}
|
101 |
}
|
|
|
6 |
use std::time::Duration;
|
7 |
use tokio::sync::Mutex;
|
8 |
|
9 |
+
use crate::results::aggregation_models::SearchResults;
|
10 |
+
|
11 |
use super::{error::PoolError, redis_cacher::RedisCache};
|
12 |
|
13 |
/// Different implementations for caching, currently it is possible to cache in-memory or in Redis.
|
|
|
16 |
/// Encapsulates the Redis based cache
|
17 |
Redis(RedisCache),
|
18 |
/// Contains the in-memory cache.
|
19 |
+
InMemory(MokaCache<String, SearchResults>),
|
20 |
}
|
21 |
|
22 |
impl Cache {
|
|
|
39 |
/// # Arguments
|
40 |
///
|
41 |
/// * `url` - It takes an url as a string.
|
42 |
+
pub async fn cached_json(&mut self, url: &str) -> Result<SearchResults, Report<PoolError>> {
|
43 |
match self {
|
44 |
+
Cache::Redis(redis_cache) => {
|
45 |
+
let json = redis_cache.cached_json(url).await?;
|
46 |
+
Ok(serde_json::from_str::<SearchResults>(&json)
|
47 |
+
.map_err(|_| PoolError::SerializationError)?)
|
48 |
+
}
|
49 |
Cache::InMemory(in_memory) => match in_memory.get(&url.to_string()) {
|
50 |
Some(res) => Ok(res),
|
51 |
None => Err(Report::new(PoolError::MissingValue)),
|
|
|
62 |
/// * `url` - It takes the url as a String.
|
63 |
pub async fn cache_results(
|
64 |
&mut self,
|
65 |
+
search_results: SearchResults,
|
66 |
url: &str,
|
67 |
) -> Result<(), Report<PoolError>> {
|
68 |
match self {
|
69 |
+
Cache::Redis(redis_cache) => {
|
70 |
+
let json = serde_json::to_string(&search_results)
|
71 |
+
.map_err(|_| PoolError::SerializationError)?;
|
72 |
+
redis_cache.cache_results(&json, url).await
|
73 |
+
}
|
74 |
Cache::InMemory(cache) => {
|
75 |
+
cache.insert(url.to_string(), search_results);
|
76 |
Ok(())
|
77 |
}
|
78 |
}
|
|
|
92 |
}
|
93 |
}
|
94 |
|
95 |
+
/// A function which retrieves the cached SearchResulsts from the internal cache.
|
96 |
+
pub async fn cached_json(&self, url: &str) -> Result<SearchResults, Report<PoolError>> {
|
97 |
let mut mut_cache = self.cache.lock().await;
|
98 |
mut_cache.cached_json(url).await
|
99 |
}
|
100 |
|
101 |
/// A function which caches the results by using the `url` as the key and
|
102 |
+
/// `SearchResults` as the value.
|
103 |
pub async fn cache_results(
|
104 |
&self,
|
105 |
+
search_results: SearchResults,
|
106 |
url: &str,
|
107 |
) -> Result<(), Report<PoolError>> {
|
108 |
let mut mut_cache = self.cache.lock().await;
|
109 |
+
mut_cache.cache_results(search_results, url).await
|
110 |
}
|
111 |
}
|
src/cache/error.rs
CHANGED
@@ -12,6 +12,7 @@ pub enum PoolError {
|
|
12 |
/// This variant handles the errors which occurs when all the connections
|
13 |
/// in the connection pool return a connection dropped redis error.
|
14 |
PoolExhaustionWithConnectionDropError,
|
|
|
15 |
MissingValue,
|
16 |
}
|
17 |
|
@@ -34,6 +35,9 @@ impl fmt::Display for PoolError {
|
|
34 |
PoolError::MissingValue => {
|
35 |
write!(f, "The value is missing from the cache")
|
36 |
}
|
|
|
|
|
|
|
37 |
}
|
38 |
}
|
39 |
}
|
|
|
12 |
/// This variant handles the errors which occurs when all the connections
|
13 |
/// in the connection pool return a connection dropped redis error.
|
14 |
PoolExhaustionWithConnectionDropError,
|
15 |
+
SerializationError,
|
16 |
MissingValue,
|
17 |
}
|
18 |
|
|
|
35 |
PoolError::MissingValue => {
|
36 |
write!(f, "The value is missing from the cache")
|
37 |
}
|
38 |
+
PoolError::SerializationError => {
|
39 |
+
write!(f, "Unable to serialize, deserialize from the cache")
|
40 |
+
}
|
41 |
}
|
42 |
}
|
43 |
}
|
src/lib.rs
CHANGED
@@ -44,7 +44,8 @@ use handler::paths::{file_path, FileType};
|
|
44 |
///
|
45 |
/// let config = Config::parse(true).unwrap();
|
46 |
/// let listener = TcpListener::bind("127.0.0.1:8080").expect("Failed to bind address");
|
47 |
-
/// let
|
|
|
48 |
/// ```
|
49 |
pub fn run(listener: TcpListener, config: Config, cache: Cache) -> std::io::Result<Server> {
|
50 |
let mut handlebars: Handlebars<'_> = Handlebars::new();
|
|
|
44 |
///
|
45 |
/// let config = Config::parse(true).unwrap();
|
46 |
/// let listener = TcpListener::bind("127.0.0.1:8080").expect("Failed to bind address");
|
47 |
+
/// let cache = Cache::new_in_memory();
|
48 |
+
/// let server = run(listener,config,cache).expect("Failed to start server");
|
49 |
/// ```
|
50 |
pub fn run(listener: TcpListener, config: Config, cache: Cache) -> std::io::Result<Server> {
|
51 |
let mut handlebars: Handlebars<'_> = Handlebars::new();
|
src/models/aggregation_models.rs
CHANGED
@@ -102,7 +102,7 @@ impl EngineErrorInfo {
|
|
102 |
/// A named struct to store, serialize, deserialize the all the search results scraped and
|
103 |
/// aggregated from the upstream search engines.
|
104 |
/// `SearchResult` structs.
|
105 |
-
#[derive(Serialize, Deserialize, Default)]
|
106 |
#[serde(rename_all = "camelCase")]
|
107 |
pub struct SearchResults {
|
108 |
/// Stores the individual serializable `SearchResult` struct into a vector of
|
|
|
102 |
/// A named struct to store, serialize, deserialize the all the search results scraped and
|
103 |
/// aggregated from the upstream search engines.
|
104 |
/// `SearchResult` structs.
|
105 |
+
#[derive(Serialize, Deserialize, Default, Clone)]
|
106 |
#[serde(rename_all = "camelCase")]
|
107 |
pub struct SearchResults {
|
108 |
/// Stores the individual serializable `SearchResult` struct into a vector of
|
src/server/routes/search.rs
CHANGED
@@ -192,12 +192,12 @@ async fn results(
|
|
192 |
safe_search: u8,
|
193 |
) -> Result<SearchResults, Box<dyn std::error::Error>> {
|
194 |
// fetch the cached results json.
|
195 |
-
let
|
196 |
// check if fetched cache results was indeed fetched or it was an error and if so
|
197 |
// handle the data accordingly.
|
198 |
-
match
|
199 |
-
|
200 |
-
|
201 |
if safe_search == 4 {
|
202 |
let mut results: SearchResults = SearchResults::default();
|
203 |
let mut _flag: bool =
|
@@ -208,9 +208,7 @@ async fn results(
|
|
208 |
results.set_disallowed();
|
209 |
results.add_style(&config.style);
|
210 |
results.set_page_query(query);
|
211 |
-
cache
|
212 |
-
.cache_results(serde_json::to_string(&results)?, &url)
|
213 |
-
.await?;
|
214 |
return Ok(results);
|
215 |
}
|
216 |
}
|
@@ -258,8 +256,7 @@ async fn results(
|
|
258 |
results.set_filtered();
|
259 |
}
|
260 |
results.add_style(&config.style);
|
261 |
-
|
262 |
-
cache.cache_results(json_results, &url).await?;
|
263 |
Ok(results)
|
264 |
}
|
265 |
}
|
|
|
192 |
safe_search: u8,
|
193 |
) -> Result<SearchResults, Box<dyn std::error::Error>> {
|
194 |
// fetch the cached results json.
|
195 |
+
let cached_results = cache.cached_json(&url).await;
|
196 |
// check if fetched cache results was indeed fetched or it was an error and if so
|
197 |
// handle the data accordingly.
|
198 |
+
match cached_results {
|
199 |
+
Ok(results) => Ok(results),
|
200 |
+
Err(_) => {
|
201 |
if safe_search == 4 {
|
202 |
let mut results: SearchResults = SearchResults::default();
|
203 |
let mut _flag: bool =
|
|
|
208 |
results.set_disallowed();
|
209 |
results.add_style(&config.style);
|
210 |
results.set_page_query(query);
|
211 |
+
cache.cache_results(results.clone(), &url).await?;
|
|
|
|
|
212 |
return Ok(results);
|
213 |
}
|
214 |
}
|
|
|
256 |
results.set_filtered();
|
257 |
}
|
258 |
results.add_style(&config.style);
|
259 |
+
cache.cache_results(results.clone(), &url).await?;
|
|
|
260 |
Ok(results)
|
261 |
}
|
262 |
}
|