Last active
July 30, 2020 22:36
-
-
Save dustinknopoff/b298e91a3c8577c06f7e4e339679761b to your computer and use it in GitHub Desktop.
Personal copy of https://www.reddit.com/r/rust/comments/hzfwyg/writing_a_simple_query_system_in_rust/
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
use core::hash::Hash; | |
use std::sync::Arc; | |
use core::any::Any; | |
use core::any::TypeId; | |
use std::collections::HashMap; | |
use async_trait::async_trait; // 0.1.36 | |
use tokio; // 0.2.21 | |
use tokio::sync::RwLock; | |
// Our trait. The async part is not necessary, I just wanted to see how it behaves :) | |
#[async_trait] | |
pub trait Query: 'static + Send + Sync + Hash { | |
type Output: Send + Sync; // The output of the query | |
// You dont need reference to text if system has text. | |
// IMO this method should be as clean as possible, only real [re]calculation, nothing else | |
// Whats more, we could take also &self parameter to pass... query parameters :) | |
async fn calc(&self, system: &System) -> Self::Output; | |
} | |
// Replaced Rc with Arc | |
pub struct QueryRef<T>(Arc<T>); | |
impl<T> std::ops::Deref for QueryRef<T> { | |
type Target = T; | |
fn deref(&self) -> &Self::Target { | |
&(*self.0) | |
} | |
} | |
// To store query parameter we have to make sure that queries (at the bottom): | |
// Add(2,3) and Add(3,2) are not stored in the same hashmap cell. To do so, we store **hashes** of parameters. | |
#[derive(PartialEq, Eq, Hash, Clone, Copy)] | |
pub struct QueryKey(TypeId, u64); | |
pub struct System { | |
text: String, | |
// Instead of storing in Arc<> whole Query, I store only Query::Output | |
queries: RwLock<HashMap<QueryKey, Arc<dyn Any + Send + Sync>>>, | |
} | |
impl System { | |
pub fn new(text: impl ToString) -> Self { | |
Self { | |
text: text.to_string(), | |
queries: Default::default() | |
} | |
} | |
fn query_key<Q: Query>(query: &Q) -> QueryKey { | |
use std::collections::hash_map::DefaultHasher; | |
use std::hash::Hasher; | |
let type_id = TypeId::of::<Q>(); | |
let mut hasher = DefaultHasher::new(); | |
query.hash(&mut hasher); | |
let hash = hasher.finish(); | |
QueryKey(type_id, hash) | |
} | |
// This is almost 1:1 with your code | |
pub async fn query_ref<Q>(&self, query: Q) -> QueryRef<Q::Output> | |
where Q: Query { | |
let query_key = Self::query_key(&query); | |
if !self.queries.read().await.contains_key(&query_key) { | |
let query_output = query.calc(&self).await; | |
self.queries.write().await.insert(query_key, Arc::new(query_output)); | |
} | |
let borrow = self.queries.read().await; | |
let any = borrow | |
.get(&query_key).expect("Fatal bug, query does not exist!") | |
.clone(); | |
let storage = any.downcast::<Q::Output>().expect("Couldn't downcast"); | |
QueryRef(storage) | |
} | |
// But here I decided to clone the output, just like Salsa does. | |
pub async fn query<Q>(&self, query: Q) -> Q::Output | |
where Q: Query, | |
Q::Output: Clone { | |
let query_key = Self::query_key(&query); | |
if !self.queries.read().await.contains_key(&query_key) { | |
let query_output = query.calc(&self).await; | |
self.queries.write().await.insert(query_key, Arc::new(query_output)); | |
} | |
let borrow = self.queries.read().await; | |
let any = borrow | |
.get(&query_key).expect("Fatal bug, query does not exist!"); | |
let storage = any.downcast_ref::<Q::Output>().expect("Couldn't downcast"); | |
storage.clone() | |
} | |
} | |
// Query doesn't store the output, instead you type `Output = ` and its stored in `System`. | |
// Now Lines have to implement Hash. | |
#[derive(Hash)] | |
pub struct Lines; | |
#[async_trait] | |
impl Query for Lines { | |
type Output = Vec<String>; | |
async fn calc(&self, system: &System) -> Self::Output { | |
println!("Calc lines"); | |
system.text | |
.lines() | |
.map(ToString::to_string) | |
.collect() | |
} | |
} | |
#[derive(Hash)] | |
pub struct RavenCount; | |
#[async_trait] | |
impl Query for RavenCount { | |
type Output = usize; | |
async fn calc(&self, system: &System) -> Self::Output { | |
println!("Calc raven count"); | |
system.query_ref(Lines).await | |
.iter() | |
.flat_map(|line| line.char_indices().map(move |x| (line, x)) ) | |
.filter(|(line, (idx, _))| { | |
line[*idx..] | |
.chars() | |
.zip("Raven".chars()) | |
.all(|(lhs, rhs)| lhs == rhs) | |
}) | |
.count() | |
} | |
} | |
// But we can use parameters! | |
#[derive(Hash)] | |
pub struct Add { | |
a: usize, | |
b: String | |
} | |
#[async_trait] | |
impl Query for Add { | |
type Output = String; | |
async fn calc(&self, _system: &System) -> Self::Output { | |
println!("Calc add"); | |
format!("{} + {}", self.a, self.b) | |
} | |
} | |
#[tokio::main] | |
async fn main() { | |
let text = "Foo\n Raven\n Foo"; | |
let system = System::new(text); | |
let raven_count = system.query(RavenCount).await; | |
println!("raven count: {}", raven_count); | |
let raven_count = system.query_ref(RavenCount).await; | |
println!("raven count 2: {}", *raven_count); | |
// Calc it once | |
let added = system.query_ref(Add { a: 2, b: "3".into() }).await; | |
println!("Added: {}", *added); | |
// Reuse memoized output | |
let added = system.query_ref(Add { a: 2, b: "3".into() }).await; | |
println!("Added 2: {}", *added); | |
// Different parameters means we have to calculate them again | |
let added = system.query_ref(Add { a: 3, b: "2".into() }).await; | |
println!("Added 3: {}", *added); | |
// But then still we should be able to read memoized output. | |
let added = system.query_ref(Add { a: 2, b: "3".into() }).await; | |
println!("Added 4: {}", *added); | |
} |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
use core::hash::Hash; | |
use std::sync::Arc; | |
use core::any::Any; | |
use core::any::TypeId; | |
use std::collections::HashMap; | |
use async_trait::async_trait; // 0.1.36 | |
use tokio; // 0.2.21 | |
use tokio::sync::RwLock; | |
// Our trait. The async part is not necessary, I just wanted to see how it behaves :) | |
#[async_trait] | |
pub trait Query: 'static + Send + Sync + Hash { | |
type Output: Send + Sync; // The output of the query | |
// You dont need reference to text if system has text. | |
// IMO this method should be as clean as possible, only real [re]calculation, nothing else | |
// Whats more, we could take also &self parameter to pass... query parameters :) | |
async fn calc(&self, system: &System) -> Self::Output; | |
} | |
// Replaced Rc with Arc | |
pub struct QueryRef<T>(Arc<T>); | |
impl<T> std::ops::Deref for QueryRef<T> { | |
type Target = T; | |
fn deref(&self) -> &Self::Target { | |
&(*self.0) | |
} | |
} | |
// To store query parameter we have to make sure that queries (at the bottom): | |
// Add(2,3) and Add(3,2) are not stored in the same hashmap cell. To do so, we store **hashes** of parameters. | |
#[derive(PartialEq, Eq, Hash, Clone, Copy)] | |
pub struct QueryKey(TypeId, u64); | |
pub struct System { | |
// Instead of storing in Arc<> whole Query, I store only Query::Output | |
queries: RwLock<HashMap<QueryKey, Arc<dyn Any + Send + Sync>>>, | |
} | |
impl System { | |
pub fn new() -> Self { | |
Self { | |
queries: Default::default() | |
} | |
} | |
fn query_key<Q: Query>(query: &Q) -> QueryKey { | |
use std::collections::hash_map::DefaultHasher; | |
use std::hash::Hasher; | |
let type_id = TypeId::of::<Q>(); | |
let mut hasher = DefaultHasher::new(); | |
query.hash(&mut hasher); | |
let hash = hasher.finish(); | |
QueryKey(type_id, hash) | |
} | |
// This is almost 1:1 with your code | |
pub async fn query_ref<Q>(&self, query: Q) -> QueryRef<Q::Output> | |
where Q: Query { | |
let query_key = Self::query_key(&query); | |
if !self.queries.read().await.contains_key(&query_key) { | |
let query_output = query.calc(&self).await; | |
self.queries.write().await.insert(query_key, Arc::new(query_output)); | |
} | |
let borrow = self.queries.read().await; | |
let any = borrow | |
.get(&query_key).expect("Fatal bug, query does not exist!") | |
.clone(); | |
let storage = any.downcast::<Q::Output>().expect("Couldn't downcast"); | |
QueryRef(storage) | |
} | |
// But here I decided to clone the output, just like Salsa does. | |
pub async fn query<Q>(&self, query: Q) -> Q::Output | |
where Q: Query, | |
Q::Output: Clone { | |
let query_key = Self::query_key(&query); | |
if !self.queries.read().await.contains_key(&query_key) { | |
let query_output = query.calc(&self).await; | |
self.queries.write().await.insert(query_key, Arc::new(query_output)); | |
} | |
let borrow = self.queries.read().await; | |
let any = borrow | |
.get(&query_key).expect("Fatal bug, query does not exist!"); | |
let storage = any.downcast_ref::<Q::Output>().expect("Couldn't downcast"); | |
storage.clone() | |
} | |
} | |
// Query doesn't store the output, instead you type `Output = ` and its stored in `System`. | |
// Now Lines have to implement Hash. | |
#[derive(Hash)] | |
pub struct Lines { | |
source: String, | |
} | |
#[async_trait] | |
impl Query for Lines { | |
type Output = Vec<String>; | |
async fn calc(&self, _system: &System) -> Self::Output { | |
println!("Calc lines"); | |
self.source | |
.lines() | |
.map(ToString::to_string) | |
.collect() | |
} | |
} | |
#[derive(Hash)] | |
pub struct RavenCount { | |
source: String | |
} | |
#[async_trait] | |
impl Query for RavenCount { | |
type Output = usize; | |
async fn calc(&self, system: &System) -> Self::Output { | |
println!("Calc raven count"); | |
system.query_ref(Lines{ source: self.source.clone()}).await | |
.iter() | |
.flat_map(|line| line.char_indices().map(move |x| (line, x)) ) | |
.filter(|(line, (idx, _))| { | |
line[*idx..] | |
.chars() | |
.zip("Raven".chars()) | |
.all(|(lhs, rhs)| lhs == rhs) | |
}) | |
.count() | |
} | |
} | |
// But we can use parameters! | |
#[derive(Hash)] | |
pub struct Add { | |
a: usize, | |
b: String | |
} | |
#[async_trait] | |
impl Query for Add { | |
type Output = String; | |
async fn calc(&self, _system: &System) -> Self::Output { | |
println!("Calc add"); | |
format!("{} + {}", self.a, self.b) | |
} | |
} | |
#[tokio::main] | |
async fn main() { | |
let text = "Foo\n Raven\n Foo"; | |
let system = System::new(); | |
let raven_count = system.query(RavenCount{ source: text.to_string()}).await; | |
println!("raven count: {}", raven_count); | |
let raven_count = system.query_ref(RavenCount{ source: text.to_string()}).await; | |
println!("raven count 2: {}", *raven_count); | |
// Calc it once | |
let added = system.query_ref(Add { a: 2, b: "3".into() }).await; | |
println!("Added: {}", *added); | |
// Reuse memoized output | |
let added = system.query_ref(Add { a: 2, b: "3".into() }).await; | |
println!("Added 2: {}", *added); | |
// Different parameters means we have to calculate them again | |
let added = system.query_ref(Add { a: 3, b: "2".into() }).await; | |
println!("Added 3: {}", *added); | |
// But then still we should be able to read memoized output. | |
let added = system.query_ref(Add { a: 2, b: "3".into() }).await; | |
println!("Added 4: {}", *added); | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment