Simple example of URL fetcher in Rust

simple-fetch.rs

// Ivan Prisyazhnyy <[email protected]>, 2016

// Package implements example of parallel URL fetcher counting Go entries.
// Test: echo 'https://golang.org\nhttps://golang.org\nars\nhttps://golang.org\n' | cargo run
// Test: echo 'https://golang.org\nhttps://golang.org\nars\nhttps://golang.org\n' | cargo run
// Test: echo 'https://golang.org\nhttps://golang.org\nars\nhttps://golang.org\n' | cargo run
// Test: echo 'https://golang.org\nhttps://golang.org\nars\nhttps://golang.org\n' | cargo run
// Test: echo 'https://golang.org\nhttps://golang.org\nars\nhttps://golang.org\n' | cargo run
//
// Cargo.toml
// [dependencies]
// hyper = "0.9"
extern crate hyper;

use std::io;
use std::io::prelude::{Read, BufRead};
use std::thread;
use std::sync::mpsc;
use std::sync::mpsc::{Sender, Receiver, RecvError};
use hyper::client::{Client};

#[derive(Clone, PartialEq, PartialOrd, Eq, Ord, Debug, Hash)]
struct Job {
	url: String,
	count: usize
}

pub struct Processor {
	jobs: usize,
	max: usize,

	total: usize,
	result: usize,

	sync: Sender<Job>,
	recv: Receiver<Job>
}

impl Processor {
	fn new(max: usize) -> Processor {
		let (s, r) = mpsc::channel();
		Processor {
			jobs: 0,
			max: max,
			total: 0,
			result: 0,
			sync: s,
			recv: r
		}
	}

	fn process(&mut self, s: &str) {
		if self.jobs >= self.max {
			println!("Waiting {}...", s);
			let res = self.recv.recv();
			self.read_result(res);
			self.jobs -= 1;
		}

		self.jobs += 1;
		self.total += 1;

		let tx = self.sync.clone();
		let req = s.to_string();
		let id = self.total;
		println!("Starting {}: {}...", id, s);
		thread::spawn(move || {
			println!("{}: started {}...", id, req);

			// For the sake of simplicity I will imagine client is Sync?
			let client = Client::new();
			let mut count = 0;
			match client.get(&req).send() {
				Ok(mut res) => {
					if res.status != hyper::Ok {
						println!("{}: error invalid status code {}", id, res.status);
					} else {
						let mut buf = String::new();
						res.read_to_string(&mut buf);
						count = buf.matches("Go").count();
					}
				},
				Err(e) => {
					println!("{}: error {}...", id, e);
				},
			}

			tx.send(Job { url: req, count: count }).unwrap();
			println!("{}: finished", id);
		});
	}

	fn wait(&mut self) {
		while self.jobs > 0 {
			let res = self.recv.recv();
			self.read_result(res);
			self.jobs -= 1;
		}
	}

	fn read_result(&mut self, r: Result<Job, RecvError>) {
		match r {
			Ok(job) => {
				self.result += job.count;
				println!("Got result {:?}", job);
			},
			Err(e) => println!("Got an error {:?}", e),
		}
	}
}

fn main() {
	let mut processor = Processor::new(2);

	let stdin = io::stdin();
	let mut buf = String::new();
	let mut handle = stdin.lock();
	while handle.read_line(&mut buf).unwrap() > 0 {
		processor.process(&buf.trim());
		buf.clear()
	}

	processor.wait();

	println!("Total: {}", processor.result);
}