jayhuang75’s gists

jayhuang75 / go_train_delay_rust_api_json_to_sql_unit_test.rs

Created November 8, 2021 00:12

go train delay rust api json to sql unit test

	#[actix_rt::test]
	async fn test_to_sql_query_ranges_must() {
	let query = r#"select date, departure, depart_scheduled, destination, arrival_scheduled, delay, created_at, count(*) OVER() AS total_count from delays WHERE ( date >= '"2017-01-01"' AND date <= '"2017-06-30"' )"#;
	let query_str = r#"
	{
	"query": {
	"must": {
	"ranges": [
	{"date": {"gte": "2017-01-01", "lte": "2017-06-30"}}
	]

jayhuang75 / rust-go-train-delay-ml-pipeline-trait.rs

Last active December 19, 2021 22:32

rust-go-train-delay-ml-pipeline-trait

	// Application Container
	pub struct Application {
	pub pipeline: Box<dyn MlPipeline>
	}

	// Application Interface
	#[async_trait(?Send)]
	pub trait MlPipeline{
	async fn extract(&self) -> Result<String, AppError>;
	async fn read_csv(&self, path: &Path) -> Result<PolarResult<DataFrame>, AppError>;

jayhuang75 / go-train-delay-ml-pipeline-data-extract.rs

Created December 19, 2021 23:02

go-train-delay-ml-pipeline-data-extract

	async fn extract(&self) -> Result<String, AppError> {
	let stmt = &self
	.client
	.prepare("COPY (SELECT * From delays) TO STDOUT (FORMAT CSV, HEADER TRUE, quote ' ')")
	.await?;

	let data = self
	.client
	.copy_out(stmt)
	.await?

jayhuang75 / go-train-delay-ml-pipeline-read-csv.rs

Created December 20, 2021 01:44

go-train-delay-ml-pipeline-read-csv

	async fn read_csv(&self, p: &Path) -> Result<PolarResult<DataFrame>, AppError> {
	let file = File::open(p)?;
	Ok(CsvReader::new(file).has_header(true).finish())
	}

jayhuang75 / go-train-delay-ml-pipeline-feature-target.rs

Created December 20, 2021 05:23

go-train-delay-ml-pipeline-feature-target

	async fn feature_and_target(
	&self,
	df: &DataFrame,
	) -> (PolarResult<DataFrame>, PolarResult<DataFrame>) {
	/* Read a dataframe, select the columns we need for feature training and target and return
	the two new dataframes*/
	//id,date,departure,depart_scheduled,destination,arrival_scheduled

	let features = df.select(vec![
	"date",

jayhuang75 / go-train-delay-ml-pipeline-feature-target-convert.rs

Created December 21, 2021 02:10

go-train-delay-ml-pipeline-feature-target-convert

	async fn feature_to_matrix(&self, df: &DataFrame) -> Result<DenseMatrix<f64>, AppError> {
	/* function to convert feature dataframe to a DenseMatrix, readable by smartcore*/

	let nrows = df.height();
	let ncols = df.width();
	// convert to array
	let features_res = df.to_ndarray::<Float64Type>()?;
	// create a zero matrix and populate with features
	let mut xmatrix: DenseMatrix<f64> = BaseMatrix::zeros(nrows, ncols);
	// populate the matrix

jayhuang75 / go-train-delay-ml-pipeline-lr.rs

Created December 21, 2021 03:35

go-train-delay-ml-pipeline-lr

	async fn linear_regression_model_build(
	&self,
	x: &DenseMatrix<f64>,
	y: &Vec<f64>,
	) -> Result<String, AppError> {
	// train split
	let (x_train, _, y_train, _) = train_test_split(x, y, 0.3, true);

	// model
	let linear_regression_model =

jayhuang75 / go-train-delay-ml-pipeline-main-flow.rs

Created December 21, 2021 17:37

go-train-delay-ml-pipeline-main-flow

	// convert to feature and target dataset
	let (features, target) = pg_source_app.pipeline.feature_and_target(&df).await;

	//convert feature to smartcore data matrix
	let x_matrix = pg_source_app
	.pipeline
	.feature_to_matrix(&features.unwrap())
	.await;

	// convert target to class data vec<f64>

jayhuang75 / go-train-delay-ml-pipeline-api.rs

Created December 23, 2021 16:24

go-train-delay-ml-pipeline-api

	#[post("/predict")]
	pub async fn predict(
	// _state: web::Data<AppState>,
	predict: web::Json<PredictData>,
	) -> Result<HttpResponse, AppError> {

	// Load the model
	let lr_model: LinearRegression<f64, DenseMatrix<f64>> = {
	let mut buf: Vec<u8> = Vec::new();
	File::open(&"./models/go_train_delay_lr_2021-12-22UTC.model")

jayhuang75 / url_shortener.rs

Created March 21, 2022 02:49

url_shortener

jayhuang75 jayhuang75