Compare commits

..

No commits in common. "6292fb2609e3a58074004f43e86a2946b8b44a97" and "8d8340c5799253568161de9453cf8c42c69fdaa3" have entirely different histories.

8 changed files with 20 additions and 58 deletions

5
.gitignore vendored
View file

@ -19,8 +19,3 @@ Cargo.lock
# Added by cargo # Added by cargo
/target /target
.vscode
.ipynb_checkpoints
/data

View file

@ -1,5 +0,0 @@
pipeline:
build:
image: rust:1-buster
commands:
- cargo install --path .

View file

@ -1,14 +1,7 @@
[package] [package]
name = "dr" name = "dr"
description = "Command-line data file processing in Rust" version = "0.1.0"
version = "0.2.0"
edition = "2021" edition = "2021"
include = [
"**/*.rs",
"Cargo.toml",
]
license-file = "LICENSE"
repository = "https://git.guillemborrell.es/guillem/dr"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html

View file

@ -1,6 +1,6 @@
# dr.rs # dr.rs
A toolkit to process data files (csv and parquet) using the command line, inspired by [csvkit](https://github.com/wireservice/csvkit), with blazing speed, and powered by Rust. A set of data files (mostly csv and parquet) processing utilities inspired by [csvkit](https://github.com/wireservice/csvkit) with blazing speed, powered by Rust.
You may wonder why I'm implementing this, since there's already [xsv](https://github.com/BurntSushi/xsv). There are two reasons for that: You may wonder why I'm implementing this, since there's already [xsv](https://github.com/BurntSushi/xsv). There are two reasons for that:
@ -37,10 +37,6 @@ shape: (3, 2)
└──────┴───────────┘ └──────┴───────────┘
``` ```
## Performance
## Built standing on the shoulders of giants ## Built standing on the shoulders of giants
None of this would be possible without [Polars](https://github.com/pola-rs/polars) None of this would be possible without [Polars](https://github.com/pola-rs/polars)

View file

@ -1,7 +0,0 @@
#!/usr/bin/env python3
import sys
import pandas as pd
df = pd.read_csv(sys.stdin)
print(df.groupby("Dept", as_index=False).Weekly_Sales.mean())

View file

@ -1,8 +0,0 @@
select
Dept,
avg(Weekly_Sales)
from
this
group by
Dept

View file

@ -1,6 +1,5 @@
use polars::frame::DataFrame; use polars::frame::DataFrame;
use polars::prelude::*; use polars::prelude::*;
use std::fs;
use std::io; use std::io;
use std::io::Read; use std::io::Read;
@ -27,10 +26,15 @@ pub fn dump_csv_to_stdout(df: &mut DataFrame) {
}; };
} }
/// Read parquet and return a Polars DataFrame /// Read parquet format from stdin and return a Polars DataFrame
pub fn read_parquet(path: String) -> DataFrame { pub fn load_parquet_from_stdin() -> DataFrame {
let file = fs::File::open(path).expect("Could not open file"); let mut buffer: String = String::new();
let df = match ParquetReader::new(file).finish() { let _res: () = match io::stdin().read_to_string(&mut buffer) {
Ok(_ok) => (),
Err(_e) => (),
};
let cursor = io::Cursor::new(buffer.as_bytes());
let df = match ParquetReader::new(cursor).finish() {
Ok(df) => df, Ok(df) => df,
Err(e) => { Err(e) => {
eprintln!("{e}"); eprintln!("{e}");

View file

@ -11,11 +11,7 @@ fn main() {
.arg(arg!(-d --delimiter <String> "Column delimiter").required(false)), .arg(arg!(-d --delimiter <String> "Column delimiter").required(false)),
) )
.subcommand(Command::new("print").about("Pretty prints the table")) .subcommand(Command::new("print").about("Pretty prints the table"))
.subcommand( .subcommand(Command::new("rpq").about("Read parquet file"))
Command::new("rpq")
.about("Read parquet file")
.arg(arg!([path] "Path to the parquet file")),
)
.get_matches(); .get_matches();
if let Some(matches) = matches.subcommand_matches("sql") { if let Some(matches) = matches.subcommand_matches("sql") {
@ -30,17 +26,15 @@ fn main() {
let mut df = io::load_csv_from_stdin(); let mut df = io::load_csv_from_stdin();
io::dump_csv_to_stdout(&mut df); io::dump_csv_to_stdout(&mut df);
} }
} else if let Some(_matches) = matches.subcommand_matches("print") { }
if let Some(_matches) = matches.subcommand_matches("print") {
let df = io::load_csv_from_stdin(); let df = io::load_csv_from_stdin();
println!("{}", df) println!("{}", df)
} else if let Some(matches) = matches.subcommand_matches("rpq") { }
if let Some(path) = matches.get_one::<String>("path") {
let mut df = io::read_parquet(path.to_string()); if let Some(_matches) = matches.subcommand_matches("rpq") {
io::dump_csv_to_stdout(&mut df); let mut df = io::load_parquet_from_stdin();
} else { io::dump_csv_to_stdout(&mut df);
eprintln!("File not found")
}
} else {
println!("No command provided. Please execute dr --help")
} }
} }