.gitignore

@@ -19,8 +19,3 @@ Cargo.lock
 # Added by cargo
 
 /target
- 
-.vscode
-.ipynb_checkpoints
-
-/data

.woodpecker.yml

@@ -1,5 +0,0 @@
-pipeline:
-  build:
-    image: rust:1-buster
-    commands:
-      - cargo install --path .

Cargo.toml

@@ -1,14 +1,7 @@
 [package]
 name = "dr"
-description = "Command-line data file processing in Rust"
+version = "0.1.0"
-version = "0.2.0"
 edition = "2021"
-include = [
-    "**/*.rs",
-    "Cargo.toml",
-]
-license-file = "LICENSE"
-repository = "https://git.guillemborrell.es/guillem/dr"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 

README.md

@@ -1,6 +1,6 @@
 # dr.rs
 
-A toolkit to process data files (csv and parquet) using the command line, inspired by [csvkit](https://github.com/wireservice/csvkit), with blazing speed, and powered by Rust.
+A set of data files (mostly csv and parquet) processing utilities inspired by [csvkit](https://github.com/wireservice/csvkit) with blazing speed, powered by Rust.
 
 You may wonder why I'm implementing this, since there's already [xsv](https://github.com/BurntSushi/xsv). There are two reasons for that:
 
@@ -37,10 +37,6 @@ shape: (3, 2)
 └──────┴───────────┘
 ```
 
-## Performance
-
-
-
 ## Built standing on the shoulders of giants
 
 None of this would be possible without [Polars](https://github.com/pola-rs/polars)

python/group.py

@@ -1,7 +0,0 @@
-#!/usr/bin/env python3
-
-import sys
-import pandas as pd
-
-df = pd.read_csv(sys.stdin)
-print(df.groupby("Dept", as_index=False).Weekly_Sales.mean())

queries/weekly_sales_by_dept.sql

@@ -1,8 +0,0 @@
-select 
-    Dept, 
-    avg(Weekly_Sales) 
-from 
-    this 
-group by 
-    Dept
-    

src/io.rs

@@ -1,6 +1,5 @@
 use polars::frame::DataFrame;
 use polars::prelude::*;
-use std::fs;
 use std::io;
 use std::io::Read;
 
@@ -27,10 +26,15 @@ pub fn dump_csv_to_stdout(df: &mut DataFrame) {
     };
 }
 
-/// Read parquet and return a Polars DataFrame
+/// Read parquet format from stdin and return a Polars DataFrame
-pub fn read_parquet(path: String) -> DataFrame {
+pub fn load_parquet_from_stdin() -> DataFrame {
-    let file = fs::File::open(path).expect("Could not open file");
+    let mut buffer: String = String::new();
-    let df = match ParquetReader::new(file).finish() {
+    let _res: () = match io::stdin().read_to_string(&mut buffer) {
+        Ok(_ok) => (),
+        Err(_e) => (),
+    };
+    let cursor = io::Cursor::new(buffer.as_bytes());
+    let df = match ParquetReader::new(cursor).finish() {
         Ok(df) => df,
         Err(e) => {
             eprintln!("{e}");

src/main.rs

@@ -11,11 +11,7 @@ fn main() {
                 .arg(arg!(-d --delimiter <String> "Column delimiter").required(false)),
         )
         .subcommand(Command::new("print").about("Pretty prints the table"))
-        .subcommand(
+        .subcommand(Command::new("rpq").about("Read parquet file"))
-            Command::new("rpq")
-                .about("Read parquet file")
-                .arg(arg!([path] "Path to the parquet file")),
-        )
         .get_matches();
 
     if let Some(matches) = matches.subcommand_matches("sql") {
@@ -30,17 +26,15 @@ fn main() {
             let mut df = io::load_csv_from_stdin();
             io::dump_csv_to_stdout(&mut df);
         }
-    } else if let Some(_matches) = matches.subcommand_matches("print") {
+    }
+
+    if let Some(_matches) = matches.subcommand_matches("print") {
         let df = io::load_csv_from_stdin();
         println!("{}", df)
-    } else if let Some(matches) = matches.subcommand_matches("rpq") {
+    }
-        if let Some(path) = matches.get_one::<String>("path") {
+
-            let mut df = io::read_parquet(path.to_string());
+    if let Some(_matches) = matches.subcommand_matches("rpq") {
-            io::dump_csv_to_stdout(&mut df);
+        let mut df = io::load_parquet_from_stdin();
-        } else {
+        io::dump_csv_to_stdout(&mut df);
-            eprintln!("File not found")
-        }
-    } else {
-        println!("No command provided. Please execute dr --help")
     }
 }