From 09635558ae4593e325f44a7cd77736eabff200d0 Mon Sep 17 00:00:00 2001 From: Till Wegmueller Date: Mon, 28 Mar 2022 15:07:05 -0300 Subject: [PATCH] Switch to rustls for embedded tls Refine Makefile parsing so we also parse define rules Implement Recursive Vraiable resolving Algorithm --- Cargo.lock | 129 +++++++++++++++++++++++++++++++++++++ pkg6dev/src/main.rs | 52 ++++++++------- userland/Cargo.toml | 7 +- userland/src/lib.rs | 98 ++++++++++++++++++++++++++-- userland/src/makefile.pest | 17 ++++- 5 files changed, 270 insertions(+), 33 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d097c0a..7b95ef0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -283,6 +283,18 @@ version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "365a1a1fb30ea1c03a830fdb2158f5236833ac81fa0ad12fe35b29cddc35cb04" +[[package]] +name = "futures-macro" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "668c6733a182cd7deb4f1de7ba3bf2120823835b3bcfbeacf7d2c4a773c1bb8b" +dependencies = [ + "proc-macro-hack", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "futures-sink" version = "0.3.14" @@ -303,10 +315,13 @@ checksum = "3c144ad54d60f23927f0a6b6d816e4271278b64f005ad65e4e35291d2de9c025" dependencies = [ "futures-core", "futures-io", + "futures-macro", "futures-task", "memchr", "pin-project-lite", "pin-utils", + "proc-macro-hack", + "proc-macro-nested", "slab", ] @@ -438,6 +453,21 @@ dependencies = [ "want", ] +[[package]] +name = "hyper-rustls" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f9f7a97316d44c0af9b0301e65010573a853a9fc97046d7331d7f6bc0fd5a64" +dependencies = [ + "futures-util", + "hyper", + "log", + "rustls", + "tokio", + "tokio-rustls", + "webpki", +] + [[package]] name = "hyper-tls" version = "0.5.0" @@ -839,6 +869,18 @@ dependencies = [ "version_check", ] +[[package]] +name = "proc-macro-hack" +version = "0.5.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5" + +[[package]] +name = "proc-macro-nested" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc881b2c22681370c6a780e47af9840ef841837bc98118431d4e1868bd0c1086" + [[package]] name = "proc-macro2" version = "1.0.36" @@ -946,6 +988,7 @@ dependencies = [ "http", "http-body", "hyper", + "hyper-rustls", "hyper-tls", "ipnet", "js-sys", @@ -955,18 +998,49 @@ dependencies = [ "native-tls", "percent-encoding", "pin-project-lite", + "rustls", "serde", "serde_json", "serde_urlencoded", "tokio", "tokio-native-tls", + "tokio-rustls", "url", "wasm-bindgen", "wasm-bindgen-futures", "web-sys", + "webpki-roots", "winreg", ] +[[package]] +name = "ring" +version = "0.16.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" +dependencies = [ + "cc", + "libc", + "once_cell", + "spin", + "untrusted", + "web-sys", + "winapi", +] + +[[package]] +name = "rustls" +version = "0.19.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35edb675feee39aec9c99fa5ff985081995a06d594114ae14cbe797ad7b7a6d7" +dependencies = [ + "base64", + "log", + "ring", + "sct", + "webpki", +] + [[package]] name = "ryu" version = "1.0.5" @@ -983,6 +1057,16 @@ dependencies = [ "winapi", ] +[[package]] +name = "sct" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b362b83898e0e69f38515b82ee15aa80636befe47c3b6d3d89a911e78fc228ce" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "security-framework" version = "2.2.0" @@ -1120,6 +1204,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "spin" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" + [[package]] name = "strsim" version = "0.10.0" @@ -1226,6 +1316,17 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-rustls" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc6844de72e57df1980054b38be3a9f4702aba4858be64dd700181a8a6d0e1b6" +dependencies = [ + "rustls", + "tokio", + "webpki", +] + [[package]] name = "tokio-util" version = "0.6.6" @@ -1308,6 +1409,12 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564" +[[package]] +name = "untrusted" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" + [[package]] name = "url" version = "2.2.1" @@ -1325,13 +1432,16 @@ name = "userland" version = "0.1.1" dependencies = [ "anyhow", + "lazy_static", "maplit 0.1.6", "pest", "pest_derive", + "regex", "reqwest", "semver", "serde", "serde_json", + "thiserror", ] [[package]] @@ -1446,6 +1556,25 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webpki" +version = "0.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e38c0608262c46d4a56202ebabdeb094cef7e560ca7a226c6bf055188aa4ea" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "webpki-roots" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aabe153544e473b775453675851ecc86863d2a81d786d741f6b76778f2a48940" +dependencies = [ + "webpki", +] + [[package]] name = "winapi" version = "0.3.9" diff --git a/pkg6dev/src/main.rs b/pkg6dev/src/main.rs index 7d798ea..f8dae48 100644 --- a/pkg6dev/src/main.rs +++ b/pkg6dev/src/main.rs @@ -75,53 +75,61 @@ fn diff_component(component_path: impl AsRef) -> Result<()> { fn show_component_info>(component_path: P) -> Result<()> { let makefile_path = component_path.as_ref().join("Makefile"); - println!("{:?}", makefile_path); - let makefile = Makefile::parse_file(&makefile_path)?; let mut name = String::new(); - if let Some(var) = makefile.variables.get("COMPONENT_NAME") { - println!("Name: {}", var.join(" ")); - name = var.first().unwrap().to_string(); + if let Some(var) = makefile.get("COMPONENT_NAME") { + println!("Name: {}", var.replace("\n", "\n\t")); + if let Some(component_name) = makefile.get_first("COMPONENT_NAME") { + name = component_name.clone(); + } } - if let Some(var) = makefile.variables.get("COMPONENT_VERSION") { - println!("Version: {}", var.join(" ")); + if let Some(var) = makefile.get("COMPONENT_VERSION") { + println!("Version: {}", var.replace("\n", "\n\t")); let latest_version = find_newest_version(&name); if latest_version.is_ok() { println!("Latest Version: {}", latest_version?); } else { - eprintln!("{:?}", latest_version.unwrap_err()) + println!("Error: Could not get latest version info: {:?}", latest_version.unwrap_err()) } } - if let Some(var) = makefile.variables.get("BUILD_BITS") { - println!("Build bits: {}", var.join(" ")); + if let Some(var) = makefile.get("BUILD_BITS") { + println!("Build bits: {}", var.replace("\n", "\n\t")); } - if let Some(var) = makefile.variables.get("COMPONENT_PROJECT_URL") { - println!("Project URl: {}", var.join("\t")); + if let Some(var) = makefile.get("COMPONENT_BUILD_ACTION") { + println!("Build action: {}", var.replace("\n", "\n\t")); } - if let Some(var) = makefile.variables.get("COMPONENT_ARCHIVE_URL") { - println!("Source URl: {}", var.join("\t")); + if let Some(var) = makefile.get("COMPONENT_PROJECT_URL") { + println!("Project URl: {}", var.replace("\n", "\n\t")); } - if let Some(var) = makefile.variables.get("COMPONENT_ARCHIVE_HASH") { - println!("Source Archive File Hash: {}", var.join(" ")); + if let Some(var) = makefile.get("COMPONENT_ARCHIVE_URL") { + println!("Source URl: {}", var.replace("\n", "\n\t")); } - if let Some(var) = makefile.variables.get("CONFIGURE_ENV") { - println!("Configure Environment: {}", var.join("\n\t")); + if let Some(var) = makefile.get("COMPONENT_ARCHIVE_HASH") { + println!("Source Archive File Hash: {}", var.replace("\n", "\n\t")); } - if let Some(var) = makefile.variables.get("CONFIGURE_OPTIONS") { - println!("./configure {}", var.join("\n\t")); + if let Some(var) = makefile.get("CONFIGURE_ENV") { + println!("Configure Environment: {}", var.replace("\n", "\n\t")); } - if let Some(var) = makefile.variables.get("REQUIRED_PACKAGES") { - println!("Dependencies:\n\t{}", var.join("\n\t")); + if let Some(var) = makefile.get("CONFIGURE_OPTIONS") { + println!("./configure {}", var.replace("\n", "\n\t")); + } + + if let Some(var) = makefile.get("REQUIRED_PACKAGES") { + println!("Dependencies:\n\t{}", var.replace("\n", "\n\t")); + } + + if let Some(var) = makefile.get("COMPONENT_INSTALL_ACTION") { + println!("Install Action:\n\t{}", var); } Ok(()) diff --git a/userland/Cargo.toml b/userland/Cargo.toml index 8e9f25b..ee622d0 100644 --- a/userland/Cargo.toml +++ b/userland/Cargo.toml @@ -10,10 +10,13 @@ description = "Helper tool for IPS package development" [dependencies] anyhow = "1.0.56" +thiserror = "1.0.30" pest_derive = "2.1.0" maplit = "0.1.6" pest = "2.1.0" -reqwest = { version = "0.11", features = ["blocking", "json"] } +reqwest = { version = "0.11", features = ["blocking", "json", "rustls-tls"] } semver = "0.11.0" serde = { version = "1.0", features = ["derive"] } -serde_json = "1.0" \ No newline at end of file +serde_json = "1.0" +regex = "1" +lazy_static = "1" \ No newline at end of file diff --git a/userland/src/lib.rs b/userland/src/lib.rs index ecbb850..dc2165b 100644 --- a/userland/src/lib.rs +++ b/userland/src/lib.rs @@ -7,12 +7,15 @@ extern crate maplit; extern crate pest_derive; -use anyhow::Result; +use anyhow::{anyhow, Result}; use std::collections::HashMap; use std::fs::read_to_string; use pest::iterators::{Pairs}; -use std::path::Path; +use std::path::{Path, PathBuf}; use pest::Parser; +use thiserror::Error; +use lazy_static::lazy_static; +use regex::Regex; #[derive(Parser)] #[grammar = "makefile.pest"] @@ -25,15 +28,77 @@ pub struct Makefile { // pub targets: HashMap, } +#[derive(Error, Debug)] +pub enum ParserError { + #[error("cannot parse {file}: {reason}")] + MakefileReadError { + file: PathBuf, + reason: anyhow::Error, + } +} + impl Makefile { - pub fn parse_file(path: &Path) -> Result { - let content = read_to_string(path)?; - parse_string(content) + pub fn parse_file>(path: P) -> Result { + let content = read_to_string(&path)?; + parse_string(content).map_err(|err| + anyhow!(ParserError::MakefileReadError{file: path.as_ref().to_path_buf(), reason: anyhow!(err)}) + ) } pub fn parse_string(content: String) -> Result { parse_string(content) } + + pub fn get(&self, var_name: &str) -> Option { + if let Some(var) = self.variables.get(var_name) { + let vars_resolved = self.resolve_nested_variables(var); + Some(vars_to_string(&vars_resolved)) + } else { + None + } + } + + fn resolve_nested_variables(&self, var: &Vec) -> Vec { + // Make a mutable copy of the variables so we can replace nested variables with their final strings + let mut vars_copy = var.clone(); + + // Logic to resolve all the nested Variables when we access them. + for (i, maybe_nested_var) in var.iter().enumerate() { + lazy_static! { + static ref VARRE: Regex = Regex::new(r"(?P\$\(.+?\))").unwrap(); + } + for captures in VARRE.captures_iter(maybe_nested_var) { + if let Some(nested_var) = captures.name("var_name") { + let nested_var_name = nested_var.as_str().replace("$(", "").replace(")", ""); + if let Some(resolved_nested_var) = self.get(&nested_var_name) { + let mut new_string = vars_copy[i].clone(); + new_string = new_string.replacen(nested_var.as_str(), &resolved_nested_var, 1); + vars_copy[i] = new_string; + } + } + } + } + vars_copy + } + + pub fn get_first(&self, var_name: &str) -> Option { + if let Some(var) = self.variables.get(var_name) { + let vars_resolved = self.resolve_nested_variables(var); + Some(vars_resolved.first().unwrap().clone()) + } else { + None + } + } +} + +fn vars_to_string(vars: &Vec) -> String { + if vars.len() == 0 { + String::new() + } else if vars.len() == 1 { + vars[0].clone() + } else { + vars.join("\n") + } } fn parse_string(content: String) -> Result { @@ -62,14 +127,35 @@ fn parse_makefile(pairs: Pairs, m: &mut Makefile) -> Result<()> { Rule::comment_string => (), Rule::include => (), Rule::target => (), + Rule::define => { + parse_define(p.into_inner(), m)?; + } Rule::EOI => (), - _ => panic!("unexpected rule {:?} inside makefile rule expected variable, comment, NEWLINE, include, target", p.as_rule()), + _ => panic!("unexpected rule {:?} inside makefile rule expected variable, define, comment, NEWLINE, include, target", p.as_rule()), } } Ok(()) } +fn parse_define(define_pair: Pairs, m: &mut Makefile) -> Result<()> { + let mut var = (String::new(), Vec::::new()); + for p in define_pair { + match p.as_rule() { + Rule::variable_name => { + var.0 = p.as_str().to_string(); + } + Rule::define_value => { + var.1.push(p.as_str().to_string()); + } + _ => panic!("unexpected rule {:?} inside makefile rule expected variable_name, define_value", p.as_rule()), + } + } + m.variables.insert(var.0, var.1); + + Ok(()) +} + fn parse_variable(variable_pair: Pairs, m: &mut Makefile) -> Result<()> { let mut var = (String::new(), Vec::::new()); for p in variable_pair { diff --git a/userland/src/makefile.pest b/userland/src/makefile.pest index ffdbdf1..a3c78d4 100644 --- a/userland/src/makefile.pest +++ b/userland/src/makefile.pest @@ -24,8 +24,8 @@ variable_value_character = { } variable_value = @{ variable_value_character* } -variable_set = { "=" } -variable_add = { "+=" } +variable_set = _{ "=" } +variable_add = _{ "+=" } variable = { variable_name ~ ( variable_set | variable_add ) ~ variable_value? } @@ -40,4 +40,15 @@ target = { target_name ~ ":" ~ variable_value } include = { "include" ~ variable_value } -makefile = { SOI ~ (NEWLINE | comment_string | variable | include | target )+ ~ EOI } +define_keyword = _{"define"} +define_end_keyword = _{NEWLINE ~ "endef"} + +define_value_character = { + !define_end_keyword + ~ ANY +} +define_value = @{ define_value_character* } + +define = { define_keyword ~ variable_name ~ variable_set ~ NEWLINE ~ define_value ~ define_end_keyword } + +makefile = { SOI ~ (NEWLINE | comment_string | define | variable | include | target )+ ~ EOI }