Switch to rustls for embedded tls

Refine Makefile parsing so we also parse define rules
Implement Recursive Vraiable resolving Algorithm

Cargo.lock

@@ -283,6 +283,18 @@ version = "0.3.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "365a1a1fb30ea1c03a830fdb2158f5236833ac81fa0ad12fe35b29cddc35cb04"
 
+[[package]]
+name = "futures-macro"
+version = "0.3.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "668c6733a182cd7deb4f1de7ba3bf2120823835b3bcfbeacf7d2c4a773c1bb8b"
+dependencies = [
+ "proc-macro-hack",
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
 [[package]]
 name = "futures-sink"
 version = "0.3.14"
@@ -303,10 +315,13 @@ checksum = "3c144ad54d60f23927f0a6b6d816e4271278b64f005ad65e4e35291d2de9c025"
 dependencies = [
  "futures-core",
  "futures-io",
+ "futures-macro",
  "futures-task",
  "memchr",
  "pin-project-lite",
  "pin-utils",
+ "proc-macro-hack",
+ "proc-macro-nested",
  "slab",
 ]
 
@@ -438,6 +453,21 @@ dependencies = [
  "want",
 ]
 
+[[package]]
+name = "hyper-rustls"
+version = "0.22.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5f9f7a97316d44c0af9b0301e65010573a853a9fc97046d7331d7f6bc0fd5a64"
+dependencies = [
+ "futures-util",
+ "hyper",
+ "log",
+ "rustls",
+ "tokio",
+ "tokio-rustls",
+ "webpki",
+]
+
 [[package]]
 name = "hyper-tls"
 version = "0.5.0"
@@ -839,6 +869,18 @@ dependencies = [
  "version_check",
 ]
 
+[[package]]
+name = "proc-macro-hack"
+version = "0.5.19"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5"
+
+[[package]]
+name = "proc-macro-nested"
+version = "0.1.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bc881b2c22681370c6a780e47af9840ef841837bc98118431d4e1868bd0c1086"
+
 [[package]]
 name = "proc-macro2"
 version = "1.0.36"
@@ -946,6 +988,7 @@ dependencies = [
  "http",
  "http-body",
  "hyper",
+ "hyper-rustls",
  "hyper-tls",
  "ipnet",
  "js-sys",
@@ -955,18 +998,49 @@ dependencies = [
  "native-tls",
  "percent-encoding",
  "pin-project-lite",
+ "rustls",
  "serde",
  "serde_json",
  "serde_urlencoded",
  "tokio",
  "tokio-native-tls",
+ "tokio-rustls",
  "url",
  "wasm-bindgen",
  "wasm-bindgen-futures",
  "web-sys",
+ "webpki-roots",
  "winreg",
 ]
 
+[[package]]
+name = "ring"
+version = "0.16.20"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc"
+dependencies = [
+ "cc",
+ "libc",
+ "once_cell",
+ "spin",
+ "untrusted",
+ "web-sys",
+ "winapi",
+]
+
+[[package]]
+name = "rustls"
+version = "0.19.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "35edb675feee39aec9c99fa5ff985081995a06d594114ae14cbe797ad7b7a6d7"
+dependencies = [
+ "base64",
+ "log",
+ "ring",
+ "sct",
+ "webpki",
+]
+
 [[package]]
 name = "ryu"
 version = "1.0.5"
@@ -983,6 +1057,16 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "sct"
+version = "0.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b362b83898e0e69f38515b82ee15aa80636befe47c3b6d3d89a911e78fc228ce"
+dependencies = [
+ "ring",
+ "untrusted",
+]
+
 [[package]]
 name = "security-framework"
 version = "2.2.0"
@@ -1120,6 +1204,12 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "spin"
+version = "0.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d"
+
 [[package]]
 name = "strsim"
 version = "0.10.0"
@@ -1226,6 +1316,17 @@ dependencies = [
  "tokio",
 ]
 
+[[package]]
+name = "tokio-rustls"
+version = "0.22.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bc6844de72e57df1980054b38be3a9f4702aba4858be64dd700181a8a6d0e1b6"
+dependencies = [
+ "rustls",
+ "tokio",
+ "webpki",
+]
+
 [[package]]
 name = "tokio-util"
 version = "0.6.6"
@@ -1308,6 +1409,12 @@ version = "0.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564"
 
+[[package]]
+name = "untrusted"
+version = "0.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a"
+
 [[package]]
 name = "url"
 version = "2.2.1"
@@ -1325,13 +1432,16 @@ name = "userland"
 version = "0.1.1"
 dependencies = [
  "anyhow",
+ "lazy_static",
  "maplit 0.1.6",
  "pest",
  "pest_derive",
+ "regex",
  "reqwest",
  "semver",
  "serde",
  "serde_json",
+ "thiserror",
 ]
 
 [[package]]
@@ -1446,6 +1556,25 @@ dependencies = [
  "wasm-bindgen",
 ]
 
+[[package]]
+name = "webpki"
+version = "0.21.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b8e38c0608262c46d4a56202ebabdeb094cef7e560ca7a226c6bf055188aa4ea"
+dependencies = [
+ "ring",
+ "untrusted",
+]
+
+[[package]]
+name = "webpki-roots"
+version = "0.21.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "aabe153544e473b775453675851ecc86863d2a81d786d741f6b76778f2a48940"
+dependencies = [
+ "webpki",
+]
+
 [[package]]
 name = "winapi"
 version = "0.3.9"

pkg6dev/src/main.rs

@@ -75,53 +75,61 @@ fn diff_component(component_path: impl AsRef<Path>) -> Result<()> {
 fn show_component_info<P: AsRef<Path>>(component_path: P) -> Result<()> {
     let makefile_path = component_path.as_ref().join("Makefile");
 
-    println!("{:?}", makefile_path);
-
     let makefile = Makefile::parse_file(&makefile_path)?;
 
     let mut name = String::new();
 
-    if let Some(var) = makefile.variables.get("COMPONENT_NAME") {
+    if let Some(var) = makefile.get("COMPONENT_NAME") {
-        println!("Name: {}", var.join(" "));
+        println!("Name: {}", var.replace("\n", "\n\t"));
-        name = var.first().unwrap().to_string();
+        if let Some(component_name) = makefile.get_first("COMPONENT_NAME") {
+            name = component_name.clone();
+        }
     }
 
-    if let Some(var) = makefile.variables.get("COMPONENT_VERSION") {
+    if let Some(var) = makefile.get("COMPONENT_VERSION") {
-        println!("Version: {}", var.join(" "));
+        println!("Version: {}", var.replace("\n", "\n\t"));
         let latest_version = find_newest_version(&name);
         if latest_version.is_ok() {
             println!("Latest Version: {}", latest_version?);
         } else {
-            eprintln!("{:?}", latest_version.unwrap_err())
+            println!("Error: Could not get latest version info: {:?}", latest_version.unwrap_err())
         }
     }
 
-    if let Some(var) = makefile.variables.get("BUILD_BITS") {
+    if let Some(var) = makefile.get("BUILD_BITS") {
-        println!("Build bits: {}", var.join(" "));
+        println!("Build bits: {}", var.replace("\n", "\n\t"));
     }
 
-    if let Some(var) = makefile.variables.get("COMPONENT_PROJECT_URL") {
+    if let Some(var) = makefile.get("COMPONENT_BUILD_ACTION") {
-        println!("Project URl: {}", var.join("\t"));
+        println!("Build action: {}", var.replace("\n", "\n\t"));
     }
 
-    if let Some(var) = makefile.variables.get("COMPONENT_ARCHIVE_URL") {
+    if let Some(var) = makefile.get("COMPONENT_PROJECT_URL") {
-        println!("Source URl: {}", var.join("\t"));
+        println!("Project URl: {}", var.replace("\n", "\n\t"));
     }
 
-    if let Some(var) = makefile.variables.get("COMPONENT_ARCHIVE_HASH") {
+    if let Some(var) = makefile.get("COMPONENT_ARCHIVE_URL") {
-        println!("Source Archive File Hash: {}", var.join(" "));
+        println!("Source URl: {}", var.replace("\n", "\n\t"));
     }
 
-    if let Some(var) = makefile.variables.get("CONFIGURE_ENV") {
+    if let Some(var) = makefile.get("COMPONENT_ARCHIVE_HASH") {
-        println!("Configure Environment: {}", var.join("\n\t"));
+        println!("Source Archive File Hash: {}", var.replace("\n", "\n\t"));
     }
 
-    if let Some(var) = makefile.variables.get("CONFIGURE_OPTIONS") {
+    if let Some(var) = makefile.get("CONFIGURE_ENV") {
-        println!("./configure {}", var.join("\n\t"));
+        println!("Configure Environment: {}", var.replace("\n", "\n\t"));
     }
 
-    if let Some(var) = makefile.variables.get("REQUIRED_PACKAGES") {
+    if let Some(var) = makefile.get("CONFIGURE_OPTIONS") {
-        println!("Dependencies:\n\t{}", var.join("\n\t"));
+        println!("./configure {}", var.replace("\n", "\n\t"));
+    }
+
+    if let Some(var) = makefile.get("REQUIRED_PACKAGES") {
+        println!("Dependencies:\n\t{}", var.replace("\n", "\n\t"));
+    }
+
+    if let Some(var) = makefile.get("COMPONENT_INSTALL_ACTION") {
+        println!("Install Action:\n\t{}", var);
     }
 
     Ok(())

userland/Cargo.toml

@@ -10,10 +10,13 @@ description = "Helper tool for IPS package development"
 
 [dependencies]
 anyhow = "1.0.56"
+thiserror = "1.0.30"
 pest_derive = "2.1.0"
 maplit = "0.1.6"
 pest = "2.1.0"
-reqwest = { version = "0.11", features = ["blocking", "json"] }
+reqwest = { version = "0.11", features = ["blocking", "json", "rustls-tls"] }
 semver = "0.11.0"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
+regex = "1"
+lazy_static = "1"

userland/src/lib.rs

@@ -7,12 +7,15 @@ extern crate maplit;
 extern crate pest_derive;
 
 
-use anyhow::Result;
+use anyhow::{anyhow, Result};
 use std::collections::HashMap;
 use std::fs::read_to_string;
 use pest::iterators::{Pairs};
-use std::path::Path;
+use std::path::{Path, PathBuf};
 use pest::Parser;
+use thiserror::Error;
+use lazy_static::lazy_static;
+use regex::Regex;
 
 #[derive(Parser)]
 #[grammar = "makefile.pest"]
@@ -25,15 +28,77 @@ pub struct Makefile {
     // pub targets: HashMap<String, String>,
 }
 
+#[derive(Error, Debug)]
+pub enum ParserError {
+    #[error("cannot parse {file}: {reason}")]
+    MakefileReadError {
+        file: PathBuf,
+        reason: anyhow::Error,
+    }
+}
+
 impl Makefile {
-    pub fn parse_file(path: &Path) -> Result<Self> {
+    pub fn parse_file<P: AsRef<Path>>(path: P) -> Result<Self> {
-        let content = read_to_string(path)?;
+        let content = read_to_string(&path)?;
-        parse_string(content)
+        parse_string(content).map_err(|err|
+            anyhow!(ParserError::MakefileReadError{file: path.as_ref().to_path_buf(), reason: anyhow!(err)})
+        )
     }
 
     pub fn parse_string(content: String) -> Result<Self> {
         parse_string(content)
     }
+
+    pub fn get(&self, var_name: &str) -> Option<String> {
+        if let Some(var) = self.variables.get(var_name) {
+            let vars_resolved = self.resolve_nested_variables(var);
+            Some(vars_to_string(&vars_resolved))
+        } else {
+            None
+        }
+    }
+
+    fn resolve_nested_variables(&self, var: &Vec<String>) -> Vec<String> {
+        // Make a mutable copy of the variables so we can replace nested variables with their final strings
+        let mut vars_copy = var.clone();
+
+        // Logic to resolve all the nested Variables when we access them.
+        for (i, maybe_nested_var) in var.iter().enumerate() {
+            lazy_static! {
+                    static ref VARRE: Regex = Regex::new(r"(?P<var_name>\$\(.+?\))").unwrap();
+                }
+            for captures in VARRE.captures_iter(maybe_nested_var) {
+                if let Some(nested_var) = captures.name("var_name") {
+                    let nested_var_name = nested_var.as_str().replace("$(", "").replace(")", "");
+                    if let Some(resolved_nested_var) = self.get(&nested_var_name) {
+                        let mut new_string = vars_copy[i].clone();
+                        new_string = new_string.replacen(nested_var.as_str(), &resolved_nested_var, 1);
+                        vars_copy[i] = new_string;
+                    }
+                }
+            }
+        }
+        vars_copy
+    }
+
+    pub fn get_first(&self, var_name: &str) -> Option<String> {
+        if let Some(var) = self.variables.get(var_name) {
+            let vars_resolved = self.resolve_nested_variables(var);
+            Some(vars_resolved.first().unwrap().clone())
+        } else {
+            None
+        }
+    }
+}
+
+fn vars_to_string(vars: &Vec<String>) -> String {
+    if vars.len() == 0 {
+        String::new()
+    } else if vars.len() == 1 {
+        vars[0].clone()
+    } else {
+        vars.join("\n")
+    }
 }
 
 fn parse_string(content: String) -> Result<Makefile> {
@@ -62,14 +127,35 @@ fn parse_makefile(pairs: Pairs<crate::Rule>, m: &mut Makefile) -> Result<()> {
             Rule::comment_string => (),
             Rule::include => (),
             Rule::target => (),
+            Rule::define => {
+                parse_define(p.into_inner(), m)?;
+            }
             Rule::EOI => (),
-            _ => panic!("unexpected rule {:?} inside makefile rule expected variable, comment, NEWLINE, include, target", p.as_rule()),
+            _ => panic!("unexpected rule {:?} inside makefile rule expected variable, define, comment, NEWLINE, include, target", p.as_rule()),
         }
     }
 
     Ok(())
 }
 
+fn parse_define(define_pair: Pairs<crate::Rule>, m: &mut Makefile) -> Result<()> {
+    let mut var = (String::new(), Vec::<String>::new());
+    for p in define_pair {
+        match p.as_rule() {
+            Rule::variable_name => {
+                var.0 = p.as_str().to_string();
+            }
+            Rule::define_value => {
+                var.1.push(p.as_str().to_string());
+            }
+            _ => panic!("unexpected rule {:?} inside makefile rule expected variable_name, define_value", p.as_rule()),
+        }
+    }
+    m.variables.insert(var.0, var.1);
+
+    Ok(())
+}
+
 fn parse_variable(variable_pair: Pairs<crate::Rule>, m: &mut Makefile) -> Result<()> {
     let mut var = (String::new(), Vec::<String>::new());
     for p in variable_pair {

userland/src/makefile.pest

@@ -24,8 +24,8 @@ variable_value_character = {
 }
 variable_value = @{ variable_value_character* }
 
-variable_set = { "=" }
+variable_set = _{ "=" }
-variable_add = { "+=" }
+variable_add = _{ "+=" }
 
 variable = { variable_name ~ ( variable_set | variable_add ) ~ variable_value? }
 
@@ -40,4 +40,15 @@ target = { target_name ~ ":" ~ variable_value }
 
 include = { "include" ~ variable_value }
 
-makefile = { SOI ~ (NEWLINE | comment_string | variable | include | target )+ ~ EOI }
+define_keyword = _{"define"}
+define_end_keyword = _{NEWLINE ~ "endef"}
+
+define_value_character = {
+    !define_end_keyword
+    ~ ANY
+}
+define_value = @{ define_value_character* }
+
+define = { define_keyword ~ variable_name ~ variable_set ~ NEWLINE ~ define_value ~ define_end_keyword }
+
+makefile = { SOI ~ (NEWLINE | comment_string | define | variable | include | target )+ ~ EOI }