From 5f6bcdf9588187413d43c880576d6522dd958229 Mon Sep 17 00:00:00 2001 From: Dongsu Park Date: Wed, 29 Nov 2023 16:15:23 +0100 Subject: [PATCH] add new option to fetch data from remote URL into string Add new option --payload-url to fetch data from the given URL into String. Only one of both --input-xml and --payload-url should be given, otherwil fail. Construct a fake package to verify signatures in case of a directly downloaded update payload file. Split the existing signature verification part into do_download_verify, for the fake pacakage code path to rely on the common download and verify part. --- src/bin/download_sysext.rs | 141 +++++++++++++++++++++++++++---------- 1 file changed, 105 insertions(+), 36 deletions(-) diff --git a/src/bin/download_sysext.rs b/src/bin/download_sysext.rs index 7f1b531..b77fbac 100644 --- a/src/bin/download_sysext.rs +++ b/src/bin/download_sysext.rs @@ -1,18 +1,22 @@ use std::error::Error; use std::borrow::Cow; -use std::path::{Path, PathBuf}; +use std::ffi::OsStr; use std::fs::File; use std::fs; use std::io; use std::io::BufReader; +use std::path::{Path, PathBuf}; +use std::str::FromStr; #[macro_use] extern crate log; -use anyhow::{Context, Result, bail}; +use anyhow::{Context, Result, bail, anyhow}; +use argh::FromArgs; use globset::{Glob, GlobSet, GlobSetBuilder}; use hard_xml::XmlRead; -use argh::FromArgs; +use omaha::FileSize; +use reqwest::Client; use reqwest::redirect::Policy; use url::Url; @@ -103,7 +107,7 @@ impl<'a> Package<'a> { let path = into_dir.join(&*self.name); let mut file = File::create(path.clone()).context(format!("failed to create path ({:?})", path.display()))?; - let res = match ue_rs::download_and_hash(&client, self.url.clone(), &mut file).await { + let res = match ue_rs::download_and_hash(client, self.url.clone(), &mut file).await { Ok(ok) => ok, Err(err) => { error!("Downloading failed with error {}", err); @@ -145,7 +149,7 @@ impl<'a> Package<'a> { let sigbytes = delta_update::get_signatures_bytes(upfreader, &header, &mut delta_archive_manifest).context(format!("failed to get_signatures_bytes path ({:?})", from_path.display()))?; // tmp dir == "/var/tmp/outdir/.tmp" - let tmpdirpathbuf = from_path.parent().unwrap().parent().unwrap().join(".tmp"); + let tmpdirpathbuf = from_path.parent().ok_or(anyhow!("unable to get parent dir"))?.parent().ok_or(anyhow!("unable to get parent dir"))?.join(".tmp"); let tmpdir = tmpdirpathbuf.as_path(); let datablobspath = tmpdir.join("ue_data_blobs"); @@ -238,6 +242,44 @@ fn get_pkgs_to_download<'a>(resp: &'a omaha::Response, glob_set: &GlobSet) Ok(to_download) } +// Read data from remote URL into File +async fn fetch_url_to_file<'a, U>(path: &'a Path, input_url: U, client: &'a Client) -> Result> +where + U: reqwest::IntoUrl + From + std::clone::Clone + std::fmt::Debug, + Url: From, +{ + let mut file = File::create(path).context(format!("failed to create path ({:?})", path.display()))?; + + ue_rs::download_and_hash(client, input_url.clone(), &mut file).await.context(format!("unable to download data(url {:?})", input_url))?; + + Ok(Package { + name: Cow::Borrowed(path.file_name().unwrap_or(OsStr::new("fakepackage")).to_str().unwrap_or("fakepackage")), + hash: hash_on_disk_sha256(path, None)?, + size: FileSize::from_bytes(file.metadata().context(format!("failed to get metadata, path ({:?})", path.display()))?.len() as usize), + url: input_url.into(), + status: PackageStatus::Unverified, + }) +} + +async fn do_download_verify(pkg: &mut Package<'_>, output_dir: &Path, unverified_dir: &Path, pubkey_file: &str, client: &Client) -> Result<()> { + pkg.check_download(unverified_dir)?; + + pkg.download(unverified_dir, client).await.context(format!("unable to download \"{:?}\"", pkg.name))?; + + // Unverified payload is stored in e.g. "output_dir/.unverified/oem.gz". + // Verified payload is stored in e.g. "output_dir/oem.raw". + let pkg_unverified = unverified_dir.join(&*pkg.name); + let pkg_verified = output_dir.join(pkg_unverified.with_extension("raw").file_name().unwrap_or_default()); + + let datablobspath = pkg.verify_signature_on_disk(&pkg_unverified, pubkey_file).context(format!("unable to verify signature \"{}\"", pkg.name))?; + + // write extracted data into the final data. + debug!("data blobs written into file {:?}", pkg_verified); + fs::rename(datablobspath, pkg_verified)?; + + Ok(()) +} + #[derive(FromArgs, Debug)] /// Parse an update-engine Omaha XML response to extract sysext images, then download and verify /// their signatures. @@ -248,7 +290,11 @@ struct Args { /// path to the Omaha XML file, or - to read from stdin #[argh(option, short = 'i')] - input_xml: String, + input_xml: Option, + + /// URL to fetch remote update payload + #[argh(option, short = 'u')] + payload_url: Option, /// path to the public key file #[argh(option, short = 'p')] @@ -281,14 +327,6 @@ async fn main() -> Result<(), Box> { let glob_set = args.image_match_glob_set()?; - let response_text = match &*args.input_xml { - "-" => io::read_to_string(io::stdin())?, - path => { - let file = File::open(path)?; - io::read_to_string(file)? - } - }; - let output_dir = Path::new(&*args.output_dir); if !output_dir.try_exists()? { return Err(format!("output directory `{}` does not exist", args.output_dir).into()); @@ -299,6 +337,58 @@ async fn main() -> Result<(), Box> { fs::create_dir_all(&unverified_dir)?; fs::create_dir_all(&temp_dir)?; + // The default policy of reqwest Client supports max 10 attempts on HTTP redirect. + let client = Client::builder().redirect(Policy::default()).build()?; + + // If input_xml exists, simply read it. + // If not, try to read from payload_url. + let res_local = match args.input_xml { + Some(name) => { + if name == "-" { + Some(io::read_to_string(io::stdin())?) + } else { + let file = File::open(name)?; + Some(io::read_to_string(file)?) + } + } + None => None, + }; + + match (&res_local, args.payload_url) { + (Some(_), Some(_)) => { + return Err("Only one of the options can be given, --input-xml or --payload-url.".into()); + } + (Some(res), None) => res, + (None, Some(url)) => { + let u = Url::parse(&url)?; + let fname = u.path_segments().ok_or(anyhow!("failed to get path segments, url ({:?})", u))?.next_back().ok_or(anyhow!("failed to get path segments, url ({:?})", u))?; + let mut pkg_fake: Package; + + let temp_payload_path = unverified_dir.join(fname); + pkg_fake = fetch_url_to_file( + &temp_payload_path, + Url::from_str(url.as_str()).context(anyhow!("failed to convert into url ({:?})", url))?, + &client, + ) + .await?; + do_download_verify( + &mut pkg_fake, + output_dir, + unverified_dir.as_path(), + args.pubkey_file.as_str(), + &client, + ) + .await?; + + // verify only a fake package, early exit and skip the rest. + return Ok(()); + } + (None, None) => return Err("Either --input-xml or --payload-url must be given.".into()), + }; + + let response_text = res_local.ok_or(anyhow!("failed to get response text"))?; + debug!("response_text: {:?}", response_text); + //// // parse response //// @@ -312,30 +402,9 @@ async fn main() -> Result<(), Box> { //// // download //// - // The default policy of reqwest Client supports max 10 attempts on HTTP redirect. - let client = reqwest::Client::builder().redirect(Policy::default()).build()?; for pkg in pkgs_to_dl.iter_mut() { - pkg.check_download(&unverified_dir)?; - - match pkg.download(&unverified_dir, &client).await { - Ok(_) => (), - _ => return Err(format!("unable to download \"{}\"", pkg.name).into()), - }; - - // Unverified payload is stored in e.g. "output_dir/.unverified/oem.gz". - // Verified payload is stored in e.g. "output_dir/oem.raw". - let pkg_unverified = unverified_dir.join(&*pkg.name); - let pkg_verified = output_dir.join(pkg_unverified.with_extension("raw").file_name().unwrap_or_default()); - - match pkg.verify_signature_on_disk(&pkg_unverified, &args.pubkey_file) { - Ok(datablobspath) => { - // write extracted data into the final data. - fs::rename(datablobspath, pkg_verified.clone())?; - debug!("data blobs written into file {:?}", pkg_verified); - } - _ => return Err(format!("unable to verify signature \"{}\"", pkg.name).into()), - }; + do_download_verify(pkg, output_dir, unverified_dir.as_path(), args.pubkey_file.as_str(), &client).await?; } // clean up data