add new option to fetch data from remote URL into string

Add new option --payload-url to fetch data from the given URL into
String. Only one of both --input-xml and --payload-url should be given,
otherwil fail.

Construct a fake package to verify signatures in case of a directly
downloaded update payload file. Split the existing signature
verification part into do_download_verify, for the fake pacakage
code path to rely on the common download and verify part.

src/bin/download_sysext.rs

@@ -1,18 +1,22 @@
 use std::error::Error;
 use std::borrow::Cow;
-use std::path::{Path, PathBuf};
+use std::ffi::OsStr;
 use std::fs::File;
 use std::fs;
 use std::io;
 use std::io::BufReader;
+use std::path::{Path, PathBuf};
+use std::str::FromStr;
 
 #[macro_use]
 extern crate log;
 
-use anyhow::{Context, Result, bail};
+use anyhow::{Context, Result, bail, anyhow};
+use argh::FromArgs;
 use globset::{Glob, GlobSet, GlobSetBuilder};
 use hard_xml::XmlRead;
-use argh::FromArgs;
+use omaha::FileSize;
+use reqwest::Client;
 use reqwest::redirect::Policy;
 use url::Url;
 
@@ -103,7 +107,7 @@ impl<'a> Package<'a> {
  let path = into_dir.join(&*self.name);
  let mut file = File::create(path.clone()).context(format!("failed to create path ({:?})", path.display()))?;
 
- let res = match ue_rs::download_and_hash(&client, self.url.clone(), &mut file).await {
+ let res = match ue_rs::download_and_hash(client, self.url.clone(), &mut file).await {
  Ok(ok) => ok,
  Err(err) => {
  error!("Downloading failed with error {}", err);
@@ -145,7 +149,7 @@ impl<'a> Package<'a> {
  let sigbytes = delta_update::get_signatures_bytes(upfreader, &header, &mut delta_archive_manifest).context(format!("failed to get_signatures_bytes path ({:?})", from_path.display()))?;
 
  // tmp dir == "/var/tmp/outdir/.tmp"
- let tmpdirpathbuf = from_path.parent().unwrap().parent().unwrap().join(".tmp");
+ let tmpdirpathbuf = from_path.parent().ok_or(anyhow!("unable to get parent dir"))?.parent().ok_or(anyhow!("unable to get parent dir"))?.join(".tmp");
  let tmpdir = tmpdirpathbuf.as_path();
  let datablobspath = tmpdir.join("ue_data_blobs");
 
@@ -238,6 +242,44 @@ fn get_pkgs_to_download<'a>(resp: &'a omaha::Response, glob_set: &GlobSet)
  Ok(to_download)
 }
 
+// Read data from remote URL into File
+async fn fetch_url_to_file<'a, U>(path: &'a Path, input_url: U, client: &'a Client) -> Result<Package<'a>>
+where
+ U: reqwest::IntoUrl + From<U> + std::clone::Clone + std::fmt::Debug,
+ Url: From<U>,
+{
+ let mut file = File::create(path).context(format!("failed to create path ({:?})", path.display()))?;
+
+ ue_rs::download_and_hash(client, input_url.clone(), &mut file).await.context(format!("unable to download data(url {:?})", input_url))?;
+
+ Ok(Package {
+ name: Cow::Borrowed(path.file_name().unwrap_or(OsStr::new("fakepackage")).to_str().unwrap_or("fakepackage")),
+ hash: hash_on_disk_sha256(path, None)?,
+ size: FileSize::from_bytes(file.metadata().context(format!("failed to get metadata, path ({:?})", path.display()))?.len() as usize),
+ url: input_url.into(),
+ status: PackageStatus::Unverified,
+ })
+}
+
+async fn do_download_verify(pkg: &mut Package<'_>, output_dir: &Path, unverified_dir: &Path, pubkey_file: &str, client: &Client) -> Result<()> {
+ pkg.check_download(unverified_dir)?;
+
+ pkg.download(unverified_dir, client).await.context(format!("unable to download \"{:?}\"", pkg.name))?;
+
+ // Unverified payload is stored in e.g. "output_dir/.unverified/oem.gz".
+ // Verified payload is stored in e.g. "output_dir/oem.raw".
+ let pkg_unverified = unverified_dir.join(&*pkg.name);
+ let pkg_verified = output_dir.join(pkg_unverified.with_extension("raw").file_name().unwrap_or_default());
+
+ let datablobspath = pkg.verify_signature_on_disk(&pkg_unverified, pubkey_file).context(format!("unable to verify signature \"{}\"", pkg.name))?;
+
+ // write extracted data into the final data.
+ debug!("data blobs written into file {:?}", pkg_verified);
+ fs::rename(datablobspath, pkg_verified)?;
+
+ Ok(())
+}
+
 #[derive(FromArgs, Debug)]
 /// Parse an update-engine Omaha XML response to extract sysext images, then download and verify
 /// their signatures.
@@ -248,7 +290,11 @@ struct Args {
 
  /// path to the Omaha XML file, or - to read from stdin
  #[argh(option, short = 'i')]
- input_xml: String,
+ input_xml: Option<String>,
+
+ /// URL to fetch remote update payload
+ #[argh(option, short = 'u')]
+ payload_url: Option<String>,
 
  /// path to the public key file
  #[argh(option, short = 'p')]
@@ -281,14 +327,6 @@ async fn main() -> Result<(), Box<dyn Error>> {
 
  let glob_set = args.image_match_glob_set()?;
 
- let response_text = match &*args.input_xml {
- "-" => io::read_to_string(io::stdin())?,
- path => {
- let file = File::open(path)?;
- io::read_to_string(file)?
- }
- };
-
  let output_dir = Path::new(&*args.output_dir);
  if !output_dir.try_exists()? {
  return Err(format!("output directory `{}` does not exist", args.output_dir).into());
@@ -299,6 +337,58 @@ async fn main() -> Result<(), Box<dyn Error>> {
  fs::create_dir_all(&unverified_dir)?;
  fs::create_dir_all(&temp_dir)?;
 
+ // The default policy of reqwest Client supports max 10 attempts on HTTP redirect.
+ let client = Client::builder().redirect(Policy::default()).build()?;
+
+ // If input_xml exists, simply read it.
+ // If not, try to read from payload_url.
+ let res_local = match args.input_xml {
+ Some(name) => {
+ if name == "-" {
+ Some(io::read_to_string(io::stdin())?)
+ } else {
+ let file = File::open(name)?;
+ Some(io::read_to_string(file)?)
+ }
+ }
+ None => None,
+ };
+
+ match (&res_local, args.payload_url) {
+ (Some(_), Some(_)) => {
+ return Err("Only one of the options can be given, --input-xml or --payload-url.".into());
+ }
+ (Some(res), None) => res,
+ (None, Some(url)) => {
+ let u = Url::parse(&url)?;
+ let fname = u.path_segments().ok_or(anyhow!("failed to get path segments, url ({:?})", u))?.next_back().ok_or(anyhow!("failed to get path segments, url ({:?})", u))?;
+ let mut pkg_fake: Package;
+
+ let temp_payload_path = unverified_dir.join(fname);
+ pkg_fake = fetch_url_to_file(
+ &temp_payload_path,
+ Url::from_str(url.as_str()).context(anyhow!("failed to convert into url ({:?})", url))?,
+ &client,
+ )
+ .await?;
+ do_download_verify(
+ &mut pkg_fake,
+ output_dir,
+ unverified_dir.as_path(),
+ args.pubkey_file.as_str(),
+ &client,
+ )
+ .await?;
+
+ // verify only a fake package, early exit and skip the rest.
+ return Ok(());
+ }
+ (None, None) => return Err("Either --input-xml or --payload-url must be given.".into()),
+ };
+
+ let response_text = res_local.ok_or(anyhow!("failed to get response text"))?;
+ debug!("response_text: {:?}", response_text);
+
  ////
  // parse response
  ////
@@ -312,30 +402,9 @@ async fn main() -> Result<(), Box<dyn Error>> {
  ////
  // download
  ////
- // The default policy of reqwest Client supports max 10 attempts on HTTP redirect.
- let client = reqwest::Client::builder().redirect(Policy::default()).build()?;
 
  for pkg in pkgs_to_dl.iter_mut() {
- pkg.check_download(&unverified_dir)?;
-
- match pkg.download(&unverified_dir, &client).await {
- Ok(_) => (),
- _ => return Err(format!("unable to download \"{}\"", pkg.name).into()),
- };
-
- // Unverified payload is stored in e.g. "output_dir/.unverified/oem.gz".
- // Verified payload is stored in e.g. "output_dir/oem.raw".
- let pkg_unverified = unverified_dir.join(&*pkg.name);
- let pkg_verified = output_dir.join(pkg_unverified.with_extension("raw").file_name().unwrap_or_default());
-
- match pkg.verify_signature_on_disk(&pkg_unverified, &args.pubkey_file) {
- Ok(datablobspath) => {
- // write extracted data into the final data.
- fs::rename(datablobspath, pkg_verified.clone())?;
- debug!("data blobs written into file {:?}", pkg_verified);
- }
- _ => return Err(format!("unable to verify signature \"{}\"", pkg.name).into()),
- };
+ do_download_verify(pkg, output_dir, unverified_dir.as_path(), args.pubkey_file.as_str(), &client).await?;
  }
 
  // clean up data