diff --git a/Cargo.lock b/Cargo.lock index d7857c2..24f9717 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -15,10 +15,10 @@ dependencies = [ "rppal", "serde", "serialport", + "simple-archive", "simplelog", "strum", "subprocess", - "tar", "test-case", "thiserror", "toml", @@ -248,16 +248,6 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" -[[package]] -name = "errno" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "534c5cf6194dfab3db3242765c03bbe257cf92f22b38f6bc0c58d59108a820ba" -dependencies = [ - "libc", - "windows-sys 0.52.0", -] - [[package]] name = "file-per-thread-logger" version = "0.2.0" @@ -268,18 +258,6 @@ dependencies = [ "log", ] -[[package]] -name = "filetime" -version = "0.2.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ee447700ac8aa0b2f2bd7bc4462ad686ba06baa6727ac149a2d6277f0d240fd" -dependencies = [ - "cfg-if", - "libc", - "redox_syscall 0.4.1", - "windows-sys 0.52.0", -] - [[package]] name = "filevec" version = "0.1.0" @@ -454,12 +432,6 @@ dependencies = [ "pkg-config", ] -[[package]] -name = "linux-raw-sys" -version = "0.4.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" - [[package]] name = "lock_api" version = "0.4.12" @@ -604,7 +576,7 @@ checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" dependencies = [ "cfg-if", "libc", - "redox_syscall 0.5.3", + "redox_syscall", "smallvec", "windows-targets 0.52.6", ] @@ -645,15 +617,6 @@ dependencies = [ "proc-macro2", ] -[[package]] -name = "redox_syscall" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" -dependencies = [ - "bitflags 1.3.2", -] - [[package]] name = "redox_syscall" version = "0.5.3" @@ -735,19 +698,6 @@ version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" -[[package]] -name = "rustix" -version = "0.38.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f" -dependencies = [ - "bitflags 2.6.0", - "errno", - "libc", - "linux-raw-sys", - "windows-sys 0.52.0", -] - [[package]] name = "rustversion" version = "1.0.17" @@ -913,17 +863,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "tar" -version = "0.4.41" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb797dad5fb5b76fcf519e702f4a589483b5ef06567f160c392832c1f5e44909" -dependencies = [ - "filetime", - "libc", - "xattr", -] - [[package]] name = "termcolor" version = "1.4.1" @@ -1290,17 +1229,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "xattr" -version = "1.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8da84f1a25939b27f6820d92aed108f83ff920fdf11a7b19366c27c4cda81d4f" -dependencies = [ - "libc", - "linux-raw-sys", - "rustix", -] - [[package]] name = "zerocopy" version = "0.7.35" diff --git a/scheduler/Cargo.toml b/scheduler/Cargo.toml index 0ea93e4..d346508 100644 --- a/scheduler/Cargo.toml +++ b/scheduler/Cargo.toml @@ -4,28 +4,26 @@ version = "0.1.0" edition = "2021" [dependencies] -log = "0.4.22" -simplelog = "0.12.2" -subprocess = "0.2.9" +anyhow = { version = "1.0.86", features = ["backtrace"] } crc = "3.2.1" -toml = "0.8.19" +filevec = { path = "../filevec" } +log = "0.4.22" rppal = "0.18.0" serde = { version = "1.0.204", features = ["derive"] } -strum = { version = "0.26.3", features = ["derive"] } serialport = "4.4.0" -test-case = "3.3.1" -tar = "0.4.41" +simple-archive = { path = "../simple-archive" } +simplelog = "0.12.2" +strum = { version = "0.26.3", features = ["derive"] } +subprocess = "0.2.9" thiserror = "1.0.63" -anyhow = { version = "1.0.86", features = ["backtrace"] } - -[dependencies.filevec] -path = "../filevec" +toml = "0.8.19" [features] mock = [] rpi = [] [dev-dependencies] +test-case = "3.3.1" file-per-thread-logger = "0.2.0" inquire = "0.7.5" diff --git a/scheduler/examples/cli.rs b/scheduler/examples/cli.rs index 446351f..4c00fd0 100644 --- a/scheduler/examples/cli.rs +++ b/scheduler/examples/cli.rs @@ -134,7 +134,7 @@ fn inquire_and_send_command( let program_id = inquire::Text::new("Program id:").prompt()?.parse()?; let timestamp = inquire::Text::new("Timestamp:").prompt()?.parse()?; let result_path = inquire::Text::new("File path for returned result:") - .with_default("./result.tar") + .with_default("./result") .prompt()?; edu.send_packet(&CEPPacket::Data(return_result(program_id, timestamp)))?; match edu.receive_multi_packet() { diff --git a/scheduler/src/command/execute_program.rs b/scheduler/src/command/execute_program.rs index 24a9313..1683292 100644 --- a/scheduler/src/command/execute_program.rs +++ b/scheduler/src/command/execute_program.rs @@ -7,14 +7,16 @@ use crate::{ communication::{CEPPacket, CommunicationHandle}, }; use anyhow::anyhow; +use simple_archive::Compression; use std::{ - io::ErrorKind, + fs::File, + io::{ErrorKind, Read, Write}, path::{Path, PathBuf}, time::Duration, }; use subprocess::Popen; -const MAXIMUM_FILE_SIZE: u64 = 1_000_000; +const MAXIMUM_FILE_SIZE: usize = 1_000_000; /// Executes a students program and starts a watchdog for it. The watchdog also creates entries in the /// status and result queue found in `context`. The result, including logs, is packed into @@ -145,27 +147,35 @@ fn run_until_timeout( /// the programs stdout/stderr and the schedulers log file. If any of the files is missing, the archive /// is created without them. fn build_result_archive(res: ResultId) -> Result<(), std::io::Error> { + let out_path = PathBuf::from(&format!("./data/{res}")); + let mut archive = simple_archive::Writer::new(std::fs::File::create(out_path)?); + let res_path = PathBuf::from(format!("./archives/{}/results/{}", res.program_id, res.timestamp)); let student_log_path = - PathBuf::from(format!("./data/{}_{}.log", res.program_id, res.timestamp)); - let log_path = PathBuf::from("log"); + PathBuf::from(format!("./data/{res}.log")); + let log_path = PathBuf::from("./log"); - let out_path = PathBuf::from(&format!("./data/{}_{}.tar", res.program_id, res.timestamp)); - let mut archive = tar::Builder::new(std::fs::File::create(out_path)?); + add_to_archive_if_exists(&mut archive, &res.to_string(), res_path, Compression::None)?; + add_to_archive_if_exists(&mut archive, "student_log", student_log_path, Compression::Zopfli)?; + add_to_archive_if_exists(&mut archive, "log", log_path, Compression::Zopfli)?; - for path in &[res_path, student_log_path, log_path] { - let mut file = match std::fs::File::options().read(true).write(true).open(path) { - Ok(f) => f, - Err(e) if e.kind() == ErrorKind::NotFound => continue, - Err(e) => return Err(e), - }; + Ok(()) +} - truncate_to_size(&mut file, MAXIMUM_FILE_SIZE)?; - archive.append_file(path.file_name().unwrap(), &mut file)?; +fn add_to_archive_if_exists( + archive: &mut simple_archive::Writer, + name: &str, + path: impl AsRef, + compression: simple_archive::Compression, +) -> std::io::Result<()> { + match std::fs::read(path) { + Ok(mut data) => { + data.truncate(MAXIMUM_FILE_SIZE); + archive.append_data(name, &data, compression)?; + Ok(()) + } + Err(ref e) if e.kind() == ErrorKind::NotFound => Ok(()), + Err(e) => Err(e), } - - archive.finish()?; - - Ok(()) } diff --git a/scheduler/src/command/execution_context.rs b/scheduler/src/command/execution_context.rs index 007e457..f662d04 100644 --- a/scheduler/src/command/execution_context.rs +++ b/scheduler/src/command/execution_context.rs @@ -1,8 +1,6 @@ use filevec::FileVec; use std::{ - str::FromStr, - sync::{Arc, Mutex}, - thread, + fmt::Display, str::FromStr, sync::{Arc, Mutex}, thread }; const EVENT_SEND_TRIES: u32 = 5; @@ -119,6 +117,12 @@ pub struct ResultId { pub timestamp: u32, } +impl Display for ResultId { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}_{}", self.program_id, self.timestamp) + } +} + #[derive(serde::Serialize, serde::Deserialize, Clone, Copy, PartialEq, Eq, Debug)] pub enum Event { Status(ProgramStatus), diff --git a/scheduler/src/command/return_result.rs b/scheduler/src/command/return_result.rs index 449d7e7..9262621 100644 --- a/scheduler/src/command/return_result.rs +++ b/scheduler/src/command/return_result.rs @@ -16,7 +16,7 @@ pub fn return_result( let program_id = u16::from_le_bytes([data[1], data[2]]); let timestamp = u32::from_le_bytes([data[3], data[4], data[5], data[6]]); - let result_path = format!("./data/{program_id}_{timestamp}.tar"); + let result_path = format!("./data/{program_id}_{timestamp}"); if !std::path::Path::new(&result_path).exists() { com.send_packet(&CEPPacket::Nack)?; @@ -51,7 +51,7 @@ pub fn return_result( fn delete_result(res: ResultId) -> CommandResult { let res_path = format!("./archives/{}/results/{}", res.program_id, res.timestamp); let log_path = format!("./data/{}_{}.log", res.program_id, res.timestamp); - let out_path = format!("./data/{}_{}.tar", res.program_id, res.timestamp); + let out_path = format!("./data/{}_{}", res.program_id, res.timestamp); let _ = std::fs::remove_file(res_path); let _ = std::fs::remove_file(log_path); let _ = std::fs::remove_file(out_path); diff --git a/scheduler/tests/simulation/full_run.rs b/scheduler/tests/simulation/full_run.rs index f52b5c6..92e9883 100644 --- a/scheduler/tests/simulation/full_run.rs +++ b/scheduler/tests/simulation/full_run.rs @@ -1,5 +1,5 @@ use crate::simulation::*; -use std::{io::Cursor, time::Duration}; +use std::time::Duration; #[test] fn full_run() { @@ -16,16 +16,11 @@ fn full_run() { // Check result let result = simulate_return_result(&mut com, 1, 3).unwrap(); - let mut result_archive = tar::Archive::new(Cursor::new(result)); com.send_packet(&CEPPacket::Ack).unwrap(); - let result_file = result_archive - .entries() - .unwrap() - .find(|x| x.as_ref().unwrap().header().path().unwrap().ends_with("3")) - .unwrap() - .unwrap(); - assert_eq!(result_file.bytes().map(|b| b.unwrap()).collect::>(), vec![0xde, 0xad]); + let decoded_result = simple_archive::Reader::new(&result[..]); + let result = decoded_result.map(Result::unwrap).find(|entry| entry.path == "1_3").unwrap(); + assert_eq!(result.data, vec![0xde, 0xad]); assert_eq!(simulate_get_status(&mut com).unwrap(), [0]); } diff --git a/scheduler/tests/software_tests/return_result.rs b/scheduler/tests/software_tests/return_result.rs index 460542e..f187df9 100644 --- a/scheduler/tests/software_tests/return_result.rs +++ b/scheduler/tests/software_tests/return_result.rs @@ -1,6 +1,7 @@ use crate::software_tests::common; use crate::software_tests::common::ComEvent::*; use common::*; +use simple_archive::Entry; use STS1_EDU_Scheduler::command::{self}; use STS1_EDU_Scheduler::communication::CEPPacket::*; @@ -25,7 +26,7 @@ fn returns_result_correctly() -> TestResult { Edu(Ack), Action(Box::new(|packet| { let bytes = packet.clone().serialize(); - std::fs::write("tests/tmp/7.tar", &bytes[3..bytes.len() - 4]).unwrap(); + std::fs::write("tests/tmp/7_3", &bytes[3..bytes.len() - 4]).unwrap(); })), Cobc(Ack), Edu(Eof), @@ -42,15 +43,12 @@ fn returns_result_correctly() -> TestResult { command::handle_command(&mut com, &mut exec); assert!(com.is_complete()); - let _ = std::fs::create_dir("./tests/tmp/7_unpack"); - std::process::Command::new("tar") - .current_dir("./tests/tmp/7_unpack") - .arg("xf") - .arg("../7.tar") - .status()?; - - assert_eq!(std::fs::read("./tests/tmp/7_unpack/3")?, vec![0xde, 0xad]); - assert!(std::fs::read("./tests/tmp/7_unpack/7_3.log").is_ok()); + let results = simple_archive::Reader::new(std::fs::File::open("tests/tmp/7_3")?) + .map(Result::unwrap) + .collect::>(); + dbg!(&results); + assert!(results.contains(&Entry { path: "7_3".to_string(), data: vec![0xde, 0xad] })); + assert!(results.iter().any(|e| e.path == "student_log")); common::cleanup("7"); Ok(()) @@ -77,7 +75,7 @@ fn truncate_result() -> TestResult { command::handle_command(&mut com, &mut exec); assert!(com.is_complete()); - assert!(std::fs::File::open("./data/8_5.tar")?.metadata()?.len() < 1_005_000); + assert!(std::fs::File::open("./data/8_5")?.metadata()?.len() < 1_005_000); common::cleanup("8"); Ok(()) @@ -116,7 +114,7 @@ fn result_is_not_deleted_after_corrupted_transfer() { command::handle_command(&mut com, &mut exec); assert!(com.is_complete()); - assert!(std::fs::File::open("./data/50_0.tar").is_ok()); + assert!(std::fs::File::open("./data/50_0").is_ok()); common::cleanup("50"); }