summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorjacksonfu14 <jackson@fashionunited.com>2025-11-17 08:49:07 +0100
committerJackson Moore <jacksonmoore@tuta.io>2025-11-17 08:49:07 +0100
commite37f7ba671f3e80396e7a4ac3a461365bd0ca78a (patch)
tree372bcc9bac7295fab920447fd1c8b84b257af337
parentfc43589d3a5206a9b555dc868a1db0c92af53802 (diff)
feature: ly processing batch chunking
-rw-r--r--morphology-img/src/file_gen.rs46
-rw-r--r--morphology-img/src/lib.rs76
-rw-r--r--morphology-img/tests/config.rs57
-rw-r--r--morphology-img/tests/integration.rs184
4 files changed, 358 insertions, 5 deletions
diff --git a/morphology-img/src/file_gen.rs b/morphology-img/src/file_gen.rs
index edeeeeb..dffb7bb 100644
--- a/morphology-img/src/file_gen.rs
+++ b/morphology-img/src/file_gen.rs
@@ -164,11 +164,51 @@ pub fn generate_ly_files(
sequences: &[Sequence],
temp_dir: &Path,
) -> Result<Vec<PathBuf>, RenderError> {
- let total_count = sequences.len();
- let mut paths = Vec::with_capacity(total_count);
+ generate_ly_files_with_offset(sequences, temp_dir, 0, sequences.len())
+}
+
+/// Generates .ly files for multiple sequences with custom offset and total count.
+///
+/// This function is used internally for batch processing to maintain proper
+/// global indexing across batches. The offset parameter allows each batch to
+/// start numbering from its global position, while total_count ensures
+/// consistent padding width across all batches.
+///
+/// # Arguments
+/// - `sequences`: The sequences to generate files for (batch)
+/// - `temp_dir`: The directory to write files to
+/// - `offset`: Starting index in the global sequence list
+/// - `total_count`: Total number of sequences across all batches
+///
+/// # Returns
+/// A vector of paths to the generated .ly files
+///
+/// # Examples
+/// ```no_run
+/// use morphology::{Sequence, Value};
+/// use morphology_img::file_gen::{generate_ly_files_with_offset, create_temp_dir};
+///
+/// // Second batch of a 12-sequence job
+/// let sequences = vec![
+/// Sequence::new(vec![Value::new(5, 0)]),
+/// Sequence::new(vec![Value::new(6, 0)]),
+/// ];
+///
+/// let temp_dir = create_temp_dir().unwrap();
+/// // offset=5, total_count=12 ensures filenames are sequence-05.ly, sequence-06.ly
+/// let paths = generate_ly_files_with_offset(&sequences, temp_dir.path(), 5, 12).unwrap();
+/// ```
+pub fn generate_ly_files_with_offset(
+ sequences: &[Sequence],
+ temp_dir: &Path,
+ offset: usize,
+ total_count: usize,
+) -> Result<Vec<PathBuf>, RenderError> {
+ let mut paths = Vec::with_capacity(sequences.len());
for (index, sequence) in sequences.iter().enumerate() {
- let filename = generate_filename(index, total_count);
+ let global_index = offset + index;
+ let filename = generate_filename(global_index, total_count);
let output_path = temp_dir.join(&filename);
generate_ly_file(sequence, &output_path)?;
diff --git a/morphology-img/src/lib.rs b/morphology-img/src/lib.rs
index 8e82f91..7d0825f 100644
--- a/morphology-img/src/lib.rs
+++ b/morphology-img/src/lib.rs
@@ -85,6 +85,12 @@ pub struct RenderConfig {
/// Optional callback for progress reporting during batch rendering
pub progress_callback: Option<ProgressCallback>,
+
+ /// Maximum number of sequences to process in a single batch
+ ///
+ /// If set, large batches will be split into smaller chunks to avoid
+ /// filesystem issues. If None, all sequences are processed in one batch.
+ pub max_batch_size: Option<usize>,
}
impl std::fmt::Debug for RenderConfig {
@@ -94,6 +100,7 @@ impl std::fmt::Debug for RenderConfig {
.field("keep_temp_files", &self.keep_temp_files)
.field("lilypond_binary", &self.lilypond_binary)
.field("progress_callback", &self.progress_callback.as_ref().map(|_| "<callback>"))
+ .field("max_batch_size", &self.max_batch_size)
.finish()
}
}
@@ -105,6 +112,7 @@ impl Default for RenderConfig {
keep_temp_files: false,
lilypond_binary: PathBuf::from("lilypond"),
progress_callback: None,
+ max_batch_size: None,
}
}
}
@@ -191,6 +199,27 @@ impl RenderConfig {
self.progress_callback = Some(callback);
self
}
+
+ /// Sets the maximum batch size for processing sequences.
+ ///
+ /// When set, large batches of sequences will be split into smaller chunks
+ /// to avoid overwhelming the filesystem or LilyPond. Each chunk is processed
+ /// independently with its own temporary directory.
+ ///
+ /// # Examples
+ ///
+ /// ```
+ /// use morphology_img::RenderConfig;
+ ///
+ /// let config = RenderConfig::new("output")
+ /// .max_batch_size(50);
+ ///
+ /// assert_eq!(config.max_batch_size, Some(50));
+ /// ```
+ pub fn max_batch_size(mut self, size: usize) -> Self {
+ self.max_batch_size = Some(size);
+ self
+ }
}
/// Renders multiple sequences to SVG files.
@@ -255,17 +284,60 @@ pub fn render_sequences(
sequences: &[morphology::Sequence],
config: &RenderConfig,
) -> Result<Vec<PathBuf>, RenderError> {
+ // Handle empty input
+ if sequences.is_empty() {
+ return Ok(Vec::new());
+ }
+
// Check if LilyPond is available before doing any work
render::check_lilypond_available()?;
+ // Determine batch size
+ let batch_size = config.max_batch_size.unwrap_or(sequences.len());
+ let mut all_svg_paths = Vec::with_capacity(sequences.len());
+ let total_count = sequences.len();
+
+ // Process sequences in chunks with global indexing
+ let mut global_offset = 0;
+ for chunk in sequences.chunks(batch_size) {
+ let svg_paths = render_sequences_batch(chunk, config, global_offset, total_count)?;
+ all_svg_paths.extend(svg_paths);
+ global_offset += chunk.len();
+ }
+
+ Ok(all_svg_paths)
+}
+
+/// Helper function to render a single batch of sequences.
+///
+/// This function handles the complete rendering pipeline for a batch:
+/// creating temp directory, writing support files, generating .ly files,
+/// and rendering to SVGs.
+///
+/// # Arguments
+/// - `sequences`: The batch of sequences to render
+/// - `config`: Configuration for rendering
+/// - `global_offset`: The starting index for this batch in the overall sequence list
+/// - `total_count`: The total number of sequences across all batches
+fn render_sequences_batch(
+ sequences: &[morphology::Sequence],
+ config: &RenderConfig,
+ global_offset: usize,
+ total_count: usize,
+) -> Result<Vec<PathBuf>, RenderError> {
// Create temporary directory for .ly files
let temp_dir = file_gen::create_temp_dir()?;
// Write support files (.ily) to temp directory
file_gen::write_support_files(temp_dir.path())?;
- // Generate .ly files from sequences
- let ly_files = file_gen::generate_ly_files(sequences, temp_dir.path())?;
+ // Generate .ly files from sequences with proper global indexing
+ let ly_files = file_gen::generate_ly_files_with_offset(
+ sequences,
+ temp_dir.path(),
+ global_offset,
+ total_count,
+ )?;
// Render .ly files to SVGs
let svg_paths = render::render_ly_files_to_svg(
diff --git a/morphology-img/tests/config.rs b/morphology-img/tests/config.rs
index 9b47ed5..6302cb0 100644
--- a/morphology-img/tests/config.rs
+++ b/morphology-img/tests/config.rs
@@ -183,3 +183,60 @@ fn test_new_is_equivalent_to_default_with_custom_output() {
assert_eq!(config_new.keep_temp_files, config_default.keep_temp_files);
assert_eq!(config_new.lilypond_binary, config_default.lilypond_binary);
}
+
+#[test]
+fn test_default_max_batch_size_is_none() {
+ let config = RenderConfig::default();
+ assert_eq!(config.max_batch_size, None);
+}
+
+#[test]
+fn test_builder_max_batch_size() {
+ let config = RenderConfig::new("output")
+ .max_batch_size(50);
+
+ assert_eq!(config.max_batch_size, Some(50));
+ assert_eq!(config.output_dir, PathBuf::from("output"));
+}
+
+#[test]
+fn test_builder_max_batch_size_various_values() {
+ let config1 = RenderConfig::new("output").max_batch_size(1);
+ let config10 = RenderConfig::new("output").max_batch_size(10);
+ let config100 = RenderConfig::new("output").max_batch_size(100);
+
+ assert_eq!(config1.max_batch_size, Some(1));
+ assert_eq!(config10.max_batch_size, Some(10));
+ assert_eq!(config100.max_batch_size, Some(100));
+}
+
+#[test]
+fn test_builder_with_all_options_including_max_batch_size() {
+ let config = RenderConfig::new("my-output")
+ .keep_temp_files(true)
+ .lilypond_binary("/usr/bin/lilypond")
+ .max_batch_size(25);
+
+ assert_eq!(config.output_dir, PathBuf::from("my-output"));
+ assert!(config.keep_temp_files);
+ assert_eq!(config.lilypond_binary, PathBuf::from("/usr/bin/lilypond"));
+ assert_eq!(config.max_batch_size, Some(25));
+}
+
+#[test]
+fn test_builder_max_batch_size_overwrites() {
+ let config = RenderConfig::new("output")
+ .max_batch_size(10)
+ .max_batch_size(20); // Override previous value
+
+ assert_eq!(config.max_batch_size, Some(20));
+}
+
+#[test]
+fn test_max_batch_size_field_is_public() {
+ let config = RenderConfig::new("output").max_batch_size(15);
+
+ // Should be able to access field directly
+ let _batch_size = config.max_batch_size;
+ assert_eq!(_batch_size, Some(15));
+}
diff --git a/morphology-img/tests/integration.rs b/morphology-img/tests/integration.rs
index 227e5d1..c8b60fc 100644
--- a/morphology-img/tests/integration.rs
+++ b/morphology-img/tests/integration.rs
@@ -536,3 +536,187 @@ fn test_progress_callback_tracking() {
let final_progress = last_progress.lock().unwrap();
assert_eq!(*final_progress, (5, 5), "final progress should be 5/5");
}
+
+// ============================================================================
+// Batch Chunking Tests
+// ============================================================================
+
+#[test]
+#[ignore]
+fn test_batch_chunking_basic() {
+ // Create 25 sequences
+ let sequences: Vec<Sequence> = (0..25)
+ .map(|i| {
+ Sequence::new(vec![Value::new(i, 0)])
+ })
+ .collect();
+
+ let output_dir = TempDir::new().unwrap();
+ let config = RenderConfig::new(output_dir.path())
+ .max_batch_size(10);
+
+ let svg_paths = render_sequences(&sequences, &config).unwrap();
+
+ // All 25 SVGs should be created despite batching
+ assert_eq!(svg_paths.len(), 25);
+
+ // Verify all SVGs exist
+ for path in &svg_paths {
+ assert!(path.exists(), "SVG file should exist: {}", path.display());
+ }
+}
+
+#[test]
+#[ignore]
+fn test_batch_size_exact_match() {
+ // Create exactly 10 sequences with batch size 10
+ let sequences: Vec<Sequence> = (0..10)
+ .map(|i| {
+ Sequence::new(vec![Value::new(i, 0)])
+ })
+ .collect();
+
+ let output_dir = TempDir::new().unwrap();
+ let config = RenderConfig::new(output_dir.path())
+ .max_batch_size(10);
+
+ let svg_paths = render_sequences(&sequences, &config).unwrap();
+
+ assert_eq!(svg_paths.len(), 10);
+ for path in &svg_paths {
+ assert!(path.exists());
+ }
+}
+
+#[test]
+#[ignore]
+fn test_batch_size_larger_than_total() {
+ // Batch size larger than total sequences
+ let sequences: Vec<Sequence> = (0..5)
+ .map(|i| {
+ Sequence::new(vec![Value::new(i, 0)])
+ })
+ .collect();
+
+ let output_dir = TempDir::new().unwrap();
+ let config = RenderConfig::new(output_dir.path())
+ .max_batch_size(100);
+
+ let svg_paths = render_sequences(&sequences, &config).unwrap();
+
+ // Should still render all 5 sequences in one batch
+ assert_eq!(svg_paths.len(), 5);
+ for path in &svg_paths {
+ assert!(path.exists());
+ }
+}
+
+#[test]
+#[ignore]
+fn test_no_batch_limit() {
+ // No batch limit set (None) - should render all at once
+ let sequences: Vec<Sequence> = (0..15)
+ .map(|i| {
+ Sequence::new(vec![Value::new(i, 0)])
+ })
+ .collect();
+
+ let output_dir = TempDir::new().unwrap();
+ let config = RenderConfig::new(output_dir.path());
+ // Note: not setting max_batch_size, should default to None
+
+ let svg_paths = render_sequences(&sequences, &config).unwrap();
+
+ assert_eq!(svg_paths.len(), 15);
+ for path in &svg_paths {
+ assert!(path.exists());
+ }
+}
+
+#[test]
+#[ignore]
+fn test_batch_chunking_preserves_order() {
+ // Test that chunking preserves the order of sequences
+ let sequences: Vec<Sequence> = (0..12)
+ .map(|i| {
+ Sequence::new(vec![Value::new(i, 0), Value::new(i, 1)])
+ })
+ .collect();
+
+ let output_dir = TempDir::new().unwrap();
+ let config = RenderConfig::new(output_dir.path())
+ .max_batch_size(5);
+
+ let svg_paths = render_sequences(&sequences, &config).unwrap();
+
+ assert_eq!(svg_paths.len(), 12);
+
+ // Verify files are in order (sequence-00.svg through sequence-11.svg)
+ for (i, path) in svg_paths.iter().enumerate() {
+ let expected_filename = format!("sequence-{:02}.svg", i);
+ assert_eq!(
+ path.file_name().unwrap().to_str().unwrap(),
+ expected_filename,
+ "SVG files should be in order"
+ );
+ }
+}
+
+#[test]
+#[ignore]
+fn test_batch_chunking_with_progress() {
+ // Test that progress callback works correctly with batching
+ let sequences: Vec<Sequence> = (0..8)
+ .map(|i| {
+ Sequence::new(vec![Value::new(i, 0)])
+ })
+ .collect();
+
+ let progress_calls = Arc::new(Mutex::new(Vec::new()));
+ let progress_calls_clone = progress_calls.clone();
+
+ let callback: ProgressCallback = Arc::new(move |current, total| {
+ progress_calls_clone.lock().unwrap().push((current, total));
+ });
+
+ let output_dir = TempDir::new().unwrap();
+ let config = RenderConfig::new(output_dir.path())
+ .max_batch_size(3)
+ .progress_callback(callback);
+
+ let svg_paths = render_sequences(&sequences, &config).unwrap();
+
+ assert_eq!(svg_paths.len(), 8);
+
+ // Progress should track within each batch
+ let calls = progress_calls.lock().unwrap();
+ assert!(!calls.is_empty(), "Progress callback should be called");
+
+ // Note: Progress is per-batch, so we should see:
+ // Batch 1 (3 items): (1,3), (2,3), (3,3)
+ // Batch 2 (3 items): (1,3), (2,3), (3,3)
+ // Batch 3 (2 items): (1,2), (2,2)
+ assert_eq!(calls.len(), 8, "Should have 8 progress calls total");
+}
+
+#[test]
+#[ignore]
+fn test_batch_size_one() {
+ // Edge case: batch size of 1 (process one at a time)
+ let sequences: Vec<Sequence> = (0..3)
+ .map(|i| {
+ Sequence::new(vec![Value::new(i, 0)])
+ })
+ .collect();
+
+ let output_dir = TempDir::new().unwrap();
+ let config = RenderConfig::new(output_dir.path())
+ .max_batch_size(1);
+
+ let svg_paths = render_sequences(&sequences, &config).unwrap();
+
+ assert_eq!(svg_paths.len(), 3);
+ for path in &svg_paths {
+ assert!(path.exists());
+ }
+}