summaryrefslogtreecommitdiff
path: root/morphology-img/src
diff options
context:
space:
mode:
authorjacksonfu14 <jackson@fashionunited.com>2025-11-17 08:49:07 +0100
committerJackson Moore <jacksonmoore@tuta.io>2025-11-17 08:49:07 +0100
commite37f7ba671f3e80396e7a4ac3a461365bd0ca78a (patch)
tree372bcc9bac7295fab920447fd1c8b84b257af337 /morphology-img/src
parentfc43589d3a5206a9b555dc868a1db0c92af53802 (diff)
feature: ly processing batch chunking
Diffstat (limited to 'morphology-img/src')
-rw-r--r--morphology-img/src/file_gen.rs46
-rw-r--r--morphology-img/src/lib.rs76
2 files changed, 117 insertions, 5 deletions
diff --git a/morphology-img/src/file_gen.rs b/morphology-img/src/file_gen.rs
index edeeeeb..dffb7bb 100644
--- a/morphology-img/src/file_gen.rs
+++ b/morphology-img/src/file_gen.rs
@@ -164,11 +164,51 @@ pub fn generate_ly_files(
sequences: &[Sequence],
temp_dir: &Path,
) -> Result<Vec<PathBuf>, RenderError> {
- let total_count = sequences.len();
- let mut paths = Vec::with_capacity(total_count);
+ generate_ly_files_with_offset(sequences, temp_dir, 0, sequences.len())
+}
+
+/// Generates .ly files for multiple sequences with custom offset and total count.
+///
+/// This function is used internally for batch processing to maintain proper
+/// global indexing across batches. The offset parameter allows each batch to
+/// start numbering from its global position, while total_count ensures
+/// consistent padding width across all batches.
+///
+/// # Arguments
+/// - `sequences`: The sequences to generate files for (batch)
+/// - `temp_dir`: The directory to write files to
+/// - `offset`: Starting index in the global sequence list
+/// - `total_count`: Total number of sequences across all batches
+///
+/// # Returns
+/// A vector of paths to the generated .ly files
+///
+/// # Examples
+/// ```no_run
+/// use morphology::{Sequence, Value};
+/// use morphology_img::file_gen::{generate_ly_files_with_offset, create_temp_dir};
+///
+/// // Second batch of a 12-sequence job
+/// let sequences = vec![
+/// Sequence::new(vec![Value::new(5, 0)]),
+/// Sequence::new(vec![Value::new(6, 0)]),
+/// ];
+///
+/// let temp_dir = create_temp_dir().unwrap();
+/// // offset=5, total_count=12 ensures filenames are sequence-05.ly, sequence-06.ly
+/// let paths = generate_ly_files_with_offset(&sequences, temp_dir.path(), 5, 12).unwrap();
+/// ```
+pub fn generate_ly_files_with_offset(
+ sequences: &[Sequence],
+ temp_dir: &Path,
+ offset: usize,
+ total_count: usize,
+) -> Result<Vec<PathBuf>, RenderError> {
+ let mut paths = Vec::with_capacity(sequences.len());
for (index, sequence) in sequences.iter().enumerate() {
- let filename = generate_filename(index, total_count);
+ let global_index = offset + index;
+ let filename = generate_filename(global_index, total_count);
let output_path = temp_dir.join(&filename);
generate_ly_file(sequence, &output_path)?;
diff --git a/morphology-img/src/lib.rs b/morphology-img/src/lib.rs
index 8e82f91..7d0825f 100644
--- a/morphology-img/src/lib.rs
+++ b/morphology-img/src/lib.rs
@@ -85,6 +85,12 @@ pub struct RenderConfig {
/// Optional callback for progress reporting during batch rendering
pub progress_callback: Option<ProgressCallback>,
+
+ /// Maximum number of sequences to process in a single batch
+ ///
+ /// If set, large batches will be split into smaller chunks to avoid
+ /// filesystem issues. If None, all sequences are processed in one batch.
+ pub max_batch_size: Option<usize>,
}
impl std::fmt::Debug for RenderConfig {
@@ -94,6 +100,7 @@ impl std::fmt::Debug for RenderConfig {
.field("keep_temp_files", &self.keep_temp_files)
.field("lilypond_binary", &self.lilypond_binary)
.field("progress_callback", &self.progress_callback.as_ref().map(|_| "<callback>"))
+ .field("max_batch_size", &self.max_batch_size)
.finish()
}
}
@@ -105,6 +112,7 @@ impl Default for RenderConfig {
keep_temp_files: false,
lilypond_binary: PathBuf::from("lilypond"),
progress_callback: None,
+ max_batch_size: None,
}
}
}
@@ -191,6 +199,27 @@ impl RenderConfig {
self.progress_callback = Some(callback);
self
}
+
+ /// Sets the maximum batch size for processing sequences.
+ ///
+ /// When set, large batches of sequences will be split into smaller chunks
+ /// to avoid overwhelming the filesystem or LilyPond. Each chunk is processed
+ /// independently with its own temporary directory.
+ ///
+ /// # Examples
+ ///
+ /// ```
+ /// use morphology_img::RenderConfig;
+ ///
+ /// let config = RenderConfig::new("output")
+ /// .max_batch_size(50);
+ ///
+ /// assert_eq!(config.max_batch_size, Some(50));
+ /// ```
+ pub fn max_batch_size(mut self, size: usize) -> Self {
+ self.max_batch_size = Some(size);
+ self
+ }
}
/// Renders multiple sequences to SVG files.
@@ -255,17 +284,60 @@ pub fn render_sequences(
sequences: &[morphology::Sequence],
config: &RenderConfig,
) -> Result<Vec<PathBuf>, RenderError> {
+ // Handle empty input
+ if sequences.is_empty() {
+ return Ok(Vec::new());
+ }
+
// Check if LilyPond is available before doing any work
render::check_lilypond_available()?;
+ // Determine batch size
+ let batch_size = config.max_batch_size.unwrap_or(sequences.len());
+ let mut all_svg_paths = Vec::with_capacity(sequences.len());
+ let total_count = sequences.len();
+
+ // Process sequences in chunks with global indexing
+ let mut global_offset = 0;
+ for chunk in sequences.chunks(batch_size) {
+ let svg_paths = render_sequences_batch(chunk, config, global_offset, total_count)?;
+ all_svg_paths.extend(svg_paths);
+ global_offset += chunk.len();
+ }
+
+ Ok(all_svg_paths)
+}
+
+/// Helper function to render a single batch of sequences.
+///
+/// This function handles the complete rendering pipeline for a batch:
+/// creating temp directory, writing support files, generating .ly files,
+/// and rendering to SVGs.
+///
+/// # Arguments
+/// - `sequences`: The batch of sequences to render
+/// - `config`: Configuration for rendering
+/// - `global_offset`: The starting index for this batch in the overall sequence list
+/// - `total_count`: The total number of sequences across all batches
+fn render_sequences_batch(
+ sequences: &[morphology::Sequence],
+ config: &RenderConfig,
+ global_offset: usize,
+ total_count: usize,
+) -> Result<Vec<PathBuf>, RenderError> {
// Create temporary directory for .ly files
let temp_dir = file_gen::create_temp_dir()?;
// Write support files (.ily) to temp directory
file_gen::write_support_files(temp_dir.path())?;
- // Generate .ly files from sequences
- let ly_files = file_gen::generate_ly_files(sequences, temp_dir.path())?;
+ // Generate .ly files from sequences with proper global indexing
+ let ly_files = file_gen::generate_ly_files_with_offset(
+ sequences,
+ temp_dir.path(),
+ global_offset,
+ total_count,
+ )?;
// Render .ly files to SVGs
let svg_paths = render::render_ly_files_to_svg(