wgpu_core/command/
transfer.rs

1#[cfg(feature = "trace")]
2use crate::device::trace::Command as TraceCommand;
3use crate::{
4    api_log,
5    command::{clear_texture, CommandEncoderError},
6    conv,
7    device::{Device, DeviceError, MissingDownlevelFlags},
8    global::Global,
9    id::{BufferId, CommandEncoderId, TextureId},
10    init_tracker::{
11        has_copy_partial_init_tracker_coverage, MemoryInitKind, TextureInitRange,
12        TextureInitTrackerAction,
13    },
14    resource::{
15        DestroyedResourceError, InvalidResourceError, MissingBufferUsageError,
16        MissingTextureUsageError, ParentDevice, Texture, TextureErrorDimension,
17    },
18    snatch::SnatchGuard,
19    track::TextureSelector,
20};
21
22use arrayvec::ArrayVec;
23use thiserror::Error;
24use wgt::{BufferAddress, BufferUsages, Extent3d, TextureUsages};
25
26use std::sync::Arc;
27
28use super::{ClearError, CommandBufferMutable};
29
30pub type ImageCopyBuffer = wgt::ImageCopyBuffer<BufferId>;
31pub type ImageCopyTexture = wgt::ImageCopyTexture<TextureId>;
32pub type ImageCopyTextureTagged = wgt::ImageCopyTextureTagged<TextureId>;
33
34#[derive(Clone, Copy, Debug)]
35pub enum CopySide {
36    Source,
37    Destination,
38}
39
40/// Error encountered while attempting a data transfer.
41#[derive(Clone, Debug, Error)]
42#[non_exhaustive]
43pub enum TransferError {
44    #[error("Source and destination cannot be the same buffer")]
45    SameSourceDestinationBuffer,
46    #[error(transparent)]
47    MissingBufferUsage(#[from] MissingBufferUsageError),
48    #[error(transparent)]
49    MissingTextureUsage(#[from] MissingTextureUsageError),
50    #[error("Copy of {start_offset}..{end_offset} would end up overrunning the bounds of the {side:?} buffer of size {buffer_size}")]
51    BufferOverrun {
52        start_offset: BufferAddress,
53        end_offset: BufferAddress,
54        buffer_size: BufferAddress,
55        side: CopySide,
56    },
57    #[error("Copy of {dimension:?} {start_offset}..{end_offset} would end up overrunning the bounds of the {side:?} texture of {dimension:?} size {texture_size}")]
58    TextureOverrun {
59        start_offset: u32,
60        end_offset: u32,
61        texture_size: u32,
62        dimension: TextureErrorDimension,
63        side: CopySide,
64    },
65    #[error("Unable to select texture aspect {aspect:?} from format {format:?}")]
66    InvalidTextureAspect {
67        format: wgt::TextureFormat,
68        aspect: wgt::TextureAspect,
69    },
70    #[error("Unable to select texture mip level {level} out of {total}")]
71    InvalidTextureMipLevel { level: u32, total: u32 },
72    #[error("Texture dimension must be 2D when copying from an external texture")]
73    InvalidDimensionExternal,
74    #[error("Buffer offset {0} is not aligned to block size or `COPY_BUFFER_ALIGNMENT`")]
75    UnalignedBufferOffset(BufferAddress),
76    #[error("Copy size {0} does not respect `COPY_BUFFER_ALIGNMENT`")]
77    UnalignedCopySize(BufferAddress),
78    #[error("Copy width is not a multiple of block width")]
79    UnalignedCopyWidth,
80    #[error("Copy height is not a multiple of block height")]
81    UnalignedCopyHeight,
82    #[error("Copy origin's x component is not a multiple of block width")]
83    UnalignedCopyOriginX,
84    #[error("Copy origin's y component is not a multiple of block height")]
85    UnalignedCopyOriginY,
86    #[error("Bytes per row does not respect `COPY_BYTES_PER_ROW_ALIGNMENT`")]
87    UnalignedBytesPerRow,
88    #[error("Number of bytes per row needs to be specified since more than one row is copied")]
89    UnspecifiedBytesPerRow,
90    #[error("Number of rows per image needs to be specified since more than one image is copied")]
91    UnspecifiedRowsPerImage,
92    #[error("Number of bytes per row is less than the number of bytes in a complete row")]
93    InvalidBytesPerRow,
94    #[error("Image is 1D and the copy height and depth are not both set to 1")]
95    InvalidCopySize,
96    #[error("Number of rows per image is invalid")]
97    InvalidRowsPerImage,
98    #[error("Copy source aspects must refer to all aspects of the source texture format")]
99    CopySrcMissingAspects,
100    #[error(
101        "Copy destination aspects must refer to all aspects of the destination texture format"
102    )]
103    CopyDstMissingAspects,
104    #[error("Copy aspect must refer to a single aspect of texture format")]
105    CopyAspectNotOne,
106    #[error("Copying from textures with format {format:?} and aspect {aspect:?} is forbidden")]
107    CopyFromForbiddenTextureFormat {
108        format: wgt::TextureFormat,
109        aspect: wgt::TextureAspect,
110    },
111    #[error("Copying to textures with format {format:?} and aspect {aspect:?} is forbidden")]
112    CopyToForbiddenTextureFormat {
113        format: wgt::TextureFormat,
114        aspect: wgt::TextureAspect,
115    },
116    #[error(
117        "Copying to textures with format {0:?} is forbidden when copying from external texture"
118    )]
119    ExternalCopyToForbiddenTextureFormat(wgt::TextureFormat),
120    #[error(
121        "Source format ({src_format:?}) and destination format ({dst_format:?}) are not copy-compatible (they may only differ in srgb-ness)"
122    )]
123    TextureFormatsNotCopyCompatible {
124        src_format: wgt::TextureFormat,
125        dst_format: wgt::TextureFormat,
126    },
127    #[error(transparent)]
128    MemoryInitFailure(#[from] ClearError),
129    #[error("Cannot encode this copy because of a missing downelevel flag")]
130    MissingDownlevelFlags(#[from] MissingDownlevelFlags),
131    #[error("Source texture sample count must be 1, got {sample_count}")]
132    InvalidSampleCount { sample_count: u32 },
133    #[error("Requested mip level {requested} does no exist (count: {count})")]
134    InvalidMipLevel { requested: u32, count: u32 },
135}
136
137/// Error encountered while attempting to do a copy on a command encoder.
138#[derive(Clone, Debug, Error)]
139#[non_exhaustive]
140pub enum CopyError {
141    #[error(transparent)]
142    Encoder(#[from] CommandEncoderError),
143    #[error("Copy error")]
144    Transfer(#[from] TransferError),
145    #[error(transparent)]
146    DestroyedResource(#[from] DestroyedResourceError),
147    #[error(transparent)]
148    InvalidResource(#[from] InvalidResourceError),
149}
150
151impl From<DeviceError> for CopyError {
152    fn from(err: DeviceError) -> Self {
153        CopyError::Encoder(CommandEncoderError::Device(err))
154    }
155}
156
157pub(crate) fn extract_texture_selector<T>(
158    copy_texture: &wgt::ImageCopyTexture<T>,
159    copy_size: &Extent3d,
160    texture: &Texture,
161) -> Result<(TextureSelector, hal::TextureCopyBase), TransferError> {
162    let format = texture.desc.format;
163    let copy_aspect = hal::FormatAspects::new(format, copy_texture.aspect);
164    if copy_aspect.is_empty() {
165        return Err(TransferError::InvalidTextureAspect {
166            format,
167            aspect: copy_texture.aspect,
168        });
169    }
170
171    let (layers, origin_z) = match texture.desc.dimension {
172        wgt::TextureDimension::D1 => (0..1, 0),
173        wgt::TextureDimension::D2 => (
174            copy_texture.origin.z..copy_texture.origin.z + copy_size.depth_or_array_layers,
175            0,
176        ),
177        wgt::TextureDimension::D3 => (0..1, copy_texture.origin.z),
178    };
179    let base = hal::TextureCopyBase {
180        origin: wgt::Origin3d {
181            x: copy_texture.origin.x,
182            y: copy_texture.origin.y,
183            z: origin_z,
184        },
185        // this value will be incremented per copied layer
186        array_layer: layers.start,
187        mip_level: copy_texture.mip_level,
188        aspect: copy_aspect,
189    };
190    let selector = TextureSelector {
191        mips: copy_texture.mip_level..copy_texture.mip_level + 1,
192        layers,
193    };
194
195    Ok((selector, base))
196}
197
198/// WebGPU's [validating linear texture data][vltd] algorithm.
199///
200/// Copied with some modifications from WebGPU standard.
201///
202/// If successful, returns a pair `(bytes, stride)`, where:
203/// - `bytes` is the number of buffer bytes required for this copy, and
204/// - `stride` number of bytes between array layers.
205///
206/// [vltd]: https://gpuweb.github.io/gpuweb/#abstract-opdef-validating-linear-texture-data
207pub(crate) fn validate_linear_texture_data(
208    layout: &wgt::ImageDataLayout,
209    format: wgt::TextureFormat,
210    aspect: wgt::TextureAspect,
211    buffer_size: BufferAddress,
212    buffer_side: CopySide,
213    copy_size: &Extent3d,
214    need_copy_aligned_rows: bool,
215) -> Result<(BufferAddress, BufferAddress), TransferError> {
216    // Convert all inputs to BufferAddress (u64) to avoid some of the overflow issues
217    // Note: u64 is not always enough to prevent overflow, especially when multiplying
218    // something with a potentially large depth value, so it is preferable to validate
219    // the copy size before calling this function (for example via `validate_texture_copy_range`).
220    let copy_width = copy_size.width as BufferAddress;
221    let copy_height = copy_size.height as BufferAddress;
222    let depth_or_array_layers = copy_size.depth_or_array_layers as BufferAddress;
223
224    let offset = layout.offset;
225
226    let block_size = format.block_copy_size(Some(aspect)).unwrap() as BufferAddress;
227    let (block_width, block_height) = format.block_dimensions();
228    let block_width = block_width as BufferAddress;
229    let block_height = block_height as BufferAddress;
230
231    if copy_width % block_width != 0 {
232        return Err(TransferError::UnalignedCopyWidth);
233    }
234    if copy_height % block_height != 0 {
235        return Err(TransferError::UnalignedCopyHeight);
236    }
237
238    let width_in_blocks = copy_width / block_width;
239    let height_in_blocks = copy_height / block_height;
240
241    let bytes_in_last_row = width_in_blocks * block_size;
242
243    let bytes_per_row = if let Some(bytes_per_row) = layout.bytes_per_row {
244        let bytes_per_row = bytes_per_row as BufferAddress;
245        if bytes_per_row < bytes_in_last_row {
246            return Err(TransferError::InvalidBytesPerRow);
247        }
248        bytes_per_row
249    } else {
250        if depth_or_array_layers > 1 || height_in_blocks > 1 {
251            return Err(TransferError::UnspecifiedBytesPerRow);
252        }
253        0
254    };
255    let rows_per_image = if let Some(rows_per_image) = layout.rows_per_image {
256        let rows_per_image = rows_per_image as BufferAddress;
257        if rows_per_image < height_in_blocks {
258            return Err(TransferError::InvalidRowsPerImage);
259        }
260        rows_per_image
261    } else {
262        if depth_or_array_layers > 1 {
263            return Err(TransferError::UnspecifiedRowsPerImage);
264        }
265        0
266    };
267
268    if need_copy_aligned_rows {
269        let bytes_per_row_alignment = wgt::COPY_BYTES_PER_ROW_ALIGNMENT as BufferAddress;
270
271        let mut offset_alignment = block_size;
272        if format.is_depth_stencil_format() {
273            offset_alignment = 4
274        }
275        if offset % offset_alignment != 0 {
276            return Err(TransferError::UnalignedBufferOffset(offset));
277        }
278
279        if bytes_per_row % bytes_per_row_alignment != 0 {
280            return Err(TransferError::UnalignedBytesPerRow);
281        }
282    }
283
284    let bytes_per_image = bytes_per_row * rows_per_image;
285
286    let required_bytes_in_copy = if depth_or_array_layers == 0 {
287        0
288    } else {
289        let mut required_bytes_in_copy = bytes_per_image * (depth_or_array_layers - 1);
290        if height_in_blocks > 0 {
291            required_bytes_in_copy += bytes_per_row * (height_in_blocks - 1) + bytes_in_last_row;
292        }
293        required_bytes_in_copy
294    };
295
296    if offset + required_bytes_in_copy > buffer_size {
297        return Err(TransferError::BufferOverrun {
298            start_offset: offset,
299            end_offset: offset + required_bytes_in_copy,
300            buffer_size,
301            side: buffer_side,
302        });
303    }
304
305    Ok((required_bytes_in_copy, bytes_per_image))
306}
307
308/// WebGPU's [validating texture copy range][vtcr] algorithm.
309///
310/// Copied with minor modifications from WebGPU standard.
311///
312/// Returns the HAL copy extent and the layer count.
313///
314/// [vtcr]: https://gpuweb.github.io/gpuweb/#validating-texture-copy-range
315pub(crate) fn validate_texture_copy_range<T>(
316    texture_copy_view: &wgt::ImageCopyTexture<T>,
317    desc: &wgt::TextureDescriptor<(), Vec<wgt::TextureFormat>>,
318    texture_side: CopySide,
319    copy_size: &Extent3d,
320) -> Result<(hal::CopyExtent, u32), TransferError> {
321    let (block_width, block_height) = desc.format.block_dimensions();
322
323    let extent_virtual = desc.mip_level_size(texture_copy_view.mip_level).ok_or(
324        TransferError::InvalidTextureMipLevel {
325            level: texture_copy_view.mip_level,
326            total: desc.mip_level_count,
327        },
328    )?;
329    // physical size can be larger than the virtual
330    let extent = extent_virtual.physical_size(desc.format);
331
332    /// Return `Ok` if a run `size` texels long starting at `start_offset` falls
333    /// entirely within `texture_size`. Otherwise, return an appropriate a`Err`.
334    fn check_dimension(
335        dimension: TextureErrorDimension,
336        side: CopySide,
337        start_offset: u32,
338        size: u32,
339        texture_size: u32,
340    ) -> Result<(), TransferError> {
341        // Avoid underflow in the subtraction by checking start_offset against
342        // texture_size first.
343        if start_offset <= texture_size && size <= texture_size - start_offset {
344            Ok(())
345        } else {
346            Err(TransferError::TextureOverrun {
347                start_offset,
348                end_offset: start_offset.wrapping_add(size),
349                texture_size,
350                dimension,
351                side,
352            })
353        }
354    }
355
356    check_dimension(
357        TextureErrorDimension::X,
358        texture_side,
359        texture_copy_view.origin.x,
360        copy_size.width,
361        extent.width,
362    )?;
363    check_dimension(
364        TextureErrorDimension::Y,
365        texture_side,
366        texture_copy_view.origin.y,
367        copy_size.height,
368        extent.height,
369    )?;
370    check_dimension(
371        TextureErrorDimension::Z,
372        texture_side,
373        texture_copy_view.origin.z,
374        copy_size.depth_or_array_layers,
375        extent.depth_or_array_layers,
376    )?;
377
378    if texture_copy_view.origin.x % block_width != 0 {
379        return Err(TransferError::UnalignedCopyOriginX);
380    }
381    if texture_copy_view.origin.y % block_height != 0 {
382        return Err(TransferError::UnalignedCopyOriginY);
383    }
384    if copy_size.width % block_width != 0 {
385        return Err(TransferError::UnalignedCopyWidth);
386    }
387    if copy_size.height % block_height != 0 {
388        return Err(TransferError::UnalignedCopyHeight);
389    }
390
391    let (depth, array_layer_count) = match desc.dimension {
392        wgt::TextureDimension::D1 => (1, 1),
393        wgt::TextureDimension::D2 => (1, copy_size.depth_or_array_layers),
394        wgt::TextureDimension::D3 => (copy_size.depth_or_array_layers, 1),
395    };
396
397    let copy_extent = hal::CopyExtent {
398        width: copy_size.width,
399        height: copy_size.height,
400        depth,
401    };
402    Ok((copy_extent, array_layer_count))
403}
404
405fn handle_texture_init(
406    init_kind: MemoryInitKind,
407    cmd_buf_data: &mut CommandBufferMutable,
408    device: &Device,
409    copy_texture: &ImageCopyTexture,
410    copy_size: &Extent3d,
411    texture: &Arc<Texture>,
412    snatch_guard: &SnatchGuard<'_>,
413) -> Result<(), ClearError> {
414    let init_action = TextureInitTrackerAction {
415        texture: texture.clone(),
416        range: TextureInitRange {
417            mip_range: copy_texture.mip_level..copy_texture.mip_level + 1,
418            layer_range: copy_texture.origin.z
419                ..(copy_texture.origin.z + copy_size.depth_or_array_layers),
420        },
421        kind: init_kind,
422    };
423
424    // Register the init action.
425    let immediate_inits = cmd_buf_data
426        .texture_memory_actions
427        .register_init_action(&{ init_action });
428
429    // In rare cases we may need to insert an init operation immediately onto the command buffer.
430    if !immediate_inits.is_empty() {
431        let cmd_buf_raw = cmd_buf_data.encoder.open(device)?;
432        for init in immediate_inits {
433            clear_texture(
434                &init.texture,
435                TextureInitRange {
436                    mip_range: init.mip_level..(init.mip_level + 1),
437                    layer_range: init.layer..(init.layer + 1),
438                },
439                cmd_buf_raw,
440                &mut cmd_buf_data.trackers.textures,
441                &device.alignments,
442                device.zero_buffer.as_ref(),
443                snatch_guard,
444            )?;
445        }
446    }
447
448    Ok(())
449}
450
451/// Prepare a transfer's source texture.
452///
453/// Ensure the source texture of a transfer is in the right initialization
454/// state, and record the state for after the transfer operation.
455fn handle_src_texture_init(
456    cmd_buf_data: &mut CommandBufferMutable,
457    device: &Device,
458    source: &ImageCopyTexture,
459    copy_size: &Extent3d,
460    texture: &Arc<Texture>,
461    snatch_guard: &SnatchGuard<'_>,
462) -> Result<(), TransferError> {
463    handle_texture_init(
464        MemoryInitKind::NeedsInitializedMemory,
465        cmd_buf_data,
466        device,
467        source,
468        copy_size,
469        texture,
470        snatch_guard,
471    )?;
472    Ok(())
473}
474
475/// Prepare a transfer's destination texture.
476///
477/// Ensure the destination texture of a transfer is in the right initialization
478/// state, and record the state for after the transfer operation.
479fn handle_dst_texture_init(
480    cmd_buf_data: &mut CommandBufferMutable,
481    device: &Device,
482    destination: &ImageCopyTexture,
483    copy_size: &Extent3d,
484    texture: &Arc<Texture>,
485    snatch_guard: &SnatchGuard<'_>,
486) -> Result<(), TransferError> {
487    // Attention: If we don't write full texture subresources, we need to a full
488    // clear first since we don't track subrects. This means that in rare cases
489    // even a *destination* texture of a transfer may need an immediate texture
490    // init.
491    let dst_init_kind = if has_copy_partial_init_tracker_coverage(
492        copy_size,
493        destination.mip_level,
494        &texture.desc,
495    ) {
496        MemoryInitKind::NeedsInitializedMemory
497    } else {
498        MemoryInitKind::ImplicitlyInitialized
499    };
500
501    handle_texture_init(
502        dst_init_kind,
503        cmd_buf_data,
504        device,
505        destination,
506        copy_size,
507        texture,
508        snatch_guard,
509    )?;
510    Ok(())
511}
512
513impl Global {
514    pub fn command_encoder_copy_buffer_to_buffer(
515        &self,
516        command_encoder_id: CommandEncoderId,
517        source: BufferId,
518        source_offset: BufferAddress,
519        destination: BufferId,
520        destination_offset: BufferAddress,
521        size: BufferAddress,
522    ) -> Result<(), CopyError> {
523        profiling::scope!("CommandEncoder::copy_buffer_to_buffer");
524        api_log!(
525            "CommandEncoder::copy_buffer_to_buffer {source:?} -> {destination:?} {size:?}bytes"
526        );
527
528        if source == destination {
529            return Err(TransferError::SameSourceDestinationBuffer.into());
530        }
531        let hub = &self.hub;
532
533        let cmd_buf = hub
534            .command_buffers
535            .get(command_encoder_id.into_command_buffer_id());
536        let mut cmd_buf_data = cmd_buf.try_get()?;
537        cmd_buf_data.check_recording()?;
538
539        let device = &cmd_buf.device;
540        device.check_is_valid()?;
541
542        #[cfg(feature = "trace")]
543        if let Some(ref mut list) = cmd_buf_data.commands {
544            list.push(TraceCommand::CopyBufferToBuffer {
545                src: source,
546                src_offset: source_offset,
547                dst: destination,
548                dst_offset: destination_offset,
549                size,
550            });
551        }
552
553        let snatch_guard = device.snatchable_lock.read();
554
555        let src_buffer = hub.buffers.get(source).get()?;
556
557        src_buffer.same_device_as(cmd_buf.as_ref())?;
558
559        let src_pending = cmd_buf_data
560            .trackers
561            .buffers
562            .set_single(&src_buffer, hal::BufferUses::COPY_SRC);
563
564        let src_raw = src_buffer.try_raw(&snatch_guard)?;
565        src_buffer
566            .check_usage(BufferUsages::COPY_SRC)
567            .map_err(TransferError::MissingBufferUsage)?;
568        // expecting only a single barrier
569        let src_barrier = src_pending.map(|pending| pending.into_hal(&src_buffer, &snatch_guard));
570
571        let dst_buffer = hub.buffers.get(destination).get()?;
572
573        dst_buffer.same_device_as(cmd_buf.as_ref())?;
574
575        let dst_pending = cmd_buf_data
576            .trackers
577            .buffers
578            .set_single(&dst_buffer, hal::BufferUses::COPY_DST);
579
580        let dst_raw = dst_buffer.try_raw(&snatch_guard)?;
581        dst_buffer
582            .check_usage(BufferUsages::COPY_DST)
583            .map_err(TransferError::MissingBufferUsage)?;
584        let dst_barrier = dst_pending.map(|pending| pending.into_hal(&dst_buffer, &snatch_guard));
585
586        if size % wgt::COPY_BUFFER_ALIGNMENT != 0 {
587            return Err(TransferError::UnalignedCopySize(size).into());
588        }
589        if source_offset % wgt::COPY_BUFFER_ALIGNMENT != 0 {
590            return Err(TransferError::UnalignedBufferOffset(source_offset).into());
591        }
592        if destination_offset % wgt::COPY_BUFFER_ALIGNMENT != 0 {
593            return Err(TransferError::UnalignedBufferOffset(destination_offset).into());
594        }
595        if !device
596            .downlevel
597            .flags
598            .contains(wgt::DownlevelFlags::UNRESTRICTED_INDEX_BUFFER)
599            && (src_buffer.usage.contains(BufferUsages::INDEX)
600                || dst_buffer.usage.contains(BufferUsages::INDEX))
601        {
602            let forbidden_usages = BufferUsages::VERTEX
603                | BufferUsages::UNIFORM
604                | BufferUsages::INDIRECT
605                | BufferUsages::STORAGE;
606            if src_buffer.usage.intersects(forbidden_usages)
607                || dst_buffer.usage.intersects(forbidden_usages)
608            {
609                return Err(TransferError::MissingDownlevelFlags(MissingDownlevelFlags(
610                    wgt::DownlevelFlags::UNRESTRICTED_INDEX_BUFFER,
611                ))
612                .into());
613            }
614        }
615
616        let source_end_offset = source_offset + size;
617        let destination_end_offset = destination_offset + size;
618        if source_end_offset > src_buffer.size {
619            return Err(TransferError::BufferOverrun {
620                start_offset: source_offset,
621                end_offset: source_end_offset,
622                buffer_size: src_buffer.size,
623                side: CopySide::Source,
624            }
625            .into());
626        }
627        if destination_end_offset > dst_buffer.size {
628            return Err(TransferError::BufferOverrun {
629                start_offset: destination_offset,
630                end_offset: destination_end_offset,
631                buffer_size: dst_buffer.size,
632                side: CopySide::Destination,
633            }
634            .into());
635        }
636
637        if size == 0 {
638            log::trace!("Ignoring copy_buffer_to_buffer of size 0");
639            return Ok(());
640        }
641
642        // Make sure source is initialized memory and mark dest as initialized.
643        cmd_buf_data.buffer_memory_init_actions.extend(
644            dst_buffer.initialization_status.read().create_action(
645                &dst_buffer,
646                destination_offset..(destination_offset + size),
647                MemoryInitKind::ImplicitlyInitialized,
648            ),
649        );
650        cmd_buf_data.buffer_memory_init_actions.extend(
651            src_buffer.initialization_status.read().create_action(
652                &src_buffer,
653                source_offset..(source_offset + size),
654                MemoryInitKind::NeedsInitializedMemory,
655            ),
656        );
657
658        let region = hal::BufferCopy {
659            src_offset: source_offset,
660            dst_offset: destination_offset,
661            size: wgt::BufferSize::new(size).unwrap(),
662        };
663        let cmd_buf_raw = cmd_buf_data.encoder.open(&cmd_buf.device)?;
664        let barriers = src_barrier
665            .into_iter()
666            .chain(dst_barrier)
667            .collect::<Vec<_>>();
668        unsafe {
669            cmd_buf_raw.transition_buffers(&barriers);
670            cmd_buf_raw.copy_buffer_to_buffer(src_raw, dst_raw, &[region]);
671        }
672        Ok(())
673    }
674
675    pub fn command_encoder_copy_buffer_to_texture(
676        &self,
677        command_encoder_id: CommandEncoderId,
678        source: &ImageCopyBuffer,
679        destination: &ImageCopyTexture,
680        copy_size: &Extent3d,
681    ) -> Result<(), CopyError> {
682        profiling::scope!("CommandEncoder::copy_buffer_to_texture");
683        api_log!(
684            "CommandEncoder::copy_buffer_to_texture {:?} -> {:?} {copy_size:?}",
685            source.buffer,
686            destination.texture
687        );
688
689        let hub = &self.hub;
690
691        let cmd_buf = hub
692            .command_buffers
693            .get(command_encoder_id.into_command_buffer_id());
694        let mut cmd_buf_data = cmd_buf.try_get()?;
695        cmd_buf_data.check_recording()?;
696
697        let device = &cmd_buf.device;
698        device.check_is_valid()?;
699
700        #[cfg(feature = "trace")]
701        if let Some(ref mut list) = cmd_buf_data.commands {
702            list.push(TraceCommand::CopyBufferToTexture {
703                src: *source,
704                dst: *destination,
705                size: *copy_size,
706            });
707        }
708
709        if copy_size.width == 0 || copy_size.height == 0 || copy_size.depth_or_array_layers == 0 {
710            log::trace!("Ignoring copy_buffer_to_texture of size 0");
711            return Ok(());
712        }
713
714        let dst_texture = hub.textures.get(destination.texture).get()?;
715
716        dst_texture.same_device_as(cmd_buf.as_ref())?;
717
718        let (hal_copy_size, array_layer_count) = validate_texture_copy_range(
719            destination,
720            &dst_texture.desc,
721            CopySide::Destination,
722            copy_size,
723        )?;
724
725        let (dst_range, dst_base) = extract_texture_selector(destination, copy_size, &dst_texture)?;
726
727        let snatch_guard = device.snatchable_lock.read();
728
729        // Handle texture init *before* dealing with barrier transitions so we
730        // have an easier time inserting "immediate-inits" that may be required
731        // by prior discards in rare cases.
732        handle_dst_texture_init(
733            &mut cmd_buf_data,
734            device,
735            destination,
736            copy_size,
737            &dst_texture,
738            &snatch_guard,
739        )?;
740
741        let src_buffer = hub.buffers.get(source.buffer).get()?;
742
743        src_buffer.same_device_as(cmd_buf.as_ref())?;
744
745        let src_pending = cmd_buf_data
746            .trackers
747            .buffers
748            .set_single(&src_buffer, hal::BufferUses::COPY_SRC);
749
750        let src_raw = src_buffer.try_raw(&snatch_guard)?;
751        src_buffer
752            .check_usage(BufferUsages::COPY_SRC)
753            .map_err(TransferError::MissingBufferUsage)?;
754        let src_barrier = src_pending.map(|pending| pending.into_hal(&src_buffer, &snatch_guard));
755
756        let dst_pending = cmd_buf_data.trackers.textures.set_single(
757            &dst_texture,
758            dst_range,
759            hal::TextureUses::COPY_DST,
760        );
761        let dst_raw = dst_texture.try_raw(&snatch_guard)?;
762        dst_texture
763            .check_usage(TextureUsages::COPY_DST)
764            .map_err(TransferError::MissingTextureUsage)?;
765        let dst_barrier = dst_pending
766            .map(|pending| pending.into_hal(dst_raw))
767            .collect::<Vec<_>>();
768
769        if !dst_base.aspect.is_one() {
770            return Err(TransferError::CopyAspectNotOne.into());
771        }
772
773        if !conv::is_valid_copy_dst_texture_format(dst_texture.desc.format, destination.aspect) {
774            return Err(TransferError::CopyToForbiddenTextureFormat {
775                format: dst_texture.desc.format,
776                aspect: destination.aspect,
777            }
778            .into());
779        }
780
781        let (required_buffer_bytes_in_copy, bytes_per_array_layer) = validate_linear_texture_data(
782            &source.layout,
783            dst_texture.desc.format,
784            destination.aspect,
785            src_buffer.size,
786            CopySide::Source,
787            copy_size,
788            true,
789        )?;
790
791        if dst_texture.desc.format.is_depth_stencil_format() {
792            device
793                .require_downlevel_flags(wgt::DownlevelFlags::DEPTH_TEXTURE_AND_BUFFER_COPIES)
794                .map_err(TransferError::from)?;
795        }
796
797        cmd_buf_data.buffer_memory_init_actions.extend(
798            src_buffer.initialization_status.read().create_action(
799                &src_buffer,
800                source.layout.offset..(source.layout.offset + required_buffer_bytes_in_copy),
801                MemoryInitKind::NeedsInitializedMemory,
802            ),
803        );
804
805        let regions = (0..array_layer_count)
806            .map(|rel_array_layer| {
807                let mut texture_base = dst_base.clone();
808                texture_base.array_layer += rel_array_layer;
809                let mut buffer_layout = source.layout;
810                buffer_layout.offset += rel_array_layer as u64 * bytes_per_array_layer;
811                hal::BufferTextureCopy {
812                    buffer_layout,
813                    texture_base,
814                    size: hal_copy_size,
815                }
816            })
817            .collect::<Vec<_>>();
818
819        let cmd_buf_raw = cmd_buf_data.encoder.open(&cmd_buf.device)?;
820        unsafe {
821            cmd_buf_raw.transition_textures(&dst_barrier);
822            cmd_buf_raw.transition_buffers(src_barrier.as_slice());
823            cmd_buf_raw.copy_buffer_to_texture(src_raw, dst_raw, &regions);
824        }
825        Ok(())
826    }
827
828    pub fn command_encoder_copy_texture_to_buffer(
829        &self,
830        command_encoder_id: CommandEncoderId,
831        source: &ImageCopyTexture,
832        destination: &ImageCopyBuffer,
833        copy_size: &Extent3d,
834    ) -> Result<(), CopyError> {
835        profiling::scope!("CommandEncoder::copy_texture_to_buffer");
836        api_log!(
837            "CommandEncoder::copy_texture_to_buffer {:?} -> {:?} {copy_size:?}",
838            source.texture,
839            destination.buffer
840        );
841
842        let hub = &self.hub;
843
844        let cmd_buf = hub
845            .command_buffers
846            .get(command_encoder_id.into_command_buffer_id());
847        let mut cmd_buf_data = cmd_buf.try_get()?;
848        cmd_buf_data.check_recording()?;
849
850        let device = &cmd_buf.device;
851        device.check_is_valid()?;
852
853        #[cfg(feature = "trace")]
854        if let Some(ref mut list) = cmd_buf_data.commands {
855            list.push(TraceCommand::CopyTextureToBuffer {
856                src: *source,
857                dst: *destination,
858                size: *copy_size,
859            });
860        }
861
862        if copy_size.width == 0 || copy_size.height == 0 || copy_size.depth_or_array_layers == 0 {
863            log::trace!("Ignoring copy_texture_to_buffer of size 0");
864            return Ok(());
865        }
866
867        let src_texture = hub.textures.get(source.texture).get()?;
868
869        src_texture.same_device_as(cmd_buf.as_ref())?;
870
871        let (hal_copy_size, array_layer_count) =
872            validate_texture_copy_range(source, &src_texture.desc, CopySide::Source, copy_size)?;
873
874        let (src_range, src_base) = extract_texture_selector(source, copy_size, &src_texture)?;
875
876        let snatch_guard = device.snatchable_lock.read();
877
878        // Handle texture init *before* dealing with barrier transitions so we
879        // have an easier time inserting "immediate-inits" that may be required
880        // by prior discards in rare cases.
881        handle_src_texture_init(
882            &mut cmd_buf_data,
883            device,
884            source,
885            copy_size,
886            &src_texture,
887            &snatch_guard,
888        )?;
889
890        let src_pending = cmd_buf_data.trackers.textures.set_single(
891            &src_texture,
892            src_range,
893            hal::TextureUses::COPY_SRC,
894        );
895        let src_raw = src_texture.try_raw(&snatch_guard)?;
896        src_texture
897            .check_usage(TextureUsages::COPY_SRC)
898            .map_err(TransferError::MissingTextureUsage)?;
899        if src_texture.desc.sample_count != 1 {
900            return Err(TransferError::InvalidSampleCount {
901                sample_count: src_texture.desc.sample_count,
902            }
903            .into());
904        }
905        if source.mip_level >= src_texture.desc.mip_level_count {
906            return Err(TransferError::InvalidMipLevel {
907                requested: source.mip_level,
908                count: src_texture.desc.mip_level_count,
909            }
910            .into());
911        }
912        let src_barrier = src_pending
913            .map(|pending| pending.into_hal(src_raw))
914            .collect::<Vec<_>>();
915
916        let dst_buffer = hub.buffers.get(destination.buffer).get()?;
917
918        dst_buffer.same_device_as(cmd_buf.as_ref())?;
919
920        let dst_pending = cmd_buf_data
921            .trackers
922            .buffers
923            .set_single(&dst_buffer, hal::BufferUses::COPY_DST);
924
925        let dst_raw = dst_buffer.try_raw(&snatch_guard)?;
926        dst_buffer
927            .check_usage(BufferUsages::COPY_DST)
928            .map_err(TransferError::MissingBufferUsage)?;
929        let dst_barrier = dst_pending.map(|pending| pending.into_hal(&dst_buffer, &snatch_guard));
930
931        if !src_base.aspect.is_one() {
932            return Err(TransferError::CopyAspectNotOne.into());
933        }
934
935        if !conv::is_valid_copy_src_texture_format(src_texture.desc.format, source.aspect) {
936            return Err(TransferError::CopyFromForbiddenTextureFormat {
937                format: src_texture.desc.format,
938                aspect: source.aspect,
939            }
940            .into());
941        }
942
943        let (required_buffer_bytes_in_copy, bytes_per_array_layer) = validate_linear_texture_data(
944            &destination.layout,
945            src_texture.desc.format,
946            source.aspect,
947            dst_buffer.size,
948            CopySide::Destination,
949            copy_size,
950            true,
951        )?;
952
953        if src_texture.desc.format.is_depth_stencil_format() {
954            device
955                .require_downlevel_flags(wgt::DownlevelFlags::DEPTH_TEXTURE_AND_BUFFER_COPIES)
956                .map_err(TransferError::from)?;
957        }
958
959        cmd_buf_data.buffer_memory_init_actions.extend(
960            dst_buffer.initialization_status.read().create_action(
961                &dst_buffer,
962                destination.layout.offset
963                    ..(destination.layout.offset + required_buffer_bytes_in_copy),
964                MemoryInitKind::ImplicitlyInitialized,
965            ),
966        );
967
968        let regions = (0..array_layer_count)
969            .map(|rel_array_layer| {
970                let mut texture_base = src_base.clone();
971                texture_base.array_layer += rel_array_layer;
972                let mut buffer_layout = destination.layout;
973                buffer_layout.offset += rel_array_layer as u64 * bytes_per_array_layer;
974                hal::BufferTextureCopy {
975                    buffer_layout,
976                    texture_base,
977                    size: hal_copy_size,
978                }
979            })
980            .collect::<Vec<_>>();
981        let cmd_buf_raw = cmd_buf_data.encoder.open(&cmd_buf.device)?;
982        unsafe {
983            cmd_buf_raw.transition_buffers(dst_barrier.as_slice());
984            cmd_buf_raw.transition_textures(&src_barrier);
985            cmd_buf_raw.copy_texture_to_buffer(
986                src_raw,
987                hal::TextureUses::COPY_SRC,
988                dst_raw,
989                &regions,
990            );
991        }
992        Ok(())
993    }
994
995    pub fn command_encoder_copy_texture_to_texture(
996        &self,
997        command_encoder_id: CommandEncoderId,
998        source: &ImageCopyTexture,
999        destination: &ImageCopyTexture,
1000        copy_size: &Extent3d,
1001    ) -> Result<(), CopyError> {
1002        profiling::scope!("CommandEncoder::copy_texture_to_texture");
1003        api_log!(
1004            "CommandEncoder::copy_texture_to_texture {:?} -> {:?} {copy_size:?}",
1005            source.texture,
1006            destination.texture
1007        );
1008
1009        let hub = &self.hub;
1010
1011        let cmd_buf = hub
1012            .command_buffers
1013            .get(command_encoder_id.into_command_buffer_id());
1014        let mut cmd_buf_data = cmd_buf.try_get()?;
1015        cmd_buf_data.check_recording()?;
1016
1017        let device = &cmd_buf.device;
1018        device.check_is_valid()?;
1019
1020        let snatch_guard = device.snatchable_lock.read();
1021
1022        #[cfg(feature = "trace")]
1023        if let Some(ref mut list) = cmd_buf_data.commands {
1024            list.push(TraceCommand::CopyTextureToTexture {
1025                src: *source,
1026                dst: *destination,
1027                size: *copy_size,
1028            });
1029        }
1030
1031        if copy_size.width == 0 || copy_size.height == 0 || copy_size.depth_or_array_layers == 0 {
1032            log::trace!("Ignoring copy_texture_to_texture of size 0");
1033            return Ok(());
1034        }
1035
1036        let src_texture = hub.textures.get(source.texture).get()?;
1037        let dst_texture = hub.textures.get(destination.texture).get()?;
1038
1039        src_texture.same_device_as(cmd_buf.as_ref())?;
1040        dst_texture.same_device_as(cmd_buf.as_ref())?;
1041
1042        // src and dst texture format must be copy-compatible
1043        // https://gpuweb.github.io/gpuweb/#copy-compatible
1044        if src_texture.desc.format.remove_srgb_suffix()
1045            != dst_texture.desc.format.remove_srgb_suffix()
1046        {
1047            return Err(TransferError::TextureFormatsNotCopyCompatible {
1048                src_format: src_texture.desc.format,
1049                dst_format: dst_texture.desc.format,
1050            }
1051            .into());
1052        }
1053
1054        let (src_copy_size, array_layer_count) =
1055            validate_texture_copy_range(source, &src_texture.desc, CopySide::Source, copy_size)?;
1056        let (dst_copy_size, _) = validate_texture_copy_range(
1057            destination,
1058            &dst_texture.desc,
1059            CopySide::Destination,
1060            copy_size,
1061        )?;
1062
1063        let (src_range, src_tex_base) = extract_texture_selector(source, copy_size, &src_texture)?;
1064        let (dst_range, dst_tex_base) =
1065            extract_texture_selector(destination, copy_size, &dst_texture)?;
1066        let src_texture_aspects = hal::FormatAspects::from(src_texture.desc.format);
1067        let dst_texture_aspects = hal::FormatAspects::from(dst_texture.desc.format);
1068        if src_tex_base.aspect != src_texture_aspects {
1069            return Err(TransferError::CopySrcMissingAspects.into());
1070        }
1071        if dst_tex_base.aspect != dst_texture_aspects {
1072            return Err(TransferError::CopyDstMissingAspects.into());
1073        }
1074
1075        // Handle texture init *before* dealing with barrier transitions so we
1076        // have an easier time inserting "immediate-inits" that may be required
1077        // by prior discards in rare cases.
1078        handle_src_texture_init(
1079            &mut cmd_buf_data,
1080            device,
1081            source,
1082            copy_size,
1083            &src_texture,
1084            &snatch_guard,
1085        )?;
1086        handle_dst_texture_init(
1087            &mut cmd_buf_data,
1088            device,
1089            destination,
1090            copy_size,
1091            &dst_texture,
1092            &snatch_guard,
1093        )?;
1094
1095        let src_pending = cmd_buf_data.trackers.textures.set_single(
1096            &src_texture,
1097            src_range,
1098            hal::TextureUses::COPY_SRC,
1099        );
1100        let src_raw = src_texture.try_raw(&snatch_guard)?;
1101        src_texture
1102            .check_usage(TextureUsages::COPY_SRC)
1103            .map_err(TransferError::MissingTextureUsage)?;
1104
1105        //TODO: try to avoid this the collection. It's needed because both
1106        // `src_pending` and `dst_pending` try to hold `trackers.textures` mutably.
1107        let mut barriers: ArrayVec<_, 2> = src_pending
1108            .map(|pending| pending.into_hal(src_raw))
1109            .collect();
1110
1111        let dst_pending = cmd_buf_data.trackers.textures.set_single(
1112            &dst_texture,
1113            dst_range,
1114            hal::TextureUses::COPY_DST,
1115        );
1116        let dst_raw = dst_texture.try_raw(&snatch_guard)?;
1117        dst_texture
1118            .check_usage(TextureUsages::COPY_DST)
1119            .map_err(TransferError::MissingTextureUsage)?;
1120
1121        barriers.extend(dst_pending.map(|pending| pending.into_hal(dst_raw)));
1122
1123        let hal_copy_size = hal::CopyExtent {
1124            width: src_copy_size.width.min(dst_copy_size.width),
1125            height: src_copy_size.height.min(dst_copy_size.height),
1126            depth: src_copy_size.depth.min(dst_copy_size.depth),
1127        };
1128        let regions = (0..array_layer_count)
1129            .map(|rel_array_layer| {
1130                let mut src_base = src_tex_base.clone();
1131                let mut dst_base = dst_tex_base.clone();
1132                src_base.array_layer += rel_array_layer;
1133                dst_base.array_layer += rel_array_layer;
1134                hal::TextureCopy {
1135                    src_base,
1136                    dst_base,
1137                    size: hal_copy_size,
1138                }
1139            })
1140            .collect::<Vec<_>>();
1141        let cmd_buf_raw = cmd_buf_data.encoder.open(&cmd_buf.device)?;
1142        unsafe {
1143            cmd_buf_raw.transition_textures(&barriers);
1144            cmd_buf_raw.copy_texture_to_texture(
1145                src_raw,
1146                hal::TextureUses::COPY_SRC,
1147                dst_raw,
1148                &regions,
1149            );
1150        }
1151
1152        Ok(())
1153    }
1154}
wgpu_core/command/transfer.rs

wgpu_core/command/
transfer.rs
