package converter

import "github.com/containerd/nydus-snapshotter/pkg/converter"

Index

Constants

const (
	ManifestOSFeatureNydus   = "nydus.remoteimage.v1"
	MediaTypeNydusBlob       = "application/vnd.oci.image.layer.nydus.blob.v1"
	BootstrapFileNameInLayer = "image/image.boot"

	ManifestNydusCache = "containerd.io/snapshot/nydus-cache"

	LayerAnnotationFSVersion          = "containerd.io/snapshot/nydus-fs-version"
	LayerAnnotationNydusBlob          = "containerd.io/snapshot/nydus-blob"
	LayerAnnotationNydusBlobDigest    = "containerd.io/snapshot/nydus-blob-digest"
	LayerAnnotationNydusBlobSize      = "containerd.io/snapshot/nydus-blob-size"
	LayerAnnotationNydusBootstrap     = "containerd.io/snapshot/nydus-bootstrap"
	LayerAnnotationNydusSourceChainID = "containerd.io/snapshot/nydus-source-chainid"
	LayerAnnotationNydusEncryptedBlob = "containerd.io/snapshot/nydus-encrypted-blob"
	LayerAnnotationNydusSourceDigest  = "containerd.io/snapshot/nydus-source-digest"
	LayerAnnotationNydusTargetDigest  = "containerd.io/snapshot/nydus-target-digest"

	LayerAnnotationNydusReferenceBlobIDs = "containerd.io/snapshot/nydus-reference-blob-ids"

	LayerAnnotationUncompressed = "containerd.io/uncompressed"
)
const EntryBlob = "image.blob"
const EntryBlobMeta = "blob.meta"
const EntryBlobMetaHeader = "blob.meta.header"
const EntryBootstrap = "image.boot"
const EntryTOC = "rafs.blob.toc"

Variables

var (
	ErrNotFound = errors.New("data not found")
)

Functions

func ConvertHookFunc

func ConvertHookFunc(opt MergeOption) converter.ConvertHookFunc

ConvertHookFunc returns a function which will be used as a callback called for each blob after conversion is done. The function only hooks the index conversion and the manifest conversion.

func IsNydusBlob

func IsNydusBlob(desc ocispec.Descriptor) bool

IsNydusBlob returns true when the specified descriptor is nydus blob layer.

func IsNydusBlobAndExists

func IsNydusBlobAndExists(ctx context.Context, cs content.Store, desc ocispec.Descriptor) bool

IsNydusBlobAndExists returns true when the specified digest of content exists in the content store and it's nydus blob format.

func IsNydusBootstrap

func IsNydusBootstrap(desc ocispec.Descriptor) bool

IsNydusBootstrap returns true when the specified descriptor is nydus bootstrap layer.

func LayerConvertFunc

func LayerConvertFunc(opt PackOption) converter.ConvertFunc

LayerConvertFunc returns a function which converts an OCI image layer to a nydus blob layer, and set the media type to "application/vnd.oci.image.layer.nydus.blob.v1".

func Merge

func Merge(ctx context.Context, layers []Layer, dest io.Writer, opt MergeOption) ([]digest.Digest, error)

Merge multiple nydus bootstraps (from each layer of image) to a final bootstrap. And due to the possibility of enabling the `ChunkDictPath` option causes the data deduplication, it will return the actual blob digests referenced by the bootstrap.

func MergeLayers

MergeLayers merges a list of nydus blob layer into a nydus bootstrap layer. The media type of the nydus bootstrap layer is "application/vnd.oci.image.layer.v1.tar+gzip".

func Pack

func Pack(ctx context.Context, dest io.Writer, opt PackOption) (io.WriteCloser, error)

Pack converts an OCI tar stream to nydus formatted stream with a tar-like structure that arranges the data as follows:

`data | tar_header | data | tar_header | [toc_entry | ... | toc_entry | tar_header]`

The caller should write OCI tar stream into the returned `io.WriteCloser`, then the Pack method will write the nydus formatted stream to `dest` provided by the caller.

Important: the caller must check `io.WriteCloser.Close() == nil` to ensure the conversion workflow is finished.

func Unpack

func Unpack(ctx context.Context, ra content.ReaderAt, dest io.Writer, opt UnpackOption) error

Unpack converts a nydus blob layer to OCI formatted tar stream.

Types

type Backend

type Backend interface {
	// Push pushes specified blob file to remote storage backend.
	Push(ctx context.Context, cs content.Store, desc ocispec.Descriptor) error
	// Check checks whether a blob exists in remote storage backend,
	// blob exists -> return (blobPath, nil)
	// blob not exists -> return ("", err)
	Check(blobDigest digest.Digest) (string, error)
	// Type returns backend type name.
	Type() string
}

Backend uploads blobs generated by nydus-image builder to a backend storage.

type Compressor

type Compressor = uint32
const (
	CompressorNone     Compressor = 0x0000_0001
	CompressorZstd     Compressor = 0x0000_0002
	CompressorLz4Block Compressor = 0x0000_0004
	CompressorMask     Compressor = 0x0000_000f
)

type Encrypter

type File

type File struct {
	Name   string
	Reader io.Reader
	Size   int64
}

type Layer

type Layer struct {
	// Digest represents the hash of whole tar blob.
	Digest digest.Digest
	// Digest represents the original OCI tar(.gz) blob.
	OriginalDigest *digest.Digest
	// ReaderAt holds the reader of whole tar blob.
	ReaderAt content.ReaderAt
}

type MergeOption

type MergeOption struct {
	// WorkDir is used as the work directory during layer merge.
	WorkDir string
	// BuilderPath holds the path of `nydus-image` binary tool.
	BuilderPath string
	// FsVersion specifies nydus RAFS format version, possible
	// values: `5`, `6` (EROFS-compatible), default is `6`.
	FsVersion string
	// ChunkDictPath holds the bootstrap path of chunk dict image.
	ChunkDictPath string
	// ParentBootstrapPath holds the bootstrap path of parent image.
	ParentBootstrapPath string
	// PrefetchPatterns holds file path pattern list want to prefetch.
	PrefetchPatterns string
	// WithTar puts bootstrap into a tar stream (no gzip).
	WithTar bool
	// OCI converts docker media types to OCI media types.
	OCI bool
	// OCIRef enables converting OCI tar(.gz) blob to nydus referenced blob.
	OCIRef bool
	// WithReferrer associates a reference to the original OCI manifest.
	// See the `subject` field description in
	// https://github.com/opencontainers/image-spec/blob/main/manifest.md#image-manifest-property-descriptions
	//
	// With this association, we can track all nydus images associated with
	// an OCI image. For example, in Harbor we can cascade to show nydus
	// images linked to an OCI image, deleting the OCI image can also delete
	// the corresponding nydus images. At runtime, nydus snapshotter can also
	// automatically upgrade an OCI image run to nydus image.
	WithReferrer bool
	// Backend uploads blobs generated by nydus-image builder to a backend storage.
	Backend Backend
	// Timeout cancels execution once exceed the specified time.
	Timeout *time.Duration
	// Encrypt encrypts the bootstrap layer if it's specified.
	Encrypt Encrypter
	// AppendFiles specifies the files that need to be appended to the bootstrap layer.
	AppendFiles []File
}

type PackOption

type PackOption struct {
	// WorkDir is used as the work directory during layer pack.
	WorkDir string
	// BuilderPath holds the path of `nydus-image` binary tool.
	BuilderPath string
	// FsVersion specifies nydus RAFS format version, possible
	// values: `5`, `6` (EROFS-compatible), default is `6`.
	FsVersion string
	// ChunkDictPath holds the bootstrap path of chunk dict image.
	ChunkDictPath string
	// PrefetchPatterns holds file path pattern list want to prefetch.
	PrefetchPatterns string
	// Compressor specifies nydus blob compression algorithm.
	Compressor string
	// OCIRef enables converting OCI tar(.gz) blob to nydus referenced blob.
	OCIRef bool
	// AlignedChunk aligns uncompressed data chunks to 4K, only for RAFS V5.
	AlignedChunk bool
	// ChunkSize sets the size of data chunks, must be power of two and between 0x1000-0x1000000.
	ChunkSize string
	// BacthSize sets the size of batch data chunks, must be power of two and between 0x1000-0x1000000 or zero.
	BatchSize string
	// Backend uploads blobs generated by nydus-image builder to a backend storage.
	Backend Backend
	// Timeout cancels execution once exceed the specified time.
	Timeout *time.Duration
	// Whether the generated Nydus blobs should be encrypted.
	Encrypt bool
	// contains filtered or unexported fields
}

type TOCEntry

type TOCEntry struct {
	// Feature flags of entry
	Flags     uint32
	Reserved1 uint32
	// Name of entry data
	Name [16]byte
	// Sha256 of uncompressed entry data
	UncompressedDigest [32]byte
	// Offset of compressed entry data
	CompressedOffset uint64
	// Size of compressed entry data
	CompressedSize uint64
	// Size of uncompressed entry data
	UncompressedSize uint64
	Reserved2        [44]byte
}

func UnpackEntry

func UnpackEntry(ra content.ReaderAt, targetName string, target io.Writer) (*TOCEntry, error)

Unpack the file from nydus formatted tar stream. The nydus formatted tar stream is a tar-like structure that arranges the data as follows:

`data | tar_header | ... | data | tar_header | [toc_entry | ... | toc_entry | tar_header]`

func (*TOCEntry) GetCompressedOffset

func (entry *TOCEntry) GetCompressedOffset() uint64

func (*TOCEntry) GetCompressedSize

func (entry *TOCEntry) GetCompressedSize() uint64

func (*TOCEntry) GetCompressor

func (entry *TOCEntry) GetCompressor() (Compressor, error)

func (*TOCEntry) GetName

func (entry *TOCEntry) GetName() string

func (*TOCEntry) GetUncompressedDigest

func (entry *TOCEntry) GetUncompressedDigest() string

func (*TOCEntry) GetUncompressedSize

func (entry *TOCEntry) GetUncompressedSize() uint64

type UnpackOption

type UnpackOption struct {
	// WorkDir is used as the work directory during layer unpack.
	WorkDir string
	// BuilderPath holds the path of `nydus-image` binary tool.
	BuilderPath string
	// Timeout cancels execution once exceed the specified time.
	Timeout *time.Duration
	// Stream enables streaming mode, which doesn't unpack the blob data to disk,
	// but setup a http server to serve the blob data.
	Stream bool
}

Source Files

constant.go convert_unix.go cs_proxy_unix.go types.go utils.go

Directories

PathSynopsis
pkg/converter/tool
Version
v0.15.2 (latest)
Published
May 26, 2025
Platform
linux/amd64
Imports
36 packages
Last checked
12 hours ago

Tools for package owners.