package npy

import "github.com/sbinet/npyio/npy"

Package npy provides read/write access to files following the NumPy data file format:

https://numpy.org/neps/nep-0001-npy-format.html

Supported types

npy supports r/w of scalars, arrays, slices and gonum/mat.Dense. Supported scalars are:

• bool,
• (u)int{8,16,32,64},
• float{32,64},
• complex{64,128}

Reading

Reading from a NumPy data file can be performed like so:

f, err := os.Open("data.npy")
var m mat.Dense
err = npy.Read(f, &m)
fmt.Printf("data = %v\n", mat.Formatted(&m, mat.Prefix("       "))))

npy can also read data directly into slices, arrays or scalars, provided the on-disk data type and the provided one match.

Example:

var data []float64
err = npy.Read(f, &data)

var data uint64
err = npy.Read(f, &data)

Writing

Writing into a NumPy data file can be done like so:

f, err := os.Create("data.npy")
var m mat.Dense = ...
err = npy.Write(f, m)

Scalars, arrays and slices are also supported:

var data []float64 = ...
err = npy.Write(f, data)

var data int64 = 42
err = npy.Write(f, data)

var data [42]complex128 = ...
err = npy.Write(f, data)

package main

import (
	"fmt"
	"log"
	"os"

	"github.com/sbinet/npyio/npy"
)

func main() {
	out, err := os.Create("data.npy")
	if err != nil {
		log.Fatal(err)
	}
	defer out.Close()

	f := []float64{0, 1, 2, 3, 4, 5}
	fmt.Printf("-- original data --\n")
	fmt.Printf("data = %v\n", f)
	err = npy.Write(out, f)
	if err != nil {
		log.Fatalf("error writing data: %v\n", err)
	}
	err = out.Close()
	if err != nil {
		log.Fatal(err)
	}

	in, err := os.Open("data.npy")
	if err != nil {
		log.Fatal(err)
	}
	defer in.Close()
	r, err := npy.NewReader(in)
	if err != nil {
		log.Fatal(err)
	}

	data := make([]float64, 3)
	err = r.Read(&data)
	if err != nil {
		log.Fatalf("error reading data: %v\n", err)
	}

	fmt.Printf("-- partial data read back --\n")
	fmt.Printf("data = %v\n", data)

	err = r.Read(&data)
	if err != nil {
		log.Fatalf("error reading data: %v\n", err)
	}

	fmt.Printf("-- rest of data read back --\n")
	fmt.Printf("data = %v\n", data)

}

-- original data --
data = [0 1 2 3 4 5]
-- partial data read back --
data = [0 1 2]
-- rest of data read back --
data = [3 4 5]

Index ¶

• Variables
• func ClassLoader(module, name string) (any, error)
• func Read(r io.Reader, ptr interface{}) error
• func TypeFrom(dtype string) reflect.Type
• func Write(w io.Writer, val interface{}) error
• type Array
• func (arr Array) Data() any
• func (arr Array) Descr() ArrayDescr
• func (arr Array) Fortran() bool
• func (*Array) PyNew(args ...any) (any, error)
• func (arr *Array) PySetState(arg any) error
• func (arr Array) Shape() []int
• func (arr Array) Strides() []int
• func (arr Array) String() string
• type ArrayDescr
• func (*ArrayDescr) Call(args ...any) (any, error)
• func (dt *ArrayDescr) PySetState(arg any) error
• func (dt ArrayDescr) String() string
• type Header
• func (h Header) String() string
• type Reader
• func NewReader(r io.Reader) (*Reader, error)
• func (r *Reader) Read(ptr interface{}) error

Examples ¶

• package (PartialRead)
• Read
• Write

Variables ¶

var (

	// ErrInvalidNumPyFormat is the error returned by NewReader when
	// the underlying io.Reader is not a valid or recognized NumPy data
	// file format.
	ErrInvalidNumPyFormat = errors.New("npy: not a valid NumPy file format")

	// ErrTypeMismatch is the error returned by Reader when the on-disk
	// data type and the user provided one do NOT match.
	ErrTypeMismatch = errors.New("npy: types don't match")

	// ErrInvalidType is the error returned by Reader and Writer when
	// confronted with a type that is not supported or can not be
	// reliably (de)serialized.
	ErrInvalidType = errors.New("npy: invalid or unsupported type")

	// Magic header present at the start of a NumPy data file format.
	// See https://numpy.org/neps/nep-0001-npy-format.html
	Magic = [6]byte{'\x93', 'N', 'U', 'M', 'P', 'Y'}
)

Functions ¶

func ClassLoader ¶

func ClassLoader(module, name string) (any, error)

ClassLoader provides a python class loader mechanism for python pickles containing numpy.dtype and numpy.ndarray values.

func Read ¶

func Read(r io.Reader, ptr interface{}) error

Read reads the data from the r NumPy data file io.Reader, into the provided pointed at value ptr. Read returns an error if the on-disk data type and the one provided don't match.

If a *mat.Dense matrix is passed to Read, the numpy-array data is loaded into the Dense matrix, honouring Fortran/C-order and dimensions/shape parameters.

Only numpy-arrays with up to 2 dimensions are supported. Only numpy-arrays with elements convertible to float64 are supported.

package main

import (
	"bytes"
	"fmt"
	"log"

	"gonum.org/v1/gonum/mat"

	"github.com/sbinet/npyio/npy"
)

func main() {
	m := mat.NewDense(2, 3, []float64{0, 1, 2, 3, 4, 5})
	fmt.Printf("-- original data --\n")
	fmt.Printf("data = %v\n", mat.Formatted(m, mat.Prefix("       ")))
	buf := new(bytes.Buffer)

	err := npy.Write(buf, m)
	if err != nil {
		log.Fatalf("error writing data: %v\n", err)
	}

	// modify original data
	m.Set(0, 0, 6)

	var data mat.Dense
	err = npy.Read(buf, &data)
	if err != nil {
		log.Fatalf("error reading data: %v\n", err)
	}

	fmt.Printf("-- data read back --\n")
	fmt.Printf("data = %v\n", mat.Formatted(&data, mat.Prefix("       ")))

	fmt.Printf("-- modified original data --\n")
	fmt.Printf("data = %v\n", mat.Formatted(m, mat.Prefix("       ")))

}

-- original data --
data = ⎡0  1  2⎤
       ⎣3  4  5⎦
-- data read back --
data = ⎡0  1  2⎤
       ⎣3  4  5⎦
-- modified original data --
data = ⎡6  1  2⎤
       ⎣3  4  5⎦

func TypeFrom ¶

func TypeFrom(dtype string) reflect.Type

TypeFrom returns the reflect.Type corresponding to the numpy-dtype string, if any.

func Write ¶

func Write(w io.Writer, val interface{}) error

Write writes 'val' into 'w' in the NumPy data format.

• if val is a scalar, it must be of a supported type (bools, (u)ints, floats and complexes)
• if val is a slice or array, it must be a slice/array of a supported type. the shape (len,) will be written out.
• if val is a mat.Dense, the correct shape will be transmitted. (ie: (nrows, ncols))

The data-array will always be written out in C-order (row-major).

package main

import (
	"bytes"
	"fmt"
	"log"

	"gonum.org/v1/gonum/mat"

	"github.com/sbinet/npyio/npy"
)

func main() {
	m := mat.NewDense(2, 3, []float64{0, 1, 2, 3, 4, 5})
	fmt.Printf("-- original data --\n")
	fmt.Printf("data = %v\n", mat.Formatted(m, mat.Prefix("       ")))
	buf := new(bytes.Buffer)

	err := npy.Write(buf, m)
	if err != nil {
		log.Fatalf("error writing data: %v\n", err)
	}

	// modify original data
	m.Set(0, 0, 6)

	var data mat.Dense
	err = npy.Read(buf, &data)
	if err != nil {
		log.Fatalf("error reading data: %v\n", err)
	}

	fmt.Printf("-- data read back --\n")
	fmt.Printf("data = %v\n", mat.Formatted(&data, mat.Prefix("       ")))

	fmt.Printf("-- modified original data --\n")
	fmt.Printf("data = %v\n", mat.Formatted(m, mat.Prefix("       ")))

}

-- original data --
data = ⎡0  1  2⎤
       ⎣3  4  5⎦
-- data read back --
data = ⎡0  1  2⎤
       ⎣3  4  5⎦
-- modified original data --
data = ⎡6  1  2⎤
       ⎣3  4  5⎦

Types ¶

type Array ¶

type Array struct {
	// contains filtered or unexported fields
}

Array is a multidimensional, homogeneous array of fixed-size items.

func (Array) Data ¶

func (arr Array) Data() any

Data returns the array's underlying data.

func (Array) Descr ¶

func (arr Array) Descr() ArrayDescr

Descr returns the array's data type descriptor.

func (Array) Fortran ¶

func (arr Array) Fortran() bool

Fortran returns whether the array's data is stored in FORTRAN-order (ie: column-major) instead of C-order (ie: row-major.)

func (*Array) PyNew ¶

func (*Array) PyNew(args ...any) (any, error)

func (*Array) PySetState ¶

func (arr *Array) PySetState(arg any) error

func (Array) Shape ¶

func (arr Array) Shape() []int

Shape returns the array's shape.

func (Array) Strides ¶

func (arr Array) Strides() []int

Strides returns the array's strides in bytes.

func (Array) String ¶

func (arr Array) String() string

type ArrayDescr ¶

type ArrayDescr struct {
	// contains filtered or unexported fields
}

ArrayDescr describes a numpy data type.

func (*ArrayDescr) Call ¶

func (*ArrayDescr) Call(args ...any) (any, error)

func (*ArrayDescr) PySetState ¶

func (dt *ArrayDescr) PySetState(arg any) error

func (ArrayDescr) String ¶

func (dt ArrayDescr) String() string

type Header ¶

type Header struct {
	Major byte // data file major version
	Minor byte // data file minor version
	Descr struct {
		Type    string // data type of array elements ('<i8', '<f4', ...)
		Fortran bool   // whether the array data is stored in Fortran-order (col-major)
		Shape   []int  // array shape (e.g. [2,3] a 2-rows, 3-cols array
	}
}

Header describes the data content of a NumPy data file.

func (Header) String ¶

func (h Header) String() string

type Reader ¶

type Reader struct {
	Header Header
	// contains filtered or unexported fields
}

Reader reads data from a NumPy data file.

func NewReader ¶

func NewReader(r io.Reader) (*Reader, error)

NewReader creates a new NumPy data file format reader.

func (*Reader) Read ¶

func (r *Reader) Read(ptr interface{}) error

Read reads the numpy-array data from the underlying NumPy file. Read returns an error if the on-disk data type and the provided one don't match.

See npy.Read() for documentation.

Source Files ¶

array.go descr.go npy.go pickle.go reader.go writer.go

Directories ¶