up follow livre

This commit is contained in:
Tykayn 2025-08-30 18:14:14 +02:00 committed by tykayn
parent b4b4398bb0
commit 3a7a3849ae
12242 changed files with 2564461 additions and 6914 deletions

View file

@ -0,0 +1,264 @@
''' Tests for fortran sequential files '''
import tempfile
import shutil
import os
from os import path
from glob import iglob
import threading
import re
from numpy.testing import assert_equal, assert_allclose
import numpy as np
import pytest
from scipy.io import (FortranFile,
_test_fortran,
FortranEOFError,
FortranFormattingError)
DATA_PATH = path.join(path.dirname(__file__), 'data')
@pytest.fixture
def io_lock():
return threading.Lock()
def test_fortranfiles_read(io_lock):
for filename in iglob(path.join(DATA_PATH, "fortran-*-*x*x*.dat")):
m = re.search(r'fortran-([^-]+)-(\d+)x(\d+)x(\d+).dat', filename, re.I)
if not m:
raise RuntimeError(f"Couldn't match {filename} filename to regex")
dims = (int(m.group(2)), int(m.group(3)), int(m.group(4)))
dtype = m.group(1).replace('s', '<')
with io_lock:
f = FortranFile(filename, 'r', '<u4')
data = f.read_record(dtype=dtype).reshape(dims, order='F')
f.close()
expected = np.arange(np.prod(dims)).reshape(dims).astype(dtype)
assert_equal(data, expected)
def test_fortranfiles_mixed_record(io_lock):
filename = path.join(DATA_PATH, "fortran-mixed.dat")
with io_lock:
with FortranFile(filename, 'r', '<u4') as f:
record = f.read_record('<i4,<f4,<i8,2<f8')
assert_equal(record['f0'][0], 1)
assert_allclose(record['f1'][0], 2.3)
assert_equal(record['f2'][0], 4)
assert_allclose(record['f3'][0], [5.6, 7.8])
def test_fortranfiles_write():
for filename in iglob(path.join(DATA_PATH, "fortran-*-*x*x*.dat")):
m = re.search(r'fortran-([^-]+)-(\d+)x(\d+)x(\d+).dat', filename, re.I)
if not m:
raise RuntimeError(f"Couldn't match {filename} filename to regex")
dims = (int(m.group(2)), int(m.group(3)), int(m.group(4)))
dtype = m.group(1).replace('s', '<')
data = np.arange(np.prod(dims)).reshape(dims).astype(dtype)
tmpdir = tempfile.mkdtemp()
try:
testFile = path.join(str(threading.get_native_id()),
tmpdir,path.basename(filename))
f = FortranFile(testFile, 'w','<u4')
f.write_record(data.T)
f.close()
originalfile = open(filename, 'rb')
newfile = open(testFile, 'rb')
assert_equal(originalfile.read(), newfile.read(),
err_msg=filename)
originalfile.close()
newfile.close()
finally:
shutil.rmtree(tmpdir)
def test_fortranfile_read_mixed_record(io_lock):
# The data file fortran-3x3d-2i.dat contains the program that
# produced it at the end.
#
# double precision :: a(3,3)
# integer :: b(2)
# ...
# open(1, file='fortran-3x3d-2i.dat', form='unformatted')
# write(1) a, b
# close(1)
#
filename = path.join(DATA_PATH, "fortran-3x3d-2i.dat")
with io_lock:
with FortranFile(filename, 'r', '<u4') as f:
record = f.read_record('(3,3)<f8', '2<i4')
ax = np.arange(3*3).reshape(3, 3).astype(np.float64)
bx = np.array([-1, -2], dtype=np.int32)
assert_equal(record[0], ax.T)
assert_equal(record[1], bx.T)
def test_fortranfile_write_mixed_record(tmpdir):
tf = path.join(str(tmpdir), str(threading.get_native_id()), 'test.dat')
os.makedirs(path.dirname(tf), exist_ok=True)
r1 = (('f4', 'f4', 'i4'), (np.float32(2), np.float32(3), np.int32(100)))
r2 = (('4f4', '(3,3)f4', '8i4'),
(np.random.randint(255, size=[4]).astype(np.float32),
np.random.randint(255, size=[3, 3]).astype(np.float32),
np.random.randint(255, size=[8]).astype(np.int32)))
records = [r1, r2]
for dtype, a in records:
with FortranFile(tf, 'w') as f:
f.write_record(*a)
with FortranFile(tf, 'r') as f:
b = f.read_record(*dtype)
assert_equal(len(a), len(b))
for aa, bb in zip(a, b):
assert_equal(bb, aa)
def test_fortran_roundtrip(tmpdir, io_lock):
filename = path.join(str(tmpdir), str(threading.get_native_id()),
'test.dat')
os.makedirs(path.dirname(filename), exist_ok=True)
rng = np.random.RandomState(1)
# double precision
m, n, k = 5, 3, 2
a = rng.randn(m, n, k)
with FortranFile(filename, 'w') as f:
f.write_record(a.T)
with io_lock:
a2 = _test_fortran.read_unformatted_double(m, n, k, filename)
with FortranFile(filename, 'r') as f:
a3 = f.read_record('(2,3,5)f8').T
assert_equal(a2, a)
assert_equal(a3, a)
# integer
m, n, k = 5, 3, 2
a = rng.randn(m, n, k).astype(np.int32)
with FortranFile(filename, 'w') as f:
f.write_record(a.T)
with io_lock:
a2 = _test_fortran.read_unformatted_int(m, n, k, filename)
with FortranFile(filename, 'r') as f:
a3 = f.read_record('(2,3,5)i4').T
assert_equal(a2, a)
assert_equal(a3, a)
# mixed
m, n, k = 5, 3, 2
a = rng.randn(m, n)
b = rng.randn(k).astype(np.intc)
with FortranFile(filename, 'w') as f:
f.write_record(a.T, b.T)
with io_lock:
a2, b2 = _test_fortran.read_unformatted_mixed(m, n, k, filename)
with FortranFile(filename, 'r') as f:
a3, b3 = f.read_record('(3,5)f8', '2i4')
a3 = a3.T
assert_equal(a2, a)
assert_equal(a3, a)
assert_equal(b2, b)
assert_equal(b3, b)
def test_fortran_eof_ok(tmpdir):
filename = path.join(str(tmpdir), str(threading.get_native_id()),
"scratch")
os.makedirs(path.dirname(filename), exist_ok=True)
rng = np.random.RandomState(1)
with FortranFile(filename, 'w') as f:
f.write_record(rng.randn(5))
f.write_record(rng.randn(3))
with FortranFile(filename, 'r') as f:
assert len(f.read_reals()) == 5
assert len(f.read_reals()) == 3
with pytest.raises(FortranEOFError):
f.read_reals()
def test_fortran_eof_broken_size(tmpdir):
filename = path.join(str(tmpdir), str(threading.get_native_id()),
"scratch")
os.makedirs(path.dirname(filename), exist_ok=True)
rng = np.random.RandomState(1)
with FortranFile(filename, 'w') as f:
f.write_record(rng.randn(5))
f.write_record(rng.randn(3))
with open(filename, "ab") as f:
f.write(b"\xff")
with FortranFile(filename, 'r') as f:
assert len(f.read_reals()) == 5
assert len(f.read_reals()) == 3
with pytest.raises(FortranFormattingError):
f.read_reals()
def test_fortran_bogus_size(tmpdir):
filename = path.join(str(tmpdir), str(threading.get_native_id()),
"scratch")
os.makedirs(path.dirname(filename), exist_ok=True)
rng = np.random.RandomState(1)
with FortranFile(filename, 'w') as f:
f.write_record(rng.randn(5))
f.write_record(rng.randn(3))
with open(filename, "w+b") as f:
f.write(b"\xff\xff")
with FortranFile(filename, 'r') as f:
with pytest.raises(FortranFormattingError):
f.read_reals()
def test_fortran_eof_broken_record(tmpdir):
filename = path.join(str(tmpdir), str(threading.get_native_id()),
"scratch")
os.makedirs(path.dirname(filename), exist_ok=True)
rng = np.random.RandomState(1)
with FortranFile(filename, 'w') as f:
f.write_record(rng.randn(5))
f.write_record(rng.randn(3))
with open(filename, "ab") as f:
f.truncate(path.getsize(filename)-20)
with FortranFile(filename, 'r') as f:
assert len(f.read_reals()) == 5
with pytest.raises(FortranFormattingError):
f.read_reals()
def test_fortran_eof_multidimensional(tmpdir):
filename = path.join(str(tmpdir), str(threading.get_native_id()),
"scratch")
os.makedirs(path.dirname(filename), exist_ok=True)
n, m, q = 3, 5, 7
dt = np.dtype([("field", np.float64, (n, m))])
a = np.zeros(q, dtype=dt)
with FortranFile(filename, 'w') as f:
f.write_record(a[0])
f.write_record(a)
f.write_record(a)
with open(filename, "ab") as f:
f.truncate(path.getsize(filename)-20)
with FortranFile(filename, 'r') as f:
assert len(f.read_record(dtype=dt)) == 1
assert len(f.read_record(dtype=dt)) == q
with pytest.raises(FortranFormattingError):
f.read_record(dtype=dt)

View file

@ -0,0 +1,483 @@
from os import path
import warnings
import numpy as np
from numpy.testing import (assert_equal, assert_array_equal,
assert_, suppress_warnings)
import pytest
from scipy.io import readsav
from scipy.io import _idl
DATA_PATH = path.join(path.dirname(__file__), 'data')
def assert_identical(a, b):
"""Assert whether value AND type are the same"""
assert_equal(a, b)
if isinstance(b, str):
assert_equal(type(a), type(b))
else:
assert_equal(np.asarray(a).dtype.type, np.asarray(b).dtype.type)
def assert_array_identical(a, b):
"""Assert whether values AND type are the same"""
assert_array_equal(a, b)
assert_equal(a.dtype.type, b.dtype.type)
# Define vectorized ID function for pointer arrays
vect_id = np.vectorize(id)
class TestIdict:
def test_idict(self):
custom_dict = {'a': np.int16(999)}
original_id = id(custom_dict)
s = readsav(path.join(DATA_PATH, 'scalar_byte.sav'),
idict=custom_dict, verbose=False)
assert_equal(original_id, id(s))
assert_('a' in s)
assert_identical(s['a'], np.int16(999))
assert_identical(s['i8u'], np.uint8(234))
class TestScalars:
# Test that scalar values are read in with the correct value and type
def test_byte(self):
s = readsav(path.join(DATA_PATH, 'scalar_byte.sav'), verbose=False)
assert_identical(s.i8u, np.uint8(234))
def test_int16(self):
s = readsav(path.join(DATA_PATH, 'scalar_int16.sav'), verbose=False)
assert_identical(s.i16s, np.int16(-23456))
def test_int32(self):
s = readsav(path.join(DATA_PATH, 'scalar_int32.sav'), verbose=False)
assert_identical(s.i32s, np.int32(-1234567890))
def test_float32(self):
s = readsav(path.join(DATA_PATH, 'scalar_float32.sav'), verbose=False)
assert_identical(s.f32, np.float32(-3.1234567e+37))
def test_float64(self):
s = readsav(path.join(DATA_PATH, 'scalar_float64.sav'), verbose=False)
assert_identical(s.f64, np.float64(-1.1976931348623157e+307))
def test_complex32(self):
s = readsav(path.join(DATA_PATH, 'scalar_complex32.sav'), verbose=False)
assert_identical(s.c32, np.complex64(3.124442e13-2.312442e31j))
def test_bytes(self):
s = readsav(path.join(DATA_PATH, 'scalar_string.sav'), verbose=False)
msg = "The quick brown fox jumps over the lazy python"
assert_identical(s.s, np.bytes_(msg))
def test_structure(self):
pass
def test_complex64(self):
s = readsav(path.join(DATA_PATH, 'scalar_complex64.sav'), verbose=False)
assert_identical(
s.c64,
np.complex128(1.1987253647623157e+112-5.1987258887729157e+307j)
)
def test_heap_pointer(self):
pass
def test_object_reference(self):
pass
def test_uint16(self):
s = readsav(path.join(DATA_PATH, 'scalar_uint16.sav'), verbose=False)
assert_identical(s.i16u, np.uint16(65511))
def test_uint32(self):
s = readsav(path.join(DATA_PATH, 'scalar_uint32.sav'), verbose=False)
assert_identical(s.i32u, np.uint32(4294967233))
def test_int64(self):
s = readsav(path.join(DATA_PATH, 'scalar_int64.sav'), verbose=False)
assert_identical(s.i64s, np.int64(-9223372036854774567))
def test_uint64(self):
s = readsav(path.join(DATA_PATH, 'scalar_uint64.sav'), verbose=False)
assert_identical(s.i64u, np.uint64(18446744073709529285))
class TestCompressed(TestScalars):
# Test that compressed .sav files can be read in
def test_compressed(self):
s = readsav(path.join(DATA_PATH, 'various_compressed.sav'), verbose=False)
assert_identical(s.i8u, np.uint8(234))
assert_identical(s.f32, np.float32(-3.1234567e+37))
assert_identical(
s.c64,
np.complex128(1.1987253647623157e+112-5.1987258887729157e+307j)
)
assert_equal(s.array5d.shape, (4, 3, 4, 6, 5))
assert_identical(s.arrays.a[0], np.array([1, 2, 3], dtype=np.int16))
assert_identical(s.arrays.b[0], np.array([4., 5., 6., 7.], dtype=np.float32))
assert_identical(s.arrays.c[0],
np.array([np.complex64(1+2j), np.complex64(7+8j)]))
assert_identical(s.arrays.d[0],
np.array([b"cheese", b"bacon", b"spam"], dtype=object))
class TestArrayDimensions:
# Test that multi-dimensional arrays are read in with the correct dimensions
def test_1d(self):
s = readsav(path.join(DATA_PATH, 'array_float32_1d.sav'), verbose=False)
assert_equal(s.array1d.shape, (123, ))
def test_2d(self):
s = readsav(path.join(DATA_PATH, 'array_float32_2d.sav'), verbose=False)
assert_equal(s.array2d.shape, (22, 12))
def test_3d(self):
s = readsav(path.join(DATA_PATH, 'array_float32_3d.sav'), verbose=False)
assert_equal(s.array3d.shape, (11, 22, 12))
def test_4d(self):
s = readsav(path.join(DATA_PATH, 'array_float32_4d.sav'), verbose=False)
assert_equal(s.array4d.shape, (4, 5, 8, 7))
def test_5d(self):
s = readsav(path.join(DATA_PATH, 'array_float32_5d.sav'), verbose=False)
assert_equal(s.array5d.shape, (4, 3, 4, 6, 5))
def test_6d(self):
s = readsav(path.join(DATA_PATH, 'array_float32_6d.sav'), verbose=False)
assert_equal(s.array6d.shape, (3, 6, 4, 5, 3, 4))
def test_7d(self):
s = readsav(path.join(DATA_PATH, 'array_float32_7d.sav'), verbose=False)
assert_equal(s.array7d.shape, (2, 1, 2, 3, 4, 3, 2))
def test_8d(self):
s = readsav(path.join(DATA_PATH, 'array_float32_8d.sav'), verbose=False)
assert_equal(s.array8d.shape, (4, 3, 2, 1, 2, 3, 5, 4))
class TestStructures:
def test_scalars(self):
s = readsav(path.join(DATA_PATH, 'struct_scalars.sav'), verbose=False)
assert_identical(s.scalars.a, np.array(np.int16(1)))
assert_identical(s.scalars.b, np.array(np.int32(2)))
assert_identical(s.scalars.c, np.array(np.float32(3.)))
assert_identical(s.scalars.d, np.array(np.float64(4.)))
assert_identical(s.scalars.e, np.array([b"spam"], dtype=object))
assert_identical(s.scalars.f, np.array(np.complex64(-1.+3j)))
def test_scalars_replicated(self):
s = readsav(path.join(DATA_PATH, 'struct_scalars_replicated.sav'),
verbose=False)
assert_identical(s.scalars_rep.a, np.repeat(np.int16(1), 5))
assert_identical(s.scalars_rep.b, np.repeat(np.int32(2), 5))
assert_identical(s.scalars_rep.c, np.repeat(np.float32(3.), 5))
assert_identical(s.scalars_rep.d, np.repeat(np.float64(4.), 5))
assert_identical(s.scalars_rep.e, np.repeat(b"spam", 5).astype(object))
assert_identical(s.scalars_rep.f, np.repeat(np.complex64(-1.+3j), 5))
def test_scalars_replicated_3d(self):
s = readsav(path.join(DATA_PATH, 'struct_scalars_replicated_3d.sav'),
verbose=False)
assert_identical(s.scalars_rep.a, np.repeat(np.int16(1), 24).reshape(4, 3, 2))
assert_identical(s.scalars_rep.b, np.repeat(np.int32(2), 24).reshape(4, 3, 2))
assert_identical(s.scalars_rep.c,
np.repeat(np.float32(3.), 24).reshape(4, 3, 2))
assert_identical(s.scalars_rep.d,
np.repeat(np.float64(4.), 24).reshape(4, 3, 2))
assert_identical(s.scalars_rep.e,
np.repeat(b"spam", 24).reshape(4, 3, 2).astype(object))
assert_identical(s.scalars_rep.f,
np.repeat(np.complex64(-1.+3j), 24).reshape(4, 3, 2))
def test_arrays(self):
s = readsav(path.join(DATA_PATH, 'struct_arrays.sav'), verbose=False)
assert_array_identical(s.arrays.a[0], np.array([1, 2, 3], dtype=np.int16))
assert_array_identical(s.arrays.b[0],
np.array([4., 5., 6., 7.], dtype=np.float32))
assert_array_identical(s.arrays.c[0],
np.array([np.complex64(1+2j), np.complex64(7+8j)]))
assert_array_identical(s.arrays.d[0],
np.array([b"cheese", b"bacon", b"spam"], dtype=object))
def test_arrays_replicated(self):
s = readsav(path.join(DATA_PATH, 'struct_arrays_replicated.sav'), verbose=False)
# Check column types
assert_(s.arrays_rep.a.dtype.type is np.object_)
assert_(s.arrays_rep.b.dtype.type is np.object_)
assert_(s.arrays_rep.c.dtype.type is np.object_)
assert_(s.arrays_rep.d.dtype.type is np.object_)
# Check column shapes
assert_equal(s.arrays_rep.a.shape, (5, ))
assert_equal(s.arrays_rep.b.shape, (5, ))
assert_equal(s.arrays_rep.c.shape, (5, ))
assert_equal(s.arrays_rep.d.shape, (5, ))
# Check values
for i in range(5):
assert_array_identical(s.arrays_rep.a[i],
np.array([1, 2, 3], dtype=np.int16))
assert_array_identical(s.arrays_rep.b[i],
np.array([4., 5., 6., 7.], dtype=np.float32))
assert_array_identical(s.arrays_rep.c[i],
np.array([np.complex64(1+2j),
np.complex64(7+8j)]))
assert_array_identical(s.arrays_rep.d[i],
np.array([b"cheese", b"bacon", b"spam"],
dtype=object))
def test_arrays_replicated_3d(self):
s = readsav(path.join(DATA_PATH, 'struct_arrays_replicated_3d.sav'),
verbose=False)
# Check column types
assert_(s.arrays_rep.a.dtype.type is np.object_)
assert_(s.arrays_rep.b.dtype.type is np.object_)
assert_(s.arrays_rep.c.dtype.type is np.object_)
assert_(s.arrays_rep.d.dtype.type is np.object_)
# Check column shapes
assert_equal(s.arrays_rep.a.shape, (4, 3, 2))
assert_equal(s.arrays_rep.b.shape, (4, 3, 2))
assert_equal(s.arrays_rep.c.shape, (4, 3, 2))
assert_equal(s.arrays_rep.d.shape, (4, 3, 2))
# Check values
for i in range(4):
for j in range(3):
for k in range(2):
assert_array_identical(s.arrays_rep.a[i, j, k],
np.array([1, 2, 3], dtype=np.int16))
assert_array_identical(s.arrays_rep.b[i, j, k],
np.array([4., 5., 6., 7.],
dtype=np.float32))
assert_array_identical(s.arrays_rep.c[i, j, k],
np.array([np.complex64(1+2j),
np.complex64(7+8j)]))
assert_array_identical(s.arrays_rep.d[i, j, k],
np.array([b"cheese", b"bacon", b"spam"],
dtype=object))
def test_inheritance(self):
s = readsav(path.join(DATA_PATH, 'struct_inherit.sav'), verbose=False)
assert_identical(s.fc.x, np.array([0], dtype=np.int16))
assert_identical(s.fc.y, np.array([0], dtype=np.int16))
assert_identical(s.fc.r, np.array([0], dtype=np.int16))
assert_identical(s.fc.c, np.array([4], dtype=np.int16))
@pytest.mark.thread_unsafe
def test_arrays_corrupt_idl80(self):
# test byte arrays with missing nbyte information from IDL 8.0 .sav file
with suppress_warnings() as sup:
sup.filter(UserWarning, "Not able to verify number of bytes from header")
s = readsav(path.join(DATA_PATH,'struct_arrays_byte_idl80.sav'),
verbose=False)
assert_identical(s.y.x[0], np.array([55,66], dtype=np.uint8))
class TestPointers:
# Check that pointers in .sav files produce references to the same object in Python
def test_pointers(self):
s = readsav(path.join(DATA_PATH, 'scalar_heap_pointer.sav'), verbose=False)
assert_identical(
s.c64_pointer1,
np.complex128(1.1987253647623157e+112-5.1987258887729157e+307j)
)
assert_identical(
s.c64_pointer2,
np.complex128(1.1987253647623157e+112-5.1987258887729157e+307j)
)
assert_(s.c64_pointer1 is s.c64_pointer2)
class TestPointerArray:
# Test that pointers in arrays are correctly read in
def test_1d(self):
s = readsav(path.join(DATA_PATH, 'array_float32_pointer_1d.sav'), verbose=False)
assert_equal(s.array1d.shape, (123, ))
assert_(np.all(s.array1d == np.float32(4.)))
assert_(np.all(vect_id(s.array1d) == id(s.array1d[0])))
def test_2d(self):
s = readsav(path.join(DATA_PATH, 'array_float32_pointer_2d.sav'), verbose=False)
assert_equal(s.array2d.shape, (22, 12))
assert_(np.all(s.array2d == np.float32(4.)))
assert_(np.all(vect_id(s.array2d) == id(s.array2d[0,0])))
def test_3d(self):
s = readsav(path.join(DATA_PATH, 'array_float32_pointer_3d.sav'), verbose=False)
assert_equal(s.array3d.shape, (11, 22, 12))
assert_(np.all(s.array3d == np.float32(4.)))
assert_(np.all(vect_id(s.array3d) == id(s.array3d[0,0,0])))
def test_4d(self):
s = readsav(path.join(DATA_PATH, 'array_float32_pointer_4d.sav'), verbose=False)
assert_equal(s.array4d.shape, (4, 5, 8, 7))
assert_(np.all(s.array4d == np.float32(4.)))
assert_(np.all(vect_id(s.array4d) == id(s.array4d[0,0,0,0])))
def test_5d(self):
s = readsav(path.join(DATA_PATH, 'array_float32_pointer_5d.sav'), verbose=False)
assert_equal(s.array5d.shape, (4, 3, 4, 6, 5))
assert_(np.all(s.array5d == np.float32(4.)))
assert_(np.all(vect_id(s.array5d) == id(s.array5d[0,0,0,0,0])))
def test_6d(self):
s = readsav(path.join(DATA_PATH, 'array_float32_pointer_6d.sav'), verbose=False)
assert_equal(s.array6d.shape, (3, 6, 4, 5, 3, 4))
assert_(np.all(s.array6d == np.float32(4.)))
assert_(np.all(vect_id(s.array6d) == id(s.array6d[0,0,0,0,0,0])))
def test_7d(self):
s = readsav(path.join(DATA_PATH, 'array_float32_pointer_7d.sav'), verbose=False)
assert_equal(s.array7d.shape, (2, 1, 2, 3, 4, 3, 2))
assert_(np.all(s.array7d == np.float32(4.)))
assert_(np.all(vect_id(s.array7d) == id(s.array7d[0,0,0,0,0,0,0])))
def test_8d(self):
s = readsav(path.join(DATA_PATH, 'array_float32_pointer_8d.sav'), verbose=False)
assert_equal(s.array8d.shape, (4, 3, 2, 1, 2, 3, 5, 4))
assert_(np.all(s.array8d == np.float32(4.)))
assert_(np.all(vect_id(s.array8d) == id(s.array8d[0,0,0,0,0,0,0,0])))
class TestPointerStructures:
# Test that structures are correctly read in
def test_scalars(self):
s = readsav(path.join(DATA_PATH, 'struct_pointers.sav'), verbose=False)
assert_identical(s.pointers.g, np.array(np.float32(4.), dtype=np.object_))
assert_identical(s.pointers.h, np.array(np.float32(4.), dtype=np.object_))
assert_(id(s.pointers.g[0]) == id(s.pointers.h[0]))
def test_pointers_replicated(self):
s = readsav(path.join(DATA_PATH, 'struct_pointers_replicated.sav'),
verbose=False)
assert_identical(s.pointers_rep.g,
np.repeat(np.float32(4.), 5).astype(np.object_))
assert_identical(s.pointers_rep.h,
np.repeat(np.float32(4.), 5).astype(np.object_))
assert_(np.all(vect_id(s.pointers_rep.g) == vect_id(s.pointers_rep.h)))
def test_pointers_replicated_3d(self):
s = readsav(path.join(DATA_PATH, 'struct_pointers_replicated_3d.sav'),
verbose=False)
s_expect = np.repeat(np.float32(4.), 24).reshape(4, 3, 2).astype(np.object_)
assert_identical(s.pointers_rep.g, s_expect)
assert_identical(s.pointers_rep.h, s_expect)
assert_(np.all(vect_id(s.pointers_rep.g) == vect_id(s.pointers_rep.h)))
def test_arrays(self):
s = readsav(path.join(DATA_PATH, 'struct_pointer_arrays.sav'), verbose=False)
assert_array_identical(s.arrays.g[0],
np.repeat(np.float32(4.), 2).astype(np.object_))
assert_array_identical(s.arrays.h[0],
np.repeat(np.float32(4.), 3).astype(np.object_))
assert_(np.all(vect_id(s.arrays.g[0]) == id(s.arrays.g[0][0])))
assert_(np.all(vect_id(s.arrays.h[0]) == id(s.arrays.h[0][0])))
assert_(id(s.arrays.g[0][0]) == id(s.arrays.h[0][0]))
def test_arrays_replicated(self):
s = readsav(path.join(DATA_PATH, 'struct_pointer_arrays_replicated.sav'),
verbose=False)
# Check column types
assert_(s.arrays_rep.g.dtype.type is np.object_)
assert_(s.arrays_rep.h.dtype.type is np.object_)
# Check column shapes
assert_equal(s.arrays_rep.g.shape, (5, ))
assert_equal(s.arrays_rep.h.shape, (5, ))
# Check values
for i in range(5):
assert_array_identical(s.arrays_rep.g[i],
np.repeat(np.float32(4.), 2).astype(np.object_))
assert_array_identical(s.arrays_rep.h[i],
np.repeat(np.float32(4.), 3).astype(np.object_))
assert_(np.all(vect_id(s.arrays_rep.g[i]) == id(s.arrays_rep.g[0][0])))
assert_(np.all(vect_id(s.arrays_rep.h[i]) == id(s.arrays_rep.h[0][0])))
def test_arrays_replicated_3d(self):
pth = path.join(DATA_PATH, 'struct_pointer_arrays_replicated_3d.sav')
s = readsav(pth, verbose=False)
# Check column types
assert_(s.arrays_rep.g.dtype.type is np.object_)
assert_(s.arrays_rep.h.dtype.type is np.object_)
# Check column shapes
assert_equal(s.arrays_rep.g.shape, (4, 3, 2))
assert_equal(s.arrays_rep.h.shape, (4, 3, 2))
# Check values
for i in range(4):
for j in range(3):
for k in range(2):
assert_array_identical(s.arrays_rep.g[i, j, k],
np.repeat(np.float32(4.), 2).astype(np.object_))
assert_array_identical(s.arrays_rep.h[i, j, k],
np.repeat(np.float32(4.), 3).astype(np.object_))
g0 = vect_id(s.arrays_rep.g[i, j, k])
g1 = id(s.arrays_rep.g[0, 0, 0][0])
assert np.all(g0 == g1)
h0 = vect_id(s.arrays_rep.h[i, j, k])
h1 = id(s.arrays_rep.h[0, 0, 0][0])
assert np.all(h0 == h1)
class TestTags:
'''Test that sav files with description tag read at all'''
def test_description(self):
s = readsav(path.join(DATA_PATH, 'scalar_byte_descr.sav'), verbose=False)
assert_identical(s.i8u, np.uint8(234))
def test_null_pointer():
# Regression test for null pointers.
s = readsav(path.join(DATA_PATH, 'null_pointer.sav'), verbose=False)
assert_identical(s.point, None)
assert_identical(s.check, np.int16(5))
@pytest.mark.thread_unsafe
def test_invalid_pointer():
# Regression test for invalid pointers (gh-4613).
# In some files in the wild, pointers can sometimes refer to a heap
# variable that does not exist. In that case, we now gracefully fail for
# that variable and replace the variable with None and emit a warning.
# Since it's difficult to artificially produce such files, the file used
# here has been edited to force the pointer reference to be invalid.
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter("always")
s = readsav(path.join(DATA_PATH, 'invalid_pointer.sav'), verbose=False)
assert_(len(w) == 1)
assert_(str(w[0].message) == ("Variable referenced by pointer not found in "
"heap: variable will be set to None"))
assert_identical(s['a'], np.array([None, None]))
def test_attrdict():
d = _idl.AttrDict({'one': 1})
assert d['one'] == 1
assert d.one == 1
with pytest.raises(KeyError):
d['two']
with pytest.raises(AttributeError, match='has no attribute'):
d.two

View file

@ -0,0 +1,831 @@
from tempfile import mkdtemp
import os
import io
import shutil
import threading
import textwrap
import numpy as np
from numpy import array, transpose, pi
from numpy.testing import (assert_equal, assert_allclose,
assert_array_equal, assert_array_almost_equal)
import pytest
from pytest import raises as assert_raises
import scipy.sparse
import scipy.io._mmio
import scipy.io._fast_matrix_market as fmm
parametrize_args = [('integer', 'int'),
('unsigned-integer', 'uint')]
pytestmark = pytest.mark.thread_unsafe
# Run the entire test suite on both _mmio and _fast_matrix_market implementations
@pytest.fixture(scope='module', params=(scipy.io._mmio, fmm), autouse=True)
def implementations(request):
global mminfo
global mmread
global mmwrite
mminfo = request.param.mminfo
mmread = request.param.mmread
mmwrite = request.param.mmwrite
class TestMMIOArray:
def setup_method(self):
self.tmpdir = mkdtemp(suffix=str(threading.get_native_id()))
self.fn = os.path.join(self.tmpdir, 'testfile.mtx')
def teardown_method(self):
shutil.rmtree(self.tmpdir)
def check(self, a, info):
mmwrite(self.fn, a)
assert_equal(mminfo(self.fn), info)
b = mmread(self.fn, spmatrix=False)
assert_array_almost_equal(a, b)
def check_exact(self, a, info):
mmwrite(self.fn, a)
assert_equal(mminfo(self.fn), info)
b = mmread(self.fn, spmatrix=False)
assert_equal(a, b)
@pytest.mark.thread_unsafe
@pytest.mark.parametrize('typeval, dtype', parametrize_args)
def test_simple_integer(self, typeval, dtype):
self.check_exact(array([[1, 2], [3, 4]], dtype=dtype),
(2, 2, 4, 'array', typeval, 'general'))
@pytest.mark.thread_unsafe
@pytest.mark.parametrize('typeval, dtype', parametrize_args)
def test_32bit_integer(self, typeval, dtype):
a = array([[2**31-1, 2**31-2], [2**31-3, 2**31-4]], dtype=dtype)
self.check_exact(a, (2, 2, 4, 'array', typeval, 'general'))
def test_64bit_integer(self):
a = array([[2**31, 2**32], [2**63-2, 2**63-1]], dtype=np.int64)
if (np.intp(0).itemsize < 8) and mmwrite == scipy.io._mmio.mmwrite:
assert_raises(OverflowError, mmwrite, self.fn, a)
else:
self.check_exact(a, (2, 2, 4, 'array', 'integer', 'general'))
def test_64bit_unsigned_integer(self):
a = array([[2**31, 2**32], [2**64-2, 2**64-1]], dtype=np.uint64)
self.check_exact(a, (2, 2, 4, 'array', 'unsigned-integer', 'general'))
@pytest.mark.parametrize('typeval, dtype', parametrize_args)
def test_simple_upper_triangle_integer(self, typeval, dtype):
self.check_exact(array([[0, 1], [0, 0]], dtype=dtype),
(2, 2, 4, 'array', typeval, 'general'))
@pytest.mark.parametrize('typeval, dtype', parametrize_args)
def test_simple_lower_triangle_integer(self, typeval, dtype):
self.check_exact(array([[0, 0], [1, 0]], dtype=dtype),
(2, 2, 4, 'array', typeval, 'general'))
@pytest.mark.parametrize('typeval, dtype', parametrize_args)
def test_simple_rectangular_integer(self, typeval, dtype):
self.check_exact(array([[1, 2, 3], [4, 5, 6]], dtype=dtype),
(2, 3, 6, 'array', typeval, 'general'))
def test_simple_rectangular_float(self):
self.check([[1, 2], [3.5, 4], [5, 6]],
(3, 2, 6, 'array', 'real', 'general'))
def test_simple_float(self):
self.check([[1, 2], [3, 4.0]],
(2, 2, 4, 'array', 'real', 'general'))
def test_simple_complex(self):
self.check([[1, 2], [3, 4j]],
(2, 2, 4, 'array', 'complex', 'general'))
@pytest.mark.parametrize('typeval, dtype', parametrize_args)
def test_simple_symmetric_integer(self, typeval, dtype):
self.check_exact(array([[1, 2], [2, 4]], dtype=dtype),
(2, 2, 4, 'array', typeval, 'symmetric'))
def test_simple_skew_symmetric_integer(self):
self.check_exact([[0, 2], [-2, 0]],
(2, 2, 4, 'array', 'integer', 'skew-symmetric'))
def test_simple_skew_symmetric_float(self):
self.check(array([[0, 2], [-2.0, 0.0]], 'f'),
(2, 2, 4, 'array', 'real', 'skew-symmetric'))
def test_simple_hermitian_complex(self):
self.check([[1, 2+3j], [2-3j, 4]],
(2, 2, 4, 'array', 'complex', 'hermitian'))
def test_random_symmetric_float(self):
sz = (20, 20)
a = np.random.random(sz)
a = a + transpose(a)
self.check(a, (20, 20, 400, 'array', 'real', 'symmetric'))
def test_random_rectangular_float(self):
sz = (20, 15)
a = np.random.random(sz)
self.check(a, (20, 15, 300, 'array', 'real', 'general'))
@pytest.mark.fail_slow(10)
def test_bad_number_of_array_header_fields(self):
s = """\
%%MatrixMarket matrix array real general
3 3 999
1.0
2.0
3.0
4.0
5.0
6.0
7.0
8.0
9.0
"""
text = textwrap.dedent(s).encode('ascii')
with pytest.raises(ValueError, match='not of length 2'):
scipy.io.mmread(io.BytesIO(text))
def test_gh13634_non_skew_symmetric_int(self):
self.check_exact(array([[1, 2], [-2, 99]], dtype=np.int32),
(2, 2, 4, 'array', 'integer', 'general'))
def test_gh13634_non_skew_symmetric_float(self):
self.check(array([[1, 2], [-2, 99.]], dtype=np.float32),
(2, 2, 4, 'array', 'real', 'general'))
class TestMMIOSparseCSR(TestMMIOArray):
def setup_method(self):
self.tmpdir = mkdtemp(suffix=str(threading.get_native_id()))
self.fn = os.path.join(self.tmpdir, 'testfile.mtx')
def teardown_method(self):
shutil.rmtree(self.tmpdir)
def check(self, a, info):
mmwrite(self.fn, a)
assert_equal(mminfo(self.fn), info)
b = mmread(self.fn, spmatrix=False)
assert_array_almost_equal(a.toarray(), b.toarray())
def check_exact(self, a, info):
mmwrite(self.fn, a)
assert_equal(mminfo(self.fn), info)
b = mmread(self.fn, spmatrix=False)
assert_equal(a.toarray(), b.toarray())
@pytest.mark.parametrize('typeval, dtype', parametrize_args)
def test_simple_integer(self, typeval, dtype):
self.check_exact(scipy.sparse.csr_array([[1, 2], [3, 4]], dtype=dtype),
(2, 2, 4, 'coordinate', typeval, 'general'))
def test_32bit_integer(self):
a = scipy.sparse.csr_array(array([[2**31-1, -2**31+2],
[2**31-3, 2**31-4]],
dtype=np.int32))
self.check_exact(a, (2, 2, 4, 'coordinate', 'integer', 'general'))
def test_64bit_integer(self):
a = scipy.sparse.csr_array(array([[2**32+1, 2**32+1],
[-2**63+2, 2**63-2]],
dtype=np.int64))
if (np.intp(0).itemsize < 8) and mmwrite == scipy.io._mmio.mmwrite:
assert_raises(OverflowError, mmwrite, self.fn, a)
else:
self.check_exact(a, (2, 2, 4, 'coordinate', 'integer', 'general'))
def test_32bit_unsigned_integer(self):
a = scipy.sparse.csr_array(array([[2**31-1, 2**31-2],
[2**31-3, 2**31-4]],
dtype=np.uint32))
self.check_exact(a, (2, 2, 4, 'coordinate', 'unsigned-integer', 'general'))
def test_64bit_unsigned_integer(self):
a = scipy.sparse.csr_array(array([[2**32+1, 2**32+1],
[2**64-2, 2**64-1]],
dtype=np.uint64))
self.check_exact(a, (2, 2, 4, 'coordinate', 'unsigned-integer', 'general'))
@pytest.mark.parametrize('typeval, dtype', parametrize_args)
def test_simple_upper_triangle_integer(self, typeval, dtype):
self.check_exact(scipy.sparse.csr_array([[0, 1], [0, 0]], dtype=dtype),
(2, 2, 1, 'coordinate', typeval, 'general'))
@pytest.mark.parametrize('typeval, dtype', parametrize_args)
def test_simple_lower_triangle_integer(self, typeval, dtype):
self.check_exact(scipy.sparse.csr_array([[0, 0], [1, 0]], dtype=dtype),
(2, 2, 1, 'coordinate', typeval, 'general'))
@pytest.mark.parametrize('typeval, dtype', parametrize_args)
def test_simple_rectangular_integer(self, typeval, dtype):
self.check_exact(scipy.sparse.csr_array([[1, 2, 3], [4, 5, 6]], dtype=dtype),
(2, 3, 6, 'coordinate', typeval, 'general'))
def test_simple_rectangular_float(self):
self.check(scipy.sparse.csr_array([[1, 2], [3.5, 4], [5, 6]]),
(3, 2, 6, 'coordinate', 'real', 'general'))
def test_simple_float(self):
self.check(scipy.sparse.csr_array([[1, 2], [3, 4.0]]),
(2, 2, 4, 'coordinate', 'real', 'general'))
def test_simple_complex(self):
self.check(scipy.sparse.csr_array([[1, 2], [3, 4j]]),
(2, 2, 4, 'coordinate', 'complex', 'general'))
@pytest.mark.parametrize('typeval, dtype', parametrize_args)
def test_simple_symmetric_integer(self, typeval, dtype):
self.check_exact(scipy.sparse.csr_array([[1, 2], [2, 4]], dtype=dtype),
(2, 2, 3, 'coordinate', typeval, 'symmetric'))
def test_simple_skew_symmetric_integer(self):
self.check_exact(scipy.sparse.csr_array([[0, 2], [-2, 0]]),
(2, 2, 1, 'coordinate', 'integer', 'skew-symmetric'))
def test_simple_skew_symmetric_float(self):
self.check(scipy.sparse.csr_array(array([[0, 2], [-2.0, 0]], 'f')),
(2, 2, 1, 'coordinate', 'real', 'skew-symmetric'))
def test_simple_hermitian_complex(self):
self.check(scipy.sparse.csr_array([[1, 2+3j], [2-3j, 4]]),
(2, 2, 3, 'coordinate', 'complex', 'hermitian'))
def test_random_symmetric_float(self):
sz = (20, 20)
a = np.random.random(sz)
a = a + transpose(a)
a = scipy.sparse.csr_array(a)
self.check(a, (20, 20, 210, 'coordinate', 'real', 'symmetric'))
def test_random_rectangular_float(self):
sz = (20, 15)
a = np.random.random(sz)
a = scipy.sparse.csr_array(a)
self.check(a, (20, 15, 300, 'coordinate', 'real', 'general'))
def test_simple_pattern(self):
a = scipy.sparse.csr_array([[0, 1.5], [3.0, 2.5]])
p = np.zeros_like(a.toarray())
p[a.toarray() > 0] = 1
info = (2, 2, 3, 'coordinate', 'pattern', 'general')
mmwrite(self.fn, a, field='pattern')
assert_equal(mminfo(self.fn), info)
b = mmread(self.fn, spmatrix=False)
assert_array_almost_equal(p, b.toarray())
assert not scipy.sparse.isspmatrix(b)
b = mmread(self.fn, spmatrix=True)
assert scipy.sparse.isspmatrix(b)
b = mmread(self.fn) # chk default
assert scipy.sparse.isspmatrix(b)
def test_gh13634_non_skew_symmetric_int(self):
a = scipy.sparse.csr_array([[1, 2], [-2, 99]], dtype=np.int32)
self.check_exact(a, (2, 2, 4, 'coordinate', 'integer', 'general'))
def test_gh13634_non_skew_symmetric_float(self):
a = scipy.sparse.csr_array([[1, 2], [-2, 99.]], dtype=np.float32)
self.check(a, (2, 2, 4, 'coordinate', 'real', 'general'))
_32bit_integer_dense_example = '''\
%%MatrixMarket matrix array integer general
2 2
2147483647
2147483646
2147483647
2147483646
'''
_32bit_integer_sparse_example = '''\
%%MatrixMarket matrix coordinate integer symmetric
2 2 2
1 1 2147483647
2 2 2147483646
'''
_64bit_integer_dense_example = '''\
%%MatrixMarket matrix array integer general
2 2
2147483648
-9223372036854775806
-2147483648
9223372036854775807
'''
_64bit_integer_sparse_general_example = '''\
%%MatrixMarket matrix coordinate integer general
2 2 3
1 1 2147483648
1 2 9223372036854775807
2 2 9223372036854775807
'''
_64bit_integer_sparse_symmetric_example = '''\
%%MatrixMarket matrix coordinate integer symmetric
2 2 3
1 1 2147483648
1 2 -9223372036854775807
2 2 9223372036854775807
'''
_64bit_integer_sparse_skew_example = '''\
%%MatrixMarket matrix coordinate integer skew-symmetric
2 2 3
1 1 2147483648
1 2 -9223372036854775807
2 2 9223372036854775807
'''
_over64bit_integer_dense_example = '''\
%%MatrixMarket matrix array integer general
2 2
2147483648
9223372036854775807
2147483648
9223372036854775808
'''
_over64bit_integer_sparse_example = '''\
%%MatrixMarket matrix coordinate integer symmetric
2 2 2
1 1 2147483648
2 2 19223372036854775808
'''
class TestMMIOReadLargeIntegers:
def setup_method(self):
self.tmpdir = mkdtemp(suffix=str(threading.get_native_id()))
self.fn = os.path.join(self.tmpdir, 'testfile.mtx')
def teardown_method(self):
shutil.rmtree(self.tmpdir)
def check_read(self, example, a, info, dense, over32, over64):
with open(self.fn, 'w') as f:
f.write(example)
assert_equal(mminfo(self.fn), info)
if ((over32 and (np.intp(0).itemsize < 8) and mmwrite == scipy.io._mmio.mmwrite)
or over64):
assert_raises(OverflowError, mmread, self.fn)
else:
b = mmread(self.fn, spmatrix=False)
if not dense:
b = b.toarray()
assert_equal(a, b)
def test_read_32bit_integer_dense(self):
a = array([[2**31-1, 2**31-1],
[2**31-2, 2**31-2]], dtype=np.int64)
self.check_read(_32bit_integer_dense_example,
a,
(2, 2, 4, 'array', 'integer', 'general'),
dense=True,
over32=False,
over64=False)
def test_read_32bit_integer_sparse(self):
a = array([[2**31-1, 0],
[0, 2**31-2]], dtype=np.int64)
self.check_read(_32bit_integer_sparse_example,
a,
(2, 2, 2, 'coordinate', 'integer', 'symmetric'),
dense=False,
over32=False,
over64=False)
def test_read_64bit_integer_dense(self):
a = array([[2**31, -2**31],
[-2**63+2, 2**63-1]], dtype=np.int64)
self.check_read(_64bit_integer_dense_example,
a,
(2, 2, 4, 'array', 'integer', 'general'),
dense=True,
over32=True,
over64=False)
def test_read_64bit_integer_sparse_general(self):
a = array([[2**31, 2**63-1],
[0, 2**63-1]], dtype=np.int64)
self.check_read(_64bit_integer_sparse_general_example,
a,
(2, 2, 3, 'coordinate', 'integer', 'general'),
dense=False,
over32=True,
over64=False)
def test_read_64bit_integer_sparse_symmetric(self):
a = array([[2**31, -2**63+1],
[-2**63+1, 2**63-1]], dtype=np.int64)
self.check_read(_64bit_integer_sparse_symmetric_example,
a,
(2, 2, 3, 'coordinate', 'integer', 'symmetric'),
dense=False,
over32=True,
over64=False)
def test_read_64bit_integer_sparse_skew(self):
a = array([[2**31, -2**63+1],
[2**63-1, 2**63-1]], dtype=np.int64)
self.check_read(_64bit_integer_sparse_skew_example,
a,
(2, 2, 3, 'coordinate', 'integer', 'skew-symmetric'),
dense=False,
over32=True,
over64=False)
def test_read_over64bit_integer_dense(self):
self.check_read(_over64bit_integer_dense_example,
None,
(2, 2, 4, 'array', 'integer', 'general'),
dense=True,
over32=True,
over64=True)
def test_read_over64bit_integer_sparse(self):
self.check_read(_over64bit_integer_sparse_example,
None,
(2, 2, 2, 'coordinate', 'integer', 'symmetric'),
dense=False,
over32=True,
over64=True)
_general_example = '''\
%%MatrixMarket matrix coordinate real general
%=================================================================================
%
% This ASCII file represents a sparse MxN matrix with L
% nonzeros in the following Matrix Market format:
%
% +----------------------------------------------+
% |%%MatrixMarket matrix coordinate real general | <--- header line
% |% | <--+
% |% comments | |-- 0 or more comment lines
% |% | <--+
% | M N L | <--- rows, columns, entries
% | I1 J1 A(I1, J1) | <--+
% | I2 J2 A(I2, J2) | |
% | I3 J3 A(I3, J3) | |-- L lines
% | . . . | |
% | IL JL A(IL, JL) | <--+
% +----------------------------------------------+
%
% Indices are 1-based, i.e. A(1,1) is the first element.
%
%=================================================================================
5 5 8
1 1 1.000e+00
2 2 1.050e+01
3 3 1.500e-02
1 4 6.000e+00
4 2 2.505e+02
4 4 -2.800e+02
4 5 3.332e+01
5 5 1.200e+01
'''
_hermitian_example = '''\
%%MatrixMarket matrix coordinate complex hermitian
5 5 7
1 1 1.0 0
2 2 10.5 0
4 2 250.5 22.22
3 3 1.5e-2 0
4 4 -2.8e2 0
5 5 12. 0
5 4 0 33.32
'''
_skew_example = '''\
%%MatrixMarket matrix coordinate real skew-symmetric
5 5 7
1 1 1.0
2 2 10.5
4 2 250.5
3 3 1.5e-2
4 4 -2.8e2
5 5 12.
5 4 0
'''
_symmetric_example = '''\
%%MatrixMarket matrix coordinate real symmetric
5 5 7
1 1 1.0
2 2 10.5
4 2 250.5
3 3 1.5e-2
4 4 -2.8e2
5 5 12.
5 4 8
'''
_symmetric_pattern_example = '''\
%%MatrixMarket matrix coordinate pattern symmetric
5 5 7
1 1
2 2
4 2
3 3
4 4
5 5
5 4
'''
# example (without comment lines) from Figure 1 in
# https://math.nist.gov/MatrixMarket/reports/MMformat.ps
_empty_lines_example = '''\
%%MatrixMarket MATRIX Coordinate Real General
5 5 8
1 1 1.0
2 2 10.5
3 3 1.5e-2
4 4 -2.8E2
5 5 12.
1 4 6
4 2 250.5
4 5 33.32
'''
class TestMMIOCoordinate:
def setup_method(self):
self.tmpdir = mkdtemp(suffix=str(threading.get_native_id()))
self.fn = os.path.join(self.tmpdir, 'testfile.mtx')
def teardown_method(self):
shutil.rmtree(self.tmpdir)
def check_read(self, example, a, info):
f = open(self.fn, 'w')
f.write(example)
f.close()
assert_equal(mminfo(self.fn), info)
b = mmread(self.fn, spmatrix=False).toarray()
assert_array_almost_equal(a, b)
def test_read_general(self):
a = [[1, 0, 0, 6, 0],
[0, 10.5, 0, 0, 0],
[0, 0, .015, 0, 0],
[0, 250.5, 0, -280, 33.32],
[0, 0, 0, 0, 12]]
self.check_read(_general_example, a,
(5, 5, 8, 'coordinate', 'real', 'general'))
def test_read_hermitian(self):
a = [[1, 0, 0, 0, 0],
[0, 10.5, 0, 250.5 - 22.22j, 0],
[0, 0, .015, 0, 0],
[0, 250.5 + 22.22j, 0, -280, -33.32j],
[0, 0, 0, 33.32j, 12]]
self.check_read(_hermitian_example, a,
(5, 5, 7, 'coordinate', 'complex', 'hermitian'))
def test_read_skew(self):
a = [[1, 0, 0, 0, 0],
[0, 10.5, 0, -250.5, 0],
[0, 0, .015, 0, 0],
[0, 250.5, 0, -280, 0],
[0, 0, 0, 0, 12]]
self.check_read(_skew_example, a,
(5, 5, 7, 'coordinate', 'real', 'skew-symmetric'))
def test_read_symmetric(self):
a = [[1, 0, 0, 0, 0],
[0, 10.5, 0, 250.5, 0],
[0, 0, .015, 0, 0],
[0, 250.5, 0, -280, 8],
[0, 0, 0, 8, 12]]
self.check_read(_symmetric_example, a,
(5, 5, 7, 'coordinate', 'real', 'symmetric'))
def test_read_symmetric_pattern(self):
a = [[1, 0, 0, 0, 0],
[0, 1, 0, 1, 0],
[0, 0, 1, 0, 0],
[0, 1, 0, 1, 1],
[0, 0, 0, 1, 1]]
self.check_read(_symmetric_pattern_example, a,
(5, 5, 7, 'coordinate', 'pattern', 'symmetric'))
def test_read_empty_lines(self):
a = [[1, 0, 0, 6, 0],
[0, 10.5, 0, 0, 0],
[0, 0, .015, 0, 0],
[0, 250.5, 0, -280, 33.32],
[0, 0, 0, 0, 12]]
self.check_read(_empty_lines_example, a,
(5, 5, 8, 'coordinate', 'real', 'general'))
def test_empty_write_read(self):
# https://github.com/scipy/scipy/issues/1410 (Trac #883)
b = scipy.sparse.coo_array((10, 10))
mmwrite(self.fn, b)
assert_equal(mminfo(self.fn),
(10, 10, 0, 'coordinate', 'real', 'symmetric'))
a = b.toarray()
b = mmread(self.fn, spmatrix=False).toarray()
assert_array_almost_equal(a, b)
def test_bzip2_py3(self):
# test if fix for #2152 works
try:
# bz2 module isn't always built when building Python.
import bz2
except ImportError:
return
I = array([0, 0, 1, 2, 3, 3, 3, 4])
J = array([0, 3, 1, 2, 1, 3, 4, 4])
V = array([1.0, 6.0, 10.5, 0.015, 250.5, -280.0, 33.32, 12.0])
b = scipy.sparse.coo_array((V, (I, J)), shape=(5, 5))
mmwrite(self.fn, b)
fn_bzip2 = f"{self.fn}.bz2"
with open(self.fn, 'rb') as f_in:
f_out = bz2.BZ2File(fn_bzip2, 'wb')
f_out.write(f_in.read())
f_out.close()
a = mmread(fn_bzip2, spmatrix=False).toarray()
assert_array_almost_equal(a, b.toarray())
def test_gzip_py3(self):
# test if fix for #2152 works
try:
# gzip module can be missing from Python installation
import gzip
except ImportError:
return
I = array([0, 0, 1, 2, 3, 3, 3, 4])
J = array([0, 3, 1, 2, 1, 3, 4, 4])
V = array([1.0, 6.0, 10.5, 0.015, 250.5, -280.0, 33.32, 12.0])
b = scipy.sparse.coo_array((V, (I, J)), shape=(5, 5))
mmwrite(self.fn, b)
fn_gzip = f"{self.fn}.gz"
with open(self.fn, 'rb') as f_in:
f_out = gzip.open(fn_gzip, 'wb')
f_out.write(f_in.read())
f_out.close()
a = mmread(fn_gzip, spmatrix=False).toarray()
assert_array_almost_equal(a, b.toarray())
def test_real_write_read(self):
I = array([0, 0, 1, 2, 3, 3, 3, 4])
J = array([0, 3, 1, 2, 1, 3, 4, 4])
V = array([1.0, 6.0, 10.5, 0.015, 250.5, -280.0, 33.32, 12.0])
b = scipy.sparse.coo_array((V, (I, J)), shape=(5, 5))
mmwrite(self.fn, b)
assert_equal(mminfo(self.fn),
(5, 5, 8, 'coordinate', 'real', 'general'))
a = b.toarray()
b = mmread(self.fn, spmatrix=False).toarray()
assert_array_almost_equal(a, b)
def test_complex_write_read(self):
I = array([0, 0, 1, 2, 3, 3, 3, 4])
J = array([0, 3, 1, 2, 1, 3, 4, 4])
V = array([1.0 + 3j, 6.0 + 2j, 10.50 + 0.9j, 0.015 + -4.4j,
250.5 + 0j, -280.0 + 5j, 33.32 + 6.4j, 12.00 + 0.8j])
b = scipy.sparse.coo_array((V, (I, J)), shape=(5, 5))
mmwrite(self.fn, b)
assert_equal(mminfo(self.fn),
(5, 5, 8, 'coordinate', 'complex', 'general'))
a = b.toarray()
b = mmread(self.fn, spmatrix=False).toarray()
assert_array_almost_equal(a, b)
def test_sparse_formats(self, tmp_path):
# Note: `tmp_path` is a pytest fixture, it handles cleanup
tmpdir = tmp_path / 'sparse_formats'
tmpdir.mkdir()
mats = []
I = array([0, 0, 1, 2, 3, 3, 3, 4])
J = array([0, 3, 1, 2, 1, 3, 4, 4])
V = array([1.0, 6.0, 10.5, 0.015, 250.5, -280.0, 33.32, 12.0])
mats.append(scipy.sparse.coo_array((V, (I, J)), shape=(5, 5)))
V = array([1.0 + 3j, 6.0 + 2j, 10.50 + 0.9j, 0.015 + -4.4j,
250.5 + 0j, -280.0 + 5j, 33.32 + 6.4j, 12.00 + 0.8j])
mats.append(scipy.sparse.coo_array((V, (I, J)), shape=(5, 5)))
for mat in mats:
expected = mat.toarray()
for fmt in ['csr', 'csc', 'coo']:
fname = tmpdir / (fmt + '.mtx')
mmwrite(fname, mat.asformat(fmt))
result = mmread(fname, spmatrix=False).toarray()
assert_array_almost_equal(result, expected)
def test_precision(self):
test_values = [pi] + [10**(i) for i in range(0, -10, -1)]
test_precisions = range(1, 10)
for value in test_values:
for precision in test_precisions:
# construct sparse matrix with test value at last main diagonal
n = 10**precision + 1
A = scipy.sparse.dok_array((n, n))
A[n-1, n-1] = value
# write matrix with test precision and read again
mmwrite(self.fn, A, precision=precision)
A = scipy.io.mmread(self.fn, spmatrix=False)
# check for right entries in matrix
assert_array_equal(A.row, [n-1])
assert_array_equal(A.col, [n-1])
assert_allclose(A.data, [float(f'{value:.{precision}g}')])
def test_bad_number_of_coordinate_header_fields(self):
s = """\
%%MatrixMarket matrix coordinate real general
5 5 8 999
1 1 1.000e+00
2 2 1.050e+01
3 3 1.500e-02
1 4 6.000e+00
4 2 2.505e+02
4 4 -2.800e+02
4 5 3.332e+01
5 5 1.200e+01
"""
text = textwrap.dedent(s).encode('ascii')
with pytest.raises(ValueError, match='not of length 3'):
scipy.io.mmread(io.BytesIO(text))
def test_gh11389():
mmread(io.StringIO("%%MatrixMarket matrix coordinate complex symmetric\n"
" 1 1 1\n"
"1 1 -2.1846000000000e+02 0.0000000000000e+00"),
spmatrix=False)
def test_gh18123(tmp_path):
lines = [" %%MatrixMarket matrix coordinate real general\n",
"5 5 3\n",
"2 3 1.0\n",
"3 4 2.0\n",
"3 5 3.0\n"]
test_file = tmp_path / "test.mtx"
with open(test_file, "w") as f:
f.writelines(lines)
mmread(test_file, spmatrix=False)
def test_mtx_append(tmp_path):
a = mmread(io.StringIO("%%MatrixMarket matrix coordinate complex symmetric\n"
" 1 1 1\n"
"1 1 -2.1846000000000e+02 0.0000000000000e+00"),
spmatrix=False)
test_writefile = tmp_path / "test_mtx"
test_readfile = tmp_path / "test_mtx.mtx"
mmwrite(test_writefile, a)
mmread(test_readfile, spmatrix=False)
def test_threadpoolctl():
try:
import threadpoolctl
if not hasattr(threadpoolctl, "register"):
pytest.skip("threadpoolctl too old")
return
except ImportError:
pytest.skip("no threadpoolctl")
return
with threadpoolctl.threadpool_limits(limits=4):
assert_equal(fmm.PARALLELISM, 4)
with threadpoolctl.threadpool_limits(limits=2, user_api='scipy'):
assert_equal(fmm.PARALLELISM, 2)
def test_gh21999_file_not_exist():
tmpdir = mkdtemp(suffix=str(threading.get_native_id()))
wrong_fn = os.path.join(tmpdir, 'not_exist_test_file.mtx')
assert_raises(FileNotFoundError, mmread, wrong_fn)

View file

@ -0,0 +1,550 @@
''' Tests for netcdf '''
import os
from os.path import join as pjoin, dirname
import shutil
import tempfile
import warnings
from io import BytesIO
from glob import glob
from contextlib import contextmanager
import numpy as np
from numpy.testing import (assert_, assert_allclose, assert_equal,
break_cycles, suppress_warnings, IS_PYPY)
import pytest
from pytest import raises as assert_raises
from scipy.io import netcdf_file
from scipy._lib._tmpdirs import in_tempdir
TEST_DATA_PATH = pjoin(dirname(__file__), 'data')
N_EG_ELS = 11 # number of elements for example variable
VARTYPE_EG = 'b' # var type for example variable
pytestmark = pytest.mark.thread_unsafe
@contextmanager
def make_simple(*args, **kwargs):
f = netcdf_file(*args, **kwargs)
f.history = 'Created for a test'
f.createDimension('time', N_EG_ELS)
time = f.createVariable('time', VARTYPE_EG, ('time',))
time[:] = np.arange(N_EG_ELS)
time.units = 'days since 2008-01-01'
f.flush()
yield f
f.close()
def check_simple(ncfileobj):
'''Example fileobj tests '''
assert_equal(ncfileobj.history, b'Created for a test')
time = ncfileobj.variables['time']
assert_equal(time.units, b'days since 2008-01-01')
assert_equal(time.shape, (N_EG_ELS,))
assert_equal(time[-1], N_EG_ELS-1)
def assert_mask_matches(arr, expected_mask):
'''
Asserts that the mask of arr is effectively the same as expected_mask.
In contrast to numpy.ma.testutils.assert_mask_equal, this function allows
testing the 'mask' of a standard numpy array (the mask in this case is treated
as all False).
Parameters
----------
arr : ndarray or MaskedArray
Array to test.
expected_mask : array_like of booleans
A list giving the expected mask.
'''
mask = np.ma.getmaskarray(arr)
assert_equal(mask, expected_mask)
def test_read_write_files():
# test round trip for example file
cwd = os.getcwd()
try:
tmpdir = tempfile.mkdtemp()
os.chdir(tmpdir)
with make_simple('simple.nc', 'w') as f:
pass
# read the file we just created in 'a' mode
with netcdf_file('simple.nc', 'a') as f:
check_simple(f)
# add something
f._attributes['appendRan'] = 1
# To read the NetCDF file we just created::
with netcdf_file('simple.nc') as f:
# Using mmap is the default (but not on pypy)
assert_equal(f.use_mmap, not IS_PYPY)
check_simple(f)
assert_equal(f._attributes['appendRan'], 1)
# Read it in append (and check mmap is off)
with netcdf_file('simple.nc', 'a') as f:
assert_(not f.use_mmap)
check_simple(f)
assert_equal(f._attributes['appendRan'], 1)
# Now without mmap
with netcdf_file('simple.nc', mmap=False) as f:
# Using mmap is the default
assert_(not f.use_mmap)
check_simple(f)
# To read the NetCDF file we just created, as file object, no
# mmap. When n * n_bytes(var_type) is not divisible by 4, this
# raised an error in pupynere 1.0.12 and scipy rev 5893, because
# calculated vsize was rounding up in units of 4 - see
# https://www.unidata.ucar.edu/software/netcdf/guide_toc.html
with open('simple.nc', 'rb') as fobj:
with netcdf_file(fobj) as f:
# by default, don't use mmap for file-like
assert_(not f.use_mmap)
check_simple(f)
# Read file from fileobj, with mmap
with suppress_warnings() as sup:
if IS_PYPY:
sup.filter(RuntimeWarning,
"Cannot close a netcdf_file opened with mmap=True.*")
with open('simple.nc', 'rb') as fobj:
with netcdf_file(fobj, mmap=True) as f:
assert_(f.use_mmap)
check_simple(f)
# Again read it in append mode (adding another att)
with open('simple.nc', 'r+b') as fobj:
with netcdf_file(fobj, 'a') as f:
assert_(not f.use_mmap)
check_simple(f)
f.createDimension('app_dim', 1)
var = f.createVariable('app_var', 'i', ('app_dim',))
var[:] = 42
# And... check that app_var made it in...
with netcdf_file('simple.nc') as f:
check_simple(f)
assert_equal(f.variables['app_var'][:], 42)
finally:
if IS_PYPY:
# windows cannot remove a dead file held by a mmap
# that has not been collected in PyPy
break_cycles()
break_cycles()
os.chdir(cwd)
shutil.rmtree(tmpdir)
def test_read_write_sio():
eg_sio1 = BytesIO()
with make_simple(eg_sio1, 'w'):
str_val = eg_sio1.getvalue()
eg_sio2 = BytesIO(str_val)
with netcdf_file(eg_sio2) as f2:
check_simple(f2)
# Test that error is raised if attempting mmap for sio
eg_sio3 = BytesIO(str_val)
assert_raises(ValueError, netcdf_file, eg_sio3, 'r', True)
# Test 64-bit offset write / read
eg_sio_64 = BytesIO()
with make_simple(eg_sio_64, 'w', version=2) as f_64:
str_val = eg_sio_64.getvalue()
eg_sio_64 = BytesIO(str_val)
with netcdf_file(eg_sio_64) as f_64:
check_simple(f_64)
assert_equal(f_64.version_byte, 2)
# also when version 2 explicitly specified
eg_sio_64 = BytesIO(str_val)
with netcdf_file(eg_sio_64, version=2) as f_64:
check_simple(f_64)
assert_equal(f_64.version_byte, 2)
def test_bytes():
raw_file = BytesIO()
f = netcdf_file(raw_file, mode='w')
# Dataset only has a single variable, dimension and attribute to avoid
# any ambiguity related to order.
f.a = 'b'
f.createDimension('dim', 1)
var = f.createVariable('var', np.int16, ('dim',))
var[0] = -9999
var.c = 'd'
f.sync()
actual = raw_file.getvalue()
expected = (b'CDF\x01'
b'\x00\x00\x00\x00'
b'\x00\x00\x00\x0a'
b'\x00\x00\x00\x01'
b'\x00\x00\x00\x03'
b'dim\x00'
b'\x00\x00\x00\x01'
b'\x00\x00\x00\x0c'
b'\x00\x00\x00\x01'
b'\x00\x00\x00\x01'
b'a\x00\x00\x00'
b'\x00\x00\x00\x02'
b'\x00\x00\x00\x01'
b'b\x00\x00\x00'
b'\x00\x00\x00\x0b'
b'\x00\x00\x00\x01'
b'\x00\x00\x00\x03'
b'var\x00'
b'\x00\x00\x00\x01'
b'\x00\x00\x00\x00'
b'\x00\x00\x00\x0c'
b'\x00\x00\x00\x01'
b'\x00\x00\x00\x01'
b'c\x00\x00\x00'
b'\x00\x00\x00\x02'
b'\x00\x00\x00\x01'
b'd\x00\x00\x00'
b'\x00\x00\x00\x03'
b'\x00\x00\x00\x04'
b'\x00\x00\x00\x78'
b'\xd8\xf1\x80\x01')
assert_equal(actual, expected)
def test_encoded_fill_value():
with netcdf_file(BytesIO(), mode='w') as f:
f.createDimension('x', 1)
var = f.createVariable('var', 'S1', ('x',))
assert_equal(var._get_encoded_fill_value(), b'\x00')
var._FillValue = b'\x01'
assert_equal(var._get_encoded_fill_value(), b'\x01')
var._FillValue = b'\x00\x00' # invalid, wrong size
assert_equal(var._get_encoded_fill_value(), b'\x00')
def test_read_example_data():
# read any example data files
for fname in glob(pjoin(TEST_DATA_PATH, '*.nc')):
with netcdf_file(fname, 'r'):
pass
with netcdf_file(fname, 'r', mmap=False):
pass
def test_itemset_no_segfault_on_readonly():
# Regression test for ticket #1202.
# Open the test file in read-only mode.
filename = pjoin(TEST_DATA_PATH, 'example_1.nc')
with suppress_warnings() as sup:
message = ("Cannot close a netcdf_file opened with mmap=True, when "
"netcdf_variables or arrays referring to its data still exist")
sup.filter(RuntimeWarning, message)
with netcdf_file(filename, 'r', mmap=True) as f:
time_var = f.variables['time']
# time_var.assignValue(42) should raise a RuntimeError--not seg. fault!
assert_raises(RuntimeError, time_var.assignValue, 42)
def test_appending_issue_gh_8625():
stream = BytesIO()
with make_simple(stream, mode='w') as f:
f.createDimension('x', 2)
f.createVariable('x', float, ('x',))
f.variables['x'][...] = 1
f.flush()
contents = stream.getvalue()
stream = BytesIO(contents)
with netcdf_file(stream, mode='a') as f:
f.variables['x'][...] = 2
def test_write_invalid_dtype():
dtypes = ['int64', 'uint64']
if np.dtype('int').itemsize == 8: # 64-bit machines
dtypes.append('int')
if np.dtype('uint').itemsize == 8: # 64-bit machines
dtypes.append('uint')
with netcdf_file(BytesIO(), 'w') as f:
f.createDimension('time', N_EG_ELS)
for dt in dtypes:
assert_raises(ValueError, f.createVariable, 'time', dt, ('time',))
def test_flush_rewind():
stream = BytesIO()
with make_simple(stream, mode='w') as f:
f.createDimension('x',4) # x is used in createVariable
v = f.createVariable('v', 'i2', ['x'])
v[:] = 1
f.flush()
len_single = len(stream.getvalue())
f.flush()
len_double = len(stream.getvalue())
assert_(len_single == len_double)
def test_dtype_specifiers():
# Numpy 1.7.0-dev had a bug where 'i2' wouldn't work.
# Specifying np.int16 or similar only works from the same commit as this
# comment was made.
with make_simple(BytesIO(), mode='w') as f:
f.createDimension('x',4)
f.createVariable('v1', 'i2', ['x'])
f.createVariable('v2', np.int16, ['x'])
f.createVariable('v3', np.dtype(np.int16), ['x'])
def test_ticket_1720():
io = BytesIO()
items = [0,0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9]
with netcdf_file(io, 'w') as f:
f.history = 'Created for a test'
f.createDimension('float_var', 10)
float_var = f.createVariable('float_var', 'f', ('float_var',))
float_var[:] = items
float_var.units = 'metres'
f.flush()
contents = io.getvalue()
io = BytesIO(contents)
with netcdf_file(io, 'r') as f:
assert_equal(f.history, b'Created for a test')
float_var = f.variables['float_var']
assert_equal(float_var.units, b'metres')
assert_equal(float_var.shape, (10,))
assert_allclose(float_var[:], items)
def test_mmaps_segfault():
filename = pjoin(TEST_DATA_PATH, 'example_1.nc')
if not IS_PYPY:
with warnings.catch_warnings():
warnings.simplefilter("error")
with netcdf_file(filename, mmap=True) as f:
x = f.variables['lat'][:]
# should not raise warnings
del x
def doit():
with netcdf_file(filename, mmap=True) as f:
return f.variables['lat'][:]
# should not crash
with suppress_warnings() as sup:
message = ("Cannot close a netcdf_file opened with mmap=True, when "
"netcdf_variables or arrays referring to its data still exist")
sup.filter(RuntimeWarning, message)
x = doit()
x.sum()
def test_zero_dimensional_var():
io = BytesIO()
with make_simple(io, 'w') as f:
v = f.createVariable('zerodim', 'i2', [])
# This is checking that .isrec returns a boolean - don't simplify it
# to 'assert not ...'
assert v.isrec is False, v.isrec
f.flush()
def test_byte_gatts():
# Check that global "string" atts work like they did before py3k
# unicode and general bytes confusion
with in_tempdir():
filename = 'g_byte_atts.nc'
f = netcdf_file(filename, 'w')
f._attributes['holy'] = b'grail'
f._attributes['witch'] = 'floats'
f.close()
f = netcdf_file(filename, 'r')
assert_equal(f._attributes['holy'], b'grail')
assert_equal(f._attributes['witch'], b'floats')
f.close()
def test_open_append():
# open 'w' put one attr
with in_tempdir():
filename = 'append_dat.nc'
f = netcdf_file(filename, 'w')
f._attributes['Kilroy'] = 'was here'
f.close()
# open again in 'a', read the att and a new one
f = netcdf_file(filename, 'a')
assert_equal(f._attributes['Kilroy'], b'was here')
f._attributes['naughty'] = b'Zoot'
f.close()
# open yet again in 'r' and check both atts
f = netcdf_file(filename, 'r')
assert_equal(f._attributes['Kilroy'], b'was here')
assert_equal(f._attributes['naughty'], b'Zoot')
f.close()
def test_append_recordDimension():
dataSize = 100
with in_tempdir():
# Create file with record time dimension
with netcdf_file('withRecordDimension.nc', 'w') as f:
f.createDimension('time', None)
f.createVariable('time', 'd', ('time',))
f.createDimension('x', dataSize)
x = f.createVariable('x', 'd', ('x',))
x[:] = np.array(range(dataSize))
f.createDimension('y', dataSize)
y = f.createVariable('y', 'd', ('y',))
y[:] = np.array(range(dataSize))
f.createVariable('testData', 'i', ('time', 'x', 'y'))
f.flush()
f.close()
for i in range(2):
# Open the file in append mode and add data
with netcdf_file('withRecordDimension.nc', 'a') as f:
f.variables['time'].data = np.append(f.variables["time"].data, i)
f.variables['testData'][i, :, :] = np.full((dataSize, dataSize), i)
f.flush()
# Read the file and check that append worked
with netcdf_file('withRecordDimension.nc') as f:
assert_equal(f.variables['time'][-1], i)
assert_equal(f.variables['testData'][-1, :, :].copy(),
np.full((dataSize, dataSize), i))
assert_equal(f.variables['time'].data.shape[0], i+1)
assert_equal(f.variables['testData'].data.shape[0], i+1)
# Read the file and check that 'data' was not saved as user defined
# attribute of testData variable during append operation
with netcdf_file('withRecordDimension.nc') as f:
with assert_raises(KeyError) as ar:
f.variables['testData']._attributes['data']
ex = ar.value
assert_equal(ex.args[0], 'data')
def test_maskandscale():
t = np.linspace(20, 30, 15)
t[3] = 100
tm = np.ma.masked_greater(t, 99)
fname = pjoin(TEST_DATA_PATH, 'example_2.nc')
with netcdf_file(fname, maskandscale=True) as f:
Temp = f.variables['Temperature']
assert_equal(Temp.missing_value, 9999)
assert_equal(Temp.add_offset, 20)
assert_equal(Temp.scale_factor, np.float32(0.01))
found = Temp[:].compressed()
del Temp # Remove ref to mmap, so file can be closed.
expected = np.round(tm.compressed(), 2)
assert_allclose(found, expected)
with in_tempdir():
newfname = 'ms.nc'
f = netcdf_file(newfname, 'w', maskandscale=True)
f.createDimension('Temperature', len(tm))
temp = f.createVariable('Temperature', 'i', ('Temperature',))
temp.missing_value = 9999
temp.scale_factor = 0.01
temp.add_offset = 20
temp[:] = tm
f.close()
with netcdf_file(newfname, maskandscale=True) as f:
Temp = f.variables['Temperature']
assert_equal(Temp.missing_value, 9999)
assert_equal(Temp.add_offset, 20)
assert_equal(Temp.scale_factor, np.float32(0.01))
expected = np.round(tm.compressed(), 2)
found = Temp[:].compressed()
del Temp
assert_allclose(found, expected)
# ------------------------------------------------------------------------
# Test reading with masked values (_FillValue / missing_value)
# ------------------------------------------------------------------------
def test_read_withValuesNearFillValue():
# Regression test for ticket #5626
fname = pjoin(TEST_DATA_PATH, 'example_3_maskedvals.nc')
with netcdf_file(fname, maskandscale=True) as f:
vardata = f.variables['var1_fillval0'][:]
assert_mask_matches(vardata, [False, True, False])
def test_read_withNoFillValue():
# For a variable with no fill value, reading data with maskandscale=True
# should return unmasked data
fname = pjoin(TEST_DATA_PATH, 'example_3_maskedvals.nc')
with netcdf_file(fname, maskandscale=True) as f:
vardata = f.variables['var2_noFillval'][:]
assert_mask_matches(vardata, [False, False, False])
assert_equal(vardata, [1,2,3])
def test_read_withFillValueAndMissingValue():
# For a variable with both _FillValue and missing_value, the _FillValue
# should be used
IRRELEVANT_VALUE = 9999
fname = pjoin(TEST_DATA_PATH, 'example_3_maskedvals.nc')
with netcdf_file(fname, maskandscale=True) as f:
vardata = f.variables['var3_fillvalAndMissingValue'][:]
assert_mask_matches(vardata, [True, False, False])
assert_equal(vardata, [IRRELEVANT_VALUE, 2, 3])
def test_read_withMissingValue():
# For a variable with missing_value but not _FillValue, the missing_value
# should be used
fname = pjoin(TEST_DATA_PATH, 'example_3_maskedvals.nc')
with netcdf_file(fname, maskandscale=True) as f:
vardata = f.variables['var4_missingValue'][:]
assert_mask_matches(vardata, [False, True, False])
def test_read_withFillValNaN():
fname = pjoin(TEST_DATA_PATH, 'example_3_maskedvals.nc')
with netcdf_file(fname, maskandscale=True) as f:
vardata = f.variables['var5_fillvalNaN'][:]
assert_mask_matches(vardata, [False, True, False])
def test_read_withChar():
fname = pjoin(TEST_DATA_PATH, 'example_3_maskedvals.nc')
with netcdf_file(fname, maskandscale=True) as f:
vardata = f.variables['var6_char'][:]
assert_mask_matches(vardata, [False, True, False])
def test_read_with2dVar():
fname = pjoin(TEST_DATA_PATH, 'example_3_maskedvals.nc')
with netcdf_file(fname, maskandscale=True) as f:
vardata = f.variables['var7_2d'][:]
assert_mask_matches(vardata, [[True, False], [False, False], [False, True]])
def test_read_withMaskAndScaleFalse():
# If a variable has a _FillValue (or missing_value) attribute, but is read
# with maskandscale set to False, the result should be unmasked
fname = pjoin(TEST_DATA_PATH, 'example_3_maskedvals.nc')
# Open file with mmap=False to avoid problems with closing a mmap'ed file
# when arrays referring to its data still exist:
with netcdf_file(fname, maskandscale=False, mmap=False) as f:
vardata = f.variables['var3_fillvalAndMissingValue'][:]
assert_mask_matches(vardata, [False, False, False])
assert_equal(vardata, [1, 2, 3])

View file

@ -0,0 +1,93 @@
"""
Ensure that we can use pathlib.Path objects in all relevant IO functions.
"""
from pathlib import Path
import numpy as np
import scipy.io
import scipy.io.wavfile
from scipy._lib._tmpdirs import tempdir
import scipy.sparse
class TestPaths:
data = np.arange(5).astype(np.int64)
def test_savemat(self):
with tempdir() as temp_dir:
path = Path(temp_dir) / 'data.mat'
scipy.io.savemat(path, {'data': self.data})
assert path.is_file()
def test_loadmat(self):
# Save data with string path, load with pathlib.Path
with tempdir() as temp_dir:
path = Path(temp_dir) / 'data.mat'
scipy.io.savemat(str(path), {'data': self.data})
mat_contents = scipy.io.loadmat(path)
assert (mat_contents['data'] == self.data).all()
def test_whosmat(self):
# Save data with string path, load with pathlib.Path
with tempdir() as temp_dir:
path = Path(temp_dir) / 'data.mat'
scipy.io.savemat(str(path), {'data': self.data})
contents = scipy.io.whosmat(path)
assert contents[0] == ('data', (1, 5), 'int64')
def test_readsav(self):
path = Path(__file__).parent / 'data/scalar_string.sav'
scipy.io.readsav(path)
def test_hb_read(self):
# Save data with string path, load with pathlib.Path
with tempdir() as temp_dir:
data = scipy.sparse.eye_array(3, format='csr')
path = Path(temp_dir) / 'data.hb'
scipy.io.hb_write(str(path), data)
data_new = scipy.io.hb_read(path, spmatrix=False)
assert (data_new != data).nnz == 0
def test_hb_write(self):
with tempdir() as temp_dir:
data = scipy.sparse.eye_array(3, format='csr')
path = Path(temp_dir) / 'data.hb'
scipy.io.hb_write(path, data)
assert path.is_file()
def test_mmio_read(self):
# Save data with string path, load with pathlib.Path
with tempdir() as temp_dir:
data = scipy.sparse.eye_array(3, format='csr')
path = Path(temp_dir) / 'data.mtx'
scipy.io.mmwrite(str(path), data)
data_new = scipy.io.mmread(path, spmatrix=False)
assert (data_new != data).nnz == 0
def test_mmio_write(self):
with tempdir() as temp_dir:
data = scipy.sparse.eye_array(3, format='csr')
path = Path(temp_dir) / 'data.mtx'
scipy.io.mmwrite(path, data)
def test_netcdf_file(self):
path = Path(__file__).parent / 'data/example_1.nc'
scipy.io.netcdf_file(path)
def test_wavfile_read(self):
path = Path(__file__).parent / 'data/test-8000Hz-le-2ch-1byteu.wav'
scipy.io.wavfile.read(path)
def test_wavfile_write(self):
# Read from str path, write to Path
input_path = Path(__file__).parent / 'data/test-8000Hz-le-2ch-1byteu.wav'
rate, data = scipy.io.wavfile.read(str(input_path))
with tempdir() as temp_dir:
output_path = Path(temp_dir) / input_path.name
scipy.io.wavfile.write(output_path, rate, data)

Some files were not shown because too many files have changed in this diff Show more