OXIESEC PANEL

Name	Size	Modified	Perms
📁 ..	-	05/25/2021 01:14:27 PM	rwxr-xr-x
📄 __init__.py	0 bytes	01/10/2015 06:49:10 AM	rw-r--r--
📁 __pycache__	-	05/25/2021 01:14:27 PM	rwxr-xr-x
📄 numpy.py	10.16 KB	04/01/2017 10:26:55 AM	rw-r--r--

Editing: numpy.py

# -*- coding: utf-8 -*-

from __future__ import absolute_import

import sys
import zlib
import warnings

import numpy as np

import ast
from ..handlers import BaseHandler, register, unregister
from ..compat import unicode
from ..util import b64decode, b64encode

__all__ = ['register_handlers', 'unregister_handlers']

native_byteorder = '<' if sys.byteorder == 'little' else '>'

def get_byteorder(arr):
    """translate equals sign to native order"""
    byteorder = arr.dtype.byteorder
    return native_byteorder if byteorder == '=' else byteorder

class NumpyBaseHandler(BaseHandler):

def flatten_dtype(self, dtype, data):
        if hasattr(dtype, 'tostring'):
            data['dtype'] = dtype.tostring()
        else:
            dtype = unicode(dtype)
            prefix = '(numpy.record, '
            if dtype.startswith(prefix):
                dtype = dtype[len(prefix):-1]
            data['dtype'] = dtype

def restore_dtype(self, data):
        dtype = data['dtype']
        if dtype.startswith(('{', '[')):
            dtype = ast.literal_eval(dtype)
        return np.dtype(dtype)

class NumpyDTypeHandler(NumpyBaseHandler):

def flatten(self, obj, data):
        self.flatten_dtype(obj, data)
        return data

def restore(self, data):
        return self.restore_dtype(data)

class NumpyGenericHandler(NumpyBaseHandler):

def flatten(self, obj, data):
        self.flatten_dtype(obj.dtype.newbyteorder('N'), data)
        data['value'] = self.context.flatten(obj.tolist(), reset=False)
        return data

def restore(self, data):
        value = self.context.restore(data['value'], reset=False)
        return self.restore_dtype(data).type(value)

class NumpyNDArrayHandler(NumpyBaseHandler):
    """Stores arrays as text representation, without regard for views
    """
    def flatten_flags(self, obj, data):
        if obj.flags.writeable is False:
            data['writeable'] = False

def restore_flags(self, data, arr):
        if not data.get('writeable', True):
            arr.flags.writeable = False

def flatten(self, obj, data):
        self.flatten_dtype(obj.dtype.newbyteorder('N'), data)
        self.flatten_flags(obj, data)
        data['values'] = self.context.flatten(obj.tolist(), reset=False)
        if 0 in obj.shape:
            # add shape information explicitly as it cannot be inferred from an empty list
            data['shape'] = obj.shape
        return data

def restore(self, data):
        values = self.context.restore(data['values'], reset=False)
        arr = np.array(
            values,
            dtype=self.restore_dtype(data),
            order=data.get('order', 'C')
        )
        shape = data.get('shape', None)
        if shape is not None:
            arr = arr.reshape(shape)

self.restore_flags(data, arr)
        return arr

class NumpyNDArrayHandlerBinary(NumpyNDArrayHandler):
    """stores arrays with size greater than 'size_treshold' as (optionally) compressed base64

Notes
    -----
    This would be easier to implement using np.save/np.load, but that would be less language-agnostic
    """

def __init__(self, size_treshold=16, compression=zlib):
        """
        :param size_treshold: nonnegative int or None
            valid values for 'size_treshold' are all nonnegative integers and None
            if size_treshold is None, values are always stored as nested lists
        :param compression: a compression module or None
            valid values for 'compression' are {zlib, bz2, None}
            if compresion is None, no compression is applied
        """
        self.size_treshold = size_treshold
        self.compression = compression

def flatten_byteorder(self, obj, data):
        byteorder = obj.dtype.byteorder
        if byteorder != '|':
            data['byteorder'] = get_byteorder(obj)

def restore_byteorder(self, data, arr):
        byteorder = data.get('byteorder', None)
        if byteorder:
            arr.dtype = arr.dtype.newbyteorder(byteorder)

def flatten(self, obj, data):
        """encode numpy to json"""
        if self.size_treshold >= obj.size or self.size_treshold is None:
            # encode as text
            data = super(NumpyNDArrayHandlerBinary, self).flatten(obj, data)
        else:
            # encode as binary
            buffer = obj.tobytes(order='a')	 # numpy docstring is lacking as of 1.11.2, but this is the option we need
            if self.compression:
                buffer = self.compression.compress(buffer)
            data['values'] = b64encode(buffer)
            data['shape'] = obj.shape
            self.flatten_dtype(obj.dtype.newbyteorder('N'), data)
            self.flatten_byteorder(obj, data)
            self.flatten_flags(obj, data)

if not obj.flags.c_contiguous:
                data['order'] = 'F'

return data

def restore(self, data):
        """decode numpy from json"""
        values = data['values']
        if isinstance(values, list):
            # decode text representation
            arr = super(NumpyNDArrayHandlerBinary, self).restore(data)
        else:
            # decode binary representation
            buffer = b64decode(values)
            if self.compression:
                buffer = self.compression.decompress(buffer)
            arr = np.ndarray(
                buffer=buffer,
                dtype=self.restore_dtype(data),
                shape=data.get('shape'),
                order=data.get('order', 'C')
            ).copy() # make a copy, to force the result to own the data
            self.restore_byteorder(data, arr)
            self.restore_flags(data, arr)

return arr

class NumpyNDArrayHandlerView(NumpyNDArrayHandlerBinary):
    """Pickles references inside ndarrays, or array-views

Notes
    -----
    The current implementation has some restrictions.

'base' arrays, or arrays which are viewed by other arrays, must be f-or-c-contiguous.
    This is not such a large restriction in practice, because all numpy array creation is c-contiguous by default.
    Relaxing this restriction would be nice though; especially if it can be done without bloating the design too much.

Furthermore, ndarrays which are views of array-like objects implementing __array_interface__,
    but which are not themselves nd-arrays, are deepcopied with a warning (by default),
    as we cannot guarantee whatever custom logic such classes implement is correctly reproduced.
    """
    def __init__(self, mode='warn', size_treshold=16, compression=zlib):
        """
        :param mode: {'warn', 'raise', 'ignore'}
            How to react when encountering array-like objects whos references we cannot safely serialize
        :param size_treshold: nonnegative int or None
            valid values for 'size_treshold' are all nonnegative integers and None
            if size_treshold is None, values are always stored as nested lists
        :param compression: a compression module or None
            valid values for 'compression' are {zlib, bz2, None}
            if compresion is None, no compression is applied
        """
        super(NumpyNDArrayHandlerView, self).__init__(size_treshold, compression)
        self.mode = mode

def flatten(self, obj, data):
        """encode numpy to json"""
        base = obj.base
        if base is None and obj.flags.forc:
            # store by value
            data = super(NumpyNDArrayHandlerView, self).flatten(obj, data)
            # ensure that views on arrays stored as text are interpreted correctly
            if not obj.flags.c_contiguous:
                data['order'] = 'F'
        elif isinstance(base, np.ndarray) and base.flags.forc:
            # store by reference
            data['base'] = self.context.flatten(base, reset=False)

offset = obj.ctypes.data - base.ctypes.data
            if offset:
                data['offset'] = offset

if not obj.flags.c_contiguous:
                data['strides'] = obj.strides

data['shape'] = obj.shape
            self.flatten_dtype(obj.dtype.newbyteorder('N'), data)
            self.flatten_flags(obj, data)

if get_byteorder(obj) != '|':
                byteorder = 'S' if get_byteorder(obj) != get_byteorder(base) else None
                if byteorder:
                    data['byteorder'] = byteorder

if self.size_treshold >= obj.size:
                # not used in restore since base is present, but include values for human-readability
                super(NumpyNDArrayHandlerBinary, self).flatten(obj, data)
        else:
            # store a deepcopy or fail
            if self.mode == 'warn':
                msg = "ndarray is defined by reference to an object we do not know how to serialize. " \
                      "A deep copy is serialized instead, breaking memory aliasing."
                warnings.warn(msg)
            elif self.mode == 'raise':
                msg = "ndarray is defined by reference to an object we do not know how to serialize."
                raise ValueError(msg)
            data = super(NumpyNDArrayHandlerView, self).flatten(obj.copy(), data)

return data

def restore(self, data):
        """decode numpy from json"""
        base = data.get('base', None)
        if base is None:
            # decode array with owndata=True
            arr = super(NumpyNDArrayHandlerView, self).restore(data)
        else:
            # decode array view, which references the data of another array
            base = self.context.restore(base, reset=False)
            assert base.flags.forc, \
                "Current implementation assumes base is C or F contiguous"

arr = np.ndarray(
                buffer=base.data,
                dtype=self.restore_dtype(data).newbyteorder(data.get('byteorder', '|')),
                shape=data.get('shape'),
                offset=data.get('offset', 0),
                strides=data.get('strides', None)
            )

self.restore_flags(data, arr)

return arr

def register_handlers():
    register(np.dtype, NumpyDTypeHandler, base=True)
    register(np.generic, NumpyGenericHandler, base=True)
    register(np.ndarray, NumpyNDArrayHandlerView(), base=True)

def unregister_handlers():
    unregister(np.dtype)
    unregister(np.generic)
    unregister(np.ndarray)