Python Bytes Reading Example

C Program

This generates a binary file for reading via Python

/**
 *  colorpoints.c
 *
 *  A C99 program to write a colored vertex
 *  structures to a binary file.
 *
 */

#include <stdio.h>

struct Vector {
    float x;
    float y;
    float z;
};

struct Color {
    unsigned short int red;
    unsigned short int green;
    unsigned short int blue;
};

struct Vertex {
    struct Vector position;
    struct Color color;
};

int main(int argc, char** argv) {
    struct Vertex vertices[] = {
        { .position = { 3323.176, 6562.231, 9351.231 },
          .color = { 3040, 34423, 54321 } },

        { .position = { 7623.982, 2542.231, 9823.121 },
          .color = { 32736, 5342, 2321 } },

        { .position = { 6729.862, 2347.212, 3421.322 },
          .color = { 45263, 36291, 36701 } },

        { .position = { 6352.121, 3432.111, 9763.232 },
          .color = { 56222, 36612, 11214 } } };

    FILE* file = fopen("colors.bin", "wb");

    if (file == NULL) {
        return -1;
    }

    fwrite(vertices, sizeof(struct Vertex), 4, file);
    fclose(file);

    return 0;
}

Python Program

This demonstrates the use of

struct.iter_unpack('@3f3Hxx', buffer)

Which allows the program to read types, in this case fffhhh into a buffer. The xx represents the packing which is required due to the use of the compiler at the time. This may vary.

This also demonstrates the outputting of bytes using python.

import struct
from pprint import pprint as pp
from binascii import hexlify

class Vector:

    def __init__(self, x, y, z):
        self.x = x
        self.y = y
        self.z = z

    def __repr__(self):
        return 'Vector({}, {}, {})'.format(self.x, self.y, self.z)


class Color:

    def __init__(self, red, green, blue):
        self.red = red
        self.green = green
        self.blue = blue

    def __repr__(self):
        return 'Color({}, {}, {})'.format(self.red, self.green, self.blue)


class Vertex:

    def __init__(self, vector, color):
        self.vector = vector
        self.color = color

    def __repr__(self):
        return 'Vertex({!r}, {!r})'.format(self.vector, self.color)


def make_colored_vertex(x, y, z, red, green, blue):
    return Vertex(Vector(x, y, z),
                  Color(red, green, blue))


def main():
    with open('colors.bin', 'rb') as f:
        buffer = f.read()

    print("buffer: {} bytes".format(len(buffer)))

    indexes = ' '.join(str(n).zfill(2) for n in range(len(buffer)))
    print(indexes)

    hex_buffer = hexlify(buffer).decode('ascii')
    hex_pairs = ' '.join(hex_buffer[i:i+2] for i in range(0, len(hex_buffer), 2))
    print(hex_pairs)

    vertices = []
    for fields in struct.iter_unpack('@3f3Hxx', buffer):
        vertex = make_colored_vertex(*fields)
        vertices.append(vertex)

    pp(vertices)

if __name__ == '__main__':
    main()

Improved Version using memoryview

memoryview offers a better way to validate binary data and provides a cast function.

"""Read binary file produced by colorpoints.c in Python.

Demonstrate use of the struct module
"""
import code

import mmap
from pprint import pprint as pp
from binascii import hexlify


class Vector:

    def __init__(self, mem_float32):
        if mem_float32.format not in "fd":
            raise TypeError("Vector: memoryview values must be floating-point numbers")
        if len(mem_float32) < 3:
            raise TypeError("Vector: memoryview must contain at least 3 floats")
        self._mem = mem_float32

    @property
    def x(self):
        return self._mem[0]

    @property
    def y(self):
        return self._mem[1]

    @property
    def z(self):
        return self._mem[2]

    def __repr__(self):
        return 'Vector({}, {}, {})'.format(self.x, self.y, self.z)


class Color:

    def __init__(self, mem_uint16):
        if mem_uint16.format not in "HILQ":
            raise TypeError("Color: memoryview values must be unsigned integers")
        if len(mem_uint16) < 3:
            raise TypeError("Color: memoryview must contain at least 3 integers")
        self._mem = mem_uint16

    @property
    def red(self):
        return self._mem[0]

    @property
    def green(self):
        return self._mem[1]

    @property
    def blue(self):
        return self._mem[2]

    def __repr__(self):
        return 'Color({}, {}, {})'.format(self.red, self.green, self.blue)


class Vertex:

    def __init__(self, vector, color):
        self.vector = vector
        self.color = color

    def __repr__(self):
        return 'Vertex({!r}, {!r})'.format(self.vector, self.color)


def make_colored_vertex(mem_vertex):
    mem_vector = mem_vertex[0:12].cast('f')
    mem_color = mem_vertex[12:18].cast('H')
    return Vertex(Vector(mem_vector),
                  Color(mem_color))


def main():
    with open('colors.bin', 'rb') as f:
        with mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ) as buffer:

            print("buffer: {} bytes".format(len(buffer)))

            indexes = ' '.join(str(n).zfill(2) for n in range(len(buffer)))
            print(indexes)

            hex_buffer = hexlify(buffer).decode('ascii')
            hex_pairs = ' '.join(hex_buffer[i:i+2] for i in range(0, len(hex_buffer), 2))
            print(hex_pairs)

            mem = memoryview(buffer)

            VERTEX_SIZE = 18
            VERTEX_STRIDE = VERTEX_SIZE + 2

            vertex_mems = (mem[i:i + VERTEX_SIZE] for i in range(0, len(mem), VERTEX_STRIDE))
            vertices = [make_colored_vertex(vertex_mem) for vertex_mem in vertex_mems]

            pp(vertices)

            del vertices
            del mem


if __name__ == '__main__':
    main()

Note the del of the following. We may need to understand why this was required.

            del vertices
            del mem

Without this we get

 BufferError: cannot close exported pointers exist

The memoryview object must be cleaned up prior to closing the mmap