How to read packed binary data in Go?

Question

I am trying to find the best way to read a packed binary in Go that Python created as follows:

import struct
f = open('tst.bin', 'wb')
fmt = 'iih' #please note this is packed binary: 4byte int, 4byte int, 2byte int
f.write(struct.pack(fmt,4, 185765, 1020))
f.write(struct.pack(fmt,4, 185765, 1022))
f.close()

I worked on some examples that I saw on Github.com and several other sources. but I can’t work normally.(update shows the working method). What is the idiomatic way of doing such things in Go? This is one of several attempts.

UPDATE & WORK

package main

    import (
            "fmt"
            "os"
            "encoding/binary"
            "io"
            )

    func main() {
            fp, err := os.Open("tst.bin")

            if err != nil {
                    panic(err)
            }

            defer fp.Close()

            lineBuf := make([]byte, 10) //4 byte int, 4 byte int, 2 byte int per line

            for true {
                _, err := fp.Read(lineBuf)

                if err == io.EOF{
                    break
                }

                aVal := int32(binary.LittleEndian.Uint32(lineBuf[0:4])) // same as: int32(uint32(b[0]) | uint32(b[1])<<8 | uint32(b[2])<<16 | uint32(b[3])<<24)
                bVal := int32(binary.LittleEndian.Uint32(lineBuf[4:8]))
                cVal := int16(binary.LittleEndian.Uint16(lineBuf[8:10])) //same as: int16(uint32(b[0]) | uint32(b[1])<<8)
                fmt.Println(aVal, bVal, cVal)
            }
    }

Answer 1

Python iih, 32- 16- (. docs). , :

type binData struct {
    A int32
    B int32
    C int16
}

func main() {
        fp, err := os.Open("tst.bin")

        if err != nil {
                panic(err)
        }

        defer fp.Close()

        for {
            thing := binData{}
            err := binary.Read(fp, binary.LittleEndian, &thing)

            if err == io.EOF{
                break
            }

            fmt.Println(thing.A, thing.B, thing.C)
        }
}

, Python , , , , little-endian, .

: ​​ main(), , .

2: , binary.Read .

Answer 2

: Google " " . , , , .

https://github.com/mwmahlberg/ProtoBufDemo

python, (pip, , ) Go

.proto

.proto . data.proto

syntax = "proto2";
package main;

message Demo {
  required uint32  A = 1;
  required uint32 B = 2;

  // A shortcomning: no 16 bit ints
  // We need to make this sure in the applications
  required uint32 C = 3;
}

protoc Python Go:

protoc --go_out=. --python_out=. data.proto

data_pb2.py data.pb.go. .

github , ,

go generate

.

Python

import data_pb2

def main():

    # We create an instance of the message type "Demo"...
    data = data_pb2.Demo()

    # ...and fill it with data
    data.A = long(5)
    data.B = long(5)
    data.C = long(2015)


    print "* Python writing to file"
    f = open('tst.bin', 'wb')

    # Note that "data.SerializeToString()" counterintuitively
    # writes binary data
    f.write(data.SerializeToString())
    f.close()

    f = open('tst.bin', 'rb')
    read = data_pb2.Demo()
    read.ParseFromString(f.read())
    f.close()

    print "* Python reading from file"
    print "\tDemo.A: %d, Demo.B: %d, Demo.C: %d" %(read.A, read.B, read.C)

if __name__ == '__main__':
    main()

, protoc, . .

Go

package main

//go:generate protoc --python_out=. data.proto
//go:generate protoc --go_out=. data.proto
import (
    "fmt"
    "os"

    "github.com/golang/protobuf/proto"
)

func main() {

    // Note that we do not handle any errors for the sake of brevity
    d := Demo{}
    f, _ := os.Open("tst.bin")
    fi, _ := f.Stat()

    // We create a buffer which is big enough to hold the entire message
    b := make([]byte,fi.Size())

    f.Read(b)

    proto.Unmarshal(b, &d)
    fmt.Println("* Go reading from file")

    // Note the explicit pointer dereference, as the fields are pointers to a pointers
    fmt.Printf("\tDemo.A: %d, Demo.B: %d, Demo.C: %d\n",*d.A,*d.B,*d.C)
}

, , data.proto main.

$ python writer.py && ./ProtoBufDemo

* Python writing to file
* Python reading from file
    Demo.A: 5, Demo.B: 5, Demo.C: 2015
* Go reading from file
    Demo.A: 5, Demo.B: 5, Demo.C: 2015

, Makefile shorcut , .go :

make run

Answer 3

, , Go, , . , 4 2 int Go int32 int16 . , , - . , - , .

package main

    import (
            "fmt"
            "os"
            "encoding/binary"
            "io"
            )

    func main() {
            fp, err := os.Open("tst.bin")

            if err != nil {
                    panic(err)
            }

            defer fp.Close()

            lineBuf := make([]byte, 10) //4 byte int, 4 byte int, 2 byte int per line

            for true {
                _, err := fp.Read(lineBuf)

                if err == io.EOF{
                    break
                }

                aVal := int32(binary.LittleEndian.Uint32(lineBuf[0:4])) // same as: int32(uint32(b[0]) | uint32(b[1])<<8 | uint32(b[2])<<16 | uint32(b[3])<<24)
                bVal := int32(binary.LittleEndian.Uint32(lineBuf[4:8]))
                cVal := int16(binary.LittleEndian.Uint16(lineBuf[8:10])) //same as: int16(uint32(b[0]) | uint32(b[1])<<8)
                fmt.Println(aVal, bVal, cVal)
            }
    }

Answer 4

binpacker libary.

:

:

buffer := new(bytes.Buffer)
packer := binpacker.NewPacker(buffer)
unpacker := binpacker.NewUnpacker(buffer)
packer.PushByte(0x01)
packer.PushUint16(math.MaxUint16)

:

var val1 byte
var val2 uint16
var err error
val1, err = unpacker.ShiftByte()
val2, err = unpacker.ShiftUint16()

:

var val1 byte
var val2 uint16
var err error
unpacker.FetchByte(&val1).FetchUint16(&val2)
unpacker.Error() // Make sure error is nil