Skip to content

Commit 4272748

Browse files
authored
Merge pull request #239 from wader/blkdat
blkdat: Add Bitcoin blk.dat decoder
2 parents 1317bb2 + 417255b commit 4272748

16 files changed

Lines changed: 1552 additions & 973 deletions

File tree

README.md

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,10 @@ avc_sei,
5353
avc_sps,
5454
[avro_ocf](doc/formats.md#avro_ocf),
5555
[bencode](doc/formats.md#bencode),
56+
bitcoin_blkdat,
57+
bitcoin_block,
58+
bitcoin_script,
59+
bitcoin_transaction,
5660
bsd_loopback_frame,
5761
[bson](doc/formats.md#bson),
5862
bzip2,

doc/formats.md

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,10 @@
2323
|`avc_sps` |H.264/AVC&nbsp;Sequence&nbsp;Parameter&nbsp;Set |<sub></sub>|
2424
|[`avro_ocf`](#avro_ocf) |Avro&nbsp;object&nbsp;container&nbsp;file |<sub></sub>|
2525
|[`bencode`](#bencode) |BitTorrent&nbsp;bencoding |<sub></sub>|
26+
|`bitcoin_blkdat` |Bitcoin&nbsp;blk.dat |<sub>`bitcoin_block`</sub>|
27+
|`bitcoin_block` |Bitcoin&nbsp;block |<sub>`bitcoin_transaction`</sub>|
28+
|`bitcoin_script` |Bitcoin&nbsp;script |<sub></sub>|
29+
|`bitcoin_transaction` |Bitcoin&nbsp;transaction |<sub>`bitcoin_script`</sub>|
2630
|`bsd_loopback_frame` |BSD&nbsp;loopback&nbsp;frame |<sub>`inet_packet`</sub>|
2731
|[`bson`](#bson) |Binary&nbsp;JSON |<sub></sub>|
2832
|`bzip2` |bzip2&nbsp;compression |<sub>`probe`</sub>|
@@ -100,7 +104,7 @@
100104
|`inet_packet` |Group |<sub>`ipv4_packet` `ipv6_packet`</sub>|
101105
|`ip_packet` |Group |<sub>`icmp` `icmpv6` `tcp_segment` `udp_datagram`</sub>|
102106
|`link_frame` |Group |<sub>`bsd_loopback_frame` `ether8023_frame` `sll2_packet` `sll_packet`</sub>|
103-
|`probe` |Group |<sub>`adts` `ar` `avro_ocf` `bzip2` `elf` `flac` `gif` `gzip` `jpeg` `json` `macho` `matroska` `mp3` `mp4` `mpeg_ts` `ogg` `pcap` `pcapng` `png` `tar` `tiff` `wav` `webp` `zip`</sub>|
107+
|`probe` |Group |<sub>`adts` `ar` `avro_ocf` `bitcoin_blkdat` `bzip2` `elf` `flac` `gif` `gzip` `jpeg` `json` `macho` `matroska` `mp3` `mp4` `mpeg_ts` `ogg` `pcap` `pcapng` `png` `tar` `tiff` `wav` `webp` `zip`</sub>|
104108
|`tcp_stream` |Group |<sub>`dns` `rtmp`</sub>|
105109
|`udp_payload` |Group |<sub>`dns`</sub>|
106110

doc/formats.svg

Lines changed: 1021 additions & 967 deletions
Loading

format/all/all.fqtest

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ $ fq -n _registry.groups.probe
33
"adts",
44
"ar",
55
"avro_ocf",
6+
"bitcoin_blkdat",
67
"bzip2",
78
"elf",
89
"flac",

format/all/all.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import (
99
_ "github.com/wader/fq/format/av1"
1010
_ "github.com/wader/fq/format/avro"
1111
_ "github.com/wader/fq/format/bencode"
12+
_ "github.com/wader/fq/format/bitcoin"
1213
_ "github.com/wader/fq/format/bson"
1314
_ "github.com/wader/fq/format/bzip2"
1415
_ "github.com/wader/fq/format/cbor"

format/all/help.fqtest

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -179,6 +179,34 @@ out # Supports torepr
179179
out ... | bencode | torepr
180180
out References and links
181181
out https://wiki.theory.org/BitTorrentSpecification#Bencoding
182+
"help(bitcoin_blkdat)"
183+
out bitcoin_blkdat: Bitcoin blk.dat decoder
184+
out Examples:
185+
out # Decode file as bitcoin_blkdat
186+
out $ fq -d bitcoin_blkdat . file
187+
out # Decode value as bitcoin_blkdat
188+
out ... | bitcoin_blkdat
189+
"help(bitcoin_block)"
190+
out bitcoin_block: Bitcoin block decoder
191+
out Examples:
192+
out # Decode file as bitcoin_block
193+
out $ fq -d bitcoin_block . file
194+
out # Decode value as bitcoin_block
195+
out ... | bitcoin_block
196+
"help(bitcoin_script)"
197+
out bitcoin_script: Bitcoin script decoder
198+
out Examples:
199+
out # Decode file as bitcoin_script
200+
out $ fq -d bitcoin_script . file
201+
out # Decode value as bitcoin_script
202+
out ... | bitcoin_script
203+
"help(bitcoin_transaction)"
204+
out bitcoin_transaction: Bitcoin transaction decoder
205+
out Examples:
206+
out # Decode file as bitcoin_transaction
207+
out $ fq -d bitcoin_transaction . file
208+
out # Decode value as bitcoin_transaction
209+
out ... | bitcoin_transaction
182210
"help(bsd_loopback_frame)"
183211
out bsd_loopback_frame: BSD loopback frame decoder
184212
out Examples:

format/bitcoin/bitcoin_blkdat.go

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
package bitcoin
2+
3+
import (
4+
"github.com/wader/fq/format"
5+
"github.com/wader/fq/format/registry"
6+
"github.com/wader/fq/pkg/decode"
7+
)
8+
9+
var bitcoinBlockFormat decode.Group
10+
11+
func init() {
12+
registry.MustRegister(decode.Format{
13+
Name: format.BITCOIN_BLKDAT,
14+
Description: "Bitcoin blk.dat",
15+
Groups: []string{format.PROBE},
16+
Dependencies: []decode.Dependency{
17+
{Names: []string{format.BITCOIN_BLOCK}, Group: &bitcoinBlockFormat},
18+
},
19+
DecodeFn: decodeBlkDat,
20+
RootArray: true,
21+
RootName: "blocks",
22+
})
23+
}
24+
25+
func decodeBlkDat(d *decode.D, in interface{}) interface{} {
26+
validBlocks := 0
27+
for !d.End() {
28+
d.FieldFormat("block", bitcoinBlockFormat, nil)
29+
validBlocks++
30+
}
31+
32+
if validBlocks == 0 {
33+
d.Fatalf("no valid blocks found")
34+
}
35+
36+
return nil
37+
}

format/bitcoin/bitcoin_block.go

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
package bitcoin
2+
3+
// https://learnmeabitcoin.com/technical/blkdat
4+
5+
import (
6+
"fmt"
7+
8+
"github.com/wader/fq/format"
9+
"github.com/wader/fq/format/registry"
10+
"github.com/wader/fq/pkg/decode"
11+
"github.com/wader/fq/pkg/scalar"
12+
)
13+
14+
var bitcoinTranscationFormat decode.Group
15+
16+
func init() {
17+
registry.MustRegister(decode.Format{
18+
Name: format.BITCOIN_BLOCK,
19+
Description: "Bitcoin block",
20+
Dependencies: []decode.Dependency{
21+
{Names: []string{format.BITCOIN_TRANSACTION}, Group: &bitcoinTranscationFormat},
22+
},
23+
DecodeFn: decodeBitcoinBlock,
24+
})
25+
}
26+
27+
var rawHexReverse = scalar.Fn(func(s scalar.S) (scalar.S, error) {
28+
return scalar.RawSym(s, -1, func(b []byte) string {
29+
decode.ReverseBytes(b)
30+
return fmt.Sprintf("%x", b)
31+
})
32+
})
33+
34+
func decodeBitcoinBlock(d *decode.D, in interface{}) interface{} {
35+
size := d.BitsLeft()
36+
37+
// TODO: move to blkdat but how to model it?
38+
switch d.PeekBits(32) {
39+
case 0xf9beb4d9,
40+
0x0b110907,
41+
0xfabfb5da:
42+
d.FieldU32("magic", scalar.UToSymStr{
43+
0xf9beb4d9: "mainnet",
44+
0x0b110907: "testnet3",
45+
0xfabfb5da: "regtest",
46+
}, scalar.ActualHex)
47+
size = int64(d.FieldU32LE("size")) * 8
48+
}
49+
50+
d.Endian = decode.LittleEndian
51+
52+
d.FramedFn(size, func(d *decode.D) {
53+
d.FieldStruct("header", func(d *decode.D) {
54+
d.FieldU32("version", scalar.ActualHex)
55+
d.FieldRawLen("previous_block_hash", 32*8, rawHexReverse)
56+
d.FieldRawLen("merkle_root", 32*8, rawHexReverse)
57+
d.FieldU32("time", scalar.DescriptionActualUUnixTime)
58+
d.FieldU32("bits", scalar.ActualHex)
59+
d.FieldU32("nonce", scalar.ActualHex)
60+
})
61+
62+
// TODO: remove? support header only decode this way?
63+
if d.BitsLeft() == 0 {
64+
return
65+
}
66+
67+
txCount := d.FieldUFn("tx_count", decodeVarInt)
68+
d.FieldArray("transactions", func(d *decode.D) {
69+
for i := uint64(0); i < txCount; i++ {
70+
d.FieldFormat("transaction", bitcoinTranscationFormat, nil)
71+
}
72+
})
73+
})
74+
75+
return nil
76+
}

format/bitcoin/bitcoin_script.go

Lines changed: 189 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,189 @@
1+
package bitcoin
2+
3+
import (
4+
"github.com/wader/fq/format"
5+
"github.com/wader/fq/format/registry"
6+
"github.com/wader/fq/pkg/decode"
7+
"github.com/wader/fq/pkg/scalar"
8+
)
9+
10+
type opcodeEntry struct {
11+
r [2]byte
12+
s scalar.S
13+
d func(d *decode.D, opcode byte)
14+
}
15+
16+
type opcodeEntries []opcodeEntry
17+
18+
func (ops opcodeEntries) lookup(u byte) (opcodeEntry, bool) {
19+
for _, fe := range ops {
20+
if u >= fe.r[0] && u <= fe.r[1] {
21+
return fe, true
22+
}
23+
}
24+
return opcodeEntry{}, false
25+
}
26+
27+
func (ops opcodeEntries) MapScalar(s scalar.S) (scalar.S, error) {
28+
u := s.ActualU()
29+
if fe, ok := ops.lookup(byte(u)); ok {
30+
s = fe.s
31+
s.Actual = u
32+
}
33+
return s, nil
34+
}
35+
36+
func init() {
37+
registry.MustRegister(decode.Format{
38+
Name: format.BITCOIN_SCRIPT,
39+
Description: "Bitcoin script",
40+
DecodeFn: decodeBitcoinScript,
41+
RootArray: true,
42+
RootName: "opcodes",
43+
})
44+
}
45+
46+
func decodeBitcoinScript(d *decode.D, in interface{}) interface{} {
47+
// based on https://en.bitcoin.it/wiki/Script
48+
opcodeEntries := opcodeEntries{
49+
{r: [2]byte{0x00, 0x00}, s: scalar.S{Sym: "false"}},
50+
// TODO: name op code?
51+
{r: [2]byte{0x01, 0x4b}, s: scalar.S{Sym: "pushself"}, d: func(d *decode.D, opcode byte) {
52+
d.FieldRawLen("arg", int64(opcode)*8)
53+
}},
54+
{r: [2]byte{0x04c, 0x4e}, s: scalar.S{Sym: "pushdata1"}, d: func(d *decode.D, opcode byte) {
55+
argLen := d.FieldU8("arg_length")
56+
d.FieldRawLen("arg", int64(argLen)*8)
57+
}},
58+
{r: [2]byte{0x04c, 0x4e}, s: scalar.S{Sym: "pushdata2"}, d: func(d *decode.D, opcode byte) {
59+
argLen := d.FieldU16("arg_length")
60+
d.FieldRawLen("arg", int64(argLen)*8)
61+
}},
62+
{r: [2]byte{0x04c, 0x4e}, s: scalar.S{Sym: "pushdata4"}, d: func(d *decode.D, opcode byte) {
63+
argLen := d.FieldU32("arg_length")
64+
d.FieldRawLen("arg", int64(argLen)*8)
65+
}},
66+
{r: [2]byte{0x4f, 0x4f}, s: scalar.S{Sym: "1negate"}},
67+
{r: [2]byte{0x51, 0x51}, s: scalar.S{Sym: "true"}},
68+
// TODO: name
69+
{r: [2]byte{0x52, 0x60}, s: scalar.S{Sym: "push"}, d: func(d *decode.D, opcode byte) {
70+
d.FieldValueU("arg", uint64(opcode-0x50))
71+
}},
72+
{r: [2]byte{0x61, 0x61}, s: scalar.S{Sym: "nop"}},
73+
{r: [2]byte{0x62, 0x62}, s: scalar.S{Sym: "ver"}},
74+
{r: [2]byte{0x63, 0x63}, s: scalar.S{Sym: "if"}},
75+
{r: [2]byte{0x64, 0x64}, s: scalar.S{Sym: "notif"}},
76+
{r: [2]byte{0x65, 0x65}, s: scalar.S{Sym: "verif"}},
77+
{r: [2]byte{0x66, 0x66}, s: scalar.S{Sym: "vernotif"}},
78+
{r: [2]byte{0x67, 0x67}, s: scalar.S{Sym: "else"}},
79+
{r: [2]byte{0x68, 0x68}, s: scalar.S{Sym: "endif"}},
80+
{r: [2]byte{0x69, 0x69}, s: scalar.S{Sym: "verify"}},
81+
{r: [2]byte{0x6a, 0x6a}, s: scalar.S{Sym: "return"}},
82+
{r: [2]byte{0x6b, 0x6b}, s: scalar.S{Sym: "toaltstack"}},
83+
{r: [2]byte{0x6c, 0x6c}, s: scalar.S{Sym: "fromaltstack"}},
84+
{r: [2]byte{0x6d, 0x6d}, s: scalar.S{Sym: "2drop"}},
85+
{r: [2]byte{0x6e, 0x6e}, s: scalar.S{Sym: "2dup"}},
86+
{r: [2]byte{0x6f, 0x6f}, s: scalar.S{Sym: "3dup"}},
87+
{r: [2]byte{0x70, 0x70}, s: scalar.S{Sym: "2over"}},
88+
{r: [2]byte{0x71, 0x71}, s: scalar.S{Sym: "2rot"}},
89+
{r: [2]byte{0x72, 0x72}, s: scalar.S{Sym: "2swap"}},
90+
{r: [2]byte{0x73, 0x73}, s: scalar.S{Sym: "ifdup"}},
91+
{r: [2]byte{0x74, 0x74}, s: scalar.S{Sym: "depth"}},
92+
{r: [2]byte{0x75, 0x75}, s: scalar.S{Sym: "drop"}},
93+
{r: [2]byte{0x76, 0x76}, s: scalar.S{Sym: "dup"}},
94+
{r: [2]byte{0x77, 0x77}, s: scalar.S{Sym: "nip"}},
95+
{r: [2]byte{0x78, 0x78}, s: scalar.S{Sym: "over"}},
96+
{r: [2]byte{0x79, 0x79}, s: scalar.S{Sym: "pick"}},
97+
{r: [2]byte{0x7a, 0x7a}, s: scalar.S{Sym: "roll"}},
98+
{r: [2]byte{0x7b, 0x7b}, s: scalar.S{Sym: "rot"}},
99+
{r: [2]byte{0x7c, 0x7c}, s: scalar.S{Sym: "swap"}},
100+
{r: [2]byte{0x7d, 0x7d}, s: scalar.S{Sym: "tuck"}},
101+
{r: [2]byte{0x7e, 0x7e}, s: scalar.S{Sym: "cat"}},
102+
{r: [2]byte{0x7f, 0x7f}, s: scalar.S{Sym: "split"}},
103+
{r: [2]byte{0x80, 0x80}, s: scalar.S{Sym: "num2bin"}},
104+
{r: [2]byte{0x81, 0x81}, s: scalar.S{Sym: "bin2num"}},
105+
{r: [2]byte{0x82, 0x82}, s: scalar.S{Sym: "size"}},
106+
{r: [2]byte{0x83, 0x83}, s: scalar.S{Sym: "invert"}},
107+
{r: [2]byte{0x84, 0x84}, s: scalar.S{Sym: "and"}},
108+
{r: [2]byte{0x85, 0x85}, s: scalar.S{Sym: "or"}},
109+
{r: [2]byte{0x86, 0x86}, s: scalar.S{Sym: "xor"}},
110+
{r: [2]byte{0x87, 0x87}, s: scalar.S{Sym: "equal"}},
111+
{r: [2]byte{0x88, 0x88}, s: scalar.S{Sym: "equalverify"}},
112+
{r: [2]byte{0x89, 0x89}, s: scalar.S{Sym: "reserved1"}},
113+
{r: [2]byte{0x8a, 0x8a}, s: scalar.S{Sym: "reserved2"}},
114+
{r: [2]byte{0x8b, 0x8b}, s: scalar.S{Sym: "1add"}},
115+
{r: [2]byte{0x8c, 0x8c}, s: scalar.S{Sym: "1sub"}},
116+
{r: [2]byte{0x8d, 0x8d}, s: scalar.S{Sym: "2mul"}},
117+
{r: [2]byte{0x8e, 0x8e}, s: scalar.S{Sym: "2div"}},
118+
{r: [2]byte{0x8f, 0x8f}, s: scalar.S{Sym: "negate"}},
119+
{r: [2]byte{0x90, 0x90}, s: scalar.S{Sym: "abs"}},
120+
{r: [2]byte{0x91, 0x91}, s: scalar.S{Sym: "not"}},
121+
{r: [2]byte{0x92, 0x92}, s: scalar.S{Sym: "0notequal"}},
122+
{r: [2]byte{0x93, 0x93}, s: scalar.S{Sym: "add"}},
123+
{r: [2]byte{0x94, 0x94}, s: scalar.S{Sym: "sub"}},
124+
{r: [2]byte{0x95, 0x95}, s: scalar.S{Sym: "mul"}},
125+
{r: [2]byte{0x96, 0x96}, s: scalar.S{Sym: "div"}},
126+
{r: [2]byte{0x97, 0x97}, s: scalar.S{Sym: "mod"}},
127+
{r: [2]byte{0x98, 0x98}, s: scalar.S{Sym: "lshift"}},
128+
{r: [2]byte{0x99, 0x99}, s: scalar.S{Sym: "rshift"}},
129+
{r: [2]byte{0x9a, 0x9a}, s: scalar.S{Sym: "booland"}},
130+
{r: [2]byte{0x9b, 0x9b}, s: scalar.S{Sym: "boolor"}},
131+
{r: [2]byte{0x9c, 0x9c}, s: scalar.S{Sym: "numequal"}},
132+
{r: [2]byte{0x9d, 0x9d}, s: scalar.S{Sym: "numequalverify"}},
133+
{r: [2]byte{0x9e, 0x9e}, s: scalar.S{Sym: "numnotequal"}},
134+
{r: [2]byte{0x9f, 0x9f}, s: scalar.S{Sym: "lessthan"}},
135+
{r: [2]byte{0xa0, 0xa0}, s: scalar.S{Sym: "greaterthan"}},
136+
{r: [2]byte{0xa1, 0xa1}, s: scalar.S{Sym: "lessthanorequal"}},
137+
{r: [2]byte{0xa2, 0xa2}, s: scalar.S{Sym: "greaterthanorequal"}},
138+
{r: [2]byte{0xa3, 0xa3}, s: scalar.S{Sym: "min"}},
139+
{r: [2]byte{0xa4, 0xa4}, s: scalar.S{Sym: "max"}},
140+
{r: [2]byte{0xa5, 0xa5}, s: scalar.S{Sym: "within"}},
141+
{r: [2]byte{0xa6, 0xa6}, s: scalar.S{Sym: "ripemd160"}},
142+
{r: [2]byte{0xa7, 0xa7}, s: scalar.S{Sym: "sha1"}},
143+
{r: [2]byte{0xa8, 0xa8}, s: scalar.S{Sym: "sha256"}},
144+
{r: [2]byte{0xa9, 0xa9}, s: scalar.S{Sym: "hash160"}},
145+
{r: [2]byte{0xaa, 0xaa}, s: scalar.S{Sym: "hash256"}},
146+
{r: [2]byte{0xab, 0xab}, s: scalar.S{Sym: "codeseparator"}},
147+
{r: [2]byte{0xac, 0xac}, s: scalar.S{Sym: "checksig"}},
148+
{r: [2]byte{0xad, 0xad}, s: scalar.S{Sym: "checksigverify"}},
149+
{r: [2]byte{0xae, 0xae}, s: scalar.S{Sym: "checkmultisig"}},
150+
{r: [2]byte{0xaf, 0xaf}, s: scalar.S{Sym: "checkmultisigverify"}},
151+
{r: [2]byte{0xb0, 0xb0}, s: scalar.S{Sym: "nop1"}},
152+
{r: [2]byte{0xb1, 0xb1}, s: scalar.S{Sym: "nop2"}},
153+
{r: [2]byte{0xb1, 0xb1}, s: scalar.S{Sym: "checklocktimeverify"}},
154+
{r: [2]byte{0xb2, 0xb2}, s: scalar.S{Sym: "nop3"}},
155+
{r: [2]byte{0xb2, 0xb2}, s: scalar.S{Sym: "checksequenceverify"}},
156+
{r: [2]byte{0xb3, 0xb3}, s: scalar.S{Sym: "nop4"}},
157+
{r: [2]byte{0xb4, 0xb4}, s: scalar.S{Sym: "nop5"}},
158+
{r: [2]byte{0xb5, 0xb5}, s: scalar.S{Sym: "nop6"}},
159+
{r: [2]byte{0xb6, 0xb6}, s: scalar.S{Sym: "nop7"}},
160+
{r: [2]byte{0xb7, 0xb7}, s: scalar.S{Sym: "nop8"}},
161+
{r: [2]byte{0xb8, 0xb8}, s: scalar.S{Sym: "nop9"}},
162+
{r: [2]byte{0xb9, 0xb9}, s: scalar.S{Sym: "nop10"}},
163+
{r: [2]byte{0xba, 0xba}, s: scalar.S{Sym: "checkdatasig"}},
164+
{r: [2]byte{0xbb, 0xbb}, s: scalar.S{Sym: "checkdatasigverif"}},
165+
{r: [2]byte{0xfa, 0xfa}, s: scalar.S{Sym: "smallinteger"}},
166+
{r: [2]byte{0xfb, 0xfb}, s: scalar.S{Sym: "pubkeys"}},
167+
{r: [2]byte{0xfc, 0xfc}, s: scalar.S{Sym: "unknown252"}},
168+
{r: [2]byte{0xfd, 0xfd}, s: scalar.S{Sym: "pubkeyhash"}},
169+
{r: [2]byte{0xfe, 0xfe}, s: scalar.S{Sym: "pubkey"}},
170+
{r: [2]byte{0xff, 0xff}, s: scalar.S{Sym: "invalidopcode"}},
171+
}
172+
173+
for !d.End() {
174+
opcode := byte(d.PeekBits(8))
175+
ope, ok := opcodeEntries.lookup(opcode)
176+
if !ok {
177+
d.Fatalf("unknown opcode %x", opcode)
178+
}
179+
180+
d.FieldStruct("opcode", func(d *decode.D) {
181+
d.FieldU8("op", opcodeEntries)
182+
if ope.d != nil {
183+
ope.d(d, opcode)
184+
}
185+
})
186+
}
187+
188+
return nil
189+
}

0 commit comments

Comments
 (0)