inlet/flow: replace raw data test files with PCAP files

Raw data files can be converted with Scapy:

```python
from scapy.all import *
wrpcap("data-1140.pcap",
  Ether(src="00:53:00:11:22:33",dst="00:53:00:44:55:66")/
  IP(src="192.0.2.100", dst="192.0.2.101")/
  UDP(sport=47873,dport=6343)/
  open("data-1140.data", "rb").read())
```

common/helpers/tests_pcap.go

@@ -0,0 +1,37 @@
+// SPDX-FileCopyrightText: 2022 Free Mobile
+// SPDX-License-Identifier: AGPL-3.0-only
+
+//go:build !release
+
+package helpers
+
+import (
+	"bytes"
+	"os"
+	"testing"
+
+	"github.com/google/gopacket"
+	"github.com/google/gopacket/layers"
+	"github.com/google/gopacket/pcapgo"
+)
+
+// ReadPcapPayload reads and parses a PCAP file and return the payload (after Layer 4).
+func ReadPcapPayload(t *testing.T, pcapfile string) []byte {
+	t.Helper()
+	f, err := os.Open(pcapfile)
+	if err != nil {
+		t.Fatalf("Open(%q) error:\n%+v", pcapfile, err)
+	}
+	defer f.Close()
+
+	reader, err := pcapgo.NewReader(f)
+	if err != nil {
+		t.Fatalf("NewReader(%q) error:\n%+v", pcapfile, err)
+	}
+	payload := bytes.NewBuffer([]byte{})
+	source := gopacket.NewPacketSource(reader, layers.LayerTypeEthernet)
+	for packet := range source.Packets() {
+		payload.Write(packet.TransportLayer().LayerPayload())
+	}
+	return payload.Bytes()
+}

go.mod

@@ -17,12 +17,14 @@ require (
 	github.com/go-playground/validator/v10 v10.11.0
 	github.com/golang/mock v1.6.0
 	github.com/golang/protobuf v1.5.2
+	github.com/google/gopacket v1.1.19
 	github.com/gosnmp/gosnmp v1.35.0
 	github.com/kentik/patricia v1.2.0
 	github.com/kylelemons/godebug v1.1.0
 	github.com/mattn/go-isatty v0.0.16
 	github.com/mitchellh/mapstructure v1.5.0
 	github.com/netsampler/goflow2 v1.1.0
+	github.com/opencontainers/image-spec v1.0.2
 	github.com/oschwald/maxminddb-golang v1.10.0
 	github.com/prometheus/client_golang v1.13.0
 	github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475
@@ -35,6 +37,7 @@ require (
 	github.com/yuin/goldmark-highlighting v0.0.0-20220208100518-594be1970594
 	golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab
 	golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9
+	google.golang.org/protobuf v1.28.1
 	gopkg.in/tomb.v2 v2.0.0-20161208151619-d5d1b5820637
 	gopkg.in/yaml.v2 v2.4.0
 	gorm.io/driver/sqlite v1.3.6
@@ -88,7 +91,6 @@ require (
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/morikuni/aec v1.0.0 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
-	github.com/opencontainers/image-spec v1.0.2 // indirect
 	github.com/paulmach/orb v0.7.1 // indirect
 	github.com/pelletier/go-toml/v2 v2.0.1 // indirect
 	github.com/pierrec/lz4 v2.6.1+incompatible // indirect
@@ -114,7 +116,6 @@ require (
 	golang.org/x/crypto v0.0.0-20220321153916-2c7772ba3064 // indirect
 	golang.org/x/net v0.0.0-20220225172249-27dd8689420f // indirect
 	golang.org/x/text v0.3.7 // indirect
-	google.golang.org/protobuf v1.28.1 // indirect
 	gotest.tools/v3 v3.3.0 // indirect
 )
 

go.sum

@@ -200,6 +200,8 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
 github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8=
+github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
 github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=

inlet/flow/decoder/netflow/root_test.go

@@ -4,7 +4,6 @@
 package netflow
 
 import (
-	"io/ioutil"
 	"net"
 	"path/filepath"
 	"testing"
@@ -19,10 +18,7 @@ func TestDecode(t *testing.T) {
 	nfdecoder := New(r)
 
 	// Send an option template
-	template, err := ioutil.ReadFile(filepath.Join("testdata", "options-template-257.data"))
+	template := helpers.ReadPcapPayload(t, filepath.Join("testdata", "options-template-257.pcap"))
-	if err != nil {
-		panic(err)
-	}
 	got := nfdecoder.Decode(decoder.RawFlow{Payload: template, Source: net.ParseIP("127.0.0.1")})
 	if got == nil {
 		t.Fatalf("Decode() error on options template")
@@ -44,10 +40,7 @@ func TestDecode(t *testing.T) {
 	}
 
 	// Send option data
-	data, err := ioutil.ReadFile(filepath.Join("testdata", "options-data-257.data"))
+	data := helpers.ReadPcapPayload(t, filepath.Join("testdata", "options-data-257.pcap"))
-	if err != nil {
-		panic(err)
-	}
 	got = nfdecoder.Decode(decoder.RawFlow{Payload: data, Source: net.ParseIP("127.0.0.1")})
 	if got == nil {
 		t.Fatalf("Decode() error on options data")
@@ -71,10 +64,7 @@ func TestDecode(t *testing.T) {
 	}
 
 	// Send a regular template
-	template, err = ioutil.ReadFile(filepath.Join("testdata", "template-260.data"))
+	template = helpers.ReadPcapPayload(t, filepath.Join("testdata", "template-260.pcap"))
-	if err != nil {
-		panic(err)
-	}
 	got = nfdecoder.Decode(decoder.RawFlow{Payload: template, Source: net.ParseIP("127.0.0.1")})
 	if got == nil {
 		t.Fatalf("Decode() error on template")
@@ -101,10 +91,7 @@ func TestDecode(t *testing.T) {
 	}
 
 	// Send data
-	data, err = ioutil.ReadFile(filepath.Join("testdata", "data-260.data"))
+	data = helpers.ReadPcapPayload(t, filepath.Join("testdata", "data-260.pcap"))
-	if err != nil {
-		panic(err)
-	}
 	got = nfdecoder.Decode(decoder.RawFlow{Payload: data, Source: net.ParseIP("127.0.0.1")})
 	if got == nil {
 		t.Fatalf("Decode() error on data")

inlet/flow/decoder/sflow/root_test.go

@@ -4,7 +4,6 @@
 package sflow
 
 import (
-	"io/ioutil"
 	"net"
 	"path/filepath"
 	"testing"
@@ -19,10 +18,7 @@ func TestDecode(t *testing.T) {
 	sdecoder := New(r)
 
 	// Send data
-	data, err := ioutil.ReadFile(filepath.Join("testdata", "data-1140.data"))
+	data := helpers.ReadPcapPayload(t, filepath.Join("testdata", "data-1140.pcap"))
-	if err != nil {
-		panic(err)
-	}
 	got := sdecoder.Decode(decoder.RawFlow{Payload: data, Source: net.ParseIP("127.0.0.1")})
 	if got == nil {
 		t.Fatalf("Decode() error on data")

inlet/flow/root_test.go

@@ -4,11 +4,14 @@
 package flow
 
 import (
+	"fmt"
+	"os"
 	"path"
 	"runtime"
 	"testing"
 	"time"
 
+	"akvorado/common/helpers"
 	"akvorado/common/reporter"
 	"akvorado/inlet/flow/input/file"
 )
@@ -17,28 +20,30 @@ func TestFlow(t *testing.T) {
 	var nominalRate int
 	_, src, _, _ := runtime.Caller(0)
 	base := path.Join(path.Dir(src), "decoder", "netflow", "testdata")
+	outDir := t.TempDir()
+	outFiles := []string{}
+	for idx, f := range []string{
+		"options-template-257.pcap",
+		"options-data-257.pcap",
+		"template-260.pcap",
+		"data-260.pcap", "data-260.pcap", "data-260.pcap", "data-260.pcap",
+		"data-260.pcap", "data-260.pcap", "data-260.pcap", "data-260.pcap",
+		"data-260.pcap", "data-260.pcap", "data-260.pcap", "data-260.pcap",
+		"data-260.pcap", "data-260.pcap", "data-260.pcap", "data-260.pcap",
+	} {
+		outFile := path.Join(outDir, fmt.Sprintf("data-%d", idx))
+		err := os.WriteFile(outFile, helpers.ReadPcapPayload(t, path.Join(base, f)), 0666)
+		if err != nil {
+			t.Fatalf("WriteFile(%q) error:\n%+v", outFile, err)
+		}
+		outFiles = append(outFiles, outFile)
+	}
+
 	inputs := []InputConfiguration{
 		{
 			Decoder: "netflow",
 			Config: &file.Configuration{
-				Paths: []string{
+				Paths: outFiles,
-					path.Join(base, "options-template-257.data"),
-					path.Join(base, "options-data-257.data"),
-					path.Join(base, "template-260.data"),
-					path.Join(base, "data-260.data"),
-					path.Join(base, "data-260.data"),
-					path.Join(base, "data-260.data"),
-					path.Join(base, "data-260.data"),
-					path.Join(base, "data-260.data"),
-					path.Join(base, "data-260.data"),
-					path.Join(base, "data-260.data"),
-					path.Join(base, "data-260.data"),
-					path.Join(base, "data-260.data"),
-					path.Join(base, "data-260.data"),
-					path.Join(base, "data-260.data"),
-					path.Join(base, "data-260.data"),
-					path.Join(base, "data-260.data"),
-				},
 			},
 		},
 	}