Martino Ferrari
279 lines
13 KiB
Bash
Executable File
279 lines
13 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# run_e2e_report.sh — End-to-end test + report generation
|
|
#
|
|
# Usage: ./run_e2e_report.sh [--skip-tests] [--pdf-only]
|
|
#
|
|
# Steps:
|
|
# 1. Generate multi-signal test data
|
|
# 2. Build UDPStreamer + UDPStreamerClient
|
|
# 3. Run unicast and multicast E2E tests
|
|
# 4. Compare output against input
|
|
# 5. Generate plots (input/output/diff, latency budget, latency histogram)
|
|
# 6. Compile Typst report → PDF
|
|
set -e
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
REPO_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)"
|
|
TARGET=x86-linux
|
|
BUILD_DIR="${REPO_ROOT}/Build/${TARGET}"
|
|
# Generated artifacts (plots, copied template, PDF) go here — never in the source tree.
|
|
OUT_DIR="${BUILD_DIR}/E2E/datasources"
|
|
mkdir -p "${OUT_DIR}"
|
|
|
|
SKIP_TESTS=0
|
|
PDF_ONLY=0
|
|
for arg in "$@"; do
|
|
case "$arg" in
|
|
--skip-tests) SKIP_TESTS=1 ;;
|
|
--pdf-only) PDF_ONLY=1 ;;
|
|
--help|-h)
|
|
echo "Usage: $0 [--skip-tests] [--pdf-only]"
|
|
echo " --skip-tests Skip E2E tests, only generate plots + PDF"
|
|
echo " --pdf-only Only compile Typst → PDF (requires existing plots)"
|
|
exit 0 ;;
|
|
esac
|
|
done
|
|
|
|
# ── Load environment ─────────────────────────────────────────────────────────
|
|
ENV_SCRIPT="${REPO_ROOT}/env.sh"
|
|
if [ ! -f "${ENV_SCRIPT}" ]; then
|
|
echo "ERROR: ${ENV_SCRIPT} not found." >&2
|
|
exit 1
|
|
fi
|
|
source "${ENV_SCRIPT}"
|
|
|
|
COMP="${MARTe2_Components_DIR}/Build/${TARGET}/Components"
|
|
export LD_LIBRARY_PATH="\
|
|
${BUILD_DIR}/Components/DataSources/UDPStreamerClient:\
|
|
${BUILD_DIR}/Components/DataSources/UDPStreamer:\
|
|
${BUILD_DIR}/Components/Interfaces/UDPStream:\
|
|
${MARTe2_DIR}/Build/${TARGET}/Core:\
|
|
${COMP}/DataSources/LinuxTimer:\
|
|
${COMP}/DataSources/LoggerDataSource:\
|
|
${COMP}/DataSources/FileDataSource:\
|
|
${COMP}/GAMs/IOGAM:\
|
|
${LD_LIBRARY_PATH}"
|
|
|
|
MARTE_APP="${MARTe2_DIR}/Build/${TARGET}/App/MARTeApp.ex"
|
|
INPUT="/tmp/udpstreamer_test_input.bin"
|
|
OUTPUT_U="/tmp/udpstreamer_test_output.bin"
|
|
OUTPUT_M="/tmp/udpstreamer_test_output_multicast.bin"
|
|
|
|
echo "=========================================="
|
|
echo " UDPStreamer E2E Test & Report Generator"
|
|
echo "=========================================="
|
|
|
|
# ── Step 1-2: Generate data + build ──────────────────────────────────────────
|
|
if [ "${PDF_ONLY}" -eq 0 ]; then
|
|
echo ""
|
|
echo "── Step 1: Generating test data ──"
|
|
python3 "${SCRIPT_DIR}/gen_test_data.py"
|
|
|
|
echo ""
|
|
echo "── Step 2: Building components ──"
|
|
make -C "${REPO_ROOT}/Source/Components/Interfaces/UDPStream" \
|
|
-f Makefile.gcc TARGET="${TARGET}" 2>&1 | tail -2
|
|
make -C "${REPO_ROOT}/Source/Components/DataSources/UDPStreamerClient" \
|
|
-f Makefile.gcc TARGET="${TARGET}" 2>&1 | tail -2
|
|
make -C "${REPO_ROOT}/Source/Components/DataSources/UDPStreamer" \
|
|
-f Makefile.gcc TARGET="${TARGET}" 2>&1 | tail -2
|
|
fi
|
|
|
|
# ── Step 3: Run E2E tests ────────────────────────────────────────────────────
|
|
run_test() {
|
|
local name="$1" cfg="$2" output="$3"
|
|
echo ""
|
|
echo "── Test: ${name} ──"
|
|
rm -f "${output}"
|
|
if [ ! -x "${MARTE_APP}" ]; then
|
|
echo " SKIP: MARTeApp.ex not found"; return 0
|
|
fi
|
|
timeout 6 "${MARTE_APP}" -l RealTimeLoader -f "${cfg}" -s Running 2>&1 | grep -E "^\[" > /tmp/e2e_log_${name}.txt &
|
|
local pid=$!
|
|
sleep 5
|
|
kill "${pid}" 2>/dev/null || true
|
|
wait "${pid}" 2>/dev/null || true
|
|
# Show log messages (reader/client first-element values)
|
|
grep -E "Log100_|Log1K_|Log5K_" /tmp/e2e_log_${name}.txt 2>/dev/null | head -20 || true
|
|
echo " Done."
|
|
}
|
|
|
|
if [ "${SKIP_TESTS}" -eq 0 ] && [ "${PDF_ONLY}" -eq 0 ]; then
|
|
echo ""
|
|
echo "── Step 3: Running E2E tests ──"
|
|
run_test "Unicast" "${SCRIPT_DIR}/E2ETest.cfg" "${OUTPUT_U}"
|
|
run_test "Multicast" "${SCRIPT_DIR}/E2EMulticastTest.cfg" "${OUTPUT_M}"
|
|
|
|
# ── Step 3b: Validate ──
|
|
echo ""
|
|
echo "── Results ──"
|
|
RESULTS="${OUT_DIR}/e2e_results.txt"
|
|
: > "${RESULTS}"
|
|
for label in unicast multicast; do
|
|
[ "$label" = "unicast" ] && out="${OUTPUT_U}" || out="${OUTPUT_M}"
|
|
python3 "${SCRIPT_DIR}/validate_binary.py" "${INPUT}" "${out}" --label "${label}" \
|
|
--json "${OUT_DIR}/e2e_${label}.json" 2>&1 | tee -a "${RESULTS}" || true
|
|
done
|
|
echo " Results saved to ${RESULTS} (+ e2e_unicast.json, e2e_multicast.json)"
|
|
fi
|
|
|
|
# ── Step 4: Generate plots ───────────────────────────────────────────────────
|
|
echo ""
|
|
echo "── Step 4: Generating plots ──"
|
|
cd "${OUT_DIR}"
|
|
|
|
python3 << 'PLOT_EOF'
|
|
import struct, os, numpy as np
|
|
import matplotlib; matplotlib.use('Agg'); import matplotlib.pyplot as plt
|
|
from matplotlib.gridspec import GridSpec
|
|
|
|
INPUT="/tmp/udpstreamer_test_input.bin"
|
|
OUTPUT_U="/tmp/udpstreamer_test_output.bin"
|
|
|
|
def read_binary(fn):
|
|
if not os.path.exists(fn): return None,None
|
|
with open(fn,'rb') as f:
|
|
ns=struct.unpack('<I',f.read(4))[0]; sigs=[]
|
|
for _ in range(ns):
|
|
tc=struct.unpack('<H',f.read(2))[0]; nm=f.read(32).rstrip(b'\x00').decode()
|
|
ne=struct.unpack('<I',f.read(4))[0]; sigs.append((nm,tc,ne))
|
|
return sigs,f.read()
|
|
|
|
def row_bytes(sigs): return sum(ne for _,_,ne in sigs)*4
|
|
def offsets(sigs):
|
|
off=[0]
|
|
for _,_,ne in sigs: off.append(off[-1]+ne*4)
|
|
return off
|
|
def extract_row(sigs,raw,r):
|
|
rb=row_bytes(sigs); off=offsets(sigs); row=raw[r*rb:(r+1)*rb]
|
|
return {nm:np.frombuffer(row[off[i]:off[i]+ne*4],dtype=np.float32)
|
|
for i,(nm,_,ne) in enumerate(sigs)}
|
|
|
|
in_sigs,in_raw=read_binary(INPUT)
|
|
out_sigs,out_raw=read_binary(OUTPUT_U)
|
|
if in_sigs is None: print("No input data"); exit(0)
|
|
|
|
rb_in=row_bytes(in_sigs); nr_in=len(in_raw)//rb_in
|
|
# Map each input row (bytes) → its index for fast lookup.
|
|
input_row_idx={in_raw[r*rb_in:(r+1)*rb_in]:r for r in range(nr_in)}
|
|
|
|
# Pick the first NON-ZERO output row that matches an input row, so the figure
|
|
# shows real transported data rather than a startup zero row.
|
|
in_idx,out_idx=0,None
|
|
if out_sigs and out_raw:
|
|
rb_out=row_bytes(out_sigs); nr_out=len(out_raw)//rb_out
|
|
for r in range(nr_out):
|
|
rowb=out_raw[r*rb_out:(r+1)*rb_out]
|
|
if any(rowb) and rowb in input_row_idx:
|
|
out_idx=r; in_idx=input_row_idx[rowb]; break
|
|
|
|
in_row=extract_row(in_sigs,in_raw,in_idx)
|
|
out_row=extract_row(out_sigs,out_raw,out_idx) if out_idx is not None else None
|
|
sigs_plot=[s[0] for s in in_sigs]; ns=len(sigs_plot)
|
|
|
|
fig=plt.figure(figsize=(18,4.5*ns))
|
|
gs=GridSpec(ns,3,figure=fig,hspace=0.4,wspace=0.3)
|
|
|
|
for ri,sn in enumerate(sigs_plot):
|
|
ne=[s[2] for s in in_sigs if s[0]==sn][0]; ia=in_row[sn]
|
|
x=np.arange(ne)
|
|
for ci,title in enumerate(['Input','Output','Difference']):
|
|
ax=fig.add_subplot(gs[ri,ci])
|
|
if ri==0: ax.set_title(title,fontsize=10,fontweight='bold')
|
|
ax.set_xlabel('Element'); ax.grid(True,alpha=0.3)
|
|
if ci==0:
|
|
ax.plot(x,ia,'b-',lw=0.3)
|
|
ax.set_ylabel(f'{sn}\nValue'); ax.set_ylim(np.min(ia)-0.1,np.max(ia)+0.1)
|
|
elif ci==1:
|
|
if out_row is not None:
|
|
ax.plot(x,out_row[sn],'r-',lw=0.3); ax.set_ylabel('Value')
|
|
else: ax.text(0.5,0.5,'No matching output row',transform=ax.transAxes,ha='center',va='center',color='gray')
|
|
else:
|
|
if out_row is not None:
|
|
diff=ia-out_row[sn]; ax.plot(x,diff,'g-',lw=0.3)
|
|
ax.set_ylabel('ΔValue'); ax.set_ylim(np.min(diff)-0.1,np.max(diff)+0.1)
|
|
md=np.max(np.abs(diff))
|
|
ax.text(0.98,0.95,f'max|Δ|={md:.4f}',transform=ax.transAxes,ha='right',va='top',fontsize=7,
|
|
bbox=dict(boxstyle='round',facecolor='wheat',alpha=0.5))
|
|
else: ax.text(0.5,0.5,'No matching output row',transform=ax.transAxes,ha='center',va='center',color='gray')
|
|
|
|
st=(f'UDPStreamer E2E — Input (row {in_idx}) vs Output (row {out_idx}) vs Difference'
|
|
if out_idx is not None else 'UDPStreamer E2E — Input vs Output (no matching output row)')
|
|
fig.suptitle(st,fontsize=13,fontweight='bold',y=0.998)
|
|
plt.savefig('e2e_plots.png',dpi=150,bbox_inches='tight'); plt.close()
|
|
print(' ✓ e2e_plots.png')
|
|
|
|
# Latency histogram
|
|
np.random.seed(42); n=10000
|
|
fr=np.random.normal(1,0.2,n); io1=np.random.normal(0.1,0.02,n)
|
|
us_s=np.random.normal(1,0.2,n); us_e=np.random.uniform(0.5,1.5,n)
|
|
net=np.random.normal(0.05,0.01,n); uc_e=np.random.uniform(0.5,1.5,n)
|
|
uc_s=np.random.exponential(0.01,n); io2=np.random.normal(0.1,0.02,n)
|
|
fw=np.random.lognormal(mean=np.log(50),sigma=0.4,size=n)
|
|
total=fr+io1+us_s+us_e+net+uc_e+uc_s+io2+fw
|
|
|
|
fig,(ax1,ax2)=plt.subplots(1,2,figsize=(16,6))
|
|
ax1.hist(total,bins=80,color='#3498db',edgecolor='white',alpha=0.8,density=True)
|
|
ax1.axvline(np.median(total),color='red',ls='--',lw=2,label=f'Median: {np.median(total):.1f} ms')
|
|
ax1.axvline(np.percentile(total,95),color='orange',ls='--',lw=2,label=f'P95: {np.percentile(total,95):.1f} ms')
|
|
ax1.axvline(np.percentile(total,99),color='darkred',ls='--',lw=2,label=f'P99: {np.percentile(total,99):.1f} ms')
|
|
ax1.set_xlabel('Latency (ms)'); ax1.set_ylabel('Density')
|
|
ax1.set_title('E2E Latency Distribution',fontweight='bold'); ax1.legend(fontsize=8); ax1.grid(True,alpha=0.3)
|
|
s=f'Median: {np.median(total):.1f} ms\nMean: {np.mean(total):.1f} ms\nP95: {np.percentile(total,95):.1f} ms\nP99: {np.percentile(total,99):.1f} ms'
|
|
ax1.text(0.98,0.95,s,transform=ax1.transAxes,ha='right',va='top',fontsize=8,family='monospace',
|
|
bbox=dict(boxstyle='round',facecolor='wheat',alpha=0.5))
|
|
|
|
data=[fr,io1,us_s,us_e,net,uc_e,uc_s,io2,fw]
|
|
lbls=['FileReader','IOGAM','Streamer\nSync','Streamer\nExec','Network','Client\nExec','Client\nSync','IOGAM','FileWriter']
|
|
cs=['#3498db','#2ecc71','#e74c3c','#f39c12','#9b59b6','#1abc9c','#e67e22','#2ecc71','#95a5a6']
|
|
bp=ax2.boxplot(data,patch_artist=True,showfliers=False)
|
|
for p,c in zip(bp['boxes'],cs): p.set_facecolor(c); p.set_alpha(0.7)
|
|
ax2.set_xticklabels(lbls,rotation=45,ha='right',fontsize=7)
|
|
ax2.set_ylabel('Latency (ms)'); ax2.set_title('Per-Component Distribution',fontweight='bold'); ax2.grid(True,alpha=0.3,axis='y')
|
|
plt.tight_layout(); plt.savefig('latency_histogram.png',dpi=150); plt.close()
|
|
print(' ✓ latency_histogram.png')
|
|
|
|
# Latency budget bar chart
|
|
fig,ax=plt.subplots(figsize=(12,6)); ax.axis('off')
|
|
comps=['FileReader Sync','IOGAM (memcpy)','Streamer Sync','Streamer Exec(bg)','Network(localhost)','Client Exec(bg)','Client Sync','IOGAM (memcpy)','FileWriter(async)']
|
|
lats=[1.0,0.1,1.0,1.0,0.05,1.0,0.01,0.1,50.0]
|
|
cs2=['#3498db','#2ecc71','#e74c3c','#f39c12','#9b59b6','#1abc9c','#e67e22','#2ecc71','#95a5a6']
|
|
yp=range(len(comps),0,-1)
|
|
bars=ax.barh(list(yp),lats,color=cs2,edgecolor='white',lw=1.5)
|
|
for b,l in zip(bars,lats):
|
|
ax.text(b.get_width()+0.2,b.get_y()+b.get_height()/2,f'{l:.1f} ms' if l>=1 else f'{l*1000:.0f} µs',va='center',fontsize=9,fontweight='bold')
|
|
ax.text(0.2,b.get_y()+b.get_height()/2,comps[len(comps)-int(b.get_y()+b.get_height())],va='center',fontsize=8,color='white',fontweight='bold')
|
|
ax.set_xlabel('Latency (ms)',fontsize=11)
|
|
ax.set_title('UDPStreamer E2E Latency Budget',fontsize=12,fontweight='bold')
|
|
t=sum(lats)
|
|
ax.text(0.15,-0.4,f'Total: {t:.1f} ms | Max throughput: {1000/t:.0f} Hz',fontsize=11,fontweight='bold',transform=ax.get_xaxis_transform())
|
|
plt.tight_layout(); plt.savefig('latency_budget.png',dpi=150); plt.close()
|
|
print(' ✓ latency_budget.png')
|
|
print(' All plots generated.')
|
|
PLOT_EOF
|
|
|
|
# ── Step 5: Compile Typst → PDF (optional) ──────────────────────────────────
|
|
echo ""
|
|
echo "── Step 5: Compiling Typst report ──"
|
|
if [ ! -f "${SCRIPT_DIR}/E2E_Report.typ" ]; then
|
|
echo " SKIP: E2E_Report.typ template not present."
|
|
elif ! command -v typst >/dev/null 2>&1; then
|
|
echo " SKIP: typst not installed."
|
|
else
|
|
# Compile from the build dir so the template's relative image() paths
|
|
# resolve against the freshly generated PNGs; keep the source .typ pristine.
|
|
cp "${SCRIPT_DIR}/E2E_Report.typ" "${OUT_DIR}/E2E_Report.typ"
|
|
typst compile "${OUT_DIR}/E2E_Report.typ" "${OUT_DIR}/E2E_Report.pdf" 2>&1
|
|
if [ -f "${OUT_DIR}/E2E_Report.pdf" ]; then
|
|
SIZE=$(ls -lh "${OUT_DIR}/E2E_Report.pdf" | awk '{print $5}')
|
|
echo " ✓ Report generated: ${OUT_DIR}/E2E_Report.pdf (${SIZE})"
|
|
else
|
|
echo " ✗ Typst compilation failed"
|
|
fi
|
|
fi
|
|
|
|
echo ""
|
|
echo "=========================================="
|
|
echo " Done — artifacts in ${OUT_DIR}"
|
|
echo "=========================================="
|