diff --git a/CHANGELOG.md b/CHANGELOG.md index 014ee3817..271647a1d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,6 +47,7 @@ OpenTelemetry Go Automatic Instrumentation adheres to [Semantic Versioning](http - Support `google.golang.org/grpc` `1.67.1`. ([#1143](https://github.com/open-telemetry/opentelemetry-go-instrumentation/pull/1143)) - Support Go `1.22.8`. ([#1143](https://github.com/open-telemetry/opentelemetry-go-instrumentation/pull/1143)) - Support Go `1.23.2`. ([#1143](https://github.com/open-telemetry/opentelemetry-go-instrumentation/pull/1143)) +- Add gRPC status code attribute for server spans (`rpc.grpc.status_code`). ([#1127](https://github.com/open-telemetry/opentelemetry-go-instrumentation/pull/1127)) ### Changed diff --git a/go.mod b/go.mod index a7379bafa..094896416 100644 --- a/go.mod +++ b/go.mod @@ -30,6 +30,7 @@ require ( go.opentelemetry.io/otel/trace v1.31.0 golang.org/x/arch v0.11.0 golang.org/x/sys v0.26.0 + google.golang.org/grpc v1.67.1 gopkg.in/yaml.v3 v3.0.1 ) @@ -75,6 +76,5 @@ require ( golang.org/x/text v0.19.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20241007155032-5fefd90f89a9 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20241007155032-5fefd90f89a9 // indirect - google.golang.org/grpc v1.67.1 // indirect google.golang.org/protobuf v1.35.1 // indirect ) diff --git a/internal/pkg/instrumentation/bpf/google.golang.org/grpc/server/bpf/probe.bpf.c b/internal/pkg/instrumentation/bpf/google.golang.org/grpc/server/bpf/probe.bpf.c index c33e96040..04b1c6fd2 100644 --- a/internal/pkg/instrumentation/bpf/google.golang.org/grpc/server/bpf/probe.bpf.c +++ b/internal/pkg/instrumentation/bpf/google.golang.org/grpc/server/bpf/probe.bpf.c @@ -19,6 +19,7 @@ struct grpc_request_t { BASE_SPAN_PROPERTIES char method[MAX_SIZE]; + u32 status_code; }; struct @@ -59,6 +60,8 @@ volatile const u64 frame_stream_id_pod; volatile const u64 stream_id_pos; volatile const u64 stream_ctx_pos; volatile const bool is_new_frame_pos; +volatile const u64 status_s_pos; +volatile const u64 status_code_pos; static __always_inline long dummy_extract_span_context_from_headers(void *stream_id, struct span_context *parent_span_context) { return 0; @@ -113,14 +116,14 @@ int uprobe_server_handleStream(struct pt_regs *ctx) if (!get_go_string_from_user_ptr((void *)(stream_ptr + stream_method_ptr_pos), grpcReq->method, sizeof(grpcReq->method))) { bpf_printk("Failed to read gRPC method from stream"); - goto done; + bpf_map_delete_elem(&streamid_to_grpc_events, &stream_id); + return 0; } // Write event bpf_map_update_elem(&grpc_events, &key, grpcReq, 0); start_tracking_span(go_context.data, &grpcReq->sc); -done: - bpf_map_delete_elem(&streamid_to_grpc_events, &stream_id); + return 0; } @@ -167,3 +170,65 @@ int uprobe_http2Server_operateHeader(struct pt_regs *ctx) return 0; } + +// func (ht *http2Server) WriteStatus(s *Stream, st *status.Status) +// https://github.com/grpc/grpc-go/blob/bcf9171a20e44ed81a6eb152e3ca9e35b2c02c5d/internal/transport/http2_server.go#L1049 +SEC("uprobe/http2Server_WriteStatus") +int uprobe_http2Server_WriteStatus(struct pt_regs *ctx) { + struct go_iface go_context = {0}; + get_Go_context(ctx, 2, stream_ctx_pos, true, &go_context); + void *key = get_consistent_key(ctx, go_context.data); + + // Get parent context if exists + void *stream_ptr = get_argument(ctx, 2); + u32 stream_id = 0; + bpf_probe_read(&stream_id, sizeof(stream_id), (void *)(stream_ptr + stream_id_pos)); + struct grpc_request_t *grpcReq = bpf_map_lookup_elem(&streamid_to_grpc_events, &stream_id); + if (grpcReq == NULL) { + // No parent span context, generate new span context + u32 zero = 0; + grpcReq = bpf_map_lookup_elem(&grpc_storage_map, &zero); + if (grpcReq == NULL) { + bpf_printk("failed to get grpcReq from storage map"); + return -1; + } + } + + void *grpcReq_event_ptr = bpf_map_lookup_elem(&grpc_events, &key); + // if grpcReq_event is null, then handleStream probe didn't run. Try starting a new span here + if (grpcReq_event_ptr == NULL) + { + grpcReq->start_time = bpf_ktime_get_ns(); + + start_span_params_t start_span_params = { + .ctx = ctx, + .sc = &grpcReq->sc, + .psc = &grpcReq->psc, + .go_context = &go_context, + // The parent span context is set by operateHeader probe + .get_parent_span_context_fn = dummy_extract_span_context_from_headers, + .get_parent_span_context_arg = NULL, + }; + start_span(&start_span_params); + + // Set attributes + if (!get_go_string_from_user_ptr((void *)(stream_ptr + stream_method_ptr_pos), grpcReq->method, sizeof(grpcReq->method))) + { + bpf_printk("Failed to read gRPC method from stream"); + bpf_map_delete_elem(&streamid_to_grpc_events, &stream_id); + return 0; + } + } + + void *status_ptr = get_argument(ctx, 3); + void *s_ptr = 0; + bpf_probe_read_user(&s_ptr, sizeof(s_ptr), (void *)(status_ptr + status_s_pos)); + // Get status code from Status.s pointer + bpf_probe_read_user(&grpcReq->status_code, sizeof(grpcReq->status_code), (void *)(s_ptr + status_code_pos)); + + bpf_map_update_elem(&grpc_events, &key, grpcReq, 0); + bpf_map_delete_elem(&streamid_to_grpc_events, &stream_id); + return 0; +} + +UPROBE_RETURN(http2Server_WriteStatus, struct grpc_request_t, grpc_events, events, 2, stream_ctx_pos, false) diff --git a/internal/pkg/instrumentation/bpf/google.golang.org/grpc/server/bpf_arm64_bpfel.go b/internal/pkg/instrumentation/bpf/google.golang.org/grpc/server/bpf_arm64_bpfel.go index 9317e8cb7..b7ef4e073 100644 --- a/internal/pkg/instrumentation/bpf/google.golang.org/grpc/server/bpf_arm64_bpfel.go +++ b/internal/pkg/instrumentation/bpf/google.golang.org/grpc/server/bpf_arm64_bpfel.go @@ -13,12 +13,12 @@ import ( ) type bpfGrpcRequestT struct { - StartTime uint64 - EndTime uint64 - Sc bpfSpanContext - Psc bpfSpanContext - Method [100]int8 - _ [4]byte + StartTime uint64 + EndTime uint64 + Sc bpfSpanContext + Psc bpfSpanContext + Method [100]int8 + StatusCode uint32 } type bpfSliceArrayBuff struct{ Buff [1024]uint8 } @@ -71,9 +71,11 @@ type bpfSpecs struct { // // It can be passed ebpf.CollectionSpec.Assign. type bpfProgramSpecs struct { - UprobeHttp2ServerOperateHeader *ebpf.ProgramSpec `ebpf:"uprobe_http2Server_operateHeader"` - UprobeServerHandleStream *ebpf.ProgramSpec `ebpf:"uprobe_server_handleStream"` - UprobeServerHandleStreamReturns *ebpf.ProgramSpec `ebpf:"uprobe_server_handleStream_Returns"` + UprobeHttp2ServerWriteStatus *ebpf.ProgramSpec `ebpf:"uprobe_http2Server_WriteStatus"` + UprobeHttp2ServerWriteStatusReturns *ebpf.ProgramSpec `ebpf:"uprobe_http2Server_WriteStatus_Returns"` + UprobeHttp2ServerOperateHeader *ebpf.ProgramSpec `ebpf:"uprobe_http2Server_operateHeader"` + UprobeServerHandleStream *ebpf.ProgramSpec `ebpf:"uprobe_server_handleStream"` + UprobeServerHandleStreamReturns *ebpf.ProgramSpec `ebpf:"uprobe_server_handleStream_Returns"` } // bpfMapSpecs contains maps before they are loaded into the kernel. @@ -136,13 +138,17 @@ func (m *bpfMaps) Close() error { // // It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign. type bpfPrograms struct { - UprobeHttp2ServerOperateHeader *ebpf.Program `ebpf:"uprobe_http2Server_operateHeader"` - UprobeServerHandleStream *ebpf.Program `ebpf:"uprobe_server_handleStream"` - UprobeServerHandleStreamReturns *ebpf.Program `ebpf:"uprobe_server_handleStream_Returns"` + UprobeHttp2ServerWriteStatus *ebpf.Program `ebpf:"uprobe_http2Server_WriteStatus"` + UprobeHttp2ServerWriteStatusReturns *ebpf.Program `ebpf:"uprobe_http2Server_WriteStatus_Returns"` + UprobeHttp2ServerOperateHeader *ebpf.Program `ebpf:"uprobe_http2Server_operateHeader"` + UprobeServerHandleStream *ebpf.Program `ebpf:"uprobe_server_handleStream"` + UprobeServerHandleStreamReturns *ebpf.Program `ebpf:"uprobe_server_handleStream_Returns"` } func (p *bpfPrograms) Close() error { return _BpfClose( + p.UprobeHttp2ServerWriteStatus, + p.UprobeHttp2ServerWriteStatusReturns, p.UprobeHttp2ServerOperateHeader, p.UprobeServerHandleStream, p.UprobeServerHandleStreamReturns, diff --git a/internal/pkg/instrumentation/bpf/google.golang.org/grpc/server/bpf_x86_bpfel.go b/internal/pkg/instrumentation/bpf/google.golang.org/grpc/server/bpf_x86_bpfel.go index 88c219259..f035dad9f 100644 --- a/internal/pkg/instrumentation/bpf/google.golang.org/grpc/server/bpf_x86_bpfel.go +++ b/internal/pkg/instrumentation/bpf/google.golang.org/grpc/server/bpf_x86_bpfel.go @@ -13,12 +13,12 @@ import ( ) type bpfGrpcRequestT struct { - StartTime uint64 - EndTime uint64 - Sc bpfSpanContext - Psc bpfSpanContext - Method [100]int8 - _ [4]byte + StartTime uint64 + EndTime uint64 + Sc bpfSpanContext + Psc bpfSpanContext + Method [100]int8 + StatusCode uint32 } type bpfSliceArrayBuff struct{ Buff [1024]uint8 } @@ -71,9 +71,11 @@ type bpfSpecs struct { // // It can be passed ebpf.CollectionSpec.Assign. type bpfProgramSpecs struct { - UprobeHttp2ServerOperateHeader *ebpf.ProgramSpec `ebpf:"uprobe_http2Server_operateHeader"` - UprobeServerHandleStream *ebpf.ProgramSpec `ebpf:"uprobe_server_handleStream"` - UprobeServerHandleStreamReturns *ebpf.ProgramSpec `ebpf:"uprobe_server_handleStream_Returns"` + UprobeHttp2ServerWriteStatus *ebpf.ProgramSpec `ebpf:"uprobe_http2Server_WriteStatus"` + UprobeHttp2ServerWriteStatusReturns *ebpf.ProgramSpec `ebpf:"uprobe_http2Server_WriteStatus_Returns"` + UprobeHttp2ServerOperateHeader *ebpf.ProgramSpec `ebpf:"uprobe_http2Server_operateHeader"` + UprobeServerHandleStream *ebpf.ProgramSpec `ebpf:"uprobe_server_handleStream"` + UprobeServerHandleStreamReturns *ebpf.ProgramSpec `ebpf:"uprobe_server_handleStream_Returns"` } // bpfMapSpecs contains maps before they are loaded into the kernel. @@ -136,13 +138,17 @@ func (m *bpfMaps) Close() error { // // It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign. type bpfPrograms struct { - UprobeHttp2ServerOperateHeader *ebpf.Program `ebpf:"uprobe_http2Server_operateHeader"` - UprobeServerHandleStream *ebpf.Program `ebpf:"uprobe_server_handleStream"` - UprobeServerHandleStreamReturns *ebpf.Program `ebpf:"uprobe_server_handleStream_Returns"` + UprobeHttp2ServerWriteStatus *ebpf.Program `ebpf:"uprobe_http2Server_WriteStatus"` + UprobeHttp2ServerWriteStatusReturns *ebpf.Program `ebpf:"uprobe_http2Server_WriteStatus_Returns"` + UprobeHttp2ServerOperateHeader *ebpf.Program `ebpf:"uprobe_http2Server_operateHeader"` + UprobeServerHandleStream *ebpf.Program `ebpf:"uprobe_server_handleStream"` + UprobeServerHandleStreamReturns *ebpf.Program `ebpf:"uprobe_server_handleStream_Returns"` } func (p *bpfPrograms) Close() error { return _BpfClose( + p.UprobeHttp2ServerWriteStatus, + p.UprobeHttp2ServerWriteStatusReturns, p.UprobeHttp2ServerOperateHeader, p.UprobeServerHandleStream, p.UprobeServerHandleStreamReturns, diff --git a/internal/pkg/instrumentation/bpf/google.golang.org/grpc/server/probe.go b/internal/pkg/instrumentation/bpf/google.golang.org/grpc/server/probe.go index 107d0e83b..2504c845c 100644 --- a/internal/pkg/instrumentation/bpf/google.golang.org/grpc/server/probe.go +++ b/internal/pkg/instrumentation/bpf/google.golang.org/grpc/server/probe.go @@ -8,10 +8,13 @@ import ( "log/slog" "github.com/hashicorp/go-version" + "golang.org/x/sys/unix" + "google.golang.org/grpc/codes" + "go.opentelemetry.io/otel/attribute" + otelcodes "go.opentelemetry.io/otel/codes" semconv "go.opentelemetry.io/otel/semconv/v1.26.0" "go.opentelemetry.io/otel/trace" - "golang.org/x/sys/unix" "go.opentelemetry.io/auto/internal/pkg/inject" "go.opentelemetry.io/auto/internal/pkg/instrumentation/context" @@ -60,6 +63,14 @@ func New(logger *slog.Logger) probe.Probe { Key: "frame_stream_id_pod", Val: structfield.NewID("golang.org/x/net", "golang.org/x/net/http2", "FrameHeader", "StreamID"), }, + probe.StructFieldConst{ + Key: "status_s_pos", + Val: structfield.NewID("google.golang.org/grpc", "google.golang.org/grpc/internal/status", "Status", "s"), + }, + probe.StructFieldConst{ + Key: "status_code_pos", + Val: structfield.NewID("google.golang.org/grpc", "google.golang.org/genproto/googleapis/rpc/status", "Status", "Code"), + }, framePosConst{}, }, Uprobes: []probe.Uprobe{ @@ -72,6 +83,11 @@ func New(logger *slog.Logger) probe.Probe { Sym: "google.golang.org/grpc/internal/transport.(*http2Server).operateHeaders", EntryProbe: "uprobe_http2Server_operateHeader", }, + { + Sym: "google.golang.org/grpc/internal/transport.(*http2Server).WriteStatus", + EntryProbe: "uprobe_http2Server_WriteStatus", + ReturnProbe: "uprobe_http2Server_WriteStatus_Returns", + }, }, SpecFn: loadBpf, ProcessFn: convertEvent, @@ -100,7 +116,8 @@ func (c framePosConst) InjectOption(td *process.TargetDetails) (inject.Option, e // event represents an event in the gRPC server during a gRPC request. type event struct { context.BaseSpanProperties - Method [100]byte + Method [100]byte + StatusCode int32 } func convertEvent(e *event) []*probe.SpanEvent { @@ -125,18 +142,29 @@ func convertEvent(e *event) []*probe.SpanEvent { pscPtr = nil } - return []*probe.SpanEvent{ - { - SpanName: method, - StartTime: utils.BootOffsetToTime(e.StartTime), - EndTime: utils.BootOffsetToTime(e.EndTime), - Attributes: []attribute.KeyValue{ - semconv.RPCSystemKey.String("grpc"), - semconv.RPCServiceKey.String(method), - }, - ParentSpanContext: pscPtr, - SpanContext: &sc, - TracerSchema: semconv.SchemaURL, + event := &probe.SpanEvent{ + SpanName: method, + StartTime: utils.BootOffsetToTime(e.StartTime), + EndTime: utils.BootOffsetToTime(e.EndTime), + Attributes: []attribute.KeyValue{ + semconv.RPCSystemKey.String("grpc"), + semconv.RPCServiceKey.String(method), + semconv.RPCGRPCStatusCodeKey.Int(int(e.StatusCode)), }, + ParentSpanContext: pscPtr, + SpanContext: &sc, + TracerSchema: semconv.SchemaURL, + } + + // Set server status codes per semconv: + // See https://github.com/open-telemetry/semantic-conventions/blob/02ecf0c71e9fa74d09d81c48e04a132db2b7060b/docs/rpc/grpc.md#grpc-status + if e.StatusCode == int32(codes.Unknown) || + e.StatusCode == int32(codes.DeadlineExceeded) || + e.StatusCode == int32(codes.Unimplemented) || + e.StatusCode == int32(codes.Internal) || + e.StatusCode == int32(codes.Unavailable) || + e.StatusCode == int32(codes.DataLoss) { + event.Status = probe.Status{Code: otelcodes.Error} } + return []*probe.SpanEvent{event} } diff --git a/internal/test/e2e/grpc/main.go b/internal/test/e2e/grpc/main.go index 3540139a3..23565f6f5 100644 --- a/internal/test/e2e/grpc/main.go +++ b/internal/test/e2e/grpc/main.go @@ -14,8 +14,10 @@ import ( "time" "google.golang.org/grpc" + "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials/insecure" pb "google.golang.org/grpc/examples/helloworld/helloworld" + "google.golang.org/grpc/status" ) const port = 1701 @@ -26,6 +28,9 @@ type server struct { func (s *server) SayHello(ctx context.Context, in *pb.HelloRequest) (*pb.HelloReply, error) { log.Printf("Received: %v", in.GetName()) + if in.GetName() == "unimplemented" { + return nil, status.Error(codes.Unimplemented, "unimplmented") + } return &pb.HelloReply{Message: "Hello " + in.GetName()}, nil } @@ -68,6 +73,13 @@ func main() { } log.Printf("Greeting: %s", r.GetMessage()) + // Contact the server expecting a server error + _, err = c.SayHello(ctx, &pb.HelloRequest{Name: "unimplemented"}) + if err == nil { + log.Fatalf("expected an error but none was received") + } + log.Printf("received expected error: %+v", err) + s.GracefulStop() <-done diff --git a/internal/test/e2e/grpc/traces.json b/internal/test/e2e/grpc/traces.json index 99b24f7a1..fda40427b 100644 --- a/internal/test/e2e/grpc/traces.json +++ b/internal/test/e2e/grpc/traces.json @@ -69,6 +69,12 @@ "value": { "stringValue": "/helloworld.Greeter/SayHello" } + }, + { + "key": "rpc.grpc.status_code", + "value": { + "intValue": "0" + } } ], "flags": 768, @@ -79,6 +85,37 @@ "status": {}, "traceId": "xxxxx" }, + { + "attributes": [ + { + "key": "rpc.system", + "value": { + "stringValue": "grpc" + } + }, + { + "key": "rpc.service", + "value": { + "stringValue": "/helloworld.Greeter/SayHello" + } + }, + { + "key": "rpc.grpc.status_code", + "value": { + "intValue": "12" + } + } + ], + "flags": 768, + "kind": 2, + "name": "/helloworld.Greeter/SayHello", + "parentSpanId": "xxxxx", + "spanId": "xxxxx", + "status": { + "code": 2 + }, + "traceId": "xxxxx" + }, { "attributes": [ { @@ -120,6 +157,49 @@ "status": {}, "traceId": "xxxxx" }, + { + "attributes": [ + { + "key": "network.peer.port", + "value": { + "intValue": "xxxxx" + } + }, + { + "key": "rpc.system", + "value": { + "stringValue": "grpc" + } + }, + { + "key": "rpc.service", + "value": { + "stringValue": "/helloworld.Greeter/SayHello" + } + }, + { + "key": "server.address", + "value": { + "stringValue": "localhost" + } + }, + { + "key": "rpc.grpc.status_code", + "value": { + "intValue": "12" + } + } + ], + "flags": 256, + "kind": 3, + "name": "/helloworld.Greeter/SayHello", + "parentSpanId": "", + "spanId": "xxxxx", + "status": { + "code": 2 + }, + "traceId": "xxxxx" + }, { "attributes": [ { diff --git a/internal/test/e2e/grpc/verify.bats b/internal/test/e2e/grpc/verify.bats index d4514520b..03c6c52ee 100644 --- a/internal/test/e2e/grpc/verify.bats +++ b/internal/test/e2e/grpc/verify.bats @@ -10,47 +10,68 @@ SCOPE="go.opentelemetry.io/auto/google.golang.org/grpc" } @test "server :: emits a span name 'SayHello'" { - result=$(server_span_names_for ${SCOPE}) + result=$(server_span_names_for ${SCOPE} | uniq) assert_equal "$result" '"/helloworld.Greeter/SayHello"' } @test "server :: includes rpc.system attribute" { - result=$(server_span_attributes_for ${SCOPE} | jq "select(.key == \"rpc.system\").value.stringValue") + result=$(server_span_attributes_for ${SCOPE} | jq "select(.key == \"rpc.system\").value.stringValue" | uniq) assert_equal "$result" '"grpc"' } @test "server :: includes rpc.service attribute" { - result=$(server_span_attributes_for ${SCOPE} | jq "select(.key == \"rpc.service\").value.stringValue") + result=$(server_span_attributes_for ${SCOPE} | jq "select(.key == \"rpc.service\").value.stringValue" | uniq) assert_equal "$result" '"/helloworld.Greeter/SayHello"' } @test "server :: trace ID present and valid in all spans" { - trace_id=$(server_spans_from_scope_named ${SCOPE} | jq ".traceId") + trace_id=$(server_spans_from_scope_named ${SCOPE} | jq ".traceId" | jq -Rn '[inputs]' | jq -r .[0]) + assert_regex "$trace_id" ${MATCH_A_TRACE_ID} + trace_id=$(server_spans_from_scope_named ${SCOPE} | jq ".traceId" | jq -Rn '[inputs]' | jq -r .[1]) assert_regex "$trace_id" ${MATCH_A_TRACE_ID} } @test "server :: span ID present and valid in all spans" { - span_id=$(server_spans_from_scope_named ${SCOPE} | jq ".spanId") + span_id=$(server_spans_from_scope_named ${SCOPE} | jq ".spanId" | jq -Rn '[inputs]' | jq -r .[0]) + assert_regex "$span_id" ${MATCH_A_SPAN_ID} + span_id=$(server_spans_from_scope_named ${SCOPE} | jq ".spanId" | jq -Rn '[inputs]' | jq -r .[1]) assert_regex "$span_id" ${MATCH_A_SPAN_ID} } @test "server :: parent span ID present and valid in all spans" { - parent_span_id=$(server_spans_from_scope_named ${SCOPE} | jq ".parentSpanId") + parent_span_id=$(server_spans_from_scope_named ${SCOPE} | jq ".parentSpanId" | jq -Rn '[inputs]' | jq -r .[0]) + assert_regex "$parent_span_id" ${MATCH_A_SPAN_ID} + parent_span_id=$(server_spans_from_scope_named ${SCOPE} | jq ".parentSpanId" | jq -Rn '[inputs]' | jq -r .[1]) assert_regex "$parent_span_id" ${MATCH_A_SPAN_ID} } +@test "server :: error code is present for unsuccessful span" { + # gRPC error code 12 - Unimplemented + result=$(server_span_attributes_for ${SCOPE} | jq 'select(.key == "rpc.grpc.status_code" and .value.intValue == "12")') + assert_not_empty "$result" +} + @test "client, server :: spans have same trace ID" { - # only check the first client span (the 2nd is an error) + # only check the first and 2nd client span (the 3rd is an error) client_trace_id=$(client_spans_from_scope_named ${SCOPE} | jq ".traceId" | jq -Rn '[inputs]' | jq -r .[0]) - server_trace_id=$(server_spans_from_scope_named ${SCOPE} | jq ".traceId") + server_trace_id=$(server_spans_from_scope_named ${SCOPE} | jq ".traceId" | jq -Rn '[inputs]' | jq -r .[0]) + assert_equal "$server_trace_id" "$client_trace_id" + + client_trace_id=$(client_spans_from_scope_named ${SCOPE} | jq ".traceId" | jq -Rn '[inputs]' | jq -r .[0]) + server_trace_id=$(server_spans_from_scope_named ${SCOPE} | jq ".traceId" | jq -Rn '[inputs]' | jq -r .[0]) assert_equal "$server_trace_id" "$client_trace_id" } @test "client, server :: server span has client span as parent" { - server_parent_span_id=$(server_spans_from_scope_named ${SCOPE} | jq ".parentSpanId") - # only check the first client span (the 2nd is an error) + server_parent_span_id=$(server_spans_from_scope_named ${SCOPE} | jq ".parentSpanId" | jq -Rn '[inputs]' | jq -r .[0]) + # only check the first and 2nd client span (the 3rd is an error) client_span_id=$(client_spans_from_scope_named ${SCOPE} | jq ".spanId"| jq -Rn '[inputs]' | jq -r .[0]) assert_equal "$client_span_id" "$server_parent_span_id" + + server_parent_span_id=$(server_spans_from_scope_named ${SCOPE} | jq ".parentSpanId" | jq -Rn '[inputs]' | jq -r .[1]) + # only check the first and 2nd client span (the 3rd is an error) + client_span_id=$(client_spans_from_scope_named ${SCOPE} | jq ".spanId"| jq -Rn '[inputs]' | jq -r .[1]) + assert_equal "$client_span_id" "$server_parent_span_id" } @test "server :: expected (redacted) trace output" {