Move gRPC driver to a subpackage and add an HTTP driver (#1420)

* Move grpc stuff to separate package

* Drop duplicated retryable status code

* Set default port to 4317

This is what the specification says for both gRPC and HTTP.

* Document gRPC option type

* Add an HTTP protocol driver for OTLP exporter

Currently it supports only binary protobuf payloads.

* Move end to end test to a separate package

It also adds some common code mock collectors can use. This will be
useful for testing the HTTP driver.

* Move export data creators to otlptest

It also extends the one record checkpointer a bit. This will be useful
for testing the HTTP driver.

* Add an HTTP mock collector and tests for HTTP driver

* Update changelog

* Do not depend on DefaultTransport

We create our own instance of the transport, which is based on
golang's DefaultTransport. That way we sidestep the issue of the
DefaultTransport being modified/overwritten. We won't have any panics
at init. The cost of it is to keep the transport fields in sync with
DefaultTransport.

* Read the whole response body before closing it

This may help with connection reuse.

* Change options to conform to our style guide

* Add jitter to backoff time

* Test TLS option

* Test extra headers

* Fix a comment

* Increase coverage

* Add a source of the backoff strategy

Author: krnowak

CHANGELOG.md

@@ -16,6 +16,7 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
 - `NewSplitDriver` for OTLP exporter that allows sending traces and metrics to different endpoints. (#1418)
 - Add codeql worfklow to GitHub Actions (#1428)
 - Added Gosec workflow to GitHub Actions (#1429)
+- A new HTTP driver for OTLP exporter in `exporters/otlp/otlphttp`. Currently it only supports the binary protobuf payloads. (#1420)
 
 ### Changed
 
@@ -29,6 +30,7 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
 - Unify endpoint API that related to OTel exporter. (#1401)
 - Metric aggregator Count() and histogram Bucket.Counts are consistently `uint64`. (1430)
 - `SamplingResult` now passed a `Tracestate` from the parent `SpanContext` (#1432)
+- Moved gRPC driver for OTLP exporter to `exporters/otlp/otlpgrpc`. (#1420)
 
 ### Removed
 

example/otel-collector/main.go

@@ -27,6 +27,7 @@ import (
 
 	"go.opentelemetry.io/otel"
 	"go.opentelemetry.io/otel/exporters/otlp"
+	"go.opentelemetry.io/otel/exporters/otlp/otlpgrpc"
 	"go.opentelemetry.io/otel/label"
 	"go.opentelemetry.io/otel/metric"
 	"go.opentelemetry.io/otel/propagation"
@@ -49,10 +50,10 @@ func initProvider() func() {
 	// `localhost:30080` endpoint. Otherwise, replace `localhost` with the
 	// endpoint of your cluster. If you run the app inside k8s, then you can
 	// probably connect directly to the service through dns
-	driver := otlp.NewGRPCDriver(
-		otlp.WithInsecure(),
-		otlp.WithEndpoint("localhost:30080"),
-		otlp.WithGRPCDialOption(grpc.WithBlock()), // useful for testing
+	driver := otlpgrpc.NewDriver(
+		otlpgrpc.WithInsecure(),
+		otlpgrpc.WithEndpoint("localhost:30080"),
+		otlpgrpc.WithDialOption(grpc.WithBlock()), // useful for testing
 	)
 	exp, err := otlp.NewExporter(ctx, driver)
 	handleErr(err, "failed to create exporter")

exporters/otlp/internal/otlptest/collector.go

@@ -0,0 +1,137 @@
+// Copyright The OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package otlptest
+
+import (
+	"sort"
+
+	collectormetricpb "go.opentelemetry.io/otel/exporters/otlp/internal/opentelemetry-proto-gen/collector/metrics/v1"
+	collectortracepb "go.opentelemetry.io/otel/exporters/otlp/internal/opentelemetry-proto-gen/collector/trace/v1"
+	commonpb "go.opentelemetry.io/otel/exporters/otlp/internal/opentelemetry-proto-gen/common/v1"
+	metricpb "go.opentelemetry.io/otel/exporters/otlp/internal/opentelemetry-proto-gen/metrics/v1"
+	resourcepb "go.opentelemetry.io/otel/exporters/otlp/internal/opentelemetry-proto-gen/resource/v1"
+	tracepb "go.opentelemetry.io/otel/exporters/otlp/internal/opentelemetry-proto-gen/trace/v1"
+)
+
+// Collector is an interface that mock collectors should implements,
+// so they can be used for the end-to-end testing.
+type Collector interface {
+	Stop() error
+	GetResourceSpans() []*tracepb.ResourceSpans
+	GetMetrics() []*metricpb.Metric
+}
+
+// SpansStorage stores the spans. Mock collectors could use it to
+// store spans they have received.
+type SpansStorage struct {
+	rsm       map[string]*tracepb.ResourceSpans
+	spanCount int
+}
+
+// MetricsStorage stores the metrics. Mock collectors could use it to
+// store metrics they have received.
+type MetricsStorage struct {
+	metrics []*metricpb.Metric
+}
+
+// NewSpansStorage creates a new spans storage.
+func NewSpansStorage() SpansStorage {
+	return SpansStorage{
+		rsm: make(map[string]*tracepb.ResourceSpans),
+	}
+}
+
+// AddSpans adds spans to the spans storage.
+func (s *SpansStorage) AddSpans(request *collectortracepb.ExportTraceServiceRequest) {
+	for _, rs := range request.GetResourceSpans() {
+		rstr := resourceString(rs.Resource)
+		if existingRs, ok := s.rsm[rstr]; !ok {
+			s.rsm[rstr] = rs
+			// TODO (rghetia): Add support for library Info.
+			if len(rs.InstrumentationLibrarySpans) == 0 {
+				rs.InstrumentationLibrarySpans = []*tracepb.InstrumentationLibrarySpans{
+					{
+						Spans: []*tracepb.Span{},
+					},
+				}
+			}
+			s.spanCount += len(rs.InstrumentationLibrarySpans[0].Spans)
+		} else {
+			if len(rs.InstrumentationLibrarySpans) > 0 {
+				newSpans := rs.InstrumentationLibrarySpans[0].GetSpans()
+				existingRs.InstrumentationLibrarySpans[0].Spans =
+					append(existingRs.InstrumentationLibrarySpans[0].Spans,
+						newSpans...)
+				s.spanCount += len(newSpans)
+			}
+		}
+	}
+}
+
+// GetSpans returns the stored spans.
+func (s *SpansStorage) GetSpans() []*tracepb.Span {
+	spans := make([]*tracepb.Span, 0, s.spanCount)
+	for _, rs := range s.rsm {
+		spans = append(spans, rs.InstrumentationLibrarySpans[0].Spans...)
+	}
+	return spans
+}
+
+// GetResourceSpans returns the stored resource spans.
+func (s *SpansStorage) GetResourceSpans() []*tracepb.ResourceSpans {
+	rss := make([]*tracepb.ResourceSpans, 0, len(s.rsm))
+	for _, rs := range s.rsm {
+		rss = append(rss, rs)
+	}
+	return rss
+}
+
+// NewMetricsStorage creates a new metrics storage.
+func NewMetricsStorage() MetricsStorage {
+	return MetricsStorage{}
+}
+
+// AddMetrics adds metrics to the metrics storage.
+func (s *MetricsStorage) AddMetrics(request *collectormetricpb.ExportMetricsServiceRequest) {
+	for _, rm := range request.GetResourceMetrics() {
+		// TODO (rghetia) handle multiple resource and library info.
+		if len(rm.InstrumentationLibraryMetrics) > 0 {
+			s.metrics = append(s.metrics, rm.InstrumentationLibraryMetrics[0].Metrics...)
+		}
+	}
+}
+
+// GetMetrics returns the stored metrics.
+func (s *MetricsStorage) GetMetrics() []*metricpb.Metric {
+	// copy in order to not change.
+	m := make([]*metricpb.Metric, 0, len(s.metrics))
+	return append(m, s.metrics...)
+}
+
+func resourceString(res *resourcepb.Resource) string {
+	sAttrs := sortedAttributes(res.GetAttributes())
+	rstr := ""
+	for _, attr := range sAttrs {
+		rstr = rstr + attr.String()
+	}
+	return rstr
+}
+
+func sortedAttributes(attrs []*commonpb.KeyValue) []*commonpb.KeyValue {
+	sort.Slice(attrs[:], func(i, j int) bool {
+		return attrs[i].Key < attrs[j].Key
+	})
+	return attrs
+}

exporters/otlp/internal/otlptest/data.go

@@ -0,0 +1,138 @@
+// Copyright The OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package otlptest
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"go.opentelemetry.io/otel/codes"
+	"go.opentelemetry.io/otel/label"
+	"go.opentelemetry.io/otel/metric"
+	"go.opentelemetry.io/otel/metric/number"
+	exportmetric "go.opentelemetry.io/otel/sdk/export/metric"
+	exporttrace "go.opentelemetry.io/otel/sdk/export/trace"
+	"go.opentelemetry.io/otel/sdk/instrumentation"
+	"go.opentelemetry.io/otel/sdk/metric/aggregator/sum"
+	"go.opentelemetry.io/otel/sdk/resource"
+	"go.opentelemetry.io/otel/trace"
+)
+
+// Used to avoid implementing locking functions for test
+// checkpointsets.
+type noopLocker struct{}
+
+// Lock implements sync.Locker, which is needed for
+// exportmetric.CheckpointSet.
+func (noopLocker) Lock() {}
+
+// Unlock implements sync.Locker, which is needed for
+// exportmetric.CheckpointSet.
+func (noopLocker) Unlock() {}
+
+// RLock implements exportmetric.CheckpointSet.
+func (noopLocker) RLock() {}
+
+// RUnlock implements exportmetric.CheckpointSet.
+func (noopLocker) RUnlock() {}
+
+// OneRecordCheckpointSet is a CheckpointSet that returns just one
+// filled record. It may be useful for testing driver's metrics
+// export.
+type OneRecordCheckpointSet struct {
+	noopLocker
+}
+
+var _ exportmetric.CheckpointSet = OneRecordCheckpointSet{}
+
+// ForEach implements exportmetric.CheckpointSet. It always invokes
+// the callback once with always the same record.
+func (OneRecordCheckpointSet) ForEach(kindSelector exportmetric.ExportKindSelector, recordFunc func(exportmetric.Record) error) error {
+	desc := metric.NewDescriptor(
+		"foo",
+		metric.CounterInstrumentKind,
+		number.Int64Kind,
+	)
+	res := resource.NewWithAttributes(label.String("a", "b"))
+	agg := sum.New(1)
+	if err := agg[0].Update(context.Background(), number.NewInt64Number(42), &desc); err != nil {
+		return err
+	}
+	start := time.Date(2020, time.December, 8, 19, 15, 0, 0, time.UTC)
+	end := time.Date(2020, time.December, 8, 19, 16, 0, 0, time.UTC)
+	labels := label.NewSet(label.String("abc", "def"), label.Int64("one", 1))
+	rec := exportmetric.NewRecord(&desc, &labels, res, agg[0].Aggregation(), start, end)
+	return recordFunc(rec)
+}
+
+// SingleSpanSnapshot returns a one-element slice with a snapshot. It
+// may be useful for testing driver's trace export.
+func SingleSpanSnapshot() []*exporttrace.SpanSnapshot {
+	sd := &exporttrace.SpanSnapshot{
+		SpanContext: trace.SpanContext{
+			TraceID:    trace.TraceID{2, 3, 4, 5, 6, 7, 8, 9, 2, 3, 4, 5, 6, 7, 8, 9},
+			SpanID:     trace.SpanID{3, 4, 5, 6, 7, 8, 9, 0},
+			TraceFlags: trace.FlagsSampled,
+		},
+		ParentSpanID:             trace.SpanID{1, 2, 3, 4, 5, 6, 7, 8},
+		SpanKind:                 trace.SpanKindInternal,
+		Name:                     "foo",
+		StartTime:                time.Date(2020, time.December, 8, 20, 23, 0, 0, time.UTC),
+		EndTime:                  time.Date(2020, time.December, 0, 20, 24, 0, 0, time.UTC),
+		Attributes:               []label.KeyValue{},
+		MessageEvents:            []exporttrace.Event{},
+		Links:                    []trace.Link{},
+		StatusCode:               codes.Ok,
+		StatusMessage:            "",
+		HasRemoteParent:          false,
+		DroppedAttributeCount:    0,
+		DroppedMessageEventCount: 0,
+		DroppedLinkCount:         0,
+		ChildSpanCount:           0,
+		Resource:                 resource.NewWithAttributes(label.String("a", "b")),
+		InstrumentationLibrary: instrumentation.Library{
+			Name:    "bar",
+			Version: "0.0.0",
+		},
+	}
+	return []*exporttrace.SpanSnapshot{sd}
+}
+
+// EmptyCheckpointSet is a checkpointer that has no records at all.
+type EmptyCheckpointSet struct {
+	noopLocker
+}
+
+var _ exportmetric.CheckpointSet = EmptyCheckpointSet{}
+
+// ForEach implements exportmetric.CheckpointSet. It never invokes the
+// callback.
+func (EmptyCheckpointSet) ForEach(kindSelector exportmetric.ExportKindSelector, recordFunc func(exportmetric.Record) error) error {
+	return nil
+}
+
+// FailCheckpointSet is a checkpointer that returns an error during
+// ForEach.
+type FailCheckpointSet struct {
+	noopLocker
+}
+
+var _ exportmetric.CheckpointSet = FailCheckpointSet{}
+
+// ForEach implements exportmetric.CheckpointSet. It always fails.
+func (FailCheckpointSet) ForEach(kindSelector exportmetric.ExportKindSelector, recordFunc func(exportmetric.Record) error) error {
+	return fmt.Errorf("fail")
+}