Merge branch '3.1.x'

pull/35914/head
Moritz Halbritter 1 year ago
commit 5bad242bfb

@ -16,7 +16,6 @@
package org.springframework.boot.actuate.autoconfigure.tracing;
import java.util.Collections;
import java.util.List;
import brave.CurrentSpanCustomizer;
@ -35,7 +34,6 @@ import brave.baggage.CorrelationScopeCustomizer;
import brave.baggage.CorrelationScopeDecorator;
import brave.context.slf4j.MDCScopeDecorator;
import brave.handler.SpanHandler;
import brave.propagation.B3Propagation;
import brave.propagation.CurrentTraceContext;
import brave.propagation.CurrentTraceContext.ScopeDecorator;
import brave.propagation.CurrentTraceContextCustomizer;
@ -48,7 +46,6 @@ import io.micrometer.tracing.brave.bridge.BravePropagator;
import io.micrometer.tracing.brave.bridge.BraveSpanCustomizer;
import io.micrometer.tracing.brave.bridge.BraveTracer;
import io.micrometer.tracing.brave.bridge.CompositeSpanHandler;
import io.micrometer.tracing.brave.bridge.W3CPropagation;
import io.micrometer.tracing.exporter.SpanExportingPredicate;
import io.micrometer.tracing.exporter.SpanFilter;
import io.micrometer.tracing.exporter.SpanReporter;
@ -103,10 +100,22 @@ public class BraveAutoConfiguration {
public Tracing braveTracing(Environment environment, TracingProperties properties, List<SpanHandler> spanHandlers,
List<TracingCustomizer> tracingCustomizers, CurrentTraceContext currentTraceContext,
Factory propagationFactory, Sampler sampler) {
if (properties.getPropagation().getType() == PropagationType.W3C
&& properties.getBrave().isSpanJoiningSupported()) {
throw new IncompatibleConfigurationException("management.tracing.propagation.type",
"management.tracing.brave.span-joining-supported");
if (properties.getBrave().isSpanJoiningSupported()) {
if (properties.getPropagation().getType() != null
&& properties.getPropagation().getType().contains(PropagationType.W3C)) {
throw new IncompatibleConfigurationException("management.tracing.propagation.type",
"management.tracing.brave.span-joining-supported");
}
if (properties.getPropagation().getType() == null
&& properties.getPropagation().getProduce().contains(PropagationType.W3C)) {
throw new IncompatibleConfigurationException("management.tracing.propagation.produce",
"management.tracing.brave.span-joining-supported");
}
if (properties.getPropagation().getType() == null
&& properties.getPropagation().getConsume().contains(PropagationType.W3C)) {
throw new IncompatibleConfigurationException("management.tracing.propagation.consume",
"management.tracing.brave.span-joining-supported");
}
}
String applicationName = environment.getProperty("spring.application.name", DEFAULT_APPLICATION_NAME);
Builder builder = Tracing.newBuilder()
@ -177,12 +186,9 @@ public class BraveAutoConfiguration {
@Bean
@ConditionalOnMissingBean
Factory propagationFactory(TracingProperties tracing) {
return switch (tracing.getPropagation().getType()) {
case B3 ->
B3Propagation.newFactoryBuilder().injectFormat(B3Propagation.Format.SINGLE_NO_PARENT).build();
case W3C -> new W3CPropagation();
};
Factory propagationFactory(TracingProperties properties) {
return CompositePropagationFactory.create(properties.getPropagation().getEffectiveProducedTypes(),
properties.getPropagation().getEffectiveConsumedTypes());
}
}
@ -201,11 +207,9 @@ public class BraveAutoConfiguration {
@ConditionalOnMissingBean
BaggagePropagation.FactoryBuilder propagationFactoryBuilder(
ObjectProvider<BaggagePropagationCustomizer> baggagePropagationCustomizers) {
Factory delegate = switch (this.tracingProperties.getPropagation().getType()) {
case B3 ->
B3Propagation.newFactoryBuilder().injectFormat(B3Propagation.Format.SINGLE_NO_PARENT).build();
case W3C -> new W3CPropagation(BRAVE_BAGGAGE_MANAGER, Collections.emptyList());
};
Factory delegate = CompositePropagationFactory.create(BRAVE_BAGGAGE_MANAGER,
this.tracingProperties.getPropagation().getEffectiveProducedTypes(),
this.tracingProperties.getPropagation().getEffectiveConsumedTypes());
FactoryBuilder builder = BaggagePropagation.newFactoryBuilder(delegate);
baggagePropagationCustomizers.orderedStream().forEach((customizer) -> customizer.customize(builder));
return builder;

@ -0,0 +1,201 @@
/*
* Copyright 2012-2023 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.boot.actuate.autoconfigure.tracing;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.stream.Stream;
import brave.internal.propagation.StringPropagationAdapter;
import brave.propagation.B3Propagation;
import brave.propagation.Propagation;
import brave.propagation.TraceContext;
import brave.propagation.TraceContextOrSamplingFlags;
import io.micrometer.tracing.BaggageManager;
import io.micrometer.tracing.brave.bridge.W3CPropagation;
/**
* {@link Factory} which supports multiple tracing formats. It is able to configure
* different formats for injecting and for extracting.
*
* @author Marcin Grzejszczak
* @author Moritz Halbritter
*/
class CompositePropagationFactory extends Propagation.Factory implements Propagation<String> {
private final Collection<Propagation.Factory> injectorFactories;
private final Collection<Propagation.Factory> extractorFactories;
private final List<Propagation<String>> injectors;
private final List<Propagation<String>> extractors;
private final boolean supportsJoin;
private final boolean requires128BitTraceId;
private final List<String> keys;
CompositePropagationFactory(Collection<Factory> injectorFactories, Collection<Factory> extractorFactories) {
this.injectorFactories = injectorFactories;
this.extractorFactories = extractorFactories;
this.injectors = this.injectorFactories.stream().map(Factory::get).toList();
this.extractors = this.extractorFactories.stream().map(Factory::get).toList();
this.supportsJoin = Stream.concat(this.injectorFactories.stream(), this.extractorFactories.stream())
.allMatch(Factory::supportsJoin);
this.requires128BitTraceId = Stream.concat(this.injectorFactories.stream(), this.extractorFactories.stream())
.anyMatch(Factory::requires128BitTraceId);
this.keys = Stream.concat(this.injectors.stream(), this.extractors.stream())
.flatMap((entry) -> entry.keys().stream())
.distinct()
.toList();
}
Collection<Factory> getInjectorFactories() {
return this.injectorFactories;
}
@Override
public List<String> keys() {
return this.keys;
}
@Override
public <R> TraceContext.Injector<R> injector(Setter<R, String> setter) {
return (traceContext, request) -> {
for (Propagation<String> injector : this.injectors) {
injector.injector(setter).inject(traceContext, request);
}
};
}
@Override
public <R> TraceContext.Extractor<R> extractor(Getter<R, String> getter) {
return (request) -> {
for (Propagation<String> extractor : this.extractors) {
TraceContextOrSamplingFlags extract = extractor.extractor(getter).extract(request);
if (extract != TraceContextOrSamplingFlags.EMPTY) {
return extract;
}
}
return TraceContextOrSamplingFlags.EMPTY;
};
}
@Override
@SuppressWarnings("deprecation")
public <K> Propagation<K> create(KeyFactory<K> keyFactory) {
return StringPropagationAdapter.create(this, keyFactory);
}
@Override
public boolean supportsJoin() {
return this.supportsJoin;
}
@Override
public boolean requires128BitTraceId() {
return this.requires128BitTraceId;
}
@Override
public TraceContext decorate(TraceContext context) {
for (Factory injectorFactory : this.injectorFactories) {
TraceContext decorated = injectorFactory.decorate(context);
if (decorated != context) {
return decorated;
}
}
for (Factory extractorFactory : this.extractorFactories) {
TraceContext decorated = extractorFactory.decorate(context);
if (decorated != context) {
return decorated;
}
}
return super.decorate(context);
}
/**
* Creates a new {@link CompositePropagationFactory}, which uses the given
* {@code injectionTypes} for injection and {@code extractionTypes} for extraction.
* @param baggageManager the baggage manager to use, or {@code null}
* @param injectionTypes the propagation types for injection
* @param extractionTypes the propagation types for extraction
* @return the {@link CompositePropagationFactory}
*/
static CompositePropagationFactory create(BaggageManager baggageManager,
Collection<TracingProperties.Propagation.PropagationType> injectionTypes,
Collection<TracingProperties.Propagation.PropagationType> extractionTypes) {
List<Factory> injectors = injectionTypes.stream()
.map((injection) -> factoryForType(baggageManager, injection))
.toList();
List<Factory> extractors = extractionTypes.stream()
.map((extraction) -> factoryForType(baggageManager, extraction))
.toList();
return new CompositePropagationFactory(injectors, extractors);
}
/**
* Creates a new {@link CompositePropagationFactory}, which uses the given
* {@code injectionTypes} for injection and {@code extractionTypes} for extraction.
* @param injectionTypes the propagation types for injection
* @param extractionTypes the propagation types for extraction
* @return the {@link CompositePropagationFactory}
*/
static CompositePropagationFactory create(Collection<TracingProperties.Propagation.PropagationType> injectionTypes,
Collection<TracingProperties.Propagation.PropagationType> extractionTypes) {
return create(null, injectionTypes, extractionTypes);
}
private static Factory factoryForType(BaggageManager baggageManager,
TracingProperties.Propagation.PropagationType type) {
return switch (type) {
case B3 -> b3Single();
case B3_MULTI -> b3Multi();
case W3C -> w3c(baggageManager);
};
}
/**
* Creates a new B3 propagation factory using a single B3 header.
* @return the B3 propagation factory
*/
private static Factory b3Single() {
return B3Propagation.newFactoryBuilder().injectFormat(B3Propagation.Format.SINGLE_NO_PARENT).build();
}
/**
* Creates a new B3 propagation factory using multiple B3 headers.
* @return the B3 propagation factory
*/
private static Factory b3Multi() {
return B3Propagation.newFactoryBuilder().injectFormat(B3Propagation.Format.MULTI).build();
}
/**
* Creates a new W3C propagation factory.
* @param baggageManager baggage manager to use, or {@code null}
* @return the W3C propagation factory
*/
private static W3CPropagation w3c(BaggageManager baggageManager) {
return (baggageManager != null) ? new W3CPropagation(baggageManager, Collections.emptyList())
: new W3CPropagation();
}
}

@ -0,0 +1,194 @@
/*
* Copyright 2012-2023 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.boot.actuate.autoconfigure.tracing;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import io.opentelemetry.api.baggage.propagation.W3CBaggagePropagator;
import io.opentelemetry.api.trace.propagation.W3CTraceContextPropagator;
import io.opentelemetry.context.Context;
import io.opentelemetry.context.propagation.TextMapGetter;
import io.opentelemetry.context.propagation.TextMapPropagator;
import io.opentelemetry.context.propagation.TextMapSetter;
import io.opentelemetry.extension.trace.propagation.B3Propagator;
/**
* {@link TextMapPropagator} which supports multiple tracing formats. It is able to
* configure different formats for injecting and for extracting.
*
* @author Moritz Halbritter
*/
class CompositeTextMapPropagator implements TextMapPropagator {
private final Collection<TextMapPropagator> injectors;
private final Collection<TextMapPropagator> mutuallyExclusiveExtractors;
private final Collection<TextMapPropagator> alwaysRunningExtractors;
private final Set<String> fields;
/**
* Creates a new {@link CompositeTextMapPropagator}.
* @param injectors the injectors
* @param mutuallyExclusiveExtractors the mutually exclusive extractors. They are
* applied in order, and as soon as an extractor extracts a context, the other
* extractors after it are no longer invoked
* @param alwaysRunningExtractors the always running extractors. They always run in
* order, regardless of the mutually exclusive extractors or whether the extractor
* before it has already extracted a context
*/
CompositeTextMapPropagator(Collection<TextMapPropagator> injectors,
Collection<TextMapPropagator> mutuallyExclusiveExtractors,
Collection<TextMapPropagator> alwaysRunningExtractors) {
this.injectors = injectors;
this.mutuallyExclusiveExtractors = mutuallyExclusiveExtractors;
this.alwaysRunningExtractors = alwaysRunningExtractors;
this.fields = concat(this.injectors, this.mutuallyExclusiveExtractors, this.alwaysRunningExtractors)
.flatMap((entry) -> entry.fields().stream())
.collect(Collectors.toSet());
}
Collection<TextMapPropagator> getInjectors() {
return this.injectors;
}
@Override
public Collection<String> fields() {
return this.fields;
}
@Override
public <C> void inject(Context context, C carrier, TextMapSetter<C> setter) {
if (context == null || setter == null) {
return;
}
for (TextMapPropagator injector : this.injectors) {
injector.inject(context, carrier, setter);
}
}
@Override
public <C> Context extract(Context context, C carrier, TextMapGetter<C> getter) {
if (context == null) {
return Context.root();
}
if (getter == null) {
return context;
}
Context currentContext = context;
for (TextMapPropagator extractor : this.mutuallyExclusiveExtractors) {
Context extractedContext = extractor.extract(currentContext, carrier, getter);
if (extractedContext != currentContext) {
currentContext = extractedContext;
break;
}
}
for (TextMapPropagator extractor : this.alwaysRunningExtractors) {
currentContext = extractor.extract(currentContext, carrier, getter);
}
return currentContext;
}
/**
* Creates a new {@link CompositeTextMapPropagator}, which uses the given
* {@code injectionTypes} for injection and {@code extractionTypes} for extraction.
* @param injectionTypes the propagation types for injection
* @param extractionTypes the propagation types for extraction
* @return the {@link CompositeTextMapPropagator}
*/
static TextMapPropagator create(Collection<TracingProperties.Propagation.PropagationType> injectionTypes,
Collection<TracingProperties.Propagation.PropagationType> extractionTypes) {
return create(null, injectionTypes, extractionTypes);
}
/**
* Creates a new {@link CompositeTextMapPropagator}, which uses the given
* {@code injectionTypes} for injection and {@code extractionTypes} for extraction.
* @param baggagePropagator the baggage propagator to use, or {@code null}
* @param injectionTypes the propagation types for injection
* @param extractionTypes the propagation types for extraction
* @return the {@link CompositeTextMapPropagator}
*/
static CompositeTextMapPropagator create(TextMapPropagator baggagePropagator,
Collection<TracingProperties.Propagation.PropagationType> injectionTypes,
Collection<TracingProperties.Propagation.PropagationType> extractionTypes) {
List<TextMapPropagator> injectors = injectionTypes.stream()
.map((injection) -> forType(injection, baggagePropagator != null))
.collect(Collectors.toCollection(ArrayList::new));
if (baggagePropagator != null) {
injectors.add(baggagePropagator);
}
List<TextMapPropagator> extractors = extractionTypes.stream()
.map((extraction) -> forType(extraction, baggagePropagator != null))
.toList();
return new CompositeTextMapPropagator(injectors, extractors,
(baggagePropagator != null) ? List.of(baggagePropagator) : Collections.emptyList());
}
@SafeVarargs
private static <T> Stream<T> concat(Collection<T>... collections) {
Stream<T> result = Stream.empty();
for (Collection<T> collection : collections) {
result = Stream.concat(result, collection.stream());
}
return result;
}
/**
* Creates a new B3 propagator using a single B3 header.
* @return the B3 propagator
*/
private static TextMapPropagator b3Single() {
return B3Propagator.injectingSingleHeader();
}
/**
* Creates a new B3 propagator using multiple B3 headers.
* @return the B3 propagator
*/
private static TextMapPropagator b3Multi() {
return B3Propagator.injectingMultiHeaders();
}
/**
* Creates a new W3C propagator.
* @param baggage whether baggage propagation should be supported
* @return the W3C propagator
*/
private static TextMapPropagator w3c(boolean baggage) {
if (!baggage) {
return W3CTraceContextPropagator.getInstance();
}
return TextMapPropagator.composite(W3CTraceContextPropagator.getInstance(), W3CBaggagePropagator.getInstance());
}
private static TextMapPropagator forType(TracingProperties.Propagation.PropagationType type, boolean baggage) {
return switch (type) {
case B3 -> b3Single();
case B3_MULTI -> b3Multi();
case W3C -> w3c(baggage);
};
}
}

@ -36,14 +36,11 @@ import io.micrometer.tracing.otel.bridge.Slf4JBaggageEventListener;
import io.micrometer.tracing.otel.bridge.Slf4JEventListener;
import io.micrometer.tracing.otel.propagation.BaggageTextMapPropagator;
import io.opentelemetry.api.OpenTelemetry;
import io.opentelemetry.api.baggage.propagation.W3CBaggagePropagator;
import io.opentelemetry.api.common.Attributes;
import io.opentelemetry.api.trace.Tracer;
import io.opentelemetry.api.trace.propagation.W3CTraceContextPropagator;
import io.opentelemetry.context.ContextStorage;
import io.opentelemetry.context.propagation.ContextPropagators;
import io.opentelemetry.context.propagation.TextMapPropagator;
import io.opentelemetry.extension.trace.propagation.B3Propagator;
import io.opentelemetry.sdk.OpenTelemetrySdk;
import io.opentelemetry.sdk.resources.Resource;
import io.opentelemetry.sdk.trace.SdkTracerProvider;
@ -70,6 +67,7 @@ import org.springframework.core.env.Environment;
* {@link EnableAutoConfiguration Auto-configuration} for OpenTelemetry.
*
* @author Moritz Halbritter
* @author Marcin Grzejszczak
* @author Yanming Zhou
* @since 3.0.0
*/
@ -193,22 +191,13 @@ public class OpenTelemetryAutoConfiguration {
}
@Bean
@ConditionalOnProperty(prefix = "management.tracing.propagation", name = "type", havingValue = "W3C",
matchIfMissing = true)
TextMapPropagator w3cTextMapPropagatorWithBaggage(OtelCurrentTraceContext otelCurrentTraceContext) {
List<String> remoteFields = this.tracingProperties.getBaggage().getRemoteFields();
return TextMapPropagator.composite(W3CTraceContextPropagator.getInstance(),
W3CBaggagePropagator.getInstance(), new BaggageTextMapPropagator(remoteFields,
new OtelBaggageManager(otelCurrentTraceContext, remoteFields, Collections.emptyList())));
}
@Bean
@ConditionalOnProperty(prefix = "management.tracing.propagation", name = "type", havingValue = "B3")
TextMapPropagator b3BaggageTextMapPropagator(OtelCurrentTraceContext otelCurrentTraceContext) {
TextMapPropagator textMapPropagatorWithBaggage(OtelCurrentTraceContext otelCurrentTraceContext) {
List<String> remoteFields = this.tracingProperties.getBaggage().getRemoteFields();
return TextMapPropagator.composite(B3Propagator.injectingSingleHeader(),
new BaggageTextMapPropagator(remoteFields,
new OtelBaggageManager(otelCurrentTraceContext, remoteFields, Collections.emptyList())));
BaggageTextMapPropagator baggagePropagator = new BaggageTextMapPropagator(remoteFields,
new OtelBaggageManager(otelCurrentTraceContext, remoteFields, Collections.emptyList()));
return CompositeTextMapPropagator.create(baggagePropagator,
this.tracingProperties.getPropagation().getEffectiveProducedTypes(),
this.tracingProperties.getPropagation().getEffectiveConsumedTypes());
}
@Bean
@ -226,18 +215,9 @@ public class OpenTelemetryAutoConfiguration {
static class NoBaggageConfiguration {
@Bean
@ConditionalOnMissingBean
@ConditionalOnProperty(prefix = "management.tracing.propagation", name = "type", havingValue = "B3")
B3Propagator b3TextMapPropagator() {
return B3Propagator.injectingSingleHeader();
}
@Bean
@ConditionalOnMissingBean
@ConditionalOnProperty(prefix = "management.tracing.propagation", name = "type", havingValue = "W3C",
matchIfMissing = true)
W3CTraceContextPropagator w3cTextMapPropagatorWithoutBaggage() {
return W3CTraceContextPropagator.getInstance();
TextMapPropagator textMapPropagator(TracingProperties properties) {
return CompositeTextMapPropagator.create(properties.getPropagation().getEffectiveProducedTypes(),
properties.getPropagation().getEffectiveConsumedTypes());
}
}

@ -176,29 +176,91 @@ public class TracingProperties {
public static class Propagation {
/**
* Tracing context propagation type.
* Tracing context propagation types produced and consumed by the application.
* Setting this property overrides the more fine-grained propagation type
* properties.
*/
private PropagationType type = PropagationType.W3C;
private List<PropagationType> type;
public PropagationType getType() {
/**
* Tracing context propagation types produced by the application.
*/
private List<PropagationType> produce = List.of(PropagationType.W3C);
/**
* Tracing context propagation types consumed by the application.
*/
private List<PropagationType> consume = List.of(PropagationType.values());
public void setType(List<PropagationType> type) {
this.type = type;
}
public void setProduce(List<PropagationType> produce) {
this.produce = produce;
}
public void setConsume(List<PropagationType> consume) {
this.consume = consume;
}
public List<PropagationType> getType() {
return this.type;
}
public void setType(PropagationType type) {
this.type = type;
public List<PropagationType> getProduce() {
return this.produce;
}
public List<PropagationType> getConsume() {
return this.consume;
}
/**
* Returns the effective context propagation types produced by the application.
* This will be {@link #getType()} if set or {@link #getProduce()} otherwise.
* @return the effective context propagation types produced by the application
*/
List<PropagationType> getEffectiveProducedTypes() {
if (this.type != null) {
return this.type;
}
return this.produce;
}
/**
* Returns the effective context propagation types consumed by the application.
* This will be {@link #getType()} if set or {@link #getConsume()} otherwise.
* @return the effective context propagation types consumed by the application
*/
List<PropagationType> getEffectiveConsumedTypes() {
if (this.type != null) {
return this.type;
}
return this.consume;
}
/**
* Supported propagation types. The declared order of the values matter.
*/
enum PropagationType {
/**
* B3 propagation type.
* <a href="https://www.w3.org/TR/trace-context/">W3C</a> propagation.
*/
W3C,
/**
* <a href="https://github.com/openzipkin/b3-propagation#single-header">B3
* single header</a> propagation.
*/
B3,
/**
* W3C propagation type.
* <a href="https://github.com/openzipkin/b3-propagation#multiple-headers">B3
* multiple headers</a> propagation.
*/
W3C
B3_MULTI;
}

@ -2180,8 +2180,18 @@
}
},
{
"name": "management.tracing.propagation.type",
"defaultValue": "W3C"
"name": "management.tracing.propagation.consume",
"defaultValue": [
"W3C",
"B3",
"B3_MULTI"
]
},
{
"name": "management.tracing.propagation.produce",
"defaultValue": [
"W3C"
]
}
],
"hints": [

@ -138,7 +138,7 @@ class BaggagePropagationIntegrationTests {
enum AutoConfig implements Supplier<ApplicationContextRunner> {
BRAVE_W3C {
BRAVE_DEFAULT {
@Override
public ApplicationContextRunner get() {
return new ApplicationContextRunner()
@ -148,7 +148,7 @@ class BaggagePropagationIntegrationTests {
}
},
OTEL_W3C {
OTEL_DEFAULT {
@Override
public ApplicationContextRunner get() {
return new ApplicationContextRunner()
@ -158,6 +158,28 @@ class BaggagePropagationIntegrationTests {
}
},
BRAVE_W3C {
@Override
public ApplicationContextRunner get() {
return new ApplicationContextRunner()
.withConfiguration(AutoConfigurations.of(BraveAutoConfiguration.class))
.withPropertyValues("management.tracing.propagation.type=W3C",
"management.tracing.baggage.remote-fields=x-vcap-request-id,country-code,bp",
"management.tracing.baggage.correlation.fields=country-code,bp");
}
},
OTEL_W3C {
@Override
public ApplicationContextRunner get() {
return new ApplicationContextRunner()
.withConfiguration(AutoConfigurations.of(OpenTelemetryAutoConfiguration.class))
.withPropertyValues("management.tracing.propagation.type=W3C",
"management.tracing.baggage.remote-fields=x-vcap-request-id,country-code,bp",
"management.tracing.baggage.correlation.fields=country-code,bp");
}
},
BRAVE_B3 {
@Override
public ApplicationContextRunner get() {
@ -169,6 +191,17 @@ class BaggagePropagationIntegrationTests {
}
},
BRAVE_B3_MULTI {
@Override
public ApplicationContextRunner get() {
return new ApplicationContextRunner()
.withConfiguration(AutoConfigurations.of(BraveAutoConfiguration.class))
.withPropertyValues("management.tracing.propagation.type=B3_MULTI",
"management.tracing.baggage.remote-fields=x-vcap-request-id,country-code,bp",
"management.tracing.baggage.correlation.fields=country-code,bp");
}
},
OTEL_B3 {
@Override
public ApplicationContextRunner get() {
@ -178,6 +211,17 @@ class BaggagePropagationIntegrationTests {
"management.tracing.baggage.remote-fields=x-vcap-request-id,country-code,bp",
"management.tracing.baggage.correlation.fields=country-code,bp");
}
},
OTEL_B3_MULTI {
@Override
public ApplicationContextRunner get() {
return new ApplicationContextRunner()
.withConfiguration(AutoConfigurations.of(OpenTelemetryAutoConfiguration.class))
.withPropertyValues("management.tracing.propagation.type=B3_MULTI",
"management.tracing.baggage.remote-fields=x-vcap-request-id,country-code,bp",
"management.tracing.baggage.correlation.fields=country-code,bp");
}
}
}

@ -17,6 +17,7 @@
package org.springframework.boot.actuate.autoconfigure.tracing;
import java.util.Collections;
import java.util.List;
import brave.Span;
import brave.SpanCustomizer;
@ -38,6 +39,7 @@ import io.micrometer.tracing.brave.bridge.W3CPropagation;
import io.micrometer.tracing.exporter.SpanExportingPredicate;
import io.micrometer.tracing.exporter.SpanFilter;
import io.micrometer.tracing.exporter.SpanReporter;
import org.assertj.core.api.Assertions;
import org.assertj.core.api.InstanceOfAssertFactories;
import org.junit.jupiter.api.Test;
@ -131,7 +133,9 @@ class BraveAutoConfigurationTests {
void shouldSupplyW3CPropagationFactoryByDefault() {
this.contextRunner.run((context) -> {
assertThat(context).hasBean("propagationFactory");
assertThat(context).hasSingleBean(W3CPropagation.class);
Factory factory = context.getBean(Factory.class);
List<Factory> injectors = getInjectors(factory);
assertThat(injectors).extracting(Factory::getClass).containsExactly(W3CPropagation.class);
assertThat(context).hasSingleBean(BaggagePropagation.FactoryBuilder.class);
});
}
@ -140,7 +144,9 @@ class BraveAutoConfigurationTests {
void shouldSupplyB3PropagationFactoryViaProperty() {
this.contextRunner.withPropertyValues("management.tracing.propagation.type=B3").run((context) -> {
assertThat(context).hasBean("propagationFactory");
assertThat(context.getBean(Factory.class)).hasToString("B3Propagation");
Factory factory = context.getBean(Factory.class);
List<Factory> injectors = getInjectors(factory);
assertThat(injectors).extracting(Factory::toString).containsExactly("B3Propagation");
assertThat(context).hasSingleBean(BaggagePropagation.FactoryBuilder.class);
});
}
@ -161,7 +167,9 @@ class BraveAutoConfigurationTests {
void shouldSupplyW3CWithoutBaggageByDefaultIfBaggageDisabled() {
this.contextRunner.withPropertyValues("management.tracing.baggage.enabled=false").run((context) -> {
assertThat(context).hasBean("propagationFactory");
assertThat(context).hasSingleBean(W3CPropagation.class);
Factory factory = context.getBean(Factory.class);
List<Factory> injectors = getInjectors(factory);
assertThat(injectors).extracting(Factory::getClass).containsExactly(W3CPropagation.class);
assertThat(context).doesNotHaveBean(BaggagePropagation.FactoryBuilder.class);
});
}
@ -172,7 +180,9 @@ class BraveAutoConfigurationTests {
.withPropertyValues("management.tracing.baggage.enabled=false", "management.tracing.propagation.type=B3")
.run((context) -> {
assertThat(context).hasBean("propagationFactory");
assertThat(context.getBean(Factory.class)).hasToString("B3Propagation");
Factory factory = context.getBean(Factory.class);
List<Factory> injectors = getInjectors(factory);
assertThat(injectors).extracting(Factory::toString).containsExactly("B3Propagation");
assertThat(context).doesNotHaveBean(BaggagePropagation.FactoryBuilder.class);
});
}
@ -248,7 +258,7 @@ class BraveAutoConfigurationTests {
}
@Test
void shouldFailIfSupportJoinedSpansIsEnabledAndW3cIsChosen() {
void shouldFailIfSupportJoinedSpansIsEnabledAndW3cIsChosenAsType() {
this.contextRunner
.withPropertyValues("management.tracing.propagation.type=W3C",
"management.tracing.brave.span-joining-supported=true")
@ -258,6 +268,26 @@ class BraveAutoConfigurationTests {
"The following configuration properties have incompatible values: [management.tracing.propagation.type, management.tracing.brave.span-joining-supported]"));
}
@Test
void shouldFailIfSupportJoinedSpansIsEnabledAndW3cIsChosenAsConsume() {
this.contextRunner.withPropertyValues("management.tracing.propagation.produce=B3",
"management.tracing.propagation.consume=W3C", "management.tracing.brave.span-joining-supported=true")
.run((context) -> assertThatThrownBy(() -> context.getBean(Tracing.class)).rootCause()
.isExactlyInstanceOf(IncompatibleConfigurationException.class)
.hasMessage(
"The following configuration properties have incompatible values: [management.tracing.propagation.consume, management.tracing.brave.span-joining-supported]"));
}
@Test
void shouldFailIfSupportJoinedSpansIsEnabledAndW3cIsChosenAsProduce() {
this.contextRunner.withPropertyValues("management.tracing.propagation.consume=B3",
"management.tracing.propagation.produce=W3C", "management.tracing.brave.span-joining-supported=true")
.run((context) -> assertThatThrownBy(() -> context.getBean(Tracing.class)).rootCause()
.isExactlyInstanceOf(IncompatibleConfigurationException.class)
.hasMessage(
"The following configuration properties have incompatible values: [management.tracing.propagation.produce, management.tracing.brave.span-joining-supported]"));
}
@Test
@SuppressWarnings("rawtypes")
void compositeSpanHandlerShouldBeFirstSpanHandler() {
@ -288,6 +318,15 @@ class BraveAutoConfigurationTests {
});
}
private List<Factory> getInjectors(Factory factory) {
assertThat(factory).as("factory").isNotNull();
if (factory instanceof CompositePropagationFactory compositePropagationFactory) {
return compositePropagationFactory.getInjectorFactories().stream().toList();
}
Assertions.fail("Expected CompositePropagationFactory, found %s".formatted(factory.getClass()));
throw new AssertionError("Unreachable");
}
@Configuration(proxyBeanMethods = false)
static class CompositeSpanHandlerComponentsConfiguration {

@ -0,0 +1,162 @@
/*
* Copyright 2012-2023 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.boot.actuate.autoconfigure.tracing;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import brave.internal.propagation.StringPropagationAdapter;
import brave.propagation.Propagation;
import brave.propagation.TraceContext;
import brave.propagation.TraceContextOrSamplingFlags;
import org.junit.jupiter.api.Test;
import org.mockito.Mockito;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.entry;
import static org.mockito.BDDMockito.given;
/**
* Tests for {@link CompositePropagationFactory}.
*
* @author Moritz Halbritter
*/
class CompositePropagationFactoryTests {
@Test
void returnsAllKeys() {
CompositePropagationFactory factory = new CompositePropagationFactory(List.of(field("a")), List.of(field("b")));
assertThat(factory.keys()).containsExactly("a", "b");
}
@Test
void supportsJoin() {
Propagation.Factory supportsJoin = Mockito.mock(Propagation.Factory.class);
given(supportsJoin.supportsJoin()).willReturn(true);
given(supportsJoin.get()).willReturn(new DummyPropagation("a"));
Propagation.Factory doesNotSupportsJoin = Mockito.mock(Propagation.Factory.class);
given(doesNotSupportsJoin.supportsJoin()).willReturn(false);
given(doesNotSupportsJoin.get()).willReturn(new DummyPropagation("a"));
CompositePropagationFactory factory = new CompositePropagationFactory(List.of(supportsJoin),
List.of(doesNotSupportsJoin));
assertThat(factory.supportsJoin()).isFalse();
}
@Test
void requires128BitTraceId() {
Propagation.Factory requires128BitTraceId = Mockito.mock(Propagation.Factory.class);
given(requires128BitTraceId.requires128BitTraceId()).willReturn(true);
given(requires128BitTraceId.get()).willReturn(new DummyPropagation("a"));
Propagation.Factory doesNotRequire128BitTraceId = Mockito.mock(Propagation.Factory.class);
given(doesNotRequire128BitTraceId.requires128BitTraceId()).willReturn(false);
given(doesNotRequire128BitTraceId.get()).willReturn(new DummyPropagation("a"));
CompositePropagationFactory factory = new CompositePropagationFactory(List.of(requires128BitTraceId),
List.of(doesNotRequire128BitTraceId));
assertThat(factory.requires128BitTraceId()).isTrue();
}
@Test
void inject() {
CompositePropagationFactory factory = new CompositePropagationFactory(List.of(field("a"), field("b")),
List.of(field("c")));
TraceContext context = context();
Map<String, String> request = new HashMap<>();
factory.injector(new MapSetter()).inject(context, request);
assertThat(request).containsOnly(entry("a", "a-value"), entry("b", "b-value"));
}
@Test
void extractorStopsAfterSuccessfulExtraction() {
CompositePropagationFactory factory = new CompositePropagationFactory(Collections.emptyList(),
List.of(field("a"), field("b")));
Map<String, String> request = Map.of("a", "a-value", "b", "b-value");
TraceContextOrSamplingFlags context = factory.extractor(new MapGetter()).extract(request);
assertThat(context.context().extra()).containsExactly("a");
}
@Test
void returnsEmptyContextWhenNoExtractorMatches() {
CompositePropagationFactory factory = new CompositePropagationFactory(Collections.emptyList(),
Collections.emptyList());
Map<String, String> request = Collections.emptyMap();
TraceContextOrSamplingFlags context = factory.extractor(new MapGetter()).extract(request);
assertThat(context.context()).isNull();
}
private static TraceContext context() {
return TraceContext.newBuilder().traceId(1).spanId(2).build();
}
private static DummyPropagation field(String field) {
return new DummyPropagation(field);
}
private static final class MapSetter implements Propagation.Setter<Map<String, String>, String> {
@Override
public void put(Map<String, String> request, String key, String value) {
request.put(key, value);
}
}
private static final class MapGetter implements Propagation.Getter<Map<String, String>, String> {
@Override
public String get(Map<String, String> request, String key) {
return request.get(key);
}
}
private static final class DummyPropagation extends Propagation.Factory implements Propagation<String> {
private final String field;
private DummyPropagation(String field) {
this.field = field;
}
@Override
@SuppressWarnings("deprecation")
public <K> Propagation<K> create(Propagation.KeyFactory<K> keyFactory) {
return StringPropagationAdapter.create(this, keyFactory);
}
@Override
public List<String> keys() {
return List.of(this.field);
}
@Override
public <R> TraceContext.Injector<R> injector(Propagation.Setter<R, String> setter) {
return (traceContext, request) -> setter.put(request, this.field, this.field + "-value");
}
@Override
public <R> TraceContext.Extractor<R> extractor(Propagation.Getter<R, String> getter) {
return (request) -> {
TraceContext context = TraceContext.newBuilder().traceId(1).spanId(2).addExtra(this.field).build();
return TraceContextOrSamplingFlags.create(context);
};
}
}
}

@ -0,0 +1,174 @@
/*
* Copyright 2012-2023 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.boot.actuate.autoconfigure.tracing;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import io.opentelemetry.context.Context;
import io.opentelemetry.context.ContextKey;
import io.opentelemetry.context.propagation.TextMapGetter;
import io.opentelemetry.context.propagation.TextMapPropagator;
import io.opentelemetry.context.propagation.TextMapSetter;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.mockito.InOrder;
import org.mockito.Mockito;
import static org.assertj.core.api.Assertions.assertThat;
/**
* Tests for {@link CompositeTextMapPropagator}.
*
* @author Moritz Halbritter
*/
class CompositeTextMapPropagatorTests {
private ContextKeyRegistry contextKeyRegistry;
@BeforeEach
void setUp() {
this.contextKeyRegistry = new ContextKeyRegistry();
}
@Test
void collectsAllFields() {
CompositeTextMapPropagator propagator = new CompositeTextMapPropagator(List.of(field("a")), List.of(field("b")),
List.of(field("c")));
assertThat(propagator.fields()).containsExactly("a", "b", "c");
}
@Test
void injectAllFields() {
CompositeTextMapPropagator propagator = new CompositeTextMapPropagator(List.of(field("a"), field("b")),
Collections.emptyList(), Collections.emptyList());
TextMapSetter<Object> setter = setter();
Object carrier = carrier();
propagator.inject(context(), carrier, setter);
InOrder inOrder = Mockito.inOrder(setter);
inOrder.verify(setter).set(carrier, "a", "a-value");
inOrder.verify(setter).set(carrier, "b", "b-value");
}
@Test
void extractMutuallyExclusive() {
CompositeTextMapPropagator propagator = new CompositeTextMapPropagator(Collections.emptyList(),
List.of(field("a"), field("b")), Collections.emptyList());
Context context = context();
Map<String, String> carrier = Map.of("a", "a-value", "b", "b-value");
context = propagator.extract(context, carrier, new MapTextMapGetter());
Object a = context.get(getObjectContextKey("a"));
assertThat(a).isEqualTo("a-value");
Object b = context.get(getObjectContextKey("b"));
assertThat(b).isNull();
}
@Test
void extractAlwaysRunning() {
CompositeTextMapPropagator propagator = new CompositeTextMapPropagator(Collections.emptyList(),
List.of(field("a"), field("b")), List.of(field("c")));
Context context = context();
Map<String, String> carrier = Map.of("a", "a-value", "b", "b-value", "c", "c-value");
context = propagator.extract(context, carrier, new MapTextMapGetter());
Object c = context.get(getObjectContextKey("c"));
assertThat(c).isEqualTo("c-value");
}
private DummyTextMapPropagator field(String field) {
return new DummyTextMapPropagator(field, this.contextKeyRegistry);
}
private ContextKey<Object> getObjectContextKey(String name) {
return this.contextKeyRegistry.get(name);
}
@SuppressWarnings("unchecked")
private static <T> TextMapSetter<T> setter() {
return Mockito.mock(TextMapSetter.class);
}
private static Object carrier() {
return new Object();
}
private static Context context() {
return Context.current();
}
private static final class ContextKeyRegistry {
private final Map<String, ContextKey<Object>> contextKeys = new HashMap<>();
private ContextKey<Object> get(String name) {
return this.contextKeys.computeIfAbsent(name, (ignore) -> ContextKey.named(name));
}
}
private static final class MapTextMapGetter implements TextMapGetter<Map<String, String>> {
@Override
public Iterable<String> keys(Map<String, String> carrier) {
return carrier.keySet();
}
@Override
public String get(Map<String, String> carrier, String key) {
if (carrier == null) {
return null;
}
return carrier.get(key);
}
}
private static final class DummyTextMapPropagator implements TextMapPropagator {
private final String field;
private final ContextKeyRegistry contextKeyRegistry;
private DummyTextMapPropagator(String field, ContextKeyRegistry contextKeyRegistry) {
this.field = field;
this.contextKeyRegistry = contextKeyRegistry;
}
@Override
public Collection<String> fields() {
return List.of(this.field);
}
@Override
public <C> void inject(Context context, C carrier, TextMapSetter<C> setter) {
setter.set(carrier, this.field, this.field + "-value");
}
@Override
public <C> Context extract(Context context, C carrier, TextMapGetter<C> getter) {
String value = getter.get(carrier, this.field);
if (value != null) {
return context.with(this.contextKeyRegistry.get(this.field), value);
}
return context;
}
}
}

@ -16,7 +16,7 @@
package org.springframework.boot.actuate.autoconfigure.tracing;
import java.util.Collection;
import java.util.ArrayList;
import java.util.List;
import io.micrometer.tracing.SpanCustomizer;
@ -27,6 +27,7 @@ import io.micrometer.tracing.otel.bridge.OtelTracer;
import io.micrometer.tracing.otel.bridge.OtelTracer.EventPublisher;
import io.micrometer.tracing.otel.bridge.Slf4JBaggageEventListener;
import io.micrometer.tracing.otel.bridge.Slf4JEventListener;
import io.micrometer.tracing.otel.propagation.BaggageTextMapPropagator;
import io.opentelemetry.api.OpenTelemetry;
import io.opentelemetry.api.trace.Tracer;
import io.opentelemetry.api.trace.propagation.W3CTraceContextPropagator;
@ -49,6 +50,7 @@ import org.springframework.context.annotation.Configuration;
import org.springframework.core.annotation.Order;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.fail;
import static org.mockito.Mockito.mock;
/**
@ -174,8 +176,10 @@ class OpenTelemetryAutoConfigurationTests {
@Test
void shouldSupplyB3PropagationIfPropagationPropertySet() {
this.contextRunner.withPropertyValues("management.tracing.propagation.type=B3").run((context) -> {
assertThat(context).hasBean("b3BaggageTextMapPropagator");
assertThat(context).doesNotHaveBean(W3CTraceContextPropagator.class);
TextMapPropagator propagator = context.getBean(TextMapPropagator.class);
List<TextMapPropagator> injectors = getInjectors(propagator);
assertThat(injectors).extracting(TextMapPropagator::getClass)
.containsExactly(B3Propagator.class, BaggageTextMapPropagator.class);
});
}
@ -184,26 +188,42 @@ class OpenTelemetryAutoConfigurationTests {
this.contextRunner
.withPropertyValues("management.tracing.propagation.type=B3", "management.tracing.baggage.enabled=false")
.run((context) -> {
assertThat(context).hasSingleBean(B3Propagator.class);
assertThat(context).hasBean("b3TextMapPropagator");
assertThat(context).doesNotHaveBean(W3CTraceContextPropagator.class);
TextMapPropagator propagator = context.getBean(TextMapPropagator.class);
List<TextMapPropagator> injectors = getInjectors(propagator);
assertThat(injectors).extracting(TextMapPropagator::getClass).containsExactly(B3Propagator.class);
});
}
@Test
void shouldSupplyW3CPropagationWithBaggageByDefault() {
this.contextRunner.withPropertyValues("management.tracing.baggage.remote-fields=foo").run((context) -> {
assertThat(context).hasBean("w3cTextMapPropagatorWithBaggage");
Collection<String> allFields = context.getBean("w3cTextMapPropagatorWithBaggage", TextMapPropagator.class)
.fields();
assertThat(allFields).containsExactly("traceparent", "tracestate", "baggage", "foo");
TextMapPropagator propagator = context.getBean(TextMapPropagator.class);
List<TextMapPropagator> injectors = getInjectors(propagator);
List<String> fields = new ArrayList<>();
for (TextMapPropagator injector : injectors) {
fields.addAll(injector.fields());
}
assertThat(fields).containsExactly("traceparent", "tracestate", "baggage", "foo");
});
}
@Test
void shouldSupplyW3CPropagationWithoutBaggageWhenDisabled() {
this.contextRunner.withPropertyValues("management.tracing.baggage.enabled=false")
.run((context) -> assertThat(context).hasBean("w3cTextMapPropagatorWithoutBaggage"));
this.contextRunner.withPropertyValues("management.tracing.baggage.enabled=false").run((context) -> {
TextMapPropagator propagator = context.getBean(TextMapPropagator.class);
List<TextMapPropagator> injectors = getInjectors(propagator);
assertThat(injectors).extracting(TextMapPropagator::getClass)
.containsExactly(W3CTraceContextPropagator.class);
});
}
private List<TextMapPropagator> getInjectors(TextMapPropagator propagator) {
assertThat(propagator).as("propagator").isNotNull();
if (propagator instanceof CompositeTextMapPropagator compositePropagator) {
return compositePropagator.getInjectors().stream().toList();
}
fail("Expected CompositeTextMapPropagator, found %s".formatted(propagator.getClass()));
throw new AssertionError("Unreachable");
}
@Test

@ -0,0 +1,66 @@
/*
* Copyright 2012-2023 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.boot.actuate.autoconfigure.tracing;
import java.util.List;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
/**
* Tests for {@link TracingProperties}.
*
* @author Moritz Halbritter
*/
class TracingPropertiesTests {
@Test
void propagationTypeShouldOverrideProduceTypes() {
TracingProperties.Propagation propagation = new TracingProperties.Propagation();
propagation.setProduce(List.of(TracingProperties.Propagation.PropagationType.W3C));
propagation.setType(List.of(TracingProperties.Propagation.PropagationType.B3));
assertThat(propagation.getEffectiveProducedTypes())
.containsExactly(TracingProperties.Propagation.PropagationType.B3);
}
@Test
void propagationTypeShouldOverrideConsumeTypes() {
TracingProperties.Propagation propagation = new TracingProperties.Propagation();
propagation.setConsume(List.of(TracingProperties.Propagation.PropagationType.W3C));
propagation.setType(List.of(TracingProperties.Propagation.PropagationType.B3));
assertThat(propagation.getEffectiveConsumedTypes())
.containsExactly(TracingProperties.Propagation.PropagationType.B3);
}
@Test
void getEffectiveConsumeTypes() {
TracingProperties.Propagation propagation = new TracingProperties.Propagation();
propagation.setConsume(List.of(TracingProperties.Propagation.PropagationType.W3C));
assertThat(propagation.getEffectiveConsumedTypes())
.containsExactly(TracingProperties.Propagation.PropagationType.W3C);
}
@Test
void getEffectiveProduceTypes() {
TracingProperties.Propagation propagation = new TracingProperties.Propagation();
propagation.setProduce(List.of(TracingProperties.Propagation.PropagationType.W3C));
assertThat(propagation.getEffectiveProducedTypes())
.containsExactly(TracingProperties.Propagation.PropagationType.W3C);
}
}
Loading…
Cancel
Save