From 84874422952ccbce696c14056306969481664270 Mon Sep 17 00:00:00 2001 From: Jorik Schellekens Date: Thu, 11 Jul 2019 17:01:12 +0100 Subject: [PATCH 1/9] Opentracing survival guide --- docs/opentracing.rst | 179 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 179 insertions(+) create mode 100644 docs/opentracing.rst diff --git a/docs/opentracing.rst b/docs/opentracing.rst new file mode 100644 index 000000000000..02eba904dbbf --- /dev/null +++ b/docs/opentracing.rst @@ -0,0 +1,179 @@ +=========== +Opentracing +=========== + +Background +---------- + +Opentracing is semi-standard being addopted by a number of distributed tracing +platforms. It is a standardised api for facilitating vendor agnostic tracing +instrumentation. That is, we can use the opentracing api and select one of a +number of tracer implementations to do the heavy lifting in the background. +Our current selected implementation is Jaeger. + +Opentracing concepts can be found at +https://opentracing.io/docs/overview/what-is-tracing/ + +Python specific tracing concepts are at https://opentracing.io/guides/python/. +Note that synapse wraps opentracing in a small library in order to make the +opentracing dependency optional. That means that the access patterns are +different to those demonstrated here. However, it is still usefull to know. +Especially if opentracing is included as a full dependency in the future or if +you are modifying synapse's opentracing lib. + +For more information about Jaeger's implementation see +https://www.jaegertracing.io/docs/ + +================= +Setup opentracing +================= + +To receive opentracing spans start up a Jaeger server using docker like so + +.. code-block:: bash + + docker run -d --name jaeger \ -e COLLECTOR_ZIPKIN_HTTP_PORT=9411 \ + -p 5775:5775/udp \ + -p 6831:6831/udp \ + -p 6832:6832/udp \ + -p 5778:5778 \ + -p 16686:16686 \ + -p 14268:14268 \ + -p 9411:9411 \ + jaegertracing/all-in-one:1.13 + +Latest documentation is probably at +https://www.jaegertracing.io/docs/1.13/getting-started/ + + +Enable opentracing in synapse +----------------------------- + +Opentracing is not enabled by default. It must be enabled in the homeserver +config by uncommenting the config options under ``opentracing``. For example: + +.. code-block:: yaml + + opentracing: + # Enable / disable tracer + tracer_enabled: true + # The list of homeservers we wish to expose our current traces to. + # The list is a list of regexes which are matched against the + # servername of the homeserver + homeserver_whitelist: + - ".*" + +Homeserver whitelisting +----------------------- + +The homeserver whitelist is configured using regex. A list of regexes can be +given and their union will be compared when propagating any spans through a +carrier. Most of the whitelist checks are encapsulated in the lib's injection +and extraction method but be aware that using custom carriers or crossing +unchartered waters will require the enforcement of this whitelist. + +``logging/opentracing.py`` has a ``whitelisted_homeserver`` method which takes +in a destination and compares it to the whitelist. + +============================ +Using opentracing in synapse +============================ + +Access to the opentracing api is mediated through the +``logging/opentracing.py`` lib. Opentracing is encapsulated such that +no statefull spans from opentracing are used in synapses code. This allows +opentracing to be easily disabled in synapse and thereby have opentracing as +an optional dependency. This does however limit the number of modifyable spans +at any point in the code to one. From here out references to opentracing refer +to the lib implemented in synapse. + +Tracing +------- + +In synapse it is not possible to start a non-active span. Spans can be started +using the ``opentracing.start_active_span`` method. This returns a scope (see +opentracing docs) which is a context manager that needs to be entered and +exited. This is usually done by using ``with``. + +.. code-block:: python + + with start_active_span("operation name"): + # Do something we want to tracer + +Forgetting to enter or exit a scope will result in some mysterious grevious log +context errors. + +At anytime where there is an active span ``opentracing.set_tag`` can be used to +set a tag on the current active span. + +Tracing functions +----------------- + +Functions can be easily traced using decorators. There is a decorator for +'normal' function and for functions which are actually deferreds. The name of +function becomes the operation name for the span. + +.. code-block:: python + + # Start a span using 'normal_function' as the operation name + @trace_function + def normal_function(*args, **kwargs): + # Does all kinds of cool and expected things + return something_usual_and_useful + + # Start a span using 'deferred_function' as the operation name + @trace_defered_function + # Yes, there is a typo in the lib. I will fix this + def deferred_function(*args, **kwargs): + # We start + yield we_wait + # we finish + defer.returnValue(something_usual_and_useful) + +Operation names can be explicitely set for functions by using +``trace_function_using_operation_name`` and +``trace_defered_function_using_operation_name`` + +.. code-block:: python + + @trace_function_using_operation_name("A *much* better operation name") + def normal_function(*args, **kwargs): + # Does all kinds of cool and expected things + return something_usual_and_useful + + @trace_defered_function_using_operation_name("An operation name that fixes the typo!") + # Yes, there is a typo in the lib. I will fix this + def deferred_function(*args, **kwargs): + # We start + yield we_wait + # we finish + defer.returnValue(something_usual_and_useful) + +Contexts and carriers +--------------------- + +There are a selection of wrappers for injecting and extracting contexts from +carriers provided. Unfortunately opentracing's standard three are not adequate +in the majority of cases. Also note that the binnary encoding format mandated +by opentracing is not actually implemented by Jaeger and it will silently noop. +Please refer to the the end of ``logging/opentracing.py`` for the available +injection and extraction methods. + +================== +Configuring Jaeger +================== + +Sampling strategies can be set as in this document: +https://www.jaegertracing.io/docs/1.13/sampling/ + +======= +Gotchas +======= + +- Checking whitelists on span propagation +- Inserting pii +- Forgetting to enter or exit a scope +- Span source: make sure that the span you expect to be active across a + function call really will be that one. Does the current function have more + than one caller? Will all of those calling functions have be in a context + with an active span? From 1345ac85010faeef87a8454e7cb1a621b021c9de Mon Sep 17 00:00:00 2001 From: Jorik Schellekens Date: Fri, 19 Jul 2019 13:47:08 +0100 Subject: [PATCH 2/9] Update decorator names in doc --- docs/opentracing.rst | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/opentracing.rst b/docs/opentracing.rst index 02eba904dbbf..c8133a8fb2cf 100644 --- a/docs/opentracing.rst +++ b/docs/opentracing.rst @@ -116,13 +116,13 @@ function becomes the operation name for the span. .. code-block:: python # Start a span using 'normal_function' as the operation name - @trace_function + @trace def normal_function(*args, **kwargs): # Does all kinds of cool and expected things return something_usual_and_useful # Start a span using 'deferred_function' as the operation name - @trace_defered_function + @trace_deferred # Yes, there is a typo in the lib. I will fix this def deferred_function(*args, **kwargs): # We start @@ -131,17 +131,17 @@ function becomes the operation name for the span. defer.returnValue(something_usual_and_useful) Operation names can be explicitely set for functions by using -``trace_function_using_operation_name`` and -``trace_defered_function_using_operation_name`` +``trace_using_operation_name`` and +``trace_deferred_using_operation_name`` .. code-block:: python - @trace_function_using_operation_name("A *much* better operation name") + @trace_using_operation_name("A *much* better operation name") def normal_function(*args, **kwargs): # Does all kinds of cool and expected things return something_usual_and_useful - @trace_defered_function_using_operation_name("An operation name that fixes the typo!") + @trace_deferred_using_operation_name("An operation name that fixes the typo!") # Yes, there is a typo in the lib. I will fix this def deferred_function(*args, **kwargs): # We start From ca51f38bb3947a463cb8782a51f2014196c706aa Mon Sep 17 00:00:00 2001 From: Jorik Schellekens Date: Fri, 19 Jul 2019 15:44:02 +0100 Subject: [PATCH 3/9] Doc cleanup These are all alterations as a result of comments in #5703, it includes mostly typos and clarifications. The most interesting changes are: - Split developer and user docs into two sections - Add a high level description of OpenTracing --- docs/opentracing.rst | 165 +++++++++++++++++++++++++-------------- synapse/config/tracer.py | 10 --- 2 files changed, 105 insertions(+), 70 deletions(-) diff --git a/docs/opentracing.rst b/docs/opentracing.rst index c8133a8fb2cf..979f1c759ea5 100644 --- a/docs/opentracing.rst +++ b/docs/opentracing.rst @@ -1,106 +1,133 @@ =========== -Opentracing +OpenTracing =========== Background ---------- -Opentracing is semi-standard being addopted by a number of distributed tracing -platforms. It is a standardised api for facilitating vendor agnostic tracing -instrumentation. That is, we can use the opentracing api and select one of a +OpenTracing is a semi-standard being adopted by a number of distributed tracing +platforms. It is a common api for facilitating vendor-agnostic tracing +instrumentation. That is, we can use the OpenTracing api and select one of a number of tracer implementations to do the heavy lifting in the background. Our current selected implementation is Jaeger. -Opentracing concepts can be found at -https://opentracing.io/docs/overview/what-is-tracing/ - -Python specific tracing concepts are at https://opentracing.io/guides/python/. -Note that synapse wraps opentracing in a small library in order to make the -opentracing dependency optional. That means that the access patterns are -different to those demonstrated here. However, it is still usefull to know. -Especially if opentracing is included as a full dependency in the future or if -you are modifying synapse's opentracing lib. +OpenTracing is a tool which gives an insight into the causal relationship of +work done in and between servers. The servers each track events and report them +to a centralised server - in our Synapse's case: Jaeger. The basic unit used to +represent events is the span. The span roughly represents a single piece of work +that was done and the time at which it occurred. A span can have child spans, +meaning that the work of the child had to be completed for the parent span to +complete, or it can have follow-on spans which represent work that is undertaken +as a result of the parent but is not depended on by the parent to in order to +finish. + +Since this is undertaken in a distributed environment a request to another +server, such as an RPC or a simple GET, can be considered a span (a unit or +work) for the local server. This causal link is what OpenTracing aims to +capture and visualise. In order to do this metadata about the local server's +span, i.e the 'span context', needs to be included with the request to the +remote. + +It is up to the remote server to decide what it does with the spans +it creates. This is called the sampling policy and it can be configured +through Jaeger's settings. + +For OpenTracing concepts see +https://opentracing.io/docs/overview/what-is-tracing/. For more information about Jaeger's implementation see https://www.jaegertracing.io/docs/ -================= -Setup opentracing -================= +===================== +Seting up OpenTracing +===================== -To receive opentracing spans start up a Jaeger server using docker like so +To receive OpenTracing spans, start up a Jaeger server. This can be done +using docker like so: .. code-block:: bash - docker run -d --name jaeger \ -e COLLECTOR_ZIPKIN_HTTP_PORT=9411 \ - -p 5775:5775/udp \ + docker run -d --name jaeger -p 6831:6831/udp \ -p 6832:6832/udp \ -p 5778:5778 \ -p 16686:16686 \ -p 14268:14268 \ - -p 9411:9411 \ jaegertracing/all-in-one:1.13 Latest documentation is probably at https://www.jaegertracing.io/docs/1.13/getting-started/ -Enable opentracing in synapse +Enable opentracing in Synapse ----------------------------- Opentracing is not enabled by default. It must be enabled in the homeserver -config by uncommenting the config options under ``opentracing``. For example: +config by uncommenting the config options under ``opentracing`` as shown in +the [sample config](./sample_config.yaml). For example: .. code-block:: yaml opentracing: - # Enable / disable tracer tracer_enabled: true - # The list of homeservers we wish to expose our current traces to. - # The list is a list of regexes which are matched against the - # servername of the homeserver homeserver_whitelist: - - ".*" + - "mytrustedhomeserver.org" + - "*.myotherhomeservers.com" Homeserver whitelisting ----------------------- -The homeserver whitelist is configured using regex. A list of regexes can be -given and their union will be compared when propagating any spans through a -carrier. Most of the whitelist checks are encapsulated in the lib's injection -and extraction method but be aware that using custom carriers or crossing -unchartered waters will require the enforcement of this whitelist. - -``logging/opentracing.py`` has a ``whitelisted_homeserver`` method which takes -in a destination and compares it to the whitelist. - -============================ -Using opentracing in synapse -============================ - -Access to the opentracing api is mediated through the -``logging/opentracing.py`` lib. Opentracing is encapsulated such that -no statefull spans from opentracing are used in synapses code. This allows -opentracing to be easily disabled in synapse and thereby have opentracing as -an optional dependency. This does however limit the number of modifyable spans -at any point in the code to one. From here out references to opentracing refer -to the lib implemented in synapse. +The homeserver whitelist is configured using regular expression. A list of regular +expressions can be given and their union will be compared when propagating any +spans contexts to another homeserver. + +Though it's mostly safe to send and receive span contexts to and from +untrusted users since span contexts are usually opaque ids it can lead to +two problems, namely: + +- If the span context is marked as sampled by the sending homeserver the receiver will + sample it. Therefore two homeservers with wildly disparaging sampling policies + could incur higher sampling counts than intended. +- Span baggage can be arbitrary data. For safety this has been disabled in Synapse + but that doesn't prevent another server sending you baggage which will be logged + to OpenTracing's logs. + +======================================== +Developers: Using OpenTracing in Synapse +======================================== + +Python-specific tracing concepts are at https://opentracing.io/guides/python/. +Note that Synapse wraps OpenTracing in a small module in order to make the +OpenTracing dependency optional. That means that the access patterns are +different to those demonstrated in the OpenTracing guides. However, it is +still useful to know, especially if OpenTracing is included as a full dependency +in the future or if you are modifying Synapse's `opentracing` module. + + +Access to the OpenTracing API is mediated through the +``logging/opentracing.py`` module. OpenTracing is encapsulated so that +no span objects from OpenTracing are exposed in Synapses code. This allows +OpenTracing to be easily disabled in Synapse and thereby have OpenTracing as +an optional dependency. This does however limit the number of modifiable spans +at any point in the code to one. From here out references to `opentracing` +in the code snippets refer to the Synapses module. Tracing ------- -In synapse it is not possible to start a non-active span. Spans can be started -using the ``opentracing.start_active_span`` method. This returns a scope (see -opentracing docs) which is a context manager that needs to be entered and +In Synapse it is not possible to start a non-active span. Spans can be started +using the ``start_active_span`` method. This returns a scope (see +OpenTracing docs) which is a context manager that needs to be entered and exited. This is usually done by using ``with``. .. code-block:: python + from synapse.logging.opentracing import start_active_span + with start_active_span("operation name"): # Do something we want to tracer -Forgetting to enter or exit a scope will result in some mysterious grevious log +Forgetting to enter or exit a scope will result in some mysterious and grievous log context errors. At anytime where there is an active span ``opentracing.set_tag`` can be used to @@ -111,9 +138,11 @@ Tracing functions Functions can be easily traced using decorators. There is a decorator for 'normal' function and for functions which are actually deferreds. The name of -function becomes the operation name for the span. +the function becomes the operation name for the span. .. code-block:: python + + from synapse.logging.opentracing import trace, trace_deferred # Start a span using 'normal_function' as the operation name @trace @@ -123,26 +152,31 @@ function becomes the operation name for the span. # Start a span using 'deferred_function' as the operation name @trace_deferred - # Yes, there is a typo in the lib. I will fix this + @defer.inlineCallbacks def deferred_function(*args, **kwargs): # We start yield we_wait # we finish defer.returnValue(something_usual_and_useful) -Operation names can be explicitely set for functions by using +Operation names can be explicitly set for functions by using ``trace_using_operation_name`` and ``trace_deferred_using_operation_name`` .. code-block:: python + from synapse.logging.opentracing import ( + trace_using_operation_name, + trace_deferred_using_operation_name + ) + @trace_using_operation_name("A *much* better operation name") def normal_function(*args, **kwargs): # Does all kinds of cool and expected things return something_usual_and_useful - @trace_deferred_using_operation_name("An operation name that fixes the typo!") - # Yes, there is a typo in the lib. I will fix this + @trace_deferred_using_operation_name("Another exciting operation name!") + @defer.inlineCallbacks def deferred_function(*args, **kwargs): # We start yield we_wait @@ -153,12 +187,23 @@ Contexts and carriers --------------------- There are a selection of wrappers for injecting and extracting contexts from -carriers provided. Unfortunately opentracing's standard three are not adequate -in the majority of cases. Also note that the binnary encoding format mandated -by opentracing is not actually implemented by Jaeger and it will silently noop. -Please refer to the the end of ``logging/opentracing.py`` for the available +carriers provided. Unfortunately OpenTracing's three context injection +techniques are not adequate for our inject of OpenTracing span-contexts into +Twisted's http headers, EDU contents and our database tables. Also note that +the binary encoding format mandated by OpenTracing is not actually implemented +by jaeger_client v4.0.0 - it will silently noop. +Please refer to the end of ``logging/opentracing.py`` for the available injection and extraction methods. +homeserver whitelisting +----------------------- + +Most of the whitelist checks are encapsulated in the modules's injection +and extraction method but be aware that using custom carriers or crossing +unchartered waters will require the enforcement of the whitelist. +``logging/opentracing.py`` has a ``whitelisted_homeserver`` method which takes +in a destination and compares it to the whitelist. + ================== Configuring Jaeger ================== diff --git a/synapse/config/tracer.py b/synapse/config/tracer.py index a2ce9ab3f66f..24b985b462f3 100644 --- a/synapse/config/tracer.py +++ b/synapse/config/tracer.py @@ -49,16 +49,6 @@ def generate_config_section(cls, **kwargs): # The list of homeservers we wish to send and receive span contexts and span baggage. # - # Though it's mostly safe to send and receive span contexts to and from - # untrusted users since span contexts are usually opaque ids it can lead to - # two problems, namely: - # - If the span context is marked as sampled by the sending homeserver the receiver will - # sample it. Therefore two homeservers with wildly disparaging sampling policies - # could incur higher sampling counts than intended. - # - Span baggage can be arbitrary data. For safety this has been disabled in synapse - # but that doesn't prevent another server sending you baggage which will be logged - # to opentracing logs. - # # This a list of regexes which are matched against the server_name of the # homeserver. # From 4767ca41c7752189f4a7485b4babdf08fabc2915 Mon Sep 17 00:00:00 2001 From: Jorik Schellekens Date: Wed, 17 Jul 2019 14:45:34 +0100 Subject: [PATCH 4/9] newsfile --- changelog.d/5703.misc | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/5703.misc diff --git a/changelog.d/5703.misc b/changelog.d/5703.misc new file mode 100644 index 000000000000..f2d236188dee --- /dev/null +++ b/changelog.d/5703.misc @@ -0,0 +1 @@ +Add a set of opentracing utils. From 28e422fa44cff5624a209bf19e233ef91e5a97c7 Mon Sep 17 00:00:00 2001 From: Jorik Schellekens Date: Fri, 19 Jul 2019 15:53:53 +0100 Subject: [PATCH 5/9] Move contributer specific info to docstring. --- docs/opentracing.rst | 124 -------------------------------- synapse/logging/opentracing.py | 126 +++++++++++++++++++++++++++++++++ 2 files changed, 126 insertions(+), 124 deletions(-) diff --git a/docs/opentracing.rst b/docs/opentracing.rst index 979f1c759ea5..18b43415893d 100644 --- a/docs/opentracing.rst +++ b/docs/opentracing.rst @@ -92,133 +92,9 @@ two problems, namely: but that doesn't prevent another server sending you baggage which will be logged to OpenTracing's logs. -======================================== -Developers: Using OpenTracing in Synapse -======================================== - -Python-specific tracing concepts are at https://opentracing.io/guides/python/. -Note that Synapse wraps OpenTracing in a small module in order to make the -OpenTracing dependency optional. That means that the access patterns are -different to those demonstrated in the OpenTracing guides. However, it is -still useful to know, especially if OpenTracing is included as a full dependency -in the future or if you are modifying Synapse's `opentracing` module. - - -Access to the OpenTracing API is mediated through the -``logging/opentracing.py`` module. OpenTracing is encapsulated so that -no span objects from OpenTracing are exposed in Synapses code. This allows -OpenTracing to be easily disabled in Synapse and thereby have OpenTracing as -an optional dependency. This does however limit the number of modifiable spans -at any point in the code to one. From here out references to `opentracing` -in the code snippets refer to the Synapses module. - -Tracing -------- - -In Synapse it is not possible to start a non-active span. Spans can be started -using the ``start_active_span`` method. This returns a scope (see -OpenTracing docs) which is a context manager that needs to be entered and -exited. This is usually done by using ``with``. - -.. code-block:: python - - from synapse.logging.opentracing import start_active_span - - with start_active_span("operation name"): - # Do something we want to tracer - -Forgetting to enter or exit a scope will result in some mysterious and grievous log -context errors. - -At anytime where there is an active span ``opentracing.set_tag`` can be used to -set a tag on the current active span. - -Tracing functions ------------------ - -Functions can be easily traced using decorators. There is a decorator for -'normal' function and for functions which are actually deferreds. The name of -the function becomes the operation name for the span. - -.. code-block:: python - - from synapse.logging.opentracing import trace, trace_deferred - - # Start a span using 'normal_function' as the operation name - @trace - def normal_function(*args, **kwargs): - # Does all kinds of cool and expected things - return something_usual_and_useful - - # Start a span using 'deferred_function' as the operation name - @trace_deferred - @defer.inlineCallbacks - def deferred_function(*args, **kwargs): - # We start - yield we_wait - # we finish - defer.returnValue(something_usual_and_useful) - -Operation names can be explicitly set for functions by using -``trace_using_operation_name`` and -``trace_deferred_using_operation_name`` - -.. code-block:: python - - from synapse.logging.opentracing import ( - trace_using_operation_name, - trace_deferred_using_operation_name - ) - - @trace_using_operation_name("A *much* better operation name") - def normal_function(*args, **kwargs): - # Does all kinds of cool and expected things - return something_usual_and_useful - - @trace_deferred_using_operation_name("Another exciting operation name!") - @defer.inlineCallbacks - def deferred_function(*args, **kwargs): - # We start - yield we_wait - # we finish - defer.returnValue(something_usual_and_useful) - -Contexts and carriers ---------------------- - -There are a selection of wrappers for injecting and extracting contexts from -carriers provided. Unfortunately OpenTracing's three context injection -techniques are not adequate for our inject of OpenTracing span-contexts into -Twisted's http headers, EDU contents and our database tables. Also note that -the binary encoding format mandated by OpenTracing is not actually implemented -by jaeger_client v4.0.0 - it will silently noop. -Please refer to the end of ``logging/opentracing.py`` for the available -injection and extraction methods. - -homeserver whitelisting ------------------------ - -Most of the whitelist checks are encapsulated in the modules's injection -and extraction method but be aware that using custom carriers or crossing -unchartered waters will require the enforcement of the whitelist. -``logging/opentracing.py`` has a ``whitelisted_homeserver`` method which takes -in a destination and compares it to the whitelist. - ================== Configuring Jaeger ================== Sampling strategies can be set as in this document: https://www.jaegertracing.io/docs/1.13/sampling/ - -======= -Gotchas -======= - -- Checking whitelists on span propagation -- Inserting pii -- Forgetting to enter or exit a scope -- Span source: make sure that the span you expect to be active across a - function call really will be that one. Does the current function have more - than one caller? Will all of those calling functions have be in a context - with an active span? diff --git a/synapse/logging/opentracing.py b/synapse/logging/opentracing.py index 415040f5ee00..17fa2abda634 100644 --- a/synapse/logging/opentracing.py +++ b/synapse/logging/opentracing.py @@ -24,6 +24,132 @@ # this move the methods have work very similarly to opentracing's and it should only # be a matter of few regexes to move over to opentracing's access patterns proper. +""" +============================ +Using OpenTracing in Synapse +============================ + +Python-specific tracing concepts are at https://opentracing.io/guides/python/. +Note that Synapse wraps OpenTracing in a small module in order to make the +OpenTracing dependency optional. That means that the access patterns are +different to those demonstrated in the OpenTracing guides. However, it is +still useful to know, especially if OpenTracing is included as a full dependency +in the future or if you are modifying Synapse's `opentracing` module. + + +Access to the OpenTracing API is mediated through the +``logging/opentracing.py`` module. OpenTracing is encapsulated so that +no span objects from OpenTracing are exposed in Synapses code. This allows +OpenTracing to be easily disabled in Synapse and thereby have OpenTracing as +an optional dependency. This does however limit the number of modifiable spans +at any point in the code to one. From here out references to `opentracing` +in the code snippets refer to the Synapses module. + +Tracing +------- + +In Synapse it is not possible to start a non-active span. Spans can be started +using the ``start_active_span`` method. This returns a scope (see +OpenTracing docs) which is a context manager that needs to be entered and +exited. This is usually done by using ``with``. + +.. code-block:: python + + from synapse.logging.opentracing import start_active_span + + with start_active_span("operation name"): + # Do something we want to tracer + +Forgetting to enter or exit a scope will result in some mysterious and grievous log +context errors. + +At anytime where there is an active span ``opentracing.set_tag`` can be used to +set a tag on the current active span. + +Tracing functions +----------------- + +Functions can be easily traced using decorators. There is a decorator for +'normal' function and for functions which are actually deferreds. The name of +the function becomes the operation name for the span. + +.. code-block:: python + + from synapse.logging.opentracing import trace, trace_deferred + + # Start a span using 'normal_function' as the operation name + @trace + def normal_function(*args, **kwargs): + # Does all kinds of cool and expected things + return something_usual_and_useful + + # Start a span using 'deferred_function' as the operation name + @trace_deferred + @defer.inlineCallbacks + def deferred_function(*args, **kwargs): + # We start + yield we_wait + # we finish + defer.returnValue(something_usual_and_useful) + +Operation names can be explicitly set for functions by using +``trace_using_operation_name`` and +``trace_deferred_using_operation_name`` + +.. code-block:: python + + from synapse.logging.opentracing import ( + trace_using_operation_name, + trace_deferred_using_operation_name + ) + + @trace_using_operation_name("A *much* better operation name") + def normal_function(*args, **kwargs): + # Does all kinds of cool and expected things + return something_usual_and_useful + + @trace_deferred_using_operation_name("Another exciting operation name!") + @defer.inlineCallbacks + def deferred_function(*args, **kwargs): + # We start + yield we_wait + # we finish + defer.returnValue(something_usual_and_useful) + +Contexts and carriers +--------------------- + +There are a selection of wrappers for injecting and extracting contexts from +carriers provided. Unfortunately OpenTracing's three context injection +techniques are not adequate for our inject of OpenTracing span-contexts into +Twisted's http headers, EDU contents and our database tables. Also note that +the binary encoding format mandated by OpenTracing is not actually implemented +by jaeger_client v4.0.0 - it will silently noop. +Please refer to the end of ``logging/opentracing.py`` for the available +injection and extraction methods. + +Homeserver whitelisting +----------------------- + +Most of the whitelist checks are encapsulated in the modules's injection +and extraction method but be aware that using custom carriers or crossing +unchartered waters will require the enforcement of the whitelist. +``logging/opentracing.py`` has a ``whitelisted_homeserver`` method which takes +in a destination and compares it to the whitelist. + +======= +Gotchas +======= + +- Checking whitelists on span propagation +- Inserting pii +- Forgetting to enter or exit a scope +- Span source: make sure that the span you expect to be active across a + function call really will be that one. Does the current function have more + than one caller? Will all of those calling functions have be in a context + with an active span? +""" + import contextlib import logging import re From 6755162bf6761cd9da84c756f4b759d930835b83 Mon Sep 17 00:00:00 2001 From: Jorik Schellekens Date: Fri, 19 Jul 2019 15:55:26 +0100 Subject: [PATCH 6/9] Sample config. --- docs/sample_config.yaml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/docs/sample_config.yaml b/docs/sample_config.yaml index 5b804d16a463..2bd818c2ab0a 100644 --- a/docs/sample_config.yaml +++ b/docs/sample_config.yaml @@ -1423,16 +1423,6 @@ opentracing: # The list of homeservers we wish to send and receive span contexts and span baggage. # - # Though it's mostly safe to send and receive span contexts to and from - # untrusted users since span contexts are usually opaque ids it can lead to - # two problems, namely: - # - If the span context is marked as sampled by the sending homeserver the receiver will - # sample it. Therefore two homeservers with wildly disparaging sampling policies - # could incur higher sampling counts than intended. - # - Span baggage can be arbitrary data. For safety this has been disabled in synapse - # but that doesn't prevent another server sending you baggage which will be logged - # to opentracing logs. - # # This a list of regexes which are matched against the server_name of the # homeserver. # From adf0c9cfaf1fe352d9032f761dfb6bb29818d405 Mon Sep 17 00:00:00 2001 From: Jorik Schellekens Date: Fri, 19 Jul 2019 15:59:40 +0100 Subject: [PATCH 7/9] Trailing whitespace. --- synapse/logging/opentracing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/logging/opentracing.py b/synapse/logging/opentracing.py index 17fa2abda634..06c72a14d247 100644 --- a/synapse/logging/opentracing.py +++ b/synapse/logging/opentracing.py @@ -74,7 +74,7 @@ the function becomes the operation name for the span. .. code-block:: python - + from synapse.logging.opentracing import trace, trace_deferred # Start a span using 'normal_function' as the operation name From 65d22aa714c39c41fe036774e024e4834d95a450 Mon Sep 17 00:00:00 2001 From: Jorik Schellekens Date: Fri, 19 Jul 2019 16:43:55 +0100 Subject: [PATCH 8/9] Update 5703.misc --- changelog.d/5703.misc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog.d/5703.misc b/changelog.d/5703.misc index f2d236188dee..6e9b2d734e03 100644 --- a/changelog.d/5703.misc +++ b/changelog.d/5703.misc @@ -1 +1 @@ -Add a set of opentracing utils. +Documentation for opentracing. From 15ff12e8c6058e39ab2bfd8109f093a7b925e191 Mon Sep 17 00:00:00 2001 From: Jorik Schellekens Date: Mon, 22 Jul 2019 10:06:23 +0100 Subject: [PATCH 9/9] Apply suggestions from code review Mostly just rewording parts of the docs for clarity. Co-Authored-By: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> --- docs/opentracing.rst | 14 +++++++------- docs/sample_config.yaml | 4 ++-- synapse/config/tracer.py | 4 ++-- synapse/logging/opentracing.py | 9 ++++----- 4 files changed, 15 insertions(+), 16 deletions(-) diff --git a/docs/opentracing.rst b/docs/opentracing.rst index 18b43415893d..b91a2208a8fb 100644 --- a/docs/opentracing.rst +++ b/docs/opentracing.rst @@ -13,7 +13,7 @@ Our current selected implementation is Jaeger. OpenTracing is a tool which gives an insight into the causal relationship of work done in and between servers. The servers each track events and report them -to a centralised server - in our Synapse's case: Jaeger. The basic unit used to +to a centralised server - in Synapse's case: Jaeger. The basic unit used to represent events is the span. The span roughly represents a single piece of work that was done and the time at which it occurred. A span can have child spans, meaning that the work of the child had to be completed for the parent span to @@ -59,12 +59,12 @@ Latest documentation is probably at https://www.jaegertracing.io/docs/1.13/getting-started/ -Enable opentracing in Synapse +Enable OpenTracing in Synapse ----------------------------- -Opentracing is not enabled by default. It must be enabled in the homeserver +OpenTracing is not enabled by default. It must be enabled in the homeserver config by uncommenting the config options under ``opentracing`` as shown in -the [sample config](./sample_config.yaml). For example: +the `sample config <./sample_config.yaml>`_. For example: .. code-block:: yaml @@ -77,7 +77,7 @@ the [sample config](./sample_config.yaml). For example: Homeserver whitelisting ----------------------- -The homeserver whitelist is configured using regular expression. A list of regular +The homeserver whitelist is configured using regular expressions. A list of regular expressions can be given and their union will be compared when propagating any spans contexts to another homeserver. @@ -86,9 +86,9 @@ untrusted users since span contexts are usually opaque ids it can lead to two problems, namely: - If the span context is marked as sampled by the sending homeserver the receiver will - sample it. Therefore two homeservers with wildly disparaging sampling policies + sample it. Therefore two homeservers with wildly different sampling policies could incur higher sampling counts than intended. -- Span baggage can be arbitrary data. For safety this has been disabled in Synapse +- Sending servers can attach arbitrary data to spans, known as 'baggage'. For safety this has been disabled in Synapse but that doesn't prevent another server sending you baggage which will be logged to OpenTracing's logs. diff --git a/docs/sample_config.yaml b/docs/sample_config.yaml index 2bd818c2ab0a..0a96197ca65e 100644 --- a/docs/sample_config.yaml +++ b/docs/sample_config.yaml @@ -1422,8 +1422,8 @@ opentracing: #enabled: true # The list of homeservers we wish to send and receive span contexts and span baggage. - # - # This a list of regexes which are matched against the server_name of the + # See docs/opentracing.rst + # This is a list of regexes which are matched against the server_name of the # homeserver. # # By defult, it is empty, so no servers are matched. diff --git a/synapse/config/tracer.py b/synapse/config/tracer.py index 24b985b462f3..4479454415d4 100644 --- a/synapse/config/tracer.py +++ b/synapse/config/tracer.py @@ -48,8 +48,8 @@ def generate_config_section(cls, **kwargs): #enabled: true # The list of homeservers we wish to send and receive span contexts and span baggage. - # - # This a list of regexes which are matched against the server_name of the + # See docs/opentracing.rst + # This is a list of regexes which are matched against the server_name of the # homeserver. # # By defult, it is empty, so no servers are matched. diff --git a/synapse/logging/opentracing.py b/synapse/logging/opentracing.py index 06c72a14d247..3da33d7826fe 100644 --- a/synapse/logging/opentracing.py +++ b/synapse/logging/opentracing.py @@ -30,16 +30,15 @@ ============================ Python-specific tracing concepts are at https://opentracing.io/guides/python/. -Note that Synapse wraps OpenTracing in a small module in order to make the +Note that Synapse wraps OpenTracing in a small module (this one) in order to make the OpenTracing dependency optional. That means that the access patterns are different to those demonstrated in the OpenTracing guides. However, it is still useful to know, especially if OpenTracing is included as a full dependency -in the future or if you are modifying Synapse's `opentracing` module. +in the future or if you are modifying this module. -Access to the OpenTracing API is mediated through the -``logging/opentracing.py`` module. OpenTracing is encapsulated so that -no span objects from OpenTracing are exposed in Synapses code. This allows +OpenTracing is encapsulated so that +no span objects from OpenTracing are exposed in Synapse's code. This allows OpenTracing to be easily disabled in Synapse and thereby have OpenTracing as an optional dependency. This does however limit the number of modifiable spans at any point in the code to one. From here out references to `opentracing`