From 30c004cf06fd53754b119507b7f210afb674b1b4 Mon Sep 17 00:00:00 2001 From: FranFin Date: Mon, 23 Nov 2020 12:36:52 +0100 Subject: [PATCH 1/5] Grouping memory profiling and benchmarking under the same section. --- _data/docs.yml | 7 ++----- .../benchmarking/{ => benchmarking}/index.md | 2 +- .../concepts/{ => benchmarking}/memo_prof/3mems.png | Bin .../concepts/{ => benchmarking}/memo_prof/index.md | 3 ++- .../{ => benchmarking}/memo_prof/overall.png | Bin .../{ => benchmarking}/memo_prof/pubsub_diagram.png | Bin .../{ => benchmarking}/memo_prof/rmw_history.png | Bin .../{ => benchmarking}/memo_prof/servcli.png | Bin .../memo_prof/services_diagram.png | Bin _includes/docs_nav.html | 3 --- 10 files changed, 5 insertions(+), 10 deletions(-) rename _docs/concepts/benchmarking/{ => benchmarking}/index.md (99%) rename _docs/concepts/{ => benchmarking}/memo_prof/3mems.png (100%) rename _docs/concepts/{ => benchmarking}/memo_prof/index.md (99%) rename _docs/concepts/{ => benchmarking}/memo_prof/overall.png (100%) rename _docs/concepts/{ => benchmarking}/memo_prof/pubsub_diagram.png (100%) rename _docs/concepts/{ => benchmarking}/memo_prof/rmw_history.png (100%) rename _docs/concepts/{ => benchmarking}/memo_prof/servcli.png (100%) rename _docs/concepts/{ => benchmarking}/memo_prof/services_diagram.png (100%) diff --git a/_data/docs.yml b/_data/docs.yml index 6a41f494..b5801fe4 100644 --- a/_data/docs.yml +++ b/_data/docs.yml @@ -35,11 +35,8 @@ - title: Benchmarking docs: - - concepts/benchmarking - -- title: Memory profiling - docs: - - concepts/memo_prof + - concepts/benchmarking/benchmarking + - concepts/benchmarking/memo_prof - title: Core Tutorials diff --git a/_docs/concepts/benchmarking/index.md b/_docs/concepts/benchmarking/benchmarking/index.md similarity index 99% rename from _docs/concepts/benchmarking/index.md rename to _docs/concepts/benchmarking/benchmarking/index.md index 5d9c23e3..b56a85af 100644 --- a/_docs/concepts/benchmarking/index.md +++ b/_docs/concepts/benchmarking/benchmarking/index.md @@ -1,7 +1,7 @@ --- title: Benchmarking redirect_from: /benchmarking/ -permalink: /docs/concepts/benchmarking/ +permalink: /docs/concepts/benchmarking/benchmarking/ --- ## Table of contents diff --git a/_docs/concepts/memo_prof/3mems.png b/_docs/concepts/benchmarking/memo_prof/3mems.png similarity index 100% rename from _docs/concepts/memo_prof/3mems.png rename to _docs/concepts/benchmarking/memo_prof/3mems.png diff --git a/_docs/concepts/memo_prof/index.md b/_docs/concepts/benchmarking/memo_prof/index.md similarity index 99% rename from _docs/concepts/memo_prof/index.md rename to _docs/concepts/benchmarking/memo_prof/index.md index 2802cc69..7ca16986 100644 --- a/_docs/concepts/memo_prof/index.md +++ b/_docs/concepts/benchmarking/memo_prof/index.md @@ -1,6 +1,7 @@ --- title: Memory profiling -permalink: /docs/concepts/memo_prof/ +redirect_from: /memo_prof/ +permalink: /docs/concepts/benchmarking/memo_prof/ --- ## Abstract diff --git a/_docs/concepts/memo_prof/overall.png b/_docs/concepts/benchmarking/memo_prof/overall.png similarity index 100% rename from _docs/concepts/memo_prof/overall.png rename to _docs/concepts/benchmarking/memo_prof/overall.png diff --git a/_docs/concepts/memo_prof/pubsub_diagram.png b/_docs/concepts/benchmarking/memo_prof/pubsub_diagram.png similarity index 100% rename from _docs/concepts/memo_prof/pubsub_diagram.png rename to _docs/concepts/benchmarking/memo_prof/pubsub_diagram.png diff --git a/_docs/concepts/memo_prof/rmw_history.png b/_docs/concepts/benchmarking/memo_prof/rmw_history.png similarity index 100% rename from _docs/concepts/memo_prof/rmw_history.png rename to _docs/concepts/benchmarking/memo_prof/rmw_history.png diff --git a/_docs/concepts/memo_prof/servcli.png b/_docs/concepts/benchmarking/memo_prof/servcli.png similarity index 100% rename from _docs/concepts/memo_prof/servcli.png rename to _docs/concepts/benchmarking/memo_prof/servcli.png diff --git a/_docs/concepts/memo_prof/services_diagram.png b/_docs/concepts/benchmarking/memo_prof/services_diagram.png similarity index 100% rename from _docs/concepts/memo_prof/services_diagram.png rename to _docs/concepts/benchmarking/memo_prof/services_diagram.png diff --git a/_includes/docs_nav.html b/_includes/docs_nav.html index f4821b47..08c0c044 100644 --- a/_includes/docs_nav.html +++ b/_includes/docs_nav.html @@ -16,9 +16,6 @@ {% if page.path contains "_docs/concepts" and section.title == "Benchmarking" %} {% assign should_show_this_menu = true %} {% endif %} - {% if page.path contains "_docs/concepts" and section.title == "Memory profiling" %} - {% assign should_show_this_menu = true %} - {% endif %} {% if page.path contains "_docs/tutorials" and section.title contains "Tutorials" %} {% assign should_show_this_menu = true %} {% endif %} From f18c86d3f1738e2d0d2316ea5f04497911501dbc Mon Sep 17 00:00:00 2001 From: FranFin Date: Mon, 23 Nov 2020 12:36:52 +0100 Subject: [PATCH 2/5] Grouping memory profiling and benchmarking under the same section. --- _data/docs.yml | 5 +- .../benchmarking/benchmarking/index.md | 171 ++++++++++++++++++ .../{ => benchmarking}/memo_prof/3mems.png | Bin .../{ => benchmarking}/memo_prof/index.md | 3 +- .../{ => benchmarking}/memo_prof/overall.png | Bin .../memo_prof/pubsub_diagram.png | Bin .../memo_prof/rmw_history.png | Bin .../{ => benchmarking}/memo_prof/servcli.png | Bin .../memo_prof/services_diagram.png | Bin _includes/docs_nav.html | 3 - 10 files changed, 174 insertions(+), 8 deletions(-) create mode 100644 _docs/concepts/benchmarking/benchmarking/index.md rename _docs/concepts/{ => benchmarking}/memo_prof/3mems.png (100%) rename _docs/concepts/{ => benchmarking}/memo_prof/index.md (99%) rename _docs/concepts/{ => benchmarking}/memo_prof/overall.png (100%) rename _docs/concepts/{ => benchmarking}/memo_prof/pubsub_diagram.png (100%) rename _docs/concepts/{ => benchmarking}/memo_prof/rmw_history.png (100%) rename _docs/concepts/{ => benchmarking}/memo_prof/servcli.png (100%) rename _docs/concepts/{ => benchmarking}/memo_prof/services_diagram.png (100%) diff --git a/_data/docs.yml b/_data/docs.yml index 3a01bfc6..bcbc1c39 100644 --- a/_data/docs.yml +++ b/_data/docs.yml @@ -37,10 +37,7 @@ docs: - concepts/benchmarking/concept - concepts/benchmarking/results - -- title: Memory profiling - docs: - - concepts/memo_prof + - concepts/benchmarking/memo_prof - title: Core Tutorials docs: diff --git a/_docs/concepts/benchmarking/benchmarking/index.md b/_docs/concepts/benchmarking/benchmarking/index.md new file mode 100644 index 00000000..b56a85af --- /dev/null +++ b/_docs/concepts/benchmarking/benchmarking/index.md @@ -0,0 +1,171 @@ +--- +title: Benchmarking +redirect_from: /benchmarking/ +permalink: /docs/concepts/benchmarking/benchmarking/ +--- + +## Table of contents + +* [Introduction to Benchmarking](#introduction-to-benchmarking) +* [Our benchmarking tool framework](#our-benchmarking-tool-framework) +* [Trace Framework Abstraction](#trace-framework-abstraction) +* [Shadow Builder](#shadow-builder) +* [Binary generation for instrumented code](#binary-generation-for-instrumented-code) + * [Receiving inputs](#receiving-inputs) + * [Parse and Check](#parse-and-check) + * [TFA Execution](#tfa-execution) + * [Compilation](#compilation) +* [Step to start benchmarking](#step-to-start-benchmarking) + + + +## Introduction to Benchmarking + +Developing working and stable application from the scribbles to the final +executing binary is long and hard tasks. During this process developers may come +across stabilities issues, perfomances issues. In addition to these issues, some +specified QoS might be difficult to quantify. Solving those problems without the +proper tools might be frustrating, tedious tasks leading to reduce developers +efficiency. An adapted benchmarking tool could overcome all those development +obstacles and increase development time. There are different KPI (Keep +Performance Indicators) that one might be interested into. In the framework of +this micro-ROS, the KPI can be freely chosen by the developer. In this way, the +benchmarking tool will remain flexible and allow the community to constantly add +some support for a lot of different KPI. + +The problems we want to tackle are: + + * Out there, many benchmarking tools exist. Each of targeting different KPIs. + * Different platforms (Linux/Nuttx/Baremetal et.c.). + * Too few time/resources to code benchmarking tool for each. + * Avoid code overhead: Keep code clarity. + * Avoid execution overhead: Do not want to make execution slower when benchmarking. + +## Our Benchmarking tool framework + +The benchmarking tool under development is providing a framework to allow +developers to create their own benchmarking tool. Each part a developer wants to +benchmark can be added as a plugin using the provided framework. In this way +plugins can be shared and this improves re-usability as much as possible. + + +## Trace Framework Abstraction + +The Shadow builder alone only parse comments from the application and pass it +along to the Trace Framework Abstraction (TFA) Core. The TFA core is aware of +the plugins that are available, all the plugins’ capabilities and platform +target. The process goes as explained below: + + * The line containing the functionality Benchmarking::XX::YY will be checked + against all the available plugins. + * Plugins that are capable of handling functionality will respond with a piece of + code that will be replaced with a piece of code. + * Then the output file will be added in a folder corresponding to the platform + type and benchmarking type. + +Being generic is the key for this benchmarking tool. The plugins will in +contrary bring the specific implementation needed to benchmark a specific +platform. Every plugin will provide information as requested by the parser: + + * Provide a list of supported platforms. + * Provide a list of functions that are handled. + * Provide snippets codes that will be added for benchmarking. + * Provide a list of patches and/or patch code + * Optional provide an end script to run and execute the benchmarks + + +## Shadow Builder + +This section will introduce some concept related to the shadow builder (SB). + +The Shadow builder is a tool that will transparently instrument the code to +benchmark. The tools will be able to output an “instrumented code” that will be +later be compiled as a normal code. The following steps describe what the shadow +builder process flow: + + * Get configuration file from the user (Benchmarking Configuration File). + * Get appropriate sources. + * Execute Trace Framework Abstraction Configuration file. + * Parse the sources file needed Injecting code. + * Compile the targeted binary for different platform. + * If needed, depending what type benchmark is undertaken, compile another + target binary benchmarking. + +The SB (Shadow Builder) is meant to be as transparent as possible for the user. +And if the benchmarking is not activated, it should be bypassed. Get +appropriate sources + +The SB is in charge of getting the path/git repository to the source code that +needs to be benchmarking. The benchmarking. The sources are specified by the +user in the benchmarking configuration file. Injecting code + +In order to inject code, there are some tools that allow this. CLang AST tool +will allow to inject some code. + + +## Binary generation for instrumented code + +The binary generation is the process of compiling the source code. In order to +benchmark, previously to compile the source code, it is necessary to instrument +the code. The code will be instrumented in a transparent way for the +programmer/user. Therefore, a configuration file provided by the programmer will +be parsed and code injected as described in a configuration file. + +### Receiving inputs + +The binary generation's pipeline receives two inputs to work with: + * Configuration Benchmarking file. + * Source code to benchmark. + +In short, the configuration describes: + + * What is benchmarked (sources). + * Where to benchmark. + * What type of benchmark. + * Optionally against what base line to compare (base line source) + +### Parse and Check + +Once the input received the **Shadow Builder** parses the configuration +file. From the configuration file, the Shadow builder gets: + + * The different benchmarking to be achieved. + * The targeted platforms. + +In addition to parsing, the Shadow Builder is in charge of checking +capabilities and consistency within the configuration file and the different +TFA's plugins registered in the TFA module. + +### TFA Execution + +Once parsed and checked against the TFA module capabilities, the Shadow +Builder will be in charge of translating configuration into source code. The +translated sources will also be achieved in cooperation with the TFA module. The +detailed steps of the TFA can be found here. At the end of this step, the TFA +will generate the new forged source code ready for compilation. In addition to +patched source code, the TFA will generate scripts that will the benchmarks. + +### Compilation + +The compilation will happen for every kind of benchmarks and +platforms targeted. Depending on the kind of benchmark that is being executed, +there will be one or more binaries per benchmarks session. The number of binary +generated also depends on what plugins are provided by the user to the shadow +builder. The shadow builder will retrieve capabilities of the plugins and +request from the developer, match them and generated software according to the +matches. + + +## Step to start benchmarking + +The shadow Builder will be executed as follow: + + * Software sources are passed to the Shadow Builder. + * The source are passed and upon comments containing /*Benchmarking::XX::YY*/ + (a tag) the code line is passed to the Trace Framework Abstraction module. + Using comments is preferable → No includes needed. + * All plugins that registered to the TFA the Benchmarking::XX::YY functionality + will return a piece of code that will be added to the source. + * Once all parsed, the shadow builder will compile for all the different + platforms requested either by plugins or by user configuration. + diff --git a/_docs/concepts/memo_prof/3mems.png b/_docs/concepts/benchmarking/memo_prof/3mems.png similarity index 100% rename from _docs/concepts/memo_prof/3mems.png rename to _docs/concepts/benchmarking/memo_prof/3mems.png diff --git a/_docs/concepts/memo_prof/index.md b/_docs/concepts/benchmarking/memo_prof/index.md similarity index 99% rename from _docs/concepts/memo_prof/index.md rename to _docs/concepts/benchmarking/memo_prof/index.md index 2802cc69..7ca16986 100644 --- a/_docs/concepts/memo_prof/index.md +++ b/_docs/concepts/benchmarking/memo_prof/index.md @@ -1,6 +1,7 @@ --- title: Memory profiling -permalink: /docs/concepts/memo_prof/ +redirect_from: /memo_prof/ +permalink: /docs/concepts/benchmarking/memo_prof/ --- ## Abstract diff --git a/_docs/concepts/memo_prof/overall.png b/_docs/concepts/benchmarking/memo_prof/overall.png similarity index 100% rename from _docs/concepts/memo_prof/overall.png rename to _docs/concepts/benchmarking/memo_prof/overall.png diff --git a/_docs/concepts/memo_prof/pubsub_diagram.png b/_docs/concepts/benchmarking/memo_prof/pubsub_diagram.png similarity index 100% rename from _docs/concepts/memo_prof/pubsub_diagram.png rename to _docs/concepts/benchmarking/memo_prof/pubsub_diagram.png diff --git a/_docs/concepts/memo_prof/rmw_history.png b/_docs/concepts/benchmarking/memo_prof/rmw_history.png similarity index 100% rename from _docs/concepts/memo_prof/rmw_history.png rename to _docs/concepts/benchmarking/memo_prof/rmw_history.png diff --git a/_docs/concepts/memo_prof/servcli.png b/_docs/concepts/benchmarking/memo_prof/servcli.png similarity index 100% rename from _docs/concepts/memo_prof/servcli.png rename to _docs/concepts/benchmarking/memo_prof/servcli.png diff --git a/_docs/concepts/memo_prof/services_diagram.png b/_docs/concepts/benchmarking/memo_prof/services_diagram.png similarity index 100% rename from _docs/concepts/memo_prof/services_diagram.png rename to _docs/concepts/benchmarking/memo_prof/services_diagram.png diff --git a/_includes/docs_nav.html b/_includes/docs_nav.html index f4821b47..08c0c044 100644 --- a/_includes/docs_nav.html +++ b/_includes/docs_nav.html @@ -16,9 +16,6 @@ {% if page.path contains "_docs/concepts" and section.title == "Benchmarking" %} {% assign should_show_this_menu = true %} {% endif %} - {% if page.path contains "_docs/concepts" and section.title == "Memory profiling" %} - {% assign should_show_this_menu = true %} - {% endif %} {% if page.path contains "_docs/tutorials" and section.title contains "Tutorials" %} {% assign should_show_this_menu = true %} {% endif %} From 0ed7e6f663af5741cbefbc33ad9fbebd185007ad Mon Sep 17 00:00:00 2001 From: amx-piap <48675389+amx-piap@users.noreply.github.com> Date: Fri, 27 Nov 2020 06:27:32 +0100 Subject: [PATCH 3/5] Update _docs/concepts/benchmarking/benchmarking/index.md Co-authored-by: Jose Antonio Moral --- _docs/concepts/benchmarking/benchmarking/index.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/_docs/concepts/benchmarking/benchmarking/index.md b/_docs/concepts/benchmarking/benchmarking/index.md index b56a85af..fda3827f 100644 --- a/_docs/concepts/benchmarking/benchmarking/index.md +++ b/_docs/concepts/benchmarking/benchmarking/index.md @@ -87,7 +87,7 @@ builder process flow: * Get appropriate sources. * Execute Trace Framework Abstraction Configuration file. * Parse the sources file needed Injecting code. - * Compile the targeted binary for different platform. + * Compile the targeted binary for different platforms. * If needed, depending what type benchmark is undertaken, compile another target binary benchmarking. @@ -168,4 +168,3 @@ The shadow Builder will be executed as follow: will return a piece of code that will be added to the source. * Once all parsed, the shadow builder will compile for all the different platforms requested either by plugins or by user configuration. - From 32c07a2293d26f025b084cb2f23ae57e8a7af7b6 Mon Sep 17 00:00:00 2001 From: amx-piap <48675389+amx-piap@users.noreply.github.com> Date: Fri, 27 Nov 2020 06:29:21 +0100 Subject: [PATCH 4/5] Apply suggestions from code review Co-authored-by: Jose Antonio Moral --- .../benchmarking/benchmarking/index.md | 21 +++++++++---------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/_docs/concepts/benchmarking/benchmarking/index.md b/_docs/concepts/benchmarking/benchmarking/index.md index fda3827f..54bd3e26 100644 --- a/_docs/concepts/benchmarking/benchmarking/index.md +++ b/_docs/concepts/benchmarking/benchmarking/index.md @@ -21,21 +21,21 @@ permalink: /docs/concepts/benchmarking/benchmarking/ ## Introduction to Benchmarking -Developing working and stable application from the scribbles to the final -executing binary is long and hard tasks. During this process developers may come -across stabilities issues, perfomances issues. In addition to these issues, some +Developing a working and stable application, from the scribbles to the final +executing binary, is a long and hard task. During this process, developers may come +across stability and perfomance issues. In addition to these issues, some specified QoS might be difficult to quantify. Solving those problems without the proper tools might be frustrating, tedious tasks leading to reduce developers efficiency. An adapted benchmarking tool could overcome all those development obstacles and increase development time. There are different KPI (Keep Performance Indicators) that one might be interested into. In the framework of -this micro-ROS, the KPI can be freely chosen by the developer. In this way, the +micro-ROS, the KPI can be freely chosen by the developer. In this way, the benchmarking tool will remain flexible and allow the community to constantly add -some support for a lot of different KPI. +support for a lot of different KPI. The problems we want to tackle are: - * Out there, many benchmarking tools exist. Each of targeting different KPIs. + * Out there, many benchmarking tools exist, each of them targeting different KPIs. * Different platforms (Linux/Nuttx/Baremetal et.c.). * Too few time/resources to code benchmarking tool for each. * Avoid code overhead: Keep code clarity. @@ -69,14 +69,14 @@ platform. Every plugin will provide information as requested by the parser: * Provide a list of supported platforms. * Provide a list of functions that are handled. - * Provide snippets codes that will be added for benchmarking. + * Provide snippet codes that will be added for benchmarking. * Provide a list of patches and/or patch code * Optional provide an end script to run and execute the benchmarks ## Shadow Builder -This section will introduce some concept related to the shadow builder (SB). +This section will introduce some concepts related to the shadow builder (SB). The Shadow builder is a tool that will transparently instrument the code to benchmark. The tools will be able to output an “instrumented code” that will be @@ -92,12 +92,11 @@ builder process flow: target binary benchmarking. The SB (Shadow Builder) is meant to be as transparent as possible for the user. -And if the benchmarking is not activated, it should be bypassed. Get -appropriate sources +And if the benchmarking is not activated, it should be bypassed. The SB is in charge of getting the path/git repository to the source code that needs to be benchmarking. The benchmarking. The sources are specified by the -user in the benchmarking configuration file. Injecting code +user in the benchmarking configuration file. In order to inject code, there are some tools that allow this. CLang AST tool will allow to inject some code. From 5ef7edc9976ba4c6cecf581f5960c629ba4dc3c5 Mon Sep 17 00:00:00 2001 From: FranFin Date: Fri, 27 Nov 2020 13:29:16 +0100 Subject: [PATCH 5/5] Redirected old page of memory profiling to new page. --- _docs/concepts/benchmarking/memo_prof/index.md | 1 + 1 file changed, 1 insertion(+) diff --git a/_docs/concepts/benchmarking/memo_prof/index.md b/_docs/concepts/benchmarking/memo_prof/index.md index 7ca16986..13f1c22d 100644 --- a/_docs/concepts/benchmarking/memo_prof/index.md +++ b/_docs/concepts/benchmarking/memo_prof/index.md @@ -1,6 +1,7 @@ --- title: Memory profiling redirect_from: /memo_prof/ +redirect_from: /concepts/memo_prof/ permalink: /docs/concepts/benchmarking/memo_prof/ ---