From 1b6eec7f6782399f608d9ca0d8765816abbf60f9 Mon Sep 17 00:00:00 2001 From: Thomas Bayer Date: Wed, 18 Dec 2024 15:12:01 +0100 Subject: [PATCH 1/5] - Allow exchange besides exc in Groovy and Javascript - More details in script errors --- .../core/beautifier/JSONBeautifier.java | 17 ---- .../core/interceptor/acl/Hostname.java | 2 +- .../beautifier/BeautifierInterceptor.java | 81 +++++++++++-------- .../core/interceptor/jwt/JsonWebToken.java | 14 ++-- .../templating/StaticInterceptor.java | 2 +- .../lang/spel/ExchangeEvaluationContext.java | 2 +- .../predic8/membrane/core/util/TextUtil.java | 49 ++++++----- .../membrane/core/util/URLParamUtil.java | 8 +- .../SOAPMessageValidatorInterceptorTest.java | 2 +- 9 files changed, 94 insertions(+), 83 deletions(-) diff --git a/core/src/main/java/com/predic8/membrane/core/beautifier/JSONBeautifier.java b/core/src/main/java/com/predic8/membrane/core/beautifier/JSONBeautifier.java index 6db709421a..47c939cb7c 100644 --- a/core/src/main/java/com/predic8/membrane/core/beautifier/JSONBeautifier.java +++ b/core/src/main/java/com/predic8/membrane/core/beautifier/JSONBeautifier.java @@ -44,21 +44,4 @@ public void configure() { objectMapper.configure(JsonGenerator.Feature.QUOTE_FIELD_NAMES, quoteFieldNames); objectMapper.configure(FAIL_ON_UNKNOWN_PROPERTIES, failOnUnknownProperties); } - - public void setIndentOutput(boolean indentOutput) { - this.indentOutput = indentOutput; - } - - public void setAllowedUnquotedFieldNames(boolean allowedUnquotedFieldNames) { - this.allowedUnquotedFieldNames = allowedUnquotedFieldNames; - } - - public void setQuoteFieldNames(boolean quoteFieldNames) { - this.quoteFieldNames = quoteFieldNames; - } - - public void setFailOnUnknownProperties(boolean failOnUnknownProperties) { - this.failOnUnknownProperties = failOnUnknownProperties; - } - } diff --git a/core/src/main/java/com/predic8/membrane/core/interceptor/acl/Hostname.java b/core/src/main/java/com/predic8/membrane/core/interceptor/acl/Hostname.java index aa657afdcf..1f8ac36ccc 100644 --- a/core/src/main/java/com/predic8/membrane/core/interceptor/acl/Hostname.java +++ b/core/src/main/java/com/predic8/membrane/core/interceptor/acl/Hostname.java @@ -85,7 +85,7 @@ public boolean matches(String hostname, String ip) { log.debug("CanonicalHostname for {} / {} is {}", hostname, ip, canonicalHostName); return Pattern.compile(schema).matcher(canonicalHostName).matches(); } catch (UnknownHostException e) { - log.warn("Could not reverse lookup canonical hostname for {} {}.", hostname, ip, e); + log.warn("Could not reverse lookup canonical hostname for {} {}.", hostname, ip); return false; } } diff --git a/core/src/main/java/com/predic8/membrane/core/interceptor/beautifier/BeautifierInterceptor.java b/core/src/main/java/com/predic8/membrane/core/interceptor/beautifier/BeautifierInterceptor.java index 1be889a327..4cfeb2092f 100644 --- a/core/src/main/java/com/predic8/membrane/core/interceptor/beautifier/BeautifierInterceptor.java +++ b/core/src/main/java/com/predic8/membrane/core/interceptor/beautifier/BeautifierInterceptor.java @@ -14,27 +14,27 @@ package com.predic8.membrane.core.interceptor.beautifier; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.ObjectWriter; -import com.predic8.membrane.annot.MCElement; -import com.predic8.membrane.core.exchange.Exchange; -import com.predic8.membrane.core.http.Message; -import com.predic8.membrane.core.interceptor.AbstractInterceptor; -import com.predic8.membrane.core.interceptor.Outcome; -import com.predic8.membrane.core.util.TextUtil; +import com.fasterxml.jackson.databind.*; +import com.predic8.membrane.annot.*; +import com.predic8.membrane.core.exchange.*; +import com.predic8.membrane.core.http.*; +import com.predic8.membrane.core.interceptor.*; +import org.slf4j.*; -import java.io.IOException; -import java.io.InputStreamReader; +import java.io.*; -import static com.predic8.membrane.core.interceptor.Outcome.CONTINUE; -import static java.nio.charset.StandardCharsets.UTF_8; +import static com.predic8.membrane.core.interceptor.Outcome.*; +import static com.predic8.membrane.core.util.TextUtil.*; +import static java.nio.charset.StandardCharsets.*; /** * @description Beautifies request and response bodies. Supported are the Formats: JSON, XML * @topic 4. Interceptors/Features */ -@MCElement(name="beautifier") -public class BeautifierInterceptor extends AbstractInterceptor{ +@MCElement(name = "beautifier") +public class BeautifierInterceptor extends AbstractInterceptor { + + private static final Logger log = LoggerFactory.getLogger(BeautifierInterceptor.class); private final ObjectWriter ow = new ObjectMapper().writerWithDefaultPrettyPrinter(); private final ObjectMapper om = new ObjectMapper(); @@ -46,38 +46,51 @@ public BeautifierInterceptor() { @Override public Outcome handleRequest(Exchange exc) throws Exception { - return handleInternal(exc, exc.getRequest()); + return handleInternal(exc.getRequest()); } @Override public Outcome handleResponse(Exchange exc) throws Exception { - return handleInternal(exc, exc.getResponse()); + return handleInternal(exc.getResponse()); } - private Outcome handleInternal(Exchange exc, Message msg) throws IOException { - if(msg.isJSON()) { - msg.setBodyContent( - ow.writeValueAsBytes( - om.readTree(msg.getBodyAsStreamDecoded()) - ) - ); + private Outcome handleInternal(Message msg) { + if (msg.isJSON()) { + beautifyJSON(msg); return CONTINUE; } - if(msg.isXML()) { - msg.setBodyContent( - TextUtil.formatXML( - new InputStreamReader( - msg.getBodyAsStream(), - msg.getHeader().getCharset() - ) - ).getBytes(UTF_8) - ); - return CONTINUE; + if (msg.isXML()) { + beautifyXML(msg); } - return CONTINUE; } + /** + * If it is not possible to beautify, leave body as it is. + */ + private static void beautifyXML(Message msg) { + try { + InputStreamReader reader = new InputStreamReader(msg.getBodyAsStream(), msg.getHeader().getCharset()); + msg.setBodyContent(formatXML(reader).getBytes(UTF_8)); + } catch (Exception e) { + // If it is not possible to beautify, to nothing + log.warn("Error parsing XML: {}", e.getMessage()); + } + } + + /** + * If it is not possible to beautify, leave body as it is. + */ + private void beautifyJSON(Message msg) { + try { + JsonNode node = om.readTree(msg.getBodyAsStreamDecoded()); + msg.setBodyContent(ow.writeValueAsBytes(node)); + } catch (IOException e) { + // If it is not possible to beautify, to nothing + log.warn("Error parsing JSON: {}", e.getMessage()); + } + } + @Override public String getShortDescription() { return "Pretty printing. Applies, if the body is JSON."; diff --git a/core/src/main/java/com/predic8/membrane/core/interceptor/jwt/JsonWebToken.java b/core/src/main/java/com/predic8/membrane/core/interceptor/jwt/JsonWebToken.java index 6ba309e079..c393c48e5b 100644 --- a/core/src/main/java/com/predic8/membrane/core/interceptor/jwt/JsonWebToken.java +++ b/core/src/main/java/com/predic8/membrane/core/interceptor/jwt/JsonWebToken.java @@ -13,14 +13,16 @@ limitations under the License. */ package com.predic8.membrane.core.interceptor.jwt; -import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.*; +import org.slf4j.*; -import java.io.IOException; -import java.util.Base64; -import java.util.Map; +import java.io.*; +import java.util.*; public class JsonWebToken { + private static final Logger log = LoggerFactory.getLogger(JsonWebToken.class.getName()); + public static abstract class AbstractJwtSubHolder { private final Map data; protected AbstractJwtSubHolder(Map data) { @@ -62,8 +64,10 @@ public static String ERROR_JWT_VALUE_NOT_PRESENT(String key) { public JsonWebToken(String jwt) throws JWTException { var chunks = jwt.split("\\."); - if (chunks.length < 3) + if (chunks.length < 3) { + log.warn("Less than 3 parts in JWT header: {}", jwt); throw new JWTException(ERROR_MALFORMED_COMPACT_SERIALIZATION); + } var decoder = Base64.getUrlDecoder(); var mapper = new ObjectMapper(); diff --git a/core/src/main/java/com/predic8/membrane/core/interceptor/templating/StaticInterceptor.java b/core/src/main/java/com/predic8/membrane/core/interceptor/templating/StaticInterceptor.java index 55d849390c..ff12ac53e3 100644 --- a/core/src/main/java/com/predic8/membrane/core/interceptor/templating/StaticInterceptor.java +++ b/core/src/main/java/com/predic8/membrane/core/interceptor/templating/StaticInterceptor.java @@ -90,7 +90,7 @@ String prettifyJson(String text) { try { return jsonBeautifier.beautify(text); } catch (IOException e) { - log.warn("Failed to format JSON", e); + log.warn("Failed to format JSON: {}", e.getMessage()); return text; } } diff --git a/core/src/main/java/com/predic8/membrane/core/lang/spel/ExchangeEvaluationContext.java b/core/src/main/java/com/predic8/membrane/core/lang/spel/ExchangeEvaluationContext.java index d1d628fe1f..e3432094e2 100644 --- a/core/src/main/java/com/predic8/membrane/core/lang/spel/ExchangeEvaluationContext.java +++ b/core/src/main/java/com/predic8/membrane/core/lang/spel/ExchangeEvaluationContext.java @@ -66,7 +66,7 @@ public ExchangeEvaluationContext(Exchange exc, Message message) { try { params = new SpELMap<>(URLParamUtil.getParams(new URIFactory(), exc, ERROR)); } catch (Exception e) { - log.warn("Error parsing query parameters", e); + log.warn("Error parsing query parameters: {}", e.getMessage()); } this.request = new SpELMessageWrapper(exc.getRequest()); } diff --git a/core/src/main/java/com/predic8/membrane/core/util/TextUtil.java b/core/src/main/java/com/predic8/membrane/core/util/TextUtil.java index 45f064c124..290aecd9a9 100644 --- a/core/src/main/java/com/predic8/membrane/core/util/TextUtil.java +++ b/core/src/main/java/com/predic8/membrane/core/util/TextUtil.java @@ -15,24 +15,15 @@ package com.predic8.membrane.core.util; -import com.predic8.beautifier.HtmlBeautifierFormatter; -import com.predic8.beautifier.PlainBeautifierFormatter; -import com.predic8.beautifier.XMLBeautifier; -import com.predic8.beautifier.XMLBeautifierFormatter; -import org.apache.commons.text.StringEscapeUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.xml.stream.XMLEventReader; -import javax.xml.stream.XMLInputFactory; -import javax.xml.stream.events.XMLEvent; -import java.io.IOException; -import java.io.Reader; -import java.io.StringReader; -import java.io.StringWriter; -import java.util.ArrayList; - -import static java.lang.Character.toUpperCase; +import com.predic8.beautifier.*; +import org.apache.commons.text.*; +import org.slf4j.*; + +import javax.xml.stream.*; +import javax.xml.stream.events.*; +import java.io.*; +import java.util.*; + import static javax.xml.stream.XMLInputFactory.*; @@ -50,11 +41,24 @@ public class TextUtil { } - public static String formatXML(Reader reader) { + /** + * + * @param reader + * @return + * @throws Exception + */ + public static String formatXML(Reader reader) throws Exception { return formatXML(reader, false); } - public static String formatXML(Reader reader, boolean asHTML) { + /** + * As HTML is needed for the AdminConsole + * @param reader XML + * @param asHTML Should output formatted as XML + * @return Formatted string + * @throws Exception + */ + public static String formatXML(Reader reader, boolean asHTML) throws Exception { StringWriter out = new StringWriter(); try { @@ -63,7 +67,8 @@ public static String formatXML(Reader reader, boolean asHTML) { beautifier.parse(reader); } catch (Exception e){ - log.error("", e); + log.warn("Error parsing XML: {}", e.getMessage()); + throw e; } finally { try { out.close(); @@ -121,7 +126,7 @@ public static String toEnglishList(String conjuction, String... args) { public static Object capitalize(String english) { if (english.length() == 0) return ""; - return toUpperCase(english.charAt(0)) + english.substring(1); + return (english.charAt(0) + english.substring(1)).toUpperCase(); } diff --git a/core/src/main/java/com/predic8/membrane/core/util/URLParamUtil.java b/core/src/main/java/com/predic8/membrane/core/util/URLParamUtil.java index 0c771fea44..5777d5d57b 100644 --- a/core/src/main/java/com/predic8/membrane/core/util/URLParamUtil.java +++ b/core/src/main/java/com/predic8/membrane/core/util/URLParamUtil.java @@ -31,7 +31,13 @@ public class URLParamUtil { private static final Pattern paramsPat = Pattern.compile("([^=]*)=?(.*)"); public static Map getParams(URIFactory uriFactory, Exchange exc, DuplicateKeyOrInvalidFormStrategy duplicateKeyOrInvalidFormStrategy) throws Exception { - URI jUri = uriFactory.create(exc.getRequest().getUri()); + String uri = exc.getRequest().getUri(); + + // Avoid unnecessary log entries + if (uri == null || uri.isEmpty()) + return Collections.emptyMap(); + + URI jUri = uriFactory.create(uri); String q = jUri.getRawQuery(); if (q == null) { if (hasNoFormParams(exc)) diff --git a/core/src/test/java/com/predic8/membrane/core/interceptor/schemavalidation/SOAPMessageValidatorInterceptorTest.java b/core/src/test/java/com/predic8/membrane/core/interceptor/schemavalidation/SOAPMessageValidatorInterceptorTest.java index 50ff62092d..bf1fe0ab6e 100644 --- a/core/src/test/java/com/predic8/membrane/core/interceptor/schemavalidation/SOAPMessageValidatorInterceptorTest.java +++ b/core/src/test/java/com/predic8/membrane/core/interceptor/schemavalidation/SOAPMessageValidatorInterceptorTest.java @@ -103,7 +103,7 @@ private Outcome getOutcome(Request request, Interceptor interceptor, String file return interceptor.handleRequest(exc); } - private String getContent(String fileName) { + private String getContent(String fileName) throws Exception { return TextUtil.formatXML(new InputStreamReader(requireNonNull(this.getClass().getResourceAsStream(fileName)))); } From 628640993e8a6fe6a4347e78b40d3f4a0b7a6100 Mon Sep 17 00:00:00 2001 From: Thomas Bayer Date: Wed, 18 Dec 2024 15:16:06 +0100 Subject: [PATCH 2/5] Cleanup --- .../core/beautifier/JSONBeautifier.java | 20 ++++++------------- .../predic8/membrane/core/util/TextUtil.java | 8 ++++---- 2 files changed, 10 insertions(+), 18 deletions(-) diff --git a/core/src/main/java/com/predic8/membrane/core/beautifier/JSONBeautifier.java b/core/src/main/java/com/predic8/membrane/core/beautifier/JSONBeautifier.java index 47c939cb7c..ad246717df 100644 --- a/core/src/main/java/com/predic8/membrane/core/beautifier/JSONBeautifier.java +++ b/core/src/main/java/com/predic8/membrane/core/beautifier/JSONBeautifier.java @@ -24,24 +24,16 @@ public class JSONBeautifier { - private final ObjectMapper objectMapper = new ObjectMapper(); - - private boolean indentOutput = true; - - private boolean allowedUnquotedFieldNames = true; - - private boolean quoteFieldNames = false; - - private boolean failOnUnknownProperties = true; + private final ObjectMapper om = new ObjectMapper(); public String beautify(String content) throws IOException { - return objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(objectMapper.readTree(content)); + return om.writerWithDefaultPrettyPrinter().writeValueAsString(om.readTree(content)); } public void configure() { - objectMapper.configure(INDENT_OUTPUT, indentOutput); - objectMapper.configure(JsonParser.Feature.ALLOW_UNQUOTED_FIELD_NAMES, allowedUnquotedFieldNames); - objectMapper.configure(JsonGenerator.Feature.QUOTE_FIELD_NAMES, quoteFieldNames); - objectMapper.configure(FAIL_ON_UNKNOWN_PROPERTIES, failOnUnknownProperties); + om.configure(INDENT_OUTPUT, true); + om.configure(JsonParser.Feature.ALLOW_UNQUOTED_FIELD_NAMES, true); + om.configure(JsonGenerator.Feature.QUOTE_FIELD_NAMES, false); + om.configure(FAIL_ON_UNKNOWN_PROPERTIES, true); } } diff --git a/core/src/main/java/com/predic8/membrane/core/util/TextUtil.java b/core/src/main/java/com/predic8/membrane/core/util/TextUtil.java index 290aecd9a9..def938b77d 100644 --- a/core/src/main/java/com/predic8/membrane/core/util/TextUtil.java +++ b/core/src/main/java/com/predic8/membrane/core/util/TextUtil.java @@ -81,7 +81,7 @@ public static String formatXML(Reader reader, boolean asHTML) throws Exception { } public static boolean isNullOrEmpty(String str) { - return str == null || str.length() == 0; + return str == null || str.isEmpty(); } public static String globToRegExp(String glob) { @@ -107,7 +107,7 @@ private static void appendReplacement(char c, StringBuilder buf) { public static String toEnglishList(String conjuction, String... args) { ArrayList l = new ArrayList<>(); for (String arg : args) - if (arg != null && arg.length() > 0) + if (arg != null && !arg.isEmpty()) l.add(arg); StringBuilder sb = new StringBuilder(); for (int i = 0; i < l.size(); i++) { @@ -124,7 +124,7 @@ public static String toEnglishList(String conjuction, String... args) { } public static Object capitalize(String english) { - if (english.length() == 0) + if (english.isEmpty()) return ""; return (english.charAt(0) + english.substring(1)).toUpperCase(); } @@ -167,7 +167,7 @@ public static String linkURL(String url) { public static Object removeFinalChar(String s) { StringBuilder sb = new StringBuilder(s); - if (sb.length() > 0) + if (!sb.isEmpty()) sb.deleteCharAt(sb.length()-1); return sb.toString(); } From 6d8b2d656c9a3c72ff3c0bbf47df3af2a5af981d Mon Sep 17 00:00:00 2001 From: Thomas Bayer Date: Wed, 18 Dec 2024 17:48:56 +0100 Subject: [PATCH 3/5] Table of content --- README.md | 128 ++++++++++++++++++++++++++++++++++++------------------ 1 file changed, 85 insertions(+), 43 deletions(-) diff --git a/README.md b/README.md index 812fe9d74a..4499499888 100644 --- a/README.md +++ b/README.md @@ -32,14 +32,48 @@ A versatile and lightweight **API Gateway** for **REST** and **legacy SOAP Web S # Content -- [Getting Started](#Getting-Started) -- [Basics](#Basics) Routing, rewriting -- [Scripting](#scripting) -- [Message Transformation](#message-transformation) -- [Security](#security) -- [Traffic Control](#Traffic-Control) Rate limiting, Load balancing -- [Legacy Web Services](#soap-web-services) SOAP and WSDL -- [Operation](#Operation) +1. [Getting Started](#Getting-Started) + - [Java](#java) + - [Docker](#docker) +2. [Basics](#Basics) Routing, rewriting + - [API Definition and Configuration](#API-Definition-and-Configuration) + - [Simple REST and HTTP Forwarding APIs](#simple-rest-and-http-forwarding-apis) +3. [OpenAPI Support](#openapi-support) + - [Deploy APIs with OpenAPI](#deploy-apis-with-openapi) +4. [Routing](#routing) + - [Short Circuit](#short-circuit) + - [URL Rewriting](#url-rewriting) +5. [Scripting](#scripting) + - [Groovy](#groovy-scripts) + - [Creating Responses with Groovy](#creating-responses-with-groovy) + - [Javascript](#javascript-scripts) +6. [Message Transformation](#message-transformation) + - [Manipulating HTTP Headers](#manipulating-http-headers) + - [Removing HTTP Headers](#removing-http-headers) + - [Create JSON from Query Parameters](#create-json-from-query-parameters) + - [Transform JSON into TEXT, JSON or XML with Templates](#transform-json-into-text-json-or-xml-with-templates) + - [Transform XML into Text or JSON](#transform-xml-into-text-or-json) + - [Complex Transformations using Javascript or Groovy](#complex-transformations-using-javascript-or-groovy) + - [Transformation with Computations](#transformation-with-computations) + - [JSON and XML Beautifier](#json-and-xml-beautifier) +7. [Conditionals with if](#conditionals-with-if) +8. [Security](#security) + - [API Keys](#api-keys) + - [Basic Authentication](#basic-authentication) + - [SSL/TLS](#ssltls) + - [JSON Web Tokens](#json-web-tokens) JWT + - [OAuth2](#oauth2) + - [Secure APIs with OAuth2](#secure-apis-with-oauth2) + - [Membrane as Authorization Server](#membrane-as-authorization-server) +9. [Traffic Control](#Traffic-Control) Rate limiting, Load balancing + - [Rate Limiting](#rate-limiting) + - [Load Balancing](#load-balancing) +8. [Legacy Web Services](#soap-web-services) SOAP and WSDL + - [API configuration from WSDL](#api-configuration-from-wsdl) + - [Message Validation against WSDL and XSD](#message-validation-against-wsdl-and-xsd) +9. [Operation](#Operation) + - [Logging](#log-http) + - [OpenTelemetry](#opentelemetry-integration) # Getting Started @@ -110,17 +144,19 @@ For detailed Docker setup instructions, see the [Membrane Deployment Guide](http - Check out the [SOAP API Tutorial](https://membrane-api.io/tutorials/soap/) for legacy web service integration. ### Read the Documentation + - For detailed guidance, visit the [official documentation](https://www.membrane-soa.org/service-proxy-doc/). # Basics -### Customizing Membrane -To configure Membrane, edit the `proxies.xml` file located in the `conf` folder. +### API Definition and Configuration + +To define new APIs or modify the existing configuration, edit the `proxies.xml` file located in the `conf` folder. This file serves as the central configuration point for managing API behavior and routing rules. ### Using Samples -Explore the sample configurations provided below. Copy and modify them to suit your needs, then save or restart the gateway to apply the changes. +Explore and copy the sample snippets below into the `proxies.xml` file and modify them to suit your needs. Then save or restart the gateway to apply the changes. Usually a save will trigger a reload automatically. -For even more sample have a look at the `examples` folder. +For even more samples have a look at the `examples` folder. ## Simple REST and HTTP Forwarding APIs @@ -139,7 +175,7 @@ To forward requests from the API Gateway to a backend, use a simple `api` config After adding the configuration to the `proxies.xml` file, open the following URL in your browser to test the API: [http://localhost:2000/shop/v2/](http://localhost:2000/shop/v2/) -## Using OpenAPI for Configuration & Validation +## OpenAPI Support ### Deploy APIs with OpenAPI Membrane allows you to configure APIs directly from OpenAPI documents in the `proxies.xml` file. Backend addresses and other details are automatically derived from the OpenAPI description. @@ -346,7 +382,7 @@ You can realize a load balancer by setting the destination randomly. ``` -### Create a Response with Groovy +### Creating Responses with Groovy The `groovy` plugin in Membrane allows you to dynamically generate custom responses. The result of the last line of the Groovy script is passed to the plugin. If the result is a `Response` object, it will be returned to the caller. @@ -395,7 +431,7 @@ For more information about using Groovy with Membrane, refer to: - [Groovy Plugin Reference](https://www.membrane-api.io/docs/current/groovy.html). - [Sample Project](distribution/examples/groovy) -### JavaScript Extension +### JavaScript Scripts In addition to Groovy, Membrane supports JavaScript for implementing custom behavior. This allows you to inspect, modify, or log details about requests and responses. @@ -420,6 +456,23 @@ The following example logs all HTTP headers from incoming requests and responses The `CONTINUE` keyword ensures that the request continues processing and is forwarded to the target URL. +When a JavaScript script returns a `Response` object as the last line of code, the request flow is interrupted, and the response is sent back to the client. This allows for creating custom responses dynamically. + +The following example generates a JSON response and sends it directly to the client: + +```xml + + + var body = JSON.stringify({ + foo: 7, + bar: 42 + }); + + Response.ok(body).contentType("application/json").build(); + + +``` + #### Learn More For more details about using JavaScript with Membrane, check the [JavaScript Plugin documentation](https://www.membrane-api.io/docs/current/javascript.html). @@ -643,7 +696,7 @@ This script transforms the input and adds some calculations. See [examples/javascript](distribution/examples/javascript) for a detailed explanation. The same transformation can also be realized with [Groovy](distribution/examples/groovy) -## Beautifier +## JSON and XML Beautifier You can beautify a JSON or XML using the `` plugin. @@ -669,11 +722,11 @@ Returns: ``` -# Branching and Conditionals +# Conditionals with if Replace `5XX` error messages from a backend: -```xml +```xml @@ -686,17 +739,6 @@ Replace `5XX` error messages from a backend: ``` -Check if certain scopes/roles are provided: -```xml - - - - - - - -``` - # Writing Extensions with Groovy or Javascript Dynamically manipulate and monitor messages with Groovy: @@ -909,20 +951,6 @@ Distribute workload to multiple backend nodes. [See the example](distribution/ex ``` -# Log HTTP - -Log data about requests and responses to a file or [database](distribution/examples/logging/jdbc-database) as [CSV](distribution/examples/logging/csv) -or [JSON](distribution/examples/logging/json) file. - -```xml - - - - - - -``` - # Websockets Route and intercept WebSocket traffic: @@ -966,6 +994,20 @@ The _validator_ checks SOAP messages against a WSDL document including reference # Operation +## Log HTTP + +Log data about requests and responses to a file or [database](distribution/examples/logging/jdbc-database) as [CSV](distribution/examples/logging/csv) +or [JSON](distribution/examples/logging/json) file. + +```xml + + + + + + +``` + ## Instrumentation ### OpenTelemetry Integration From aecc4a2f0e80308d44846a3aa385d93c13577da1 Mon Sep 17 00:00:00 2001 From: Thomas Bayer Date: Wed, 18 Dec 2024 18:38:52 +0100 Subject: [PATCH 4/5] Prometheus example for README.md --- README.md | 35 ++++++++++++++++++++++ distribution/examples/prometheus/README.md | 35 +++++++++++++++------- docs/ROADMAP.md | 7 +++++ 3 files changed, 67 insertions(+), 10 deletions(-) create mode 100644 docs/ROADMAP.md diff --git a/README.md b/README.md index 4499499888..4b69fd5cb8 100644 --- a/README.md +++ b/README.md @@ -73,6 +73,7 @@ A versatile and lightweight **API Gateway** for **REST** and **legacy SOAP Web S - [Message Validation against WSDL and XSD](#message-validation-against-wsdl-and-xsd) 9. [Operation](#Operation) - [Logging](#log-http) + - [Monitoring with Prometheus and Grafana](#monitoring-with-prometheus-and-grafana) - [OpenTelemetry](#opentelemetry-integration) # Getting Started @@ -1010,6 +1011,40 @@ or [JSON](distribution/examples/logging/json) file. ## Instrumentation +### Monitoring with Prometheus and Grafana + +Membrane supports seamless monitoring with Prometheus and Grafana, enabling visibility into API performance and system metrics. + +Add an API with the `prometheus` plugin to your `proxies.xml` file. This will expose metrics at the specified endpoint: + +```xml + + /metrics + + +``` + +Then you can query the metrics by navigating to: +[http://localhost:2000/metrics](http://localhost:2000/metrics). + +This endpoint provides Prometheus-compatible metrics, which you can scrape using a Prometheus server. + +For a complete configuration example with Prometheus and Grafana, refer to: +[Prometheus Example](distribution/examples/prometheus). + +### Monitoring with Prometheus and Grafana + +Add an API with the `prometheus` plugin at the top of the `proxies.xml` file. + +```xml + + /metrics + + +``` + +Then query the metrics endpoint by opening [http://localhost:2000/metrics](http://localhost:2000/metrics). Now you can setup a prometheus to scrape that endpoint. For a complete example with prometheus and Grafana have a look at [examples/prometheus](distribution/examples/prometheus). + ### OpenTelemetry Integration Membrane supports integration with **OpenTelemetry** traces using the `openTelemetry` plugin and the `W3C` propagation standard. This enables detailed tracing of requests across Membrane and backend services. diff --git a/distribution/examples/prometheus/README.md b/distribution/examples/prometheus/README.md index 406497c855..0631158eec 100644 --- a/distribution/examples/prometheus/README.md +++ b/distribution/examples/prometheus/README.md @@ -5,10 +5,13 @@ Membrane supports monitoring through the integration with Prometheus. With the Prometheus plugin, Membrane enables the observation of APIs. It gathers metrics about the processes passing through it and makes them available for scraping by Prometheus. This data can then be used for monitoring and alerting purposes. To enable monitoring for an API, simply add the Prometheus plugin to it. + ## Run the Example To monitor APIs using Prometheus and Grafana, follow the steps below: +### Providing a Metrics Endpoint for Prometheus + 1. Start the example environment with Docker Compose: ```bash @@ -17,11 +20,11 @@ To monitor APIs using Prometheus and Grafana, follow the steps below: 2. Access the following endpoints: - - [localhost:2001](http://localhost:2001) - Returns a status code of 200. - - [localhost:2002](http://localhost:2002) - Returns a status code of 404. - - [localhost:2003](http://localhost:2003) - Returns a status code of 500. +- [localhost:2001](http://localhost:2001) - Returns a status code of 200. +- [localhost:2002](http://localhost:2002) - Returns a status code of 404. +- [localhost:2003](http://localhost:2003) - Returns a status code of 500. - You can use cURL commands to access these endpoints, e.g.: +You can use cURL commands to access these endpoints, e.g.: ```bash curl -v http://localhost:2001 @@ -29,14 +32,25 @@ To monitor APIs using Prometheus and Grafana, follow the steps below: curl -v http://localhost:2003 ``` -3. After accessing the endpoints, proceed to Grafana: +3. Then, query the metrics endpoint of Membrane + + - Open [http://localhost:2000/metrics](http://localhost:2000/metrics) + +### Quering Prometheus + +1. Open Prometheus at [http://localhost:9090](http://localhost:9090) +2. Search for `membrane_count` + +### Grafana + +1. Proceed to Grafana: - - Access [localhost:3000](http://localhost:3000) in your browser. - - Log in with the default credentials: username `admin` and password `admin`. - - Click on "Explore" from the left-hand menu. - - You can now select different queries to display the collected metrics. +- Access [localhost:3000](http://localhost:3000) in your browser. +- Log in with the default credentials: username `admin` and password `admin`. +- Click on "Explore" from the left-hand menu. +- You can now select different queries to display the collected metrics. - ![Grafana example](prometheus-grafana-example.png) +![Grafana example](prometheus-grafana-example.png) **HOW IT IS DONE** @@ -45,6 +59,7 @@ Take a look at the `proxies.xml`. ```xml + /metrics diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md new file mode 100644 index 0000000000..7b26d2c0a0 --- /dev/null +++ b/docs/ROADMAP.md @@ -0,0 +1,7 @@ +# Membrane Roadmap + +# Version 6.0.0 + +- Grafana Dashboard to import in examples/prometheus + - Also provide the datasource config + - Maybe the config can be included into the docker-compose setup From 06d9a68ca65b2b3bcdbf4211797288ed6f0fcee0 Mon Sep 17 00:00:00 2001 From: Thomas Bayer Date: Wed, 18 Dec 2024 20:51:24 +0100 Subject: [PATCH 5/5] JSON and XML protection --- README.md | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/README.md b/README.md index 4b69fd5cb8..2ddd5aa068 100644 --- a/README.md +++ b/README.md @@ -65,6 +65,7 @@ A versatile and lightweight **API Gateway** for **REST** and **legacy SOAP Web S - [OAuth2](#oauth2) - [Secure APIs with OAuth2](#secure-apis-with-oauth2) - [Membrane as Authorization Server](#membrane-as-authorization-server) + - [XML and JSON Protection](#xml-and-json-protection) 9. [Traffic Control](#Traffic-Control) Rate limiting, Load balancing - [Rate Limiting](#rate-limiting) - [Load Balancing](#load-balancing) @@ -919,6 +920,49 @@ Secure endpoints with SSL/TLS: ``` +### XML and JSON Protection + +Membrane offers protection mechanisms to secure your APIs from common risks associated with XML and JSON payloads. + +#### XML Protection + +The `xmlProtection` plugin inspects incoming XML requests and mitigates risks such as: + +- External entity references (XXE attacks). +- Excessively large element names. +- High numbers of attributes or deeply nested structures. + +**Example:** +```xml + + + + +``` + +See [XML Protection Reference](https://www.membrane-api.io/docs/current/xmlProtection.html). + +#### JSON Protection + +The `jsonProtection` plugin safeguards APIs from JSON-based vulnerabilities by setting limits on: + +- **Depth**: Prevents overly nested JSON structures. +- **Key Length**: Restricts excessively long keys. +- **Object Size**: Maximum number of fields in aJSON object. +- **String Length**: Controls maximum length of string values. +- **...** + +**Example:** + +```xml + + + + +``` + +See [JSON Protection](https://www.membrane-api.io/docs/current/jsonProtection.html). + # Traffic Control ## Rate Limiting