From 336f848ec316e5109cfa955780ae635481bcfbe4 Mon Sep 17 00:00:00 2001
From: Anton Rubin <anton.rubin@eliatra.com>
Date: Mon, 14 Oct 2024 15:45:51 +0100
Subject: [PATCH] adding more examples

Signed-off-by: Anton Rubin <anton.rubin@eliatra.com>
---
 _analyzers/standard.md | 25 +++++++++++++++++++++++++
 1 file changed, 25 insertions(+)

diff --git a/_analyzers/standard.md b/_analyzers/standard.md
index 4c2f0e9d1b..284d01fd07 100644
--- a/_analyzers/standard.md
+++ b/_analyzers/standard.md
@@ -58,3 +58,28 @@ PUT /my_custom_index
 ```
 {% include copy-curl.html %}
 
+## Generated tokens
+
+Use the following request to examine the tokens generated using the created analyzer:
+
+```json
+POST /my_custom_index/_analyze
+{
+  "analyzer": "my_custom_analyzer",
+  "text": "The slow turtle swims away"
+}
+```
+{% include copy-curl.html %}
+
+The response contains the generated tokens:
+
+```json
+{
+  "tokens": [
+    {"token": "slow","start_offset": 4,"end_offset": 8,"type": "<ALPHANUM>","position": 1},
+    {"token": "turtle","start_offset": 9,"end_offset": 15,"type": "<ALPHANUM>","position": 2},
+    {"token": "swims","start_offset": 16,"end_offset": 21,"type": "<ALPHANUM>","position": 3},
+    {"token": "away","start_offset": 22,"end_offset": 26,"type": "<ALPHANUM>","position": 4}
+  ]
+}
+```