Manual ILM Policy and Template Creation

Provides the necessary definitions for manually creating log entities required by Apinizer when log management is managed manually. Step-by-step explains ILM policy and Template creation operations in the Elasticsearch cluster.

If log management will be done through Apinizer, these operations do not need to be performed, these operations are done automatically by Apinizer.

Variables

Dynamic values and their descriptions in the requests are shown in the table below.

Variable	Description
<ELASTICSEARCH_IP_ADDRESS>	Host information of the Elasticsearch cluster.
<ELASTICSEARCH_PORT>	Port information of the Elasticsearch cluster.
<INDEX_KEY>	Unique identifier for the index.
<INDEX_PATTERN_NAME>	Index pattern name (e.g.: apinizer-log-apiproxy-*).
<ILM_POLICY_NAME>	ILM policy name (e.g.: apinizer-log-ilm-policy-default).
<NUMBER_OF_SHARDS>	Number of shards for the index.
<NUMBER_OF_REPLICAS>	Number of replicas for the index.
<REFRESH_INTERVAL>	Index refresh interval (e.g.: 1s).

Creating ILM Policies

Creating ILM Policy Request Named apinizer-log-ilm-policy-default

curl -XPUT "http://<ELASTICSEARCH_IP_ADDRESS>:<ELASTICSEARCH_PORT>/_ilm/policy/apinizer-log-ilm-policy-<INDEX_KEY>" -H 'Content-Type: application/json' -d 
'{
  "policy": {
    "phases": {
      "hot": {
        "min_age": "0ms",
        "actions": {
          "rollover": {
            "max_age": "30d",
            "max_size": "30gb",
            "max_docs": 15000000
          },
          "set_priority": {
            "priority": null
          }
        }
      },
      "warm": {
        "actions": {
          "readonly": {},
          "allocate": {
            "number_of_replicas": 0,
            "include": {},
            "exclude": {}
          },
          "shrink": {
            "number_of_shards": 1
          },
          "forcemerge": {
            "max_num_segments": 1
          },
          "set_priority": {
            "priority": null
          }
        }
      },
      "cold": {
        "min_age": "90d",
        "actions": {
          "set_priority": {
            "priority": null
          }
        }
      }
    }
  }
}'

Creating Template

Creating Template Request Named apinizer-log-apiproxy-template-default

{
  "index_patterns": ["<INDEX_PATTERN_NAME>"],
  "data_stream": {},
  "template": {
    "settings": {
      "index": {
        "lifecycle": {
          "name": "<ILM_POLICY_NAME>"
        },
        "number_of_shards": "<NUMBER_OF_SHARDS>",
        "number_of_replicas": "<NUMBER_OF_REPLICAS>",
        "refresh_interval": "<REFRESH_INTERVAL>"
      }
    },
    "mappings": {
      "properties": {
        "aprp": {
          "type": "keyword"
        },
        "fcrb": {
          "type": "text"
        },
        "hr1ru": {
          "type": "keyword"
        },
        "aci": {
          "type": "keyword"
        },
        "tapt": {
          "type": "integer"
        },
        "trqpt": {
          "type": "integer"
        },
        "trspt": {
          "type": "integer"
        },
        "hr1rp": {
          "type": "keyword"
        },
        "uok": {
          "ignore_above": 50,
          "type": "keyword"
        },
        "sr2t": {
          "type": "integer"
        },
        "fcrp": {
          "type": "nested",
          "properties": {
            "k": {
              "type": "keyword"
            },
            "v": {
              "type": "keyword"
            }
          }
        },
        "api": {
          "type": "keyword"
        },
        "appi": {
          "type": "keyword"
        },
        "appin": {
          "type": "keyword"
        },
        "fcrh": {
          "type": "nested",
          "properties": {
            "k": {
              "type": "keyword"
            },
            "v": {
              "type": "keyword"
            }
          }
        },
        "apn": {
          "type": "keyword"
        },
        "pet": {
          "ignore_above": 75,
          "type": "keyword"
        },
        "apr": {
          "type": "short"
        },
        "tbah": {
          "type": "nested",
          "properties": {
            "k": {
              "type": "keyword"
            },
            "v": {
              "type": "keyword"
            }
          }
        },
        "ei": {
          "type": "keyword"
        },
        "apmi": {
          "type": "keyword"
        },
        "hr1qs": {
          "type": "keyword"
        },
        "tbap": {
          "type": "nested",
          "properties": {
            "k": {
              "type": "keyword"
            },
            "v": {
              "type": "keyword"
            }
          }
        },
        "ra": {
          "type": "keyword"
        },
        "et": {
          "ignore_above": 75,
          "type": "keyword"
        },
        "tba": {
          "type": "text"
        },
        "hr1rh": {
          "type": "keyword"
        },
        "trt": {
          "type": "integer"
        },
        "rrc": {
          "type": "integer"
        },
        "rfc": {
          "type": "integer"
        },
        "icr": {
          "type": "boolean"
        },
        "crs": {
          "type": "keyword"
        },
        "mc": {
          "type": "integer"
        },
        "msc": {
          "type": "integer"
        },
        "mfc": {
          "type": "integer"
        },
        "mtd": {
          "type": "long"
        },
        "ma": {
          "type": "keyword"
        },
        "sr1t": {
          "type": "integer"
        },
        "hr1ra": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword"
            }
          }
        },
        "loc": {
          "type": "geo_point"
        },
        "xff": {
          "type": "keyword"
        },
        "apmn": {
          "type": "keyword"
        },
        "hr1hm": {
          "ignore_above": 50,
          "type": "keyword"
        },
        "rt": {
          "ignore_above": 7,
          "type": "keyword"
        },
        "hr1lp": {
          "type": "keyword"
        },
        "hr1ln": {
          "type": "keyword"
        },
        "hr1pi": {
          "type": "keyword"
        },
        "sc": {
          "type": "short"
        },
        "fbarh": {
          "type": "nested",
          "properties": {
            "k": {
              "type": "keyword"
            },
            "v": {
              "type": "keyword"
            }
          }
        },
        "tcb": {
          "type": "text"
        },
        "hr1rsi": {
          "type": "keyword"
        },
        "tch": {
          "type": "nested",
          "properties": {
            "k": {
              "type": "keyword"
            },
            "v": {
              "type": "keyword"
            }
          }
        },
        "eat": {
          "type": "keyword"
        },
        "hr1r1u": {
          "type": "keyword"
        },
        "hr1p": {
          "type": "keyword"
        },
        "hr1s": {
          "type": "keyword"
        },
        "fbarb": {
          "type": "text"
        },
        "@timestamp": {
          "format": "yyyy-MM-dd'T'HH:mm:ss.S'Z'||yyyy-MM-dd'T'HH:mm:ss.SS'Z'||yyyy-MM-dd'T'HH:mm:ss.SSS'Z'",
          "type": "date"
        },
        "ch": {
          "type": "boolean"
        },
        "hr1ce": {
          "type": "keyword"
        },
        "hr1sp": {
          "type": "keyword"
        },
        "hr1sn": {
          "type": "keyword"
        },
        "hr1la": {
          "type": "keyword"
        },
        "hr1ct": {
          "ignore_above": 50,
          "type": "keyword"
        },
        "pi": {
          "type": "keyword"
        },
        "hr1cp": {
          "type": "keyword"
        },
        "hr1cl": {
          "type": "keyword"
        }
      }
    }
  }
}

Template Data Structure Table

Field Name	Data Stored in Field/Description	Java Mapping Type	Elastic Search Mapping Type
@timestamp	Time Stamp	Instant	date format: yyyy-MM-dd’T’HH:mm:ss.S’Z’ yyyy-MM-dd’T’HH:mm:ss.SS’Z’ yyyy-MM-dd’T’HH:mm:ss.SSS’Z’
aci	Apinizer Correlation ID	String	keyword
ei	Environment ID	String	keyword
pi	Project ID	String	keyword
api	API Proxy ID	String	keyword
appi	Application ID	String	keyword
appin	Application Name	String	keyword
apn	API Proxy Name	String	keyword
eat	API Type	Enum	keyword
apr	API Proxy Revision	Integer	short
aprp	API Proxy Relative Path	String	keyword
apmi	API Proxy Method ID	String	keyword
apmn	API Proxy Method Name	String	keyword
xff	X Forwarded For	String	keyword
hr1ra	HTTP Request Remote Address	String	text, keyword
hr1hm	HTTP Request HTTP Method	String	keyword ignore_above: 50
hr1ct	HTTP Request Content Type	String	keyword ignore_above: 50
uok	Username or Key	String	keyword ignore_above: 50
ra	Routing Address	String	keyword
rrc	Routing Retry Count	Integer	integer
rfc	Routing Failover Count	Integer	integer
trqpt	Time Request Pipeline Total	Integer	integer
trspt	Time Response Pipeline Total	Integer	integer
trt	Time Routing Total	Integer	integer
tapt	Time API Proxy Total	Integer	integer
sr1t	Size Request Total	Integer	integer
sr2t	Size Response Total	Integer	integer
ch	Cache Hit	Boolean	boolean
sc	Status Code	int	short
et	Error Type	Enum	keyword ignore_above: 75
pet	Parent Error Type	Enum	keyword ignore_above: 75
rt	Result Type	Enum	keyword ignore_above: 7
hr1pi	HTTP Request Path Info	String	keyword
hr1cp	HTTP Request Context Path	String	keyword
hr1qs	HTTP Request Query String	String	keyword
hr1ru	HTTP Request Remote User	String	keyword
hr1rsi	HTTP Request Requested SessionID	String	keyword
hr1r1u	HTTP Request Request URI	String	keyword
hr1ce	HTTP Request Character Encoding	String	keyword
hr1cl	HTTP Request Content Length	String	keyword
hr1p	HTTP Request Protocol	String	keyword
hr1s	HTTP Request Scheme	String	keyword
hr1sn	HTTP Request Server Name	String	keyword
hr1sp	HTTP Request Server Port	String	keyword
loc	Location	GeoLocation{ Double lat, Double lon }	geo_point
hr1rh	HTTP Request Remote Host	String	keyword
hr1rp	HTTP Request Remote Port	String	keyword
hr1ln	HTTP Request Local Name	String	keyword
hr1la	HTTP Request Local Addr	String	keyword
hr1lp	HTTP Request Local Port	String	keyword
fcrp	From Client Read Only Parameter	Map<String,String>	nested k: keyword v: keyword
fcrh	From Client Read Only Header	Map<String,String>	nested k: keyword v: keyword
fcrb	From Client Read Only Body	String	text
tbap	To Backend API Parameter	Map<String,String>	nested k: keyword v: keyword
tbah	To Backend API Header	Map<String,String>	nested k: keyword v: keyword
tba	To Backend API Body	String	text
fbarh	From Backend API Read Only Header	Map<String,String>	nested k: keyword v: keyword
fbarb	From Backend API Read Only Body	String	text
tch	To Client Header	Map<String,String>	nested k: keyword v: keyword
tcb	To Client Body	String	text

You can review the data example created in this data structure on the API Traffic Log Record Data Structure page.

Index Creation and Querying

When creating indices or querying with data stream, requests are sent according to the value in the index_patterns metadata within the template.

Index Creation

curl -XPOST "http://<ELASTICSEARCH_IP_ADDRESS>:<ELASTICSEARCH_PORT>/apinizer-log-apiproxy-<INDEX_KEY>/_doc" -H 'Content-Type: application/json' -d'{"@timestamp": "2023-05-31T11:40:36.376Z"}'

Running Query

curl -XGET "http://<ELASTICSEARCH_IP_ADDRESS>:<ELASTICSEARCH_PORT>/apinizer-log-apiproxy-<INDEX_KEY>/_search" -H 'Content-Type: application/json' -d
'{
  "from": 0,
  "size": 10,
  "query": {
    "bool": {
      "filter": [
        {
          "range": {
            "@timestamp": {
              "from": "now-10m/m",
              "to": "now/m",
              "include_lower": true,
              "include_upper": true
            }
          }
        }
      ],
      "adjust_pure_negative": true,
      "boost": 1
    }
  }, 
  "sort": [
    {
      "@timestamp": {
        "order": "desc"
      }
    }
  ]
}'