Elasticsearch stops refreshing - elasticsearch

in Elasticsearch 6.6.2 i'm having this weird problem
where elastic stops refreshing all indexes. Any UPDATE/INDEX/DELETE operation
returns OK result but search is not showing changes. I see no errors in logs.
If i try to manually force refresh by API this will run forever with no response.
If i restart ES it resumes working correctly and all documents indexed during blackout are there. The problem appears again after few days. This is happening on only one single instance out of many tens that we have in production with exactly the same configuration.
All indexes are open and green.
UPDATE:
this is the output of /someindex/_settings?include_defaults during problem:
{
"someindex": {
"settings": {
"index": {
"creation_date": "1627983403578",
"number_of_shards": "5",
"number_of_replicas": "1",
"uuid": "eWWHKAsHQYix3uZSTs6YXg",
"version": {
"created": "6060299"
},
"provided_name": "someindex"
}
},
"defaults": {
"index": {
"max_inner_result_window": "100",
"unassigned": {
"node_left": {
"delayed_timeout": "1m"
}
},
"max_terms_count": "65536",
"lifecycle": {
"name": "",
"rollover_alias": "",
"indexing_complete": "false"
},
"routing_partition_size": "1",
"max_docvalue_fields_search": "100",
"merge": {
"scheduler": {
"max_thread_count": "2",
"auto_throttle": "true",
"max_merge_count": "7"
},
"policy": {
"reclaim_deletes_weight": "2.0",
"floor_segment": "2mb",
"max_merge_at_once_explicit": "30",
"max_merge_at_once": "10",
"max_merged_segment": "5gb",
"expunge_deletes_allowed": "10.0",
"segments_per_tier": "10.0",
"deletes_pct_allowed": "33.0"
}
},
"max_refresh_listeners": "1000",
"max_regex_length": "1000",
"load_fixed_bitset_filters_eagerly": "true",
"number_of_routing_shards": "5",
"write": {
"wait_for_active_shards": "1"
},
"mapping": {
"coerce": "false",
"nested_fields": {
"limit": "50"
},
"depth": {
"limit": "20"
},
"ignore_malformed": "false",
"total_fields": {
"limit": "1000"
}
},
"source_only": "false",
"soft_deletes": {
"enabled": "false",
"retention": {
"operations": "0"
}
},
"max_script_fields": "32",
"query": {
"default_field": [
"*"
],
"parse": {
"allow_unmapped_fields": "true"
}
},
"format": "0",
"frozen": "false",
"sort": {
"missing": [],
"mode": [],
"field": [],
"order": []
},
"priority": "1",
"codec": "default",
"max_rescore_window": "10000",
"max_adjacency_matrix_filters": "100",
"gc_deletes": "60s",
"optimize_auto_generated_id": "true",
"max_ngram_diff": "1",
"translog": {
"generation_threshold_size": "64mb",
"flush_threshold_size": "512mb",
"sync_interval": "5s",
"retention": {
"size": "512mb",
"age": "12h"
},
"durability": "REQUEST"
},
"auto_expand_replicas": "false",
"mapper": {
"dynamic": "true"
},
"requests": {
"cache": {
"enable": "true"
}
},
"data_path": "",
"highlight": {
"max_analyzed_offset": "-1"
},
"routing": {
"rebalance": {
"enable": "all"
},
"allocation": {
"enable": "all",
"total_shards_per_node": "-1"
}
},
"search": {
"slowlog": {
"level": "TRACE",
"threshold": {
"fetch": {
"warn": "-1",
"trace": "-1",
"debug": "-1",
"info": "-1"
},
"query": {
"warn": "-1",
"trace": "-1",
"debug": "-1",
"info": "-1"
}
}
},
"throttled": "false"
},
"fielddata": {
"cache": "node"
},
"default_pipeline": "_none",
"max_slices_per_scroll": "1024",
"shard": {
"check_on_startup": "false"
},
"xpack": {
"watcher": {
"template": {
"version": ""
}
},
"version": "",
"ccr": {
"following_index": "false"
}
},
"percolator": {
"map_unmapped_fields_as_text": "false",
"map_unmapped_fields_as_string": "false"
},
"allocation": {
"max_retries": "5"
},
"refresh_interval": "1s",
"indexing": {
"slowlog": {
"reformat": "true",
"threshold": {
"index": {
"warn": "-1",
"trace": "-1",
"debug": "-1",
"info": "-1"
}
},
"source": "1000",
"level": "TRACE"
}
},
"compound_format": "0.1",
"blocks": {
"metadata": "false",
"read": "false",
"read_only_allow_delete": "false",
"read_only": "false",
"write": "false"
},
"max_result_window": "10000",
"store": {
"stats_refresh_interval": "10s",
"type": "",
"fs": {
"fs_lock": "native"
},
"preload": []
},
"queries": {
"cache": {
"enabled": "true"
}
},
"ttl": {
"disable_purge": "false"
},
"warmer": {
"enabled": "true"
},
"max_shingle_diff": "3",
"query_string": {
"lenient": "false"
}
}
}
}
}
Any hint?
Thanks

Related

Creating/Manipulating Elasticsearch Data View from filebeat.yml

I created some kibana dashboards and they are referencing to a kibana data view. This data view is currently default generated by Filebeat when i'm uploading the filebeat setup, etc. Therefore the data view gets a random Id and my dashboard gets a mising reference since those id's are changing. Now i want to configure somewhere the default created data view since my index- and template-pattern have fixed names and i could create a fixed data view. Then my Dashboards wouldn't go into "missing references". I searched the following sites:
https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-installation-configuration.html
https://github.com/elastic/kibana/issues
I created the Dashboards within the kibana UI and exported them for easier integration within my filebeat project.
My filebeat.yml file
# ============================== Filebeat modules ==============================
filebeat.config.modules:
# Glob pattern for configuration loading
path: ${path.config}/modules.d/*.yml
# Set to true to enable config reloading
reload.enabled: false
# ======================= Elasticsearch template setting =======================
setup.template.settings:
index.number_of_shards: 1
setup.template.name: pac-filebeat-%{[agent.version]}
setup.template.pattern: pac-filebeat-%{[agent.version]}
setup.template.fields: ${path.config}/fields.yml
setup.dashboards.enabled: false
setup.dashboards.directory: ${path.config}\kibana\custom
# =================================== Kibana ===================================
# Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API.
# This requires a Kibana endpoint configuration.
setup.kibana:
# Kibana Host
# Scheme and port can be left out and will be set to the default (http and 5601)
# In case you specify and additional path, the scheme is required: http://localhost:5601/path
# IPv6 addresses should always be defined as: https://[2001:db8::1]:5601
host: "localhost:5601"
# ---------------------------- Elasticsearch Output ----------------------------
output.elasticsearch:
# Array of hosts to connect to.
hosts: ["localhost:9200"]
# Protocol - either `http` (default) or `https`.
protocol: "http"
index: pac-filebeat-%{[agent.version]}
# ================================= Processors =================================
processors:
- add_host_metadata:
when.not.contains.tags: forwarded
- add_cloud_metadata: ~
- add_docker_metadata: ~
- add_kubernetes_metadata: ~
# ================================== Logging ===================================
This is one example Dashboard.
{
"attributes": {
"description": "",
"hits": 0,
"kibanaSavedObjectMeta": {
"searchSourceJSON": {
"query": {
"query": "",
"language": "kuery"
},
"filter": []
}
},
"optionsJSON": {
"useMargins": true,
"syncColors": false,
"syncTooltips": false,
"hidePanelTitles": false
},
"panelsJSON": [
{
"version": "8.5.3",
"type": "lens",
"gridData": {
"x": 0,
"y": 0,
"w": 9,
"h": 10,
"i": "70779a3d-7880-416d-9736-8a374af8ae41"
},
"panelIndex": "70779a3d-7880-416d-9736-8a374af8ae41",
"embeddableConfig": {
"attributes": {
"title": "",
"visualizationType": "lnsLegacyMetric",
"type": "lens",
"references": [
{
"type": "index-pattern",
"id": "454f0a8e-d5b8-4c8d-9dbe-a1a61782d24d",
"name": "indexpattern-datasource-layer-2a27a515-43d0-4aae-8764-467983de9faf"
},
{
"type": "index-pattern",
"name": "99c2853a-48e3-4042-badf-9c0829cec119",
"id": "454f0a8e-d5b8-4c8d-9dbe-a1a61782d24d"
}
],
"state": {
"visualization": {
"layerId": "2a27a515-43d0-4aae-8764-467983de9faf",
"accessor": "10b01daf-1c3d-46d1-a014-b381ac70019c",
"layerType": "data"
},
"query": {
"query": "",
"language": "kuery"
},
"filters": [
{
"meta": {
"index": "99c2853a-48e3-4042-badf-9c0829cec119",
"alias": "pac.log.level : ERROR",
"type": "custom",
"key": "query",
"value": {
"bool": {
"must": [],
"filter": [
{
"bool": {
"should": [
{
"match": {
"pac.log.level": "ERROR"
}
}
],
"minimum_should_match": 1
}
}
],
"should": [],
"must_not": []
}
},
"disabled": false,
"negate": false
},
"query": {
"bool": {
"must": [],
"filter": [
{
"bool": {
"should": [
{
"match": {
"pac.log.level": "ERROR"
}
}
],
"minimum_should_match": 1
}
}
],
"should": [],
"must_not": []
}
},
"$state": {
"store": "appState"
}
}
],
"datasourceStates": {
"indexpattern": {
"layers": {
"2a27a515-43d0-4aae-8764-467983de9faf": {
"columns": {
"10b01daf-1c3d-46d1-a014-b381ac70019c": {
"label": "Errors",
"customLabel": true,
"dataType": "number",
"operationType": "count",
"isBucketed": false,
"scale": "ratio",
"sourceField": "pac.log.level",
"filter": {
"query": "pac.log.level : ERROR",
"language": "kuery"
},
"params": {
"emptyAsNull": true
}
}
},
"columnOrder": [
"10b01daf-1c3d-46d1-a014-b381ac70019c"
],
"incompleteColumns": {}
}
}
}
},
"internalReferences": [],
"adHocDataViews": {}
}
},
"enhancements": {}
}
},
{
"version": "8.5.3",
"type": "lens",
"gridData": {
"x": 9,
"y": 0,
"w": 12,
"h": 10,
"i": "2bbc2f60-dfb2-4545-a725-9f9bec323e4b"
},
"panelIndex": "2bbc2f60-dfb2-4545-a725-9f9bec323e4b",
"embeddableConfig": {
"attributes": {
"title": "",
"visualizationType": "lnsXY",
"type": "lens",
"references": [
{
"type": "index-pattern",
"id": "454f0a8e-d5b8-4c8d-9dbe-a1a61782d24d",
"name": "indexpattern-datasource-layer-a0f69cd9-6726-4551-9a91-cb330cf0b16d"
}
],
"state": {
"visualization": {
"legend": {
"isVisible": true,
"position": "right"
},
"valueLabels": "hide",
"fittingFunction": "None",
"xTitle": "",
"yTitle": "",
"axisTitlesVisibilitySettings": {
"x": false,
"yLeft": false,
"yRight": true
},
"tickLabelsVisibilitySettings": {
"x": true,
"yLeft": false,
"yRight": true
},
"labelsOrientation": {
"x": 0,
"yLeft": 0,
"yRight": 0
},
"gridlinesVisibilitySettings": {
"x": true,
"yLeft": false,
"yRight": true
},
"preferredSeriesType": "bar_horizontal_percentage_stacked",
"layers": [
{
"layerId": "a0f69cd9-6726-4551-9a91-cb330cf0b16d",
"seriesType": "bar_horizontal_percentage_stacked",
"accessors": [
"cbf5323b-0d81-4411-afa3-91087f062874"
],
"yConfig": [],
"layerType": "data",
"splitAccessor": "a98ba577-d539-4ab3-b798-2bddcb2efd3f",
"palette": {
"type": "palette",
"name": "status"
}
}
]
},
"query": {
"query": "",
"language": "kuery"
},
"filters": [],
"datasourceStates": {
"indexpattern": {
"layers": {
"a0f69cd9-6726-4551-9a91-cb330cf0b16d": {
"columns": {
"a98ba577-d539-4ab3-b798-2bddcb2efd3f": {
"label": "ESA states",
"dataType": "string",
"operationType": "terms",
"scale": "ordinal",
"sourceField": "pac.log.esa.connection",
"isBucketed": true,
"params": {
"size": 3,
"orderBy": {
"type": "column",
"columnId": "cbf5323b-0d81-4411-afa3-91087f062874"
},
"orderDirection": "desc",
"otherBucket": true,
"missingBucket": false,
"parentFormat": {
"id": "terms"
}
},
"customLabel": true
},
"cbf5323b-0d81-4411-afa3-91087f062874": {
"label": "Connected",
"dataType": "number",
"operationType": "unique_count",
"scale": "ratio",
"sourceField": "pac.log.esa.connection",
"isBucketed": false,
"filter": {
"query": "pac.log.esa.connection : Connected",
"language": "kuery"
},
"params": {
"emptyAsNull": true
},
"customLabel": true
}
},
"columnOrder": [
"a98ba577-d539-4ab3-b798-2bddcb2efd3f",
"cbf5323b-0d81-4411-afa3-91087f062874"
],
"incompleteColumns": {}
}
}
}
},
"internalReferences": [],
"adHocDataViews": {}
}
},
"enhancements": {}
}
},
{
"version": "8.5.3",
"type": "lens",
"gridData": {
"x": 21,
"y": 0,
"w": 7,
"h": 10,
"i": "ef851b6b-222a-4b70-a7c3-ec6a85bdf4d1"
},
"panelIndex": "ef851b6b-222a-4b70-a7c3-ec6a85bdf4d1",
"embeddableConfig": {
"attributes": {
"title": "",
"visualizationType": "lnsLegacyMetric",
"type": "lens",
"references": [
{
"type": "index-pattern",
"id": "454f0a8e-d5b8-4c8d-9dbe-a1a61782d24d",
"name": "indexpattern-datasource-layer-d8719702-b241-4931-8bd3-d706f1403c25"
}
],
"state": {
"visualization": {
"layerId": "d8719702-b241-4931-8bd3-d706f1403c25",
"accessor": "312ded46-d0e9-4dcf-9779-b0ca10249a90",
"layerType": "data"
},
"query": {
"query": "",
"language": "kuery"
},
"filters": [],
"datasourceStates": {
"indexpattern": {
"layers": {
"d8719702-b241-4931-8bd3-d706f1403c25": {
"columns": {
"312ded46-d0e9-4dcf-9779-b0ca10249a90": {
"label": "Failed DEO Executions",
"customLabel": true,
"dataType": "number",
"operationType": "count",
"isBucketed": false,
"scale": "ratio",
"sourceField": "pac.log.deo.status",
"filter": {
"query": "pac.log.deo.status : failed",
"language": "kuery"
},
"params": {
"emptyAsNull": true
}
}
},
"columnOrder": [
"312ded46-d0e9-4dcf-9779-b0ca10249a90"
],
"incompleteColumns": {}
}
}
}
},
"internalReferences": [],
"adHocDataViews": {}
}
},
"enhancements": {}
}
},
{
"version": "8.5.3",
"type": "lens",
"gridData": {
"x": 0,
"y": 10,
"w": 14,
"h": 11,
"i": "7bcd2a3e-54c1-4b09-a0b6-d41c339c2ae6"
},
"panelIndex": "7bcd2a3e-54c1-4b09-a0b6-d41c339c2ae6",
"embeddableConfig": {
"attributes": {
"title": "",
"visualizationType": "lnsXY",
"type": "lens",
"references": [
{
"type": "index-pattern",
"id": "454f0a8e-d5b8-4c8d-9dbe-a1a61782d24d",
"name": "indexpattern-datasource-layer-cd91feb0-0927-4465-b0fd-b5bac50af734"
}
],
"state": {
"visualization": {
"title": "Empty XY chart",
"legend": {
"isVisible": true,
"position": "right",
"showSingleSeries": true
},
"valueLabels": "hide",
"preferredSeriesType": "line",
"layers": [
{
"layerId": "cd91feb0-0927-4465-b0fd-b5bac50af734",
"accessors": [
"8ccc17fc-529e-4460-9925-149e77e37dd1"
],
"position": "top",
"seriesType": "line",
"showGridlines": false,
"layerType": "data",
"xAccessor": "43aa69d1-f806-4f87-a2b8-57718e85400a"
}
],
"yTitle": "",
"axisTitlesVisibilitySettings": {
"x": true,
"yLeft": false,
"yRight": true
}
},
"query": {
"query": "",
"language": "kuery"
},
"filters": [],
"datasourceStates": {
"indexpattern": {
"layers": {
"cd91feb0-0927-4465-b0fd-b5bac50af734": {
"columns": {
"43aa69d1-f806-4f87-a2b8-57718e85400a": {
"label": "#timestamp",
"dataType": "date",
"operationType": "date_histogram",
"sourceField": "#timestamp",
"isBucketed": true,
"scale": "interval",
"params": {
"interval": "auto",
"includeEmptyRows": true,
"dropPartials": false
}
},
"8ccc17fc-529e-4460-9925-149e77e37dd1": {
"label": " System cpu usage",
"dataType": "number",
"operationType": "median",
"sourceField": "pac.log.system.monitor.cpu.pct",
"isBucketed": false,
"scale": "ratio",
"params": {
"emptyAsNull": true,
"format": {
"id": "percent",
"params": {
"decimals": 2
}
}
},
"customLabel": true
}
},
"columnOrder": [
"43aa69d1-f806-4f87-a2b8-57718e85400a",
"8ccc17fc-529e-4460-9925-149e77e37dd1"
],
"incompleteColumns": {}
}
}
}
},
"internalReferences": [],
"adHocDataViews": {}
}
},
"enhancements": {}
}
}
],
"timeRestore": false,
"title": "ESA overview",
"version": 1
},
"coreMigrationVersion": "8.5.3",
"id": "d29c4ac0-ab68-11ed-bfcf-939f8111002e",
"migrationVersion": {
"dashboard": "8.5.0"
},
"references": [
{
"id": "454f0a8e-d5b8-4c8d-9dbe-a1a61782d24d",
"name": "70779a3d-7880-416d-9736-8a374af8ae41:indexpattern-datasource-layer-2a27a515-43d0-4aae-8764-467983de9faf",
"type": "index-pattern"
},
{
"id": "454f0a8e-d5b8-4c8d-9dbe-a1a61782d24d",
"name": "70779a3d-7880-416d-9736-8a374af8ae41:99c2853a-48e3-4042-badf-9c0829cec119",
"type": "index-pattern"
},
{
"id": "454f0a8e-d5b8-4c8d-9dbe-a1a61782d24d",
"name": "2bbc2f60-dfb2-4545-a725-9f9bec323e4b:indexpattern-datasource-layer-a0f69cd9-6726-4551-9a91-cb330cf0b16d",
"type": "index-pattern"
},
{
"id": "454f0a8e-d5b8-4c8d-9dbe-a1a61782d24d",
"name": "ef851b6b-222a-4b70-a7c3-ec6a85bdf4d1:indexpattern-datasource-layer-d8719702-b241-4931-8bd3-d706f1403c25",
"type": "index-pattern"
},
{
"id": "454f0a8e-d5b8-4c8d-9dbe-a1a61782d24d",
"name": "7bcd2a3e-54c1-4b09-a0b6-d41c339c2ae6:indexpattern-datasource-layer-cd91feb0-0927-4465-b0fd-b5bac50af734",
"type": "index-pattern"
},
{
"id": "c6a11f70-ab68-11ed-bfcf-939f8111002e",
"name": "tag-c6a11f70-ab68-11ed-bfcf-939f8111002e",
"type": "tag"
}
],
"type": "dashboard",
"updated_at": "2023-02-13T07:14:45.551Z",
"version": "WzE1NTI1NiwzMl0="
}
This would be the used tag inside the above dashboard
{
"attributes": {
"color": "#52bc22",
"description": "",
"name": "custom"
},
"coreMigrationVersion": "8.5.3",
"id": "c6a11f70-ab68-11ed-bfcf-939f8111002e",
"migrationVersion": {
"tag": "8.0.0"
},
"references": [],
"type": "tag",
"updated_at": "2023-02-13T07:39:07.543Z",
"version": "WzE1NTM0NywzMl0="
}
The error message:
error dashboard asset: returned 200 to import file: 1 error: error: missing_references, asset ID=d29c4ac0-ab68-11ed-bfcf-939f8111002e; asset type=dashboard; references=[{Type:index-pattern ID:454f0a8e-d5b8-4c8d-9dbe-a1a61782d24d} {Type:index-pattern ID:454f0a8e-d5b8-4c8d-9dbe-a1a61782d24d} {Type:index-pattern ID:454f0a8e-d5b8-4c8d-9dbe-a1a61782d24d} {Type:index-pattern ID:454f0a8e-d5b8-4c8d-9dbe-a1a61782d24d} {Type:index-pattern ID:454f0a8e-d5b8-4c8d-9dbe-a1a61782d24d}]. Response: {"successCount":1,"success":false,"warnings":[],"successResults":[{"type":"tag","id":"c6a11f70-ab68-11ed-bfcf-939f8111002e","meta":{"title":"custom","icon":"tag"},"overwrite":true}],"errors":[{"id":"d29c4ac0-ab68-11ed-bfcf-939f8111002e","type":"dashboard","meta":{"title":"ESA overview","icon":"dashboardApp"},"error":{"type":"missing_references","references":[{"type":"index-pattern","id":"454f0a8e-d5b8-4c8d-9dbe-a1a61782d24d"},{"type":"index-pattern","id":"454f0a8e-d5b8-4c8d-9dbe-a1a61782d24d"},{"type":"index-pattern","id":"454f0a8e-d5b8-4c8d-9dbe-a1a61782d24d"},{"type":"index-pattern","id":"454f0a8e-d5b8-4c8d-9dbe-a1a61782d24d"},{"type":"index-pattern","id":"454f0a8e-d5b8-4c8d-9dbe-a1a61782d24d"}]},"overwrite":true}]}
kind regardes
You can configure Filebeat to use your own dashboard using the following settings:
setup.dashboards.url
setup.dashboards.file
setup.dashboards.index
setup.dashboards.string_replacements (<-- probably the one you're looking for)

Searching through objects inside nested documents provides unexpected output

I am getting unexpected result from Elasticsearch while searching though object property which are inside nested property. I am using elasticsearch-dsl python library for creating document and querying. Is this bug or I am missing something in querying and mapping parts? Below are the elasticsearch json mapping, query, unexpected result and expected result
Mapping:
{
"deal_acls": {
"type": "nested",
"properties": {
"created_at": {
"type": "date"
},
"created_by": {
"properties": {
"id": {
"type": "keyword"
},
"name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword"
}
}
},
"status": {
"type": "keyword",
"normalizer": "lowercase"
}
}
},
"permission": {
"properties": {
"CRM": {
"properties": {
"description": {
"properties": {
"created_by": {
"type": "keyword",
"normalizer": "lowercase"
},
"object": {
"type": "keyword",
"normalizer": "lowercase"
},
"object_id": {
"type": "keyword",
"normalizer": "lowercase"
},
"timestamp": {
"type": "date"
}
}
},
"permission": {
"properties": {
"delete": {
"type": "integer"
},
"edit": {
"type": "integer"
},
"manage": {
"type": "long"
},
"read": {
"type": "integer"
},
"write": {
"type": "integer"
}
}
}
}
},
"deal": {
"properties": {
"description": {
"properties": {
"created_by": {
"type": "keyword",
"normalizer": "lowercase"
},
"object": {
"type": "keyword",
"normalizer": "lowercase"
},
"object_id": {
"type": "keyword",
"normalizer": "lowercase"
},
"timestamp": {
"type": "date"
}
}
},
"permission": {
"properties": {
"delete": {
"type": "integer"
},
"edit": {
"type": "integer"
},
"manage": {
"type": "long"
},
"read": {
"type": "integer"
},
"write": {
"type": "integer"
}
}
}
}
},
"document": {
"properties": {
"description": {
"properties": {
"created_by": {
"type": "keyword",
"normalizer": "lowercase"
},
"object": {
"type": "keyword",
"normalizer": "lowercase"
},
"object_id": {
"type": "keyword",
"normalizer": "lowercase"
},
"timestamp": {
"type": "date"
}
}
},
"permission": {
"properties": {
"delete": {
"type": "integer"
},
"edit": {
"type": "integer"
},
"manage": {
"type": "long"
},
"read": {
"type": "integer"
},
"write": {
"type": "integer"
}
}
}
}
},
"external_deal_team": {
"properties": {
"description": {
"properties": {
"created_by": {
"type": "keyword",
"normalizer": "lowercase"
},
"object": {
"type": "keyword",
"normalizer": "lowercase"
},
"object_id": {
"type": "keyword",
"normalizer": "lowercase"
},
"timestamp": {
"type": "date"
}
}
},
"permission": {
"properties": {
"delete": {
"type": "integer"
},
"edit": {
"type": "integer"
},
"manage": {
"type": "long"
},
"read": {
"type": "integer"
},
"write": {
"type": "integer"
}
}
}
}
},
"internal_deal_team": {
"properties": {
"description": {
"properties": {
"created_by": {
"type": "keyword",
"normalizer": "lowercase"
},
"object": {
"type": "keyword",
"normalizer": "lowercase"
},
"object_id": {
"type": "keyword",
"normalizer": "lowercase"
},
"timestamp": {
"type": "date"
}
}
},
"permission": {
"properties": {
"delete": {
"type": "integer"
},
"edit": {
"type": "integer"
},
"manage": {
"type": "long"
},
"read": {
"type": "integer"
},
"write": {
"type": "integer"
}
}
}
}
}
}
},
"status": {
"type": "keyword",
"normalizer": "lowercase"
},
"updated_at": {
"type": "date"
},
"updated_by": {
"properties": {
"id": {
"type": "keyword"
},
"name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword"
}
}
},
"status": {
"type": "keyword",
"normalizer": "lowercase"
}
}
},
"user": {
"properties": {
"id": {
"type": "keyword"
},
"name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword"
}
}
},
"status": {
"type": "keyword",
"normalizer": "lowercase"
}
}
}
}
}
}
Query:
{
"query": {
"bool": {
"must": [
{
"nested": {
"path": "deal_acls",
"query": {
"term": {
"deal_acls.permission.deal.permission.read": 1
}
}
}
},
{
"nested": {
"path": "deal_acls",
"query": {
"terms": {
"deal_acls.user.id": [
"5f7cea05-6562-4bdd-8448-19cfbe11783a"
]
}
}
}
}
]
}
}
}
Unexpected result: Since the deal permission of user with id=5f7cea05-6562-4bdd-8448-19cfbe11783a is 0, it should be returning empty hits.
{
"took": 2,
"timed_out": false,
"_shards": {
"total": 1,
"successful": 1,
"skipped": 0,
"failed": 0
},
"hits": {
"total": {
"value": 1,
"relation": "eq"
},
"max_score": 2,
"hits": [
{
"_index": "dev.crecentric.dealvault.deals",
"_type": "_doc",
"_id": "a928838d-3422-41db-b40e-28f5c793f806",
"_score": 2,
"_source": {
"id": "a928838d-3422-41db-b40e-28f5c793f806",
"deal_acls": [
{
"user": {
"id": "5f7cea05-6562-4bdd-8448-19cfbe11783a",
"name": "testerrrrs testesssss",
"status": "active"
},
"permission": {
"deal": {
"permission": {
"edit": 0,
"read": 0,
"write": 0,
"delete": 0,
"manage": 0
},
"description": {
"object": "workspace",
"object_id": "fbc840b1-8727-4945-a070-fa1c105f9550",
"timestamp": "2022-02-10T05:46:05.140867+00:00",
"created_by": "d78411e5-6645-4b95-a98c-db6db8748580"
}
},
"external_deal_team": {
"permission": {
"edit": 0,
"read": 1,
"write": 1,
"delete": 0,
"manage": 0
},
"description": {
"object": "workspace",
"object_id": "fbc840b1-8727-4945-a070-fa1c105f9550",
"timestamp": "2022-02-10T05:46:05.140902+00:00",
"created_by": "d78411e5-6645-4b95-a98c-db6db8748580"
}
},
"internal_deal_team": {
"permission": {
"edit": 0,
"read": 1,
"write": 1,
"delete": 0,
"manage": 0
},
"description": {
"object": "workspace",
"object_id": "fbc840b1-8727-4945-a070-fa1c105f9550",
"timestamp": "2022-02-10T05:46:05.140910+00:00",
"created_by": "d78411e5-6645-4b95-a98c-db6db8748580"
}
}
},
"status": "active",
"created_at": "2022-02-10T05:40:15.727598+05:45",
"updated_at": "2022-02-10T05:46:05.177076+05:45"
},
{
"user": {
"id": "d78411e5-6645-4b95-a98c-db6db8748580",
"name": "Ramesh Pradhan",
"status": "active"
},
"permission": {
"CRM": {
"permission": {
"edit": 1,
"read": 1,
"write": 1,
"delete": 1,
"manage": 1
},
"description": {
"object": "owner",
"object_id": "d78411e5-6645-4b95-a98c-db6db8748580",
"timestamp": "2022-02-10T05:35:41.453881+00:00",
"created_by": "d78411e5-6645-4b95-a98c-db6db8748580"
}
},
"deal": {
"permission": {
"edit": 1,
"read": 1,
"write": 1,
"delete": 1,
"manage": 1
},
"description": {
"object": "owner",
"object_id": "d78411e5-6645-4b95-a98c-db6db8748580",
"timestamp": "2022-02-10T05:35:41.453881+00:00",
"created_by": "d78411e5-6645-4b95-a98c-db6db8748580"
}
},
"document": {
"permission": {
"edit": 1,
"read": 1,
"write": 1,
"delete": 1,
"manage": 1
},
"description": {
"object": "owner",
"object_id": "d78411e5-6645-4b95-a98c-db6db8748580",
"timestamp": "2022-02-10T05:35:41.453881+00:00",
"created_by": "d78411e5-6645-4b95-a98c-db6db8748580"
}
},
"external_deal_team": {
"permission": {
"edit": 1,
"read": 1,
"write": 1,
"delete": 1,
"manage": 1
},
"description": {
"object": "owner",
"object_id": "d78411e5-6645-4b95-a98c-db6db8748580",
"timestamp": "2022-02-10T05:35:41.453881+00:00",
"created_by": "d78411e5-6645-4b95-a98c-db6db8748580"
}
},
"internal_deal_team": {
"permission": {
"edit": 1,
"read": 1,
"write": 1,
"delete": 1,
"manage": 1
},
"description": {
"object": "owner",
"object_id": "d78411e5-6645-4b95-a98c-db6db8748580",
"timestamp": "2022-02-10T05:35:41.453881+00:00",
"created_by": "d78411e5-6645-4b95-a98c-db6db8748580"
}
}
},
"status": "active",
"created_at": "2022-02-10T05:35:41.453913+05:45",
"updated_at": "2022-02-10T05:35:41.462956+05:45"
}
]
}
}
]
}
}
Expected result:
{
"took": 2,
"timed_out": false,
"_shards": {
"total": 1,
"successful": 1,
"skipped": 0,
"failed": 0
},
"hits": {
"total": {
"value": 1,
"relation": "eq"
},
"max_score": 2,
"hits": []
}
}
You are using two nested queries: so the result is a document which contains "deal_acls.permission.deal.permission.read": 1 and "deal_acls.user.id": ["5f7cea05-6562-4bdd-8448-19cfbe11783a"], but not necessary in the same subobject of the nested field. Try using a terms query inside a single nested query:
{
"query": {
"nested": {
"path": "deal_acls",
"query": {
"bool": {
"must": [
{ "match": { "deal_acls.user.id": "5f7cea05-6562-4bdd-8448-19cfbe11783a" }},
{ "match": { "deal_acls.permission.deal.permission.read": 1 }}
]
}
}
}
}
}
In case if someone is searching for answer, I have fixed this using this query:
{
"query": {
"nested": {
"path": "deal_acls",
"query": {
"bool": {
"must": [
{
"term": {
"deal_acls.permission.deal.permission.read": 1
}
},
{
"terms": {
"deal_acls.user.id": [
"5f7cea05-6562-4bdd-8448-19cfbe11783a"
]
}
}
]
}
}
}
}
}

Kafka JDBC source not showing numeric values

I am deploying a Kafka Connect JDBC Source. It is connecting properly to the dabase, but the result I am getting is this:
{
"schema": {
"type": "struct",
"fields": [
{
"type": "bytes",
"optional": false,
"name": "org.apache.kafka.connect.data.Decimal",
"version": 1,
"parameters": {
"scale": "0"
},
"field": "ID"
},
{
"type": "bytes",
"optional": false,
"name": "org.apache.kafka.connect.data.Decimal",
"version": 1,
"parameters": {
"scale": "0"
},
"field": "TENANT_ID"
},
{
"type": "bytes",
"optional": false,
"name": "org.apache.kafka.connect.data.Decimal",
"version": 1,
"parameters": {
"scale": "0"
},
"field": "IS_ACTIVE"
},
{
"type": "int64",
"optional": false,
"name": "org.apache.kafka.connect.data.Timestamp",
"version": 1,
"field": "CREATION_DATE"
},
{
"type": "int64",
"optional": true,
"name": "org.apache.kafka.connect.data.Timestamp",
"version": 1,
"field": "LAST_LOGIN"
},
{
"type": "string",
"optional": true,
"field": "NAME"
},
{
"type": "string",
"optional": true,
"field": "MOBILEPHONE"
},
{
"type": "string",
"optional": true,
"field": "EMAIL"
},
{
"type": "string",
"optional": true,
"field": "USERNAME"
},
{
"type": "string",
"optional": true,
"field": "PASSWORD"
},
{
"type": "string",
"optional": true,
"field": "EXTERNAL_ID"
}
],
"optional": false
},
"payload": {
"ID": "fdo=",
"TENANT_ID": "Uw==",
"IS_ACTIVE": "AQ==",
"CREATION_DATE": 1548987456000,
"LAST_LOGIN": 1557401837030,
"NAME": " ",
"MOBILEPHONE": " ",
"EMAIL": " ",
"USERNAME": "ES00613751",
"PASSWORD": " ",
"EXTERNAL_ID": " "
}
}
As you can see, the numeric and timestamp values are not showing the value properly.
The config:
name=jdbc-teradata-source-connector
connector.class=io.confluent.connect.jdbc.JdbcSourceConnector
tasks.max=1
connection.url=...
numeric.maping=best_fit
topic.prefix=test-2
mode=timestamp+incrementing
timestamp.column.name=LAST_LOGIN
incrementing.column.name=ID
topic=test-jdbc-oracle-source
The numeric mapping does not work since it is Confluent 3.2.2
I have also tried to cast the numbers to numeric but it does not work either.
Add in your connector config numeric.mapping
"numeric.mapping":"best_fit"
you can see all explication here

JSONata, pulling data from nested objects

I'm new to JSONata and so still getting my head around it. I need to pull data out of a nested object, for example from :
{
"transaction": {
"id": "de112b4b-82e2-4172-a89f-68724c90b692"
},
"domain": {
"id": "realworld"
},
"listing": {
"spanner": {
"information": {
"type": {
"VENDOR": "Charlie"
},
"variables": {
},
"uid": "08_spanner",
"mode": {
"store": "bob"
},
"version": "1",
"name": "Harrold"
}
},
"hammer": {
"information": {
"type": {
"VENDOR": "Cliff"
},
"variables": {
},
"uid": "08_hammer",
"mode": {
"store": "steve"
},
"version": "1",
"name": "Mike"
}
},
"wrench": {
"information": {
"type": {
"VENDOR": "Dave"
},
"variables": {
},
"uid": "08_wrench",
"mode": {
"store": "bob"
},
"version": "1",
"name": "Kent"
}
}
}
}
... I need to pull out the listing data with it's "information", but without the information key. So the result would look like:
{
"spanner": {
"type": {
"VENDOR": "Charlie"
},
"variables": {
},
"uid": "08_spanner",
"mode": {
"store": "bob"
},
"version": "1",
"name": "Harrold"
},
"hammer": {
"type": {
"VENDOR": "Cliff"
},
"variables": {
},
"uid": "08_hammer",
"mode": {
"store": "steve"
},
"version": "1",
"name": "Mike"
},
"wrench": {
"type": {
"VENDOR": "Dave"
},
"variables": {
},
"uid": "08_wrench",
"mode": {
"store": "bob"
},
"version": "1",
"name": "Kent"
}
}
I have been messing around on http://try.jsonata.org/ and can see how powerful the library is but so far no success at achieving this goal. Any and all help is appreciated :).
The following expression will do this:
listing.$each(function($value, $key) {
{ $key: $value.information }
}) ~> $merge()
See http://try.jsonata.org/S17Erm85z

Kendo UI Grid Column typed date has a trouble

I have a grid which would have many columns typed date. These all grids is generated from a generic function, so that I cannot know whether the column type is date or not. There is one rule validates through all columns. It is that all columns name end with "Date" suffix. For instance, createDate, editDate, visitedDate, etc...
So that, I can understand that it can be date and I parse it like you can see at the dataSource parse function
I have trouble when I update the cell. It does not reflect its own value to Model. The date column proto function returns "Invalid Date" error. I do not understand what it happen
var dataSource = new kendo.data.DataSource({
"data": [
{
"hidden_gridColumns": "",
"id": "21632",
"projectId": "146",
"customerTypeId": "4",
"district": "0",
"fieldSize": "12",
"fieldType": "0",
"floorCoveringType": "12",
"lastChangeDate": null,
"estimatedModificationDate": null,
"latestCompany": ""
}
],
"schema": {
"model": {
"id": "id",
"fields": {
"gridColumns": {
"type": "string"
},
"id": {
"type": "string"
},
"projectId": {
"type": "string"
},
"customerTypeId": {
"type": "string"
},
"district": {
"type": "string"
},
"fieldSize": {
"type": "string"
},
"fieldType": {
"type": "string"
},
"floorCoveringType": {
"type": "string"
},
"lastChangeDate": {
"type": "date"
},
"estimatedModificationDate": {
"type": "string"
},
"latestCompany": {
"type": "string"
}
}
},
parse: function(data){
$.each(data,
function(rowNo,
row){
$.each(row,
function(colName,
column){
if(colName.indexOf("Date")>=0){
console.log(colName + " taranıyor");
row[colName] = kendo.parseDate(row[colName], "dd-MM-yyyy");
}
});
});
return data;
}
},
"batch": true
})
And this is my columns structure:
var columns = [
{
"title": "gridColumns",
"field": "gridColumns",
"hidden": true
},
{
"title": "id",
"field": "id",
"hidden": true
},
{
"title": "projectId",
"field": "projectId",
"hidden": true
},
{
"title": "Müşteri Tipi",
"field": "customerTypeId",
"hidden": false,
"width": "91",
"values": [
{
"value": "1",
"text": "Yurtiçi "
},
{
"value": "2",
"text": "Yurtdışı"
},
{
"value": "3",
"text": "Spor Kulübü"
},
{
"value": "4",
"text": "Diğer"
},
{
"value": "5",
"text": "Üniversite"
},
{
"value": "6",
"text": ""
}
]
},
{
"title": "district",
"field": "district",
"hidden": true,
"width": "132"
},
{
"title": "Saha Ölçüsü",
"field": "fieldSize",
"hidden": false,
"width": "85"
},
{
"title": "Saha Türü",
"field": "fieldType",
"hidden": false,
"values": [
{
"value": "1",
"text": "Açık"
},
{
"value": "0",
"text": "Kapalı"
}
]
},
{
"title": "Halı Cinsi",
"field": "floorCoveringType",
"hidden": false,
"width": "76"
},
{
"title": "Son Halı değişim Tarih",
"field": "lastChangeDate",
"hidden": false,
"format": "{0:dd-MM-yyyy}"
},
{
"title": "Tahmini Yenileme Tarihih",
"field": "estimatedModificationDate",
"hidden": false
},
{
"title": "Son Çalıştığı Halı Firması",
"field": "latestCompany",
"hidden": false
}
]
I solved this problem by the code below:
parse: function(response) {
var tmpData=[];
for (var i = 0; i < response.length; i++) {
var tmpRow = response[i];
$.each(tmpRow, function(colNo, colValue){
if(colNo.indexOf("Date")>-1){
tmpRow[colNo]=kendo.parseDate(new Date(tmpRow[colNo]));
}
});
tmpData.push(tmpRow);
}
return tmpData;
}

Resources