two node Elasticsearch cluster having data sync issue - elasticsearch

Question

I have a 2 node Elasticsearch cluster with both as master eligible nodes. They both are formed a cluster and named as **node1 and **node2. Couple of weeks back we have upgraded the version of ES from 7.9 to 7.17.4. Now there was an issue reported that there is some data sync issue between the nodes.
When hitting elasticsearch url we are seeing different data for an index. so we tried to hit two node urls individually as below and we observed different data on each node.
curl http://node1:9200/vehicle/_search?q=***
curl http://node2:9200/vehicle/_search?q=***
I know that data on a cluster will get synced automatically on elasticsearch but I think this is not happening in my case. Could you please let me know why this is happening ? is there anything that I can forcefully sync data between nodes.
What could be the reason to get different data on the nodes ? what should I check why there is a different data getting populated ?
Here are the curl queries and their results on two nodes. Node 2 has an additional object along with the first object with a small difference in the last line.
curl http://node1:9200/vehicle/_search?q=XYZ
{
"took": 5,
"timed_out": false,
"_shards": {
"total": 1,
"successful": 1,
"skipped": 0,
"failed": 0
},
"hits": {
"total": {
"value": 1,
"relation": "eq"
},
"max_score": 7.8380795,
"hits": [
{
"_index": "vehicle_inventory",
"_type": "_doc",
"_id": "109802",
"_score": 7.8380795,
"_source": {
"abs": "Yes",
"addressLine1": "965 ALDRIN DR",
"addressLine2": " ",
"aeroCabExtenders": "Yes",
"aeroRoof": " ",
"aeroSideFairings": "Yes",
"airConditioning": "Yes",
"areaName": " ",
"areaNumber": "0617",
"bodyMake": "NULL",
"bodySerialNumber": "NULL",
"bodyFloorType": "",
"bodyRoofType": "",
"brakeType": "NULL",
"cabStyle": "Conventional",
"category": "TRACTOR",
"chassisBumper": "AERO",
"chassisMakeCode": "FTL",
"cdlRequired": "Yes",
"city": "Eagan",
"color": "White",
"modified_color": "WHITE",
"compared": false,
"saved": false,
"country": "US",
"countryExt": "United States",
"collisionWarning": "No",
"collisionWarningModel": "",
"customerNumber": "RENTAL",
"description": "AVAILABLE JANUARY 2022",
"displayDeliveryCal": "Y",
"districtName": " ",
"driverSeatType": "AIR",
"emailAddress": "E-MAIL.GENERAL.LOCATION#XXXXXX",
"engineBrake": "Yes",
"engineMake": "Detroit",
"engineModel4": "DD15AT",
"engineHours": 0,
"faxNumber": "06516860476",
"featuredListing": "Y",
"frontAxleCapacity": "13300",
"frontWheelType": "",
"fuelType": "DIESEL",
"grossVehicleWeight": 80000,
"height": "NULL",
"horsepower": 455,
"image": "/usedtrucks/image/109802/1",
"interiorLevel": "PREMIUM",
"individualPageUrl": "",
"liftGate": "NULL",
"liftGateCapacity": "0",
"liftGateModel": " ",
"loadLength": "NULL",
"make": "Freightliner",
"managersSpecial": "false",
"managerSpecialDiscountRetail": "0",
"managerSpecialDiscountWholeSale": "0",
"mileage": "701,830 miles",
"model": "Cascadia 12564ST",
"newListing": "false",
"numberOfBunks": "2",
"numberOfDriveAxles": "2",
"numberOfTanks": "Dual",
"numericHeight": "",
"numericLoadLength": -1,
"numericWidth": "",
"offLease": "N",
"originalWholeSalePrice": "64,500",
"originalRetailPrice": "64,500",
"phoneNumber": "1-866-309-1962",
"promotionVehicle": "true",
"rearAxleCapacity": "40000",
"rearAxleMake": "Detroit",
"rearDoor": "NULL",
"rearRatio": "2.53",
"rearWheelType": "",
"reeferStandby": "",
"refrigerationHours": "",
"refrigerationMake": "NULL",
"refrigerationModel": "NULL",
"retailPrice": "54,500",
"salePutUpDate": "2021-12-17 00:00:00",
"sideConstruction": "NULL",
"sideDoor": "NULL",
"sleeperSize": "72",
"sleeperType": "Raised Roof",
"slidingFifthWheel": "Yes",
"state": "MN",
"status": "ACTIVE",
"statusComments": " ",
"sale_reason_desc": "AVAILABLE JANUARY 2022",
"saleType": " ",
"size": "",
"suspension": "Air",
"tareWeight": "18795",
"thumbnail": "usedtrucks/image/109802t/1",
"tireSize": "11R22.5",
"tireType": "S/WAY",
"totalFuelCapacity": "220",
"transmissionMake": "Detroit",
"transmissionModel": "12DA1550",
"transmissionSpeed": "12",
"transmissionType": "Automatic",
"type": "Sleeper Tractor",
"unitNumber": "109802",
"use": "RENTAL",
"vin": "3AKJGLD53FSGJ8668",
"wheelBase": "228\"",
"wheelType": "Aluminum",
"wholesalePrice": "59,500",
"width": "NULL",
"year": 2015,
"zip": "55121",
"numeric_mileage": 701830,
"numeric_retail_price": 54500,
"numeric_wholesale_price": 59500,
"dLatitude": 44.944400787353516,
"dLongitude": -93.09329986572266,
"bodyModel": null,
"promotionAmount": "10,000",
"salePrice": 0,
"location": "016510",
"jpegImage": "/usedtrucks/JpgServlet/109802_1.jpg",
"translatedCategory": "Heavy Duty Tractors",
"originalCategory": "Heavy Duty Tractors",
"monFriHours": "M-F 8:00 am to 8:00 pm ET",
"satHours": "Sat 9:00 am to 3:00 pm ET",
"mobileSitePhone": "1-800-485-9671",
"salePutUpDatesort": "2021-12-17",
"stringyear": "2015",
"range_grossVehicleWeight": null,
"range_loadLength": null,
"range_horsepower": "401 and more",
"range_cabType": "Raised Roof",
"range_retailCenter": null,
"viewCount": "222ST15",
"advertTerm": "Level 1",
"numeric_Height": 0,
"numeric_totalFuelCapacity": 0,
"numeric_Width": 0,
"promotionList": [
{
"name": "Save on Select Sleeper Tractors",
"description": "Save on Select Sleeper Tractors",
"amount": 10000,
"amountDisplayRetail": "10,000",
"amountDisplayWholeSale": "10,000",
"type": "DISCOUNT",
"rule": "viewCount != null and viewCount.equalsIgnoreCase('222ST15') and (year <= 2014 or numeric_mileage >= 700)"
}
],
"imageMaxInstance": 10,
"defaultImageBeanList": null,
"translationsMap": {
"cdlRequired": "Yes",
"countryExt": "United States",
"advertTerm": "Level 1",
"color": "White",
"range_horsepower": "401 and more",
"range_cabType": "Raised Roof",
"model": "Cascadia 12564ST",
"category": "Heavy Duty Tractors",
"type": "Sleeper Tractor"
},
"vehicle_suggestions": {},
"vehicle_locations": {
"lon": "-93.0933",
"lat": "44.9444"
},
"combo_filter": [
{
"filter_name": "range_horsepower",
"filter_value": "401 and more"
},
{
"filter_name": "range_cabType",
"filter_value": "Raised Roof"
},
{
"filter_name": "range_retailCenter",
"filter_value": null
},
{
"filter_name": "advertTerm",
"filter_value": "Level 1"
},
{
"filter_name": "airConditioning",
"filter_value": "Yes"
},
{
"filter_name": "category",
"filter_value": "Heavy Duty Tractors"
},
{
"filter_name": "color",
"filter_value": "White"
},
{
"filter_name": "engineMake",
"filter_value": "Detroit"
},
{
"filter_name": "fuelType",
"filter_value": "DIESEL"
},
{
"filter_name": "make",
"filter_value": "Freightliner"
},
{
"filter_name": "model",
"filter_value": "Cascadia 12564ST"
},
{
"filter_name": "newListing",
"filter_value": "false"
},
{
"filter_name": "numberOfBunks",
"filter_value": "2"
},
{
"filter_name": "numberOfDriveAxles",
"filter_value": "2"
},
{
"filter_name": "state",
"filter_value": "MN"
},
{
"filter_name": "transmissionMake",
"filter_value": "Detroit"
},
{
"filter_name": "transmissionSpeed",
"filter_value": "12"
},
{
"filter_name": "transmissionType",
"filter_value": "Automatic"
},
{
"filter_name": "type",
"filter_value": "Sleeper Tractor"
},
{
"filter_name": "collisionWarning",
"filter_value": "No"
},
{
"filter_name": "canada",
"filter_value": null
},
{
"filter_name": "unitedStates",
"filter_value": "MN"
},
{
"filter_name": "countryExt",
"filter_value": "United States"
},
{
"filter_name": "promotionVehicle",
"filter_value": "true"
},
{
"filter_name": "promotions",
"filter_value": "Current Deals"
},
{
"filter_name": "promotions",
"filter_value": "Save on Select Sleeper Tractors"
}
],
"slider_filter_integer": [
{
"filter_name": "numeric_mileage",
"filter_value": 701830
},
{
"filter_name": "year",
"filter_value": 2015
}
],
"slider_filter_float": [
{
"filter_name": "numeric_retail_price",
"filter_value": 54500
},
{
"filter_name": "numeric_wholesale_price",
"filter_value": 59500
}
],
"basic_filter": {},
"rowNum": 2465,
"displayManagerspecial": null,
"displayPromotions": "true",
"canada": null,
"unitedStates": "MN",
"mgrSpecialRetail": "false",
"mgrSpecialWholesale": "false",
"promotions": []
}
}
]
}
}
curl http://node2:9200/vehicle/_search?q=XYZ
{
"took": 5,
"timed_out": false,
"_shards": {
"total": 1,
"successful": 1,
"skipped": 0,
"failed": 0
},
"hits": {
"total": {
"value": 1,
"relation": "eq"
},
"max_score": 7.8380795,
"hits": [
{
"_index": "vehicle_inventory",
"_type": "_doc",
"_id": "109802",
"_score": 7.8380795,
"_source": {
"abs": "Yes",
"addressLine1": "965 ALDRIN DR",
"addressLine2": " ",
"aeroCabExtenders": "Yes",
"aeroRoof": " ",
"aeroSideFairings": "Yes",
"airConditioning": "Yes",
"areaName": " ",
"areaNumber": "0617",
"bodyMake": "NULL",
"bodySerialNumber": "NULL",
"bodyFloorType": "",
"bodyRoofType": "",
"brakeType": "NULL",
"cabStyle": "Conventional",
"category": "TRACTOR",
"chassisBumper": "AERO",
"chassisMakeCode": "FTL",
"cdlRequired": "Yes",
"city": "Eagan",
"color": "White",
"modified_color": "WHITE",
"compared": false,
"saved": false,
"country": "US",
"countryExt": "United States",
"collisionWarning": "No",
"collisionWarningModel": "",
"customerNumber": "RENTAL",
"description": "AVAILABLE JANUARY 2022",
"displayDeliveryCal": "Y",
"districtName": " ",
"driverSeatType": "AIR",
"emailAddress": "E-MAIL.GENERAL.LOCATION#XXXXXX",
"engineBrake": "Yes",
"engineMake": "Detroit",
"engineModel": "DD15AT",
"engineHours": 0,
"faxNumber": "06516860476",
"featuredListing": "Y",
"frontAxleCapacity": "13300",
"frontWheelType": "",
"fuelType": "DIESEL",
"grossVehicleWeight": 80000,
"height": "NULL",
"horsepower": 455,
"image": "/usedtrucks/image/109802/1",
"interiorLevel": "PREMIUM",
"individualPageUrl": "",
"liftGate": "NULL",
"liftGateCapacity": "0",
"liftGateModel": " ",
"loadLength": "NULL",
"make": "Freightliner",
"managersSpecial": "false",
"managerSpecialDiscountRetail": "0",
"managerSpecialDiscountWholeSale": "0",
"mileage": "701,830 miles",
"model": "Cascadia 12564ST",
"newListing": "false",
"numberOfBunks": "2",
"numberOfDriveAxles": "2",
"numberOfTanks": "Dual",
"numericHeight": "",
"numericLoadLength": -1,
"numericWidth": "",
"offLease": "N",
"originalWholeSalePrice": "64,500",
"originalRetailPrice": "64,500",
"phoneNumber": "1-866-309-1962",
"promotionVehicle": "true",
"rearAxleCapacity": "40000",
"rearAxleMake": "Detroit",
"rearDoor": "NULL",
"rearRatio": "2.53",
"rearWheelType": "",
"reeferStandby": "",
"refrigerationHours": "",
"refrigerationMake": "NULL",
"refrigerationModel": "NULL",
"retailPrice": "44,500",
"salePutUpDate": "2021-12-17 00:00:00",
"sideConstruction": "NULL",
"sideDoor": "NULL",
"sleeperSize": "72",
"sleeperType": "Raised Roof",
"slidingFifthWheel": "Yes",
"state": "MN",
"status": "ACTIVE",
"statusComments": " ",
"sale_reason_desc": "AVAILABLE JANUARY 2022",
"saleType": " ",
"size": "",
"suspension": "Air",
"tareWeight": "18795",
"thumbnail": "usedtrucks/image/109802t/1",
"tireSize": "11R22.5",
"tireType": "S/WAY",
"totalFuelCapacity": "220",
"transmissionMake": "Detroit",
"transmissionModel": "12DA1550",
"transmissionSpeed": "12",
"transmissionType": "Automatic",
"type": "Sleeper Tractor",
"unitNumber": "109802",
"use": "RENTAL",
"vin": "3AKJGLD53FSGJ8668",
"wheelBase": "228\"",
"wheelType": "Aluminum",
"wholesalePrice": "59,500",
"width": "NULL",
"year": 2015,
"zip": "55121",
"numeric_mileage": 701830,
"numeric_retail_price": 44500,
"numeric_wholesale_price": 59500,
"dLatitude": 44.944400787353516,
"dLongitude": -93.09329986572266,
"bodyModel": null,
"promotionAmount": "10,000",
"salePrice": 0,
"location": "016510",
"jpegImage": "/usedtrucks/JpgServlet/109802_1.jpg",
"translatedCategory": "Heavy Duty Tractors",
"originalCategory": "Heavy Duty Tractors",
"monFriHours": "M-F 8:00 am to 8:00 pm ET",
"satHours": "Sat 9:00 am to 3:00 pm ET",
"mobileSitePhone": "1-800-485-9671",
"salePutUpDatesort": "2021-12-17",
"stringyear": "2015",
"range_grossVehicleWeight": null,
"range_loadLength": null,
"range_horsepower": "401 and more",
"range_cabType": "Raised Roof",
"range_retailCenter": null,
"viewCount": "222ST15",
"advertTerm": "Level 1",
"numeric_Height": 0,
"numeric_totalFuelCapacity": 0,
"numeric_Width": 0,
"promotionList": [
{
"name": "Save on Select Sleeper Tractors",
"description": "Save on Select Sleeper Tractors",
"amount": 10000,
"amountDisplayRetail": "10,000",
"amountDisplayWholeSale": "10,000",
"type": "DISCOUNT",
"rule": "viewCount != null and viewCount.equalsIgnoreCase('222ST15') and (numeric_mileage < 700 or year > 2014)"
},
{
"name": "Save on Select Sleeper Tractors. Save an additional $5,000 when financing with Premier Financing",
"description": "Save on Select Sleeper Tractors. Save an additional $5,000 when financing with Premier Financing",
"amount": 10000,
"amountDisplayRetail": "10,000",
"amountDisplayWholeSale": "10,000",
"type": "DISCOUNT",
"rule": "viewCount != null and viewCount.equalsIgnoreCase('222ST15') and (numeric_mileage >= 700 or year <= 2014) "
}
],
"imageMaxInstance": 10,
"defaultImageBeanList": null,
"translationsMap": {
"cdlRequired": "Yes",
"countryExt": "United States",
"advertTerm": "Level 1",
"color": "White",
"range_horsepower": "401 and more",
"range_cabType": "Raised Roof",
"model": "Cascadia 12564ST",
"category": "Heavy Duty Tractors",
"type": "Sleeper Tractor"
},
"vehicle_suggestions": {},
"vehicle_locations": {
"lon": "-93.0933",
"lat": "44.9444"
},
"combo_filter": [
{
"filter_name": "range_horsepower",
"filter_value": "401 and more"
},
{
"filter_name": "range_cabType",
"filter_value": "Raised Roof"
},
{
"filter_name": "range_retailCenter",
"filter_value": null
},
{
"filter_name": "advertTerm",
"filter_value": "Level 1"
},
{
"filter_name": "airConditioning",
"filter_value": "Yes"
},
{
"filter_name": "category",
"filter_value": "Heavy Duty Tractors"
},
{
"filter_name": "color",
"filter_value": "White"
},
{
"filter_name": "engineMake",
"filter_value": "Detroit"
},
{
"filter_name": "fuelType",
"filter_value": "DIESEL"
},
{
"filter_name": "make",
"filter_value": "Freightliner"
},
{
"filter_name": "model",
"filter_value": "Cascadia 12564ST"
},
{
"filter_name": "newListing",
"filter_value": "false"
},
{
"filter_name": "numberOfBunks",
"filter_value": "2"
},
{
"filter_name": "numberOfDriveAxles",
"filter_value": "2"
},
{
"filter_name": "state",
"filter_value": "MN"
},
{
"filter_name": "transmissionMake",
"filter_value": "Detroit"
},
{
"filter_name": "transmissionSpeed",
"filter_value": "12"
},
{
"filter_name": "transmissionType",
"filter_value": "Automatic"
},
{
"filter_name": "type",
"filter_value": "Sleeper Tractor"
},
{
"filter_name": "collisionWarning",
"filter_value": "No"
},
{
"filter_name": "canada",
"filter_value": null
},
{
"filter_name": "unitedStates",
"filter_value": "MN"
},
{
"filter_name": "countryExt",
"filter_value": "United States"
},
{
"filter_name": "promotionVehicle",
"filter_value": "true"
},
{
"filter_name": "promotions",
"filter_value": "Current Deals"
},
{
"filter_name": "promotions",
"filter_value": "Save on Select Sleeper Tractors"
},
{
"filter_name": "promotions",
"filter_value": "Save on Select Sleeper Tractors. Save an additional $5,000 when financing with Premier Financing"
}
],
"slider_filter_integer": [
{
"filter_name": "numeric_mileage",
"filter_value": 701830
},
{
"filter_name": "year",
"filter_value": 2015
}
],
"slider_filter_float": [
{
"filter_name": "numeric_retail_price",
"filter_value": 44500
},
{
"filter_name": "numeric_wholesale_price",
"filter_value": 59500
}
],
"basic_filter": {},
"rowNum": 2465,
"displayManagerspecial": null,
"displayPromotions": "true",
"canada": null,
"unitedStates": "MN",
"mgrSpecialRetail": "false",
"mgrSpecialWholesale": "false",
"promotions": []
}
}
]
}
}
Output of GET _cat/shards/vehicle*?v from both the nodes
node1
index shard prirep state docs store ip node
vehicle_inventory 0 p STARTED 120528 9.9mb 3.178.159.250 qa-esearch-n1
vehicle_inventory 0 r UNASSIGNED
node2
index shard prirep state docs store ip node
vehicle_inventory 0 p STARTED 121065 10.1mb 3.178.159.252 qa-esearch-n2
vehicle_inventory 0 r UNASSIGNED
Please observe there is a difference in storage size (9.9mb and 10.1mb). why is the size difference.
Output of GET _cat/nodes?v from both nodes
node1
ip heap.percent ram.percent cpu load_1m load_5m load_15m node.role master name
3.178.159.250 54 96 7 2.10 1.53 1.58 cdfhilmrstw * qa-esearch-n1
node2
ip heap.percent ram.percent cpu load_1m load_5m load_15m node.role master name
3.178.159.252 30 98 6 0.35 0.21 0.17 cdfhilmrstw * qa-esearch-n2