After enabling of tls in our TYK server it runs out of filehandles and stop working - api-gateway

Question

We have enabled tls in our on premise TYK dashboard and gateway after that it runs out of filehandles and stop working
Redirecting to /bin/systemctl status tyk-dashboard.service
tyk-dashboard.service - Tyk API Dashboard Loaded: loaded
(/usr/lib/systemd/system/tyk-dashboard.service; enabled; vendor
preset: disabled) Active: active (running) since Thu 2018-10-18
11:24:32 CEST; 3h 57min ago Main PID: 13062 (tyk-analytics) CGroup:
/system.slice/tyk-dashboard.service └─13062
/opt/tyk-dashboard/tyk-analytics --conf
/opt/tyk-dashboard/tyk_analytics.conf
Oct 18 15:22:00 sktudv01tyk01.ccta.dk tyk-analytics[13062]: 2018/10/18
15:22:00 http: Accept error: accept tcp [::]:3000: accept4: too many
open files; retrying in 1s Oct 18 15:22:01 sktudv01tyk01.ccta.dk
tyk-analytics[13062]: 2018/10/18 15:22:01 http: Accept error: accept
tcp [::]:3000: accept4: too many open files; retrying in 1s
our conf files looks like this
tyk.conf
{
"listen_port": 8443,
"node_secret": "secret",
"secret": "secret",
"template_path": "/opt/tyk-gateway/templates",
"use_db_app_configs": true,
"db_app_conf_options": {
"connection_string": "https://localhost:3000",
"node_is_segmented": false,
"tags": []
},
"disable_dashboard_zeroconf": false,
"app_path": "/opt/tyk-gateway/apps",
"middleware_path": "/opt/tyk-gateway/middleware",
"storage": {
"type": "redis",
"host": "localhost",
"port": 6379,
"username": "",
"password": "",
"database": 0,
"optimisation_max_idle": 2000,
"optimisation_max_active": 4000
},
"enable_analytics": true,
"analytics_config": {
"type": "",
"ignored_ips": [],
"enable_detailed_recording": true,
"enable_geo_ip": false,
"geo_ip_db_path": "",
"normalise_urls": {
"enabled": true,
"normalise_uuids": true,
"normalise_numbers": true,
"custom_patterns": []
}
},
"health_check": {
"enable_health_checks": false,
"health_check_value_timeouts": 60
},
"optimisations_use_async_session_write": true,
"allow_master_keys": false,
"policies": {
"policy_source": "service",
"policy_connection_string": "",
"policy_record_name": "tyk_policies",
"allow_explicit_policy_id": true
},
"hash_keys": true,
"suppress_redis_signal_reload": false,
"use_redis_log": true,
"close_connections": true,
"enable_non_transactional_rate_limiter": true,
"enable_sentinel_rate_limiter": false,
"experimental_process_org_off_thread": false,
"local_session_cache": {
"disable_cached_session_state": false
},
"http_server_options": {
"enable_websockets": true,
"use_ssl": true,
"server_name": "localhost",
"certificates": [
{
"domain_name": "*.ccta.dk",
"cert_file": "/etc/pki/tls/certs/localhost.crt",
"key_file": "/etc/pki/tls/private/localhost.key"
}
],
"ssl_insecure_skip_verify": false
},
"uptime_tests": {
"disable": false,
"config": {
"enable_uptime_analytics": true,
"failure_trigger_sample_size": 2,
"time_wait": 10,
"checker_pool_size": 50
}
},
"hostname": "",
"enable_custom_domains": true,
"enable_jsvm": true,
"oauth_redirect_uri_separator": ";",
"coprocess_options": {
"enable_coprocess": false,
"coprocess_grpc_server": ""
},
"pid_file_location": "./tyk-gateway.pid",
"allow_insecure_configs": true,
"public_key_path": "",
"close_idle_connections": false,
"allow_remote_config": false,
"enable_bundle_downloader": true,
"bundle_base_url": "",
"global_session_lifetime": 100,
"force_global_session_lifetime": false,
"max_idle_connections_per_host": 500
}
our tyk_analytics.conf
{
"listen_port": 3000,
"tyk_api_config": {
"Host": "https://localhost",
"Port": "8443",
"Secret": "secret"
},
"mongo_url": "mongodb://127.0.0.1/tyk_analytics",
"mongo_use_ssl": false,
"mongo_ssl_insecure_skip_verify": false,
"page_size": 10,
"admin_secret": "secret",
"shared_node_secret": "secret",
"redis_port": 6379,
"redis_host": "localhost",
"redis_password": "",
"enable_cluster": false,
"redis_use_ssl": false,
"redis_ssl_insecure_skip_verify": false,
"force_api_defaults": false,
"notify_on_change": true,
"license_key": "secret",
"redis_database": 0,
"redis_hosts": null,
"hash_keys": true,
"email_backend": {
"enable_email_notifications": false,
"code": "",
"settings": null,
"default_from_email": "",
"default_from_name": "",
"dashboard_hostname": ""
},
"hide_listen_path": false,
"sentry_code": "",
"sentry_js_code": "",
"use_sentry": false,
"enable_master_keys": false,
"enable_duplicate_slugs": true,
"show_org_id": true,
"host_config": {
"enable_host_names": true,
"disable_org_slug_prefix": true,
"hostname": "localhost",
"override_hostname": "localhost",
"portal_domains": {},
"portal_root_path": "/portal",
"generate_secure_paths": false,
"secure_cookies": false,
"use_strict_hostmatch": false
},
"http_server_options": {
"use_ssl": true,
"servername": "localhost",
"certificates": [
{
"domain_name": "*.ccta.dk",
"cert_file": "/etc/pki/tls/certs/dev.api.data.ccta.dk.crt",
"key_file": "/etc/pki/tls/private/dev.api.data.ccta.dk.key"
}
],
"min_version": 0
},
"security": {
"allow_admin_reset_password": false,
"login_failure_username_limit": 0,
"login_failure_ip_limit": 0,
"login_failure_expiration": 0,
"audit_log_path": "/var/log/tyk/tyk-audit.log"
},
"ui": {
"languages": {
"Chinese": "cn",
"English": "en",
"French": "fr",
"Korean": "ko"
},
"hide_help": false,
"default_lang": "en",
"login_page": {},
"nav": {},
"uptime": {},
"portal_section": null,
"designer": {},
"dont_show_admin_sockets": false,
"dont_allow_license_management": false,
"dont_allow_license_management_view": false,
"cloud": false
},
"home_dir": "/opt/tyk-dashboard",
"identity_broker": {
"enabled": false,
"host": {
"connection_string": "http://localhost:3010",
"secret": "secret"
}
},
"tagging_options": {
"tag_all_apis_by_org": false
},
"use_sharded_analytics": false,
"enable_aggregate_lookups": true,
"enable_analytics_cache": false,
"aggregate_lookup_cutoff": "01/07/2016",
"maintenance_mode": false,
"allow_explicit_policy_id": false,
"private_key_path": "",
"node_schema_path": "",
"oauth_redirect_uri_separator": ";",
"statsd_connection_string": "",
"statsd_prefix": "",
"disable_parallel_sessions": false,
"dashboard_session_lifetime": 0,
"alternative_dashboard_url": "",
"sso_permission_defaults": null,
"sso_default_group_id": "",
"sso_custom_login_url": "",
"sso_custom_portal_login_url": "",
"notifications_listen_port": 5000,
"portal_session_lifetime": 0,
"enable_delete_key_by_hash": false
}
cat /proc/981/limits Limit Soft Limit
Hard Limit Units Max cpu time unlimited
unlimited seconds Max file size unlimited
unlimited bytes Max data size unlimited
unlimited bytes Max stack size 8388608
unlimited bytes Max core file size 0
unlimited bytes Max resident set unlimited
unlimited bytes Max processes 31191
31191 processes Max open files 1024
4096 files Max locked memory 65536
65536 bytes Max address space unlimited
unlimited bytes Max file locks unlimited
unlimited locks Max pending signals 31191
31191 signals Max msgqueue size 819200
819200 bytes Max nice priority 0
0 Max realtime priority 0 0 Max realtime
timeout unlimited unlimited us
Number of file handles when it failed again with Oct 23 13:04:34
sktudv01tyk01 tyk-analytics: 2018/10/23 13:04:34 http: Accept error:
accept tcp [::]:3000: accept4: too many open files; retrying in 1s
lsof | wc -l
31677
cat /usr/lib/systemd/system/tyk-gateway.service
[Unit]
Description=Tyk API Gateway
[Service]
Type=simple
User=root
Group=root
# Load env vars from /etc/default/ and /etc/sysconfig/ if they exist.
# Prefixing the path with '-' makes it try to load, but if the file doesn't
# exist, it continues onward.
EnvironmentFile=-/etc/default/tyk-gateway
EnvironmentFile=-/etc/sysconfig/tyk-gateway
ExecStart=/opt/tyk-gateway/tyk --conf /opt/tyk-gateway/tyk.conf
Restart=always
WorkingDirectory=/opt/tyk-gateway
RuntimeDirectory=tyk
RuntimeDirectoryMode=0770
LimitNOFILE=80000
[Install]
WantedBy=multi-user.target
OS settings ( centos )
# /etc/security/limits.conf
* hard maxlogins 10
* soft nproc 80000
* hard nproc 80000
* soft nofile 80000
* hard nofile 80000
root soft nproc 80000
root hard nproc 80000
root soft nofile 80000
root hard nofile 80000
and in sysctl.conf i added
fs.file-max=80000
When i restart the tyk-dashboard i can login using https and my api's respond ok on https but after some minuttes it runs out of file handles
What do i need to change here, when starting dashboard alone everything is steady and running well, no errors in my log. But when i start the gateway my numbers of open files increase every second
gateway startup log
Oct 24 08:55:36 sktudv01tyk01.ccta.dk tyk[10135]: time="Oct 24 08:55:36" level=info msg="Setting up analytics normaliser"
Oct 24 08:55:36 sktudv01tyk01.ccta.dk tyk[10135]: time="Oct 24 08:55:36" level=info msg="PIDFile location set to: ./tyk-gateway.pid"
Oct 24 08:55:36 sktudv01tyk01.ccta.dk tyk[10135]: time="Oct 24 08:55:36" level=info msg="Initialising Tyk REST API Endpoints"
Oct 24 08:55:36 sktudv01tyk01.ccta.dk tyk[10135]: time="Oct 24 08:55:36" level=warning msg="Default secret `secret` should be changed for production."
Oct 24 08:55:36 sktudv01tyk01.ccta.dk tyk[10135]: time="Oct 24 08:55:36" level=warning msg="Default node_secret `secret` should be changed for production."
Oct 24 08:55:36 sktudv01tyk01.ccta.dk tyk[10135]: time="Oct 24 08:55:36" level=info msg="Redis connection pools are ready after number of retires" currRetry=0
Oct 24 08:55:36 sktudv01tyk01.ccta.dk tyk[10135]: time="Oct 24 08:55:36" level=info msg="Redis connection pools are ready"
Oct 24 08:55:36 sktudv01tyk01.ccta.dk tyk[10135]: time="Oct 24 08:55:36" level=info msg="--> Using SSL (https)"
Oct 24 08:55:36 sktudv01tyk01.ccta.dk tyk[10135]: time="Oct 24 08:55:36" level=info msg="Setting up Server"
Oct 24 08:55:36 sktudv01tyk01.ccta.dk tyk[10135]: time="Oct 24 08:55:36" level=info msg="Registering node."
Oct 24 08:55:36 sktudv01tyk01.ccta.dk tyk[10135]: time="Oct 24 08:55:36" level=error msg="Response failed with code 404; retrying in 5s"
Oct 24 08:55:37 sktudv01tyk01.ccta.dk tyk[10135]: time="Oct 24 08:55:37" level=warning msg="Insecure configuration detected (allowing)!"
Oct 24 08:55:37 sktudv01tyk01.ccta.dk tyk[10135]: time="Oct 24 08:55:37" level=info msg="Hostname set with dashboard zeroconf signal"
Oct 24 08:55:41 sktudv01tyk01.ccta.dk tyk[10135]: time="Oct 24 08:55:41" level=error msg="Response failed with code 404; retrying in 5s"
Oct 24 08:55:46 sktudv01tyk01.ccta.dk tyk[10135]: time="Oct 24 08:55:46" level=info msg="Node registered" id=10321add-ffb6-40c5-4692-c2035ee2760d
Oct 24 08:55:46 sktudv01tyk01.ccta.dk tyk[10135]: time="Oct 24 08:55:46" level=info msg="Gateway started (v2.7.3)"
Oct 24 08:55:46 sktudv01tyk01.ccta.dk tyk[10135]: time="Oct 24 08:55:46" level=info msg="Initialising distributed rate limiter"
Oct 24 08:55:46 sktudv01tyk01.ccta.dk tyk[10135]: time="Oct 24 08:55:46" level=info msg="--> Listening on address: (open interface)"
Oct 24 08:55:46 sktudv01tyk01.ccta.dk tyk[10135]: time="Oct 24 08:55:46" level=info msg="--> Listening on port: 8443"
Oct 24 08:55:46 sktudv01tyk01.ccta.dk tyk[10135]: time="Oct 24 08:55:46" level=info msg="--> PID: 10135"
Oct 24 08:55:46 sktudv01tyk01.ccta.dk tyk[10135]: time="Oct 24 08:55:46" level=info msg="Starting gateway rate limiter notifications..."
Oct 24 08:55:46 sktudv01tyk01.ccta.dk tyk[10135]: time="Oct 24 08:55:46" level=info msg="Loading policies"
Oct 24 08:55:46 sktudv01tyk01.ccta.dk tyk[10135]: time="Oct 24 08:55:46" level=info msg="Using Policies from Dashboard Service"
Oct 24 08:55:46 sktudv01tyk01.ccta.dk tyk[10135]: time="Oct 24 08:55:46" level=info msg="Mutex lock acquired... calling"
Oct 24 08:55:46 sktudv01tyk01.ccta.dk tyk[10135]: time="Oct 24 08:55:46" level=info msg="Calling dashboard service for policy list"
Oct 24 08:55:46 sktudv01tyk01.ccta.dk tyk[10135]: time="Oct 24 08:55:46" level=info msg="Processing policy list"
Oct 24 08:55:46 sktudv01tyk01.ccta.dk tyk[10135]: time="Oct 24 08:55:46" level=info msg="Policies found (0 total):"
Oct 24 08:55:46 sktudv01tyk01.ccta.dk tyk[10135]: time="Oct 24 08:55:46" level=info msg="Detected 8 APIs"
Oct 24 08:55:46 sktudv01tyk01.ccta.dk tyk[10135]: time="Oct 24 08:55:46" level=info msg="Preparing new router"
Oct 24 08:55:46 sktudv01tyk01.ccta.dk tyk[10135]: time="Oct 24 08:55:46" level=info msg="Initialising Tyk REST API Endpoints"
Oct 24 08:55:46 sktudv01tyk01.ccta.dk tyk[10135]: time="Oct 24 08:55:46" level=info msg="Loading API configurations."
Oct 24 08:55:46 sktudv01tyk01.ccta.dk tyk[10135]: time="Oct 24 08:55:46" level=info msg="Tracking hostname" api_name="Robotics - fast excel API #rpa" domain="(no host)"
Oct 24 08:55:46 sktudv01tyk01.ccta.dk tyk[10135]: time="Oct 24 08:55:46" level=info msg="Tracking hostname" api_name="gulapi_aarsopg #Gul" domain="(no host)"
Oct 24 08:55:46 sktudv01tyk01.ccta.dk tyk[10135]: time="Oct 24 08:55:46" level=info msg="Tracking hostname" api_name="rpaqlik_prod #rpa #prod" domain="(no host)"
Oct 24 08:55:46 sktudv01tyk01.ccta.dk tyk[10135]: time="Oct 24 08:55:46" level=info msg="Tracking hostname" api_name=awsvalues domain="(no host)"
Oct 24 08:55:46 sktudv01tyk01.ccta.dk tyk[10135]: time="Oct 24 08:55:46" level=info msg="Tracking hostname" api_name="rpaqlik #rpa" domain="(no host)"
Oct 24 08:55:46 sktudv01tyk01.ccta.dk tyk[10135]: time="Oct 24 08:55:46" level=info msg="Tracking hostname" api_name=postman domain="(no host)"
Oct 24 08:55:46 sktudv01tyk01.ccta.dk tyk[10135]: time="Oct 24 08:55:46" level=info msg="Tracking hostname" api_name=simons domain="(no host)"
Oct 24 08:55:46 sktudv01tyk01.ccta.dk tyk[10135]: time="Oct 24 08:55:46" level=info msg="Tracking hostname" api_name="eboks #eboks #java #dropwizard" domain="(no host)"
Oct 24 08:55:46 sktudv01tyk01.ccta.dk tyk[10135]: time="Oct 24 08:55:46" level=info msg="Loading API" api_name="eboks #eboks #java #dropwizard"
Oct 24 08:55:46 sktudv01tyk01.ccta.dk tyk[10135]: time="Oct 24 08:55:46" level=info msg="Loading API" api_name="Robotics - fast excel API #rpa"
Oct 24 08:55:46 sktudv01tyk01.ccta.dk tyk[10135]: time="Oct 24 08:55:46" level=info msg="Loading API" api_name="gulapi_aarsopg #Gul"
lines 7537-7582/7616 100%

Answer 1

If you are using SystemD, ensure that it does not mess with your file handler settings, by checking process limits like this: cat /proc/<pid>/limits. And if you find anomalies follow this guide https://tyk.io/docs/deploy-tyk-premise-production/#file-handles
Additionally your "db_app_conf_options.connection_string" is empty, instead it should point to the dashboard.
Also, you configured Gateway to use TLS, but in Dashboard config it still points to "http", and same for Gateway port.
Hope it helps.