Change of IP doesn't get noticed by check_mk
orangutan.singlenode.bitmask.net got a new IP (37.218.240.146):
root@donkey:/etc# grep 37.218.240.146 -ir
ssh/ssh_config:Host orangutan orangutan.singlenode.bitmask.net orangutan.singlenode.bitmask.i 37.218.240.146
ssh/ssh_known_hosts:orangutan,orangutan.singlenode.bitmask.net,orangutan.singlenode.bitmask.i,37.218.240.146 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC90yKXsI4xAttXDDaV9s8ECyXV0JzCyONg+2naD92o6fdPMPL93dCqxa51rf2xO4MvFTaGLRonWF7Pze0a64X2b+FqAEanah0H7DHajN8Jlurf29LE89JieE+17HnAliejbJ9aOZGOrY0HkvrWVJRhI0l85Gg1UoBLEZLhIf88MNterBQP6JAK3TNFUPaLlNZXxlr1g6Pud/OrCpZ35NFDpez1Ju13ULpp7CULK1gvVlaCjFce4xmeKNDvzf/51IoF5XTuEu5NXWE3GNBdO/l5CDWFCdY2tj8iKQzY7ACJVIZgSDQykGBXNVv1mnh4cgX/9WJ+W1VjL+whUa5Eobb/
hosts:37.218.240.146 orangutan.singlenode.bitmask.net orangutan.singlenode.bitmask.i api.singlenode.bitmask.net nicknym.singlenode.bitmask.net singlenode.bitmask.net orangutan
leap/hiera.yaml: ip_address: "37.218.240.146"
leap/hiera.yaml: ip_address: "37.218.240.146"After a deploy to donkey, it still has it's old IP:
root@donkey:/etc# grep -A 3 orang /etc/nagios3/conf.d/check_mk/check_mk_objects.cfg | head
# orangutan.singlenode.bitmask.i
# ----------------------------------------------------
define host {
host_name orangutan.singlenode.bitmask.i
use check_mk_host
address 37.218.247.46But even after manually re-inventorizing and reconfiguring nagios doesn't help:
root@donkey:/etc# check_mk -vv -II orangutan.singlenode.bitmask.i
Discovering services on orangutan.singlenode.bitmask.i:
orangutan.singlenode.bitmask.i:
Calling external program ssh -o ConnectTimeout=5 -l root -i /etc/check_mk/.ssh/id_rsa -p 22 orangutan.singlenode.bitmask.i check_mk_agent
1 cpu.loads
- CPU load cpuload_default_levels
1 cpu.threads
- Number of threads threads_default_levels
1 df
- fs_/ {}
3 kernel
- Kernel Major Page Faults kernel_default_levels
- Kernel Context Switches kernel_default_levels
- Kernel Process Creations kernel_default_levels
1 kernel.util
- CPU utilization {}
1 lnx_if
- Interface 2 {'state': ['1'], 'speed': 0}
33 local
- Leap_MX_Queue None
- Couchdb_customers_database None
- Couchdb_identities_database None
- Couchdb_invite_codes_database None
- Couchdb_keycache_database None
- Couchdb_messages_database None
- Couchdb_shared_database None
- Couchdb_tickets_database None
- Couchdb_users_database None
- Couchdb_sessions_database None
- Couchdb_tokens_database None
- Couchdb_global_stats None
- Network/Can_connect_to_internet None
- Network/Is_stunnel_running None
- Network/Is_shorewall_running None
- Network/Are_server_certificates_valid None
- CouchDB/Are_daemons_running None
- CouchDB/Is_CouchDB_running None
- CouchDB/Do_ACL_users_exist None
- CouchDB/Do_required_databases_exist None
- CouchDB/Can_records_be_created None
- CouchDB/Delete_mail_storage_used_in_mx_tests None
- Webapp/Can_contact_couchdb None
- Webapp/Are_daemons_running None
- Webapp/Can_access_webapp None
- Webapp/Can_create_and_authenticate_and_delete_user_via_API None
- Webapp/Can_sync_Soledad None
- Mx/Can_contact_couchdb None
- Mx/Can_query_identities_db None
- Mx/Are_MX_daemons_running None
- Mx/Can_deliver_email None
- Mx/Can_postfix_query_leapmx None
- Soledad/Is_Soledad_running None
6 logwatch
- LOG /var/log/messages None
- LOG /var/log/auth.log None
- LOG /var/log/kern.log None
- LOG /var/log/leap/mx.log None
- LOG /var/log/leap/webapp.log None
- LOG /var/log/syslog None
1 mem.used
- Memory used {}
1 mounts
- Mount options of / ['attr2', 'inode64', 'noquota', 'relatime', 'rw', 'sunit=8192', 'swidth=8192']
2 mrpe
- APT None
- haveged_proc None
1 ntp.time
- NTP Time ntp_default_levels
1 postfix_mailq
- Postfix Queue postfix_mailq_default_levels
1 tcp_conn_stats
- TCP Connections tcp_conn_stats_default_levels
1 uptime
- Uptime {}
root@donkey:/etc# check_mk -O
Generating configuration for core (type nagios)...OK
Validating Nagios configuration...OK
Precompiling host checks...OK
Reloading monitoring core...OK
It turned out that check_mk does cache the IPs:
root@donkey:/etc# cat /var/lib/check_mk/ipaddresses.cache
{'snail.dev.bitmask.i': '37.218.242.215', 'dotterel.dev.bitmask.i': '37.218.240.50', 'mudskipper.cdev.bitmask.i': '1.209.122.111', 'aardwolf.unstable.bitmask.i': '199.119.112.205', 'elephant.dev.bitmask.i': '37.218.242.214', 'diplocaulus.bitmask.i': '199.119.112.150', 'ant.demo.bitmask.i': '198.252.153.82', 'newt.dev.bitmask.i': '37.218.242.145', 'alpaca.mail.bitmask.i': '199.119.112.223', 'ferret.dev.bitmask.i': '1.209.122.123', 'donkey.unstable.bitmask.i': '127.0.1.1', 'giraffe.cdev.bitmask.i': '37.218.242.191', 'otter.demo.bitmask.i': '46.165.242.166', 'lynx.cdev.bitmask.i': '37.218.242.146', 'demodex.dev.bitmask.i': '204.13.164.171', 'millipede.demo.bitmask.i': '198.252.153.83', 'tuatara.unstable.bitmask.i': '199.119.112.208', 'zebra.cdev.bitmask.i': '37.218.242.147', 'leech.demo.bitmask.i': '198.252.153.85', 'orangutan.singlenode.bitmask.i': '37.218.247.46', 'wallaby.demo.bitmask.i': '204.13.164.57', 'clam.dev.bitmask.i': '37.218.242.213', 'gecko.cdev.bitmask.i': '1.209.122.112', 'cat.unstable.bitmask.i': '199.119.112.206'}Only after manually purging the IP cache with check_mk --update-dns-cache the IP gets updated:
root@donkey:/etc# check_mk --update-dns-cache
root@donkey:/etc# cat /var/lib/check_mk/ipaddresses.cache
{'snail.dev.bitmask.i': '37.218.242.215', 'dotterel.dev.bitmask.i': '37.218.240.50', 'aardwolf.unstable.bitmask.i': '199.119.112.205', 'elephant.dev.bitmask.i': '37.218.242.214', 'ferret.dev.bitmask.i': '1.209.122.123', 'diplocaulus.bitmask.i': '199.119.112.150', 'ant.demo.bitmask.i': '198.252.153.82', 'newt.dev.bitmask.i': '37.218.242.145', 'alpaca.mail.bitmask.i': '199.119.112.223', 'mudskipper.cdev.bitmask.i': '1.209.122.111', 'donkey.unstable.bitmask.i': '127.0.1.1', 'giraffe.cdev.bitmask.i': '37.218.242.191', 'otter.demo.bitmask.i': '46.165.242.166', 'lynx.cdev.bitmask.i': '37.218.242.146', 'demodex.dev.bitmask.i': '204.13.164.171', 'millipede.demo.bitmask.i': '198.252.153.83', 'tuatara.unstable.bitmask.i': '199.119.112.208', 'zebra.cdev.bitmask.i': '37.218.242.147', 'leech.demo.bitmask.i': '198.252.153.85', 'orangutan.singlenode.bitmask.i': '37.218.240.146', 'wallaby.demo.bitmask.i': '204.13.164.57', 'clam.dev.bitmask.i': '37.218.242.213', 'gecko.cdev.bitmask.i': '1.209.122.112', 'cat.unstable.bitmask.i': '199.119.112.206'}
root@donkey:/etc# check_mk -O
Generating configuration for core (type nagios)...OK
Validating Nagios configuration...OK
Precompiling host checks...OK
Reloading monitoring core...OK
root@donkey:/etc# grep -A 3 orang /etc/nagios3/conf.d/check_mk/check_mk_objects.cfg | head
# orangutan.singlenode.bitmask.i
# ----------------------------------------------------
define host {
host_name orangutan.singlenode.bitmask.i
use check_mk_host
address 37.218.240.146
_TAGS
--
alias orangutan
So we should include check_mk --update-dns-cache into the platform deploy manifests.