Postfix TLS issue-Collection of common programming errors

I’m trying to enable TLS on Postfix but the daemon is crashing:

Sep 16 16:00:38 core postfix/master[1689]: warning: process /usr/libexec/postfix/smtpd pid 1694 killed by signal 11
Sep 16 16:00:38 core postfix/master[1689]: warning: /usr/libexec/postfix/smtpd: bad command startup -- throttling

This is OpenVZ container (CentOS 6.3 x86_64) with two IPs

# postconf -n
alias_database = hash:/etc/aliases
alias_maps = hash:/etc/aliases
broken_sasl_auth_clients = yes
command_directory = /usr/sbin
config_directory = /etc/postfix
daemon_directory = /usr/libexec/postfix
data_directory = /var/lib/postfix
debug_peer_level = 2
disable_vrfy_command = yes
home_mailbox = Maildir/
html_directory = no
inet_interfaces = all
inet_protocols = all
local_recipient_maps = 
mail_owner = postfix
mailbox_command = 
mailq_path = /usr/bin/mailq.postfix
manpage_directory = /usr/share/man
mydestination = $myhostname, localhost.$mydomain, localhost
mydomain = domain.com
myhostname = mail.domain.com
mynetworks = 127.0.0.0/8
myorigin = $mydomain
newaliases_path = /usr/bin/newaliases.postfix
queue_directory = /var/spool/postfix
readme_directory = /usr/share/doc/postfix-2.6.6/README_FILES
relay_domains = 
sample_directory = /usr/share/doc/postfix-2.6.6/samples
sendmail_path = /usr/sbin/sendmail.postfix
setgid_group = postdrop
smtp_tls_note_starttls_offer = yes
smtp_tls_session_cache_database = btree:/var/lib/postfix/smtpd_tls_cache.db
smtp_use_tls = yes
smtpd_delay_reject = yes
smtpd_error_sleep_time = 1s
smtpd_hard_error_limit = 20
smtpd_helo_required = yes
smtpd_helo_restrictions = permit_mynetworks,    reject_non_fqdn_hostname,   reject_invalid_hostname,    permit
smtpd_recipient_restrictions = permit_mynetworks,   permit_sasl_authenticated,  reject_unauth_pipelining,   reject_non_fqdn_recipient,  reject_unknown_recipient_domain,    reject_invalid_hostname,    reject_non_fqdn_hostname,   reject_non_fqdn_sender, reject_unknown_sender_domain,   reject_unauth_destination   reject_rbl_client cbl.abuseat.org,  reject_rbl_client bl.spamcop.net,   permit
smtpd_sasl_auth_enable = yes
smtpd_sasl_local_domain = $myhostname
smtpd_sasl_path = private/auth
smtpd_sasl_security_options = noanonymous
smtpd_sasl_type = dovecot
smtpd_sender_restrictions = permit_mynetworks,  reject_non_fqdn_sender, reject_unknown_sender_domain,   permit
smtpd_soft_error_limit = 10
smtpd_tls_CAfile = /etc/postfix/ssl/cacert.pem
smtpd_tls_cert_file = /etc/postfix/ssl/smtpd.crt
smtpd_tls_key_file = /etc/postfix/ssl/smtpd.key
smtpd_tls_loglevel = 1
smtpd_tls_received_header = yes
smtpd_tls_session_cache_timeout = 3600s
smtpd_use_tls = yes
tls_random_source = dev:/dev/urandom
unknown_local_recipient_reject_code = 550
# postconf -a
cyrus
dovecot
# ldd /usr/libexec/postfix/smtpd
    linux-vdso.so.1 =>  (0x00007fff10dfe000)
    libldap-2.4.so.2 => /lib64/libldap-2.4.so.2 (0x00007f8a01c2c000)
    liblber-2.4.so.2 => /lib64/liblber-2.4.so.2 (0x00007f8a01a1d000)
    libpcre.so.0 => /lib64/libpcre.so.0 (0x00007f8a017f0000)
    libmysqlclient.so.16 => /usr/lib64/libmysqlclient.so.16 (0x00007f8a014ea000)
    libm.so.6 => /lib64/libm.so.6 (0x00007f8a01266000)
    libsasl2.so.2 => /usr/lib64/libsasl2.so.2 (0x00007f8a0104b000)
    libssl.so.10 => /usr/lib64/libssl.so.10 (0x00007f8a00df0000)
    libcrypto.so.10 => /usr/lib64/libcrypto.so.10 (0x00007f8a00a56000)
    libdl.so.2 => /lib64/libdl.so.2 (0x00007f8a00851000)
    libz.so.1 => /lib64/libz.so.1 (0x00007f8a0063b000)
    libdb-4.7.so => /lib64/libdb-4.7.so (0x00007f8a002c7000)
    libnsl.so.1 => /lib64/libnsl.so.1 (0x00007f8a000ad000)
    libresolv.so.2 => /lib64/libresolv.so.2 (0x00007f89ffe93000)
    libc.so.6 => /lib64/libc.so.6 (0x00007f89ffb00000)
    libssl3.so => /usr/lib64/libssl3.so (0x00007f89ff8c3000)
    libsmime3.so => /usr/lib64/libsmime3.so (0x00007f89ff697000)
    libnss3.so => /usr/lib64/libnss3.so (0x00007f89ff35b000)
    libnssutil3.so => /usr/lib64/libnssutil3.so (0x00007f89ff134000)
    libplds4.so => /lib64/libplds4.so (0x00007f89fef30000)
    libplc4.so => /lib64/libplc4.so (0x00007f89fed2b000)
    libnspr4.so => /lib64/libnspr4.so (0x00007f89feaed000)
    libpthread.so.0 => /lib64/libpthread.so.0 (0x00007f89fe8d0000)
    libcrypt.so.1 => /lib64/libcrypt.so.1 (0x00007f89fe699000)
    libgssapi_krb5.so.2 => /lib64/libgssapi_krb5.so.2 (0x00007f89fe456000)
    libkrb5.so.3 => /lib64/libkrb5.so.3 (0x00007f89fe177000)
    libcom_err.so.2 => /lib64/libcom_err.so.2 (0x00007f89fdf73000)
    libk5crypto.so.3 => /lib64/libk5crypto.so.3 (0x00007f89fdd46000)
    /lib64/ld-linux-x86-64.so.2 (0x00007f8a02102000)
    libfreebl3.so => /lib64/libfreebl3.so (0x00007f89fdae4000)
    libkrb5support.so.0 => /lib64/libkrb5support.so.0 (0x00007f89fd8d8000)
    libkeyutils.so.1 => /lib64/libkeyutils.so.1 (0x00007f89fd6d5000)
    libselinux.so.1 => /lib64/libselinux.so.1 (0x00007f89fd4b5000)
# cat /etc/postfix/master.cf | grep -v "^#"
smtp      inet  n       -       n       -       -       smtpd -v
smtps   inet n   -   n   - - smtpd
   -o smtpd_sasl_auth_enable=yes
   -o smtpd_reject_unlisted_sender=yes
   -o smtpd_recipient_restrictions=permit_sasl_authenticated,reject
   -o broken_sasl_auth_clients=yes 
pickup    fifo  n       -       n       60      1       pickup
cleanup   unix  n       -       n       -       0       cleanup
qmgr      fifo  n       -       n       300     1       qmgr
tlsmgr    unix  -       -       n       1000?   1       tlsmgr
rewrite   unix  -       -       n       -       -       trivial-rewrite
bounce    unix  -       -       n       -       0       bounce
defer     unix  -       -       n       -       0       bounce
trace     unix  -       -       n       -       0       bounce
verify    unix  -       -       n       -       1       verify
flush     unix  n       -       n       1000?   0       flush
proxymap  unix  -       -       n       -       -       proxymap
proxywrite unix -       -       n       -       1       proxymap
smtp      unix  -       -       n       -       -       smtp
relay     unix  -       -       n       -       -       smtp
    -o smtp_fallback_relay=
showq     unix  n       -       n       -       -       showq
error     unix  -       -       n       -       -       error
retry     unix  -       -       n       -       -       error
discard   unix  -       -       n       -       -       discard
local     unix  -       n       n       -       -       local
virtual   unix  -       n       n       -       -       virtual
lmtp      unix  -       -       n       -       -       lmtp
anvil     unix  -       -       n       -       1       anvil
scache    unix  -       -       n       -       1       scache

Requested information from the comment:

# { postconf -d; postconf -d; postconf -n; } | sort | uniq -u
alias_maps = hash:/etc/aliases
broken_sasl_auth_clients = yes
disable_vrfy_command = yes
home_mailbox = Maildir/
inet_protocols = all
local_recipient_maps = 
mailq_path = /usr/bin/mailq.postfix
manpage_directory = /usr/share/man
mydestination = $myhostname, localhost.$mydomain, localhost, $mydomain
myhostname = mail.domain.com
mynetworks = 127.0.0.0/8
myorigin = $mydomain
newaliases_path = /usr/bin/newaliases.postfix
readme_directory = /usr/share/doc/postfix-2.6.6/README_FILES
relay_domains = 
sample_directory = /usr/share/doc/postfix-2.6.6/samples
sendmail_path = /usr/sbin/sendmail.postfix
smtpd_hard_error_limit = 20
smtpd_helo_required = yes
smtpd_helo_restrictions = permit_mynetworks,    reject_non_fqdn_hostname,   reject_invalid_hostname,    permit
smtpd_recipient_restrictions = permit_mynetworks,   permit_sasl_authenticated,  reject_unauth_pipelining,   reject_non_fqdn_recipient,  reject_unknown_recipient_domain,    reject_invalid_hostname,    reject_non_fqdn_hostname,   reject_non_fqdn_sender, reject_unknown_sender_domain,   reject_unauth_destination,  reject_rbl_client cbl.abuseat.org,  reject_rbl_client bl.spamcop.net,   permit
smtpd_sasl_auth_enable = yes
smtpd_sasl_local_domain = $myhostname
smtpd_sasl_path = private/auth
smtpd_sasl_type = dovecot
smtpd_sender_restrictions = permit_mynetworks,  reject_non_fqdn_sender, reject_unknown_sender_domain,   permit
smtpd_tls_auth_only = yes
smtpd_tls_CAfile = /etc/postfix/ssl/cacert.pem
smtpd_tls_cert_file = /etc/postfix/ssl/smtpd.crt
smtpd_tls_key_file = /etc/postfix/ssl/smtpd.key
smtpd_tls_loglevel = 2
smtpd_tls_received_header = yes
smtpd_tls_security_level = may
smtpd_tls_session_cache_database = btree:${data_directory}/smtpd_tls_session_cache
smtp_tls_note_starttls_offer = yes

Postfix version:

# postconf mail_version
mail_version = 2.6.6

Logs (telnet 127.0.0.1 25):

Sep 18 14:07:25 core postfix/postfix-script[4078]: starting the Postfix mail system
Sep 18 14:07:25 core postfix/master[4079]: daemon started -- version 2.6.6, configuration /etc/postfix
Sep 18 14:07:49 core postfix/smtpd[4083]: initializing the server-side TLS engine
Sep 18 14:07:49 core postfix/tlsmgr[4084]: open smtpd TLS cache btree:/var/lib/postfix/smtpd_tls_session_cache
Sep 18 14:07:49 core postfix/tlsmgr[4084]: tlsmgr_cache_run_event: start TLS smtpd session cache cleanup
Sep 18 14:07:49 core postfix/master[4079]: warning: process /usr/libexec/postfix/smtpd pid 4083 killed by signal 11
Sep 18 14:07:49 core postfix/master[4079]: warning: /usr/libexec/postfix/smtpd: bad command startup -- throttling
  1. smtp_tls_session_cache_database = btree:/var/lib/postfix/smtpd_tls_cache.db

    NOTE that smtp != smtp*d*.

    You have not set smtpD_tls_session_cache_database, and this setting has no default.

    Add it and reload.

    EDIT: meh, it’s not actually required. Oh well, perhaps somme more logging will give us a clue.

    RE-EDIT: please do the following:

    • stop postfix: postfix stop
    • clear the logs with mv /var/log/mail.log ~/mail.log.bak and restart your syslog service
    • show the output of { postconf -d; postconf -d; postconf -n; } | sort | uniq -u, as this will show us what was set in main.cf that is not a default
    • show the output of postconf mail_version
    • edit main.cf and set inet_interfaces = 127.0.0.1 and debug_peer_list = 127.0.0.1
    • start postfix: postfix start
    • make a connection using telnet to the local server: telnet 127.0.0.1 25
    • observe what happens and post the clean logs from the beginning.
  2. Okay so I’m comparing this to my working Postfix+Dovecot solution that is using TLS just fine at this point. The version is newer than you’re reporting as this is running on Ubuntu 12.04 LTS.

    # postconf mail_version
    mail_version = 2.9.3
    

    Now if I check only my non-default smtpd_* settings I have the following:

    # postconf -n |grep ^smtpd_
    smtpd_banner = $myhostname ESMTP $mail_name
    smtpd_client_restrictions = reject_rbl_client sbl.spamhaus.org, reject_rbl_client blackholes.easynet.nl, reject_rbl_client dnsbl.njabl.org
    smtpd_data_restrictions = reject_unauth_pipelining
    smtpd_delay_reject = yes
    smtpd_hard_error_limit = 12
    smtpd_helo_required = yes
    smtpd_helo_restrictions = permit_mynetworks, warn_if_reject reject_non_fqdn_hostname, reject_invalid_hostname, permit
    smtpd_recipient_limit = 16
    smtpd_recipient_restrictions = reject_unauth_pipelining, permit_mynetworks, permit_sasl_authenticated, reject_non_fqdn_recipient, reject_unknown_recipient_domain, reject_unauth_destination, check_policy_service inet:127.0.0.1:10023, permit
    smtpd_sasl_auth_enable = yes
    smtpd_sasl_authenticated_header = yes
    smtpd_sasl_local_domain =
    smtpd_sasl_path = private/auth
    smtpd_sasl_security_options = noanonymous
    smtpd_sasl_type = dovecot
    smtpd_sender_restrictions = permit_sasl_authenticated, permit_mynetworks, warn_if_reject reject_non_fqdn_sender, reject_unknown_sender_domain, reject_unauth_pipelining, permit
    smtpd_soft_error_limit = 3
    smtpd_tls_CAfile = /etc/ssl/certs/ca-certificates.crt
    smtpd_tls_cert_file = /etc/ssl/certs/server.example.net.crt
    smtpd_tls_key_file = /etc/ssl/private/server.example.net.key
    smtpd_tls_loglevel = 1
    smtpd_tls_received_header = yes
    smtpd_tls_security_level = may
    smtpd_tls_session_cache_timeout = 3600s
    smtpd_use_tls = yes
    

    Another difference between my system and yours is that I am running in a chroot so my master.cf has the following for comparison:

    smtp      inet  n       -       -       -       -       smtpd
    #smtp      inet  n       -       -       -       1       postscreen
    #smtpd     pass  -       -       -       -       -       smtpd
    #dnsblog   unix  -       -       -       -       0       dnsblog
    #tlsproxy  unix  -       -       -       -       0       tlsproxy
    submission inet n       -       -       -       -       smtpd
      -o syslog_name=postfix/submission
      -o smtpd_tls_security_level=encrypt
      -o smtpd_sasl_auth_enable=yes
      -o smtpd_client_restrictions=permit_sasl_authenticated,reject
      -o milter_macro_daemon_name=ORIGINATING
    #smtps     inet  n       -       -       -       -       smtpd
    #  -o syslog_name=postfix/smtps
    #  -o smtpd_tls_wrappermode=yes
    #  -o smtpd_sasl_auth_enable=yes
    #  -o smtpd_client_restrictions=permit_sasl_authenticated,reject
    #  -o milter_macro_daemon_name=ORIGINATING
    

    As you can tell by reading that I’m only listening on 25/tcp (smtp) and 587/tcp (submission) as there is no need for 465/tcp (smtps) as 25/tcp will support STARTTLS and 587/tcp requires STARTTLS.

    About the only real significant difference I could tell from my master.cf and yours was that you have smtp starting with smtpd -v and I simply have smtpd

    I question what caused the log entries for postfix/tlsmgr as simply telneting to port 25 should not have even initiated the TLS connection unless the STARTTLS command was given once connected and then to properly test that you would be better off testing using:

    openssl s_client -connect localhost:25 -starttls smtp -CApath /etc/ssl/certs
    

    Notes for my SSL setup: I instal the ca-certificates package and then place my certificate under /etc/ssl/certs and my key under /etc/ssl/private. I then stick my intermediate CA under /usr/local/share/ca-certificates and run update-ca-certificates. This re-generates theca-certificates.crtand creates the proper symlinks under/etc/ssl/certs`.