From 86905b8aaca913bd18af4b3f0fd51ca2b1ec18a4 Mon Sep 17 00:00:00 2001 From: Andrii Furmanets Date: Thu, 22 Oct 2026 10:49:09 +0300 Subject: [PATCH] Fix CRAM-MD5 authentication with non-ASCII secrets cram_secret mixed the UTF-8 secret with the binary challenge and digest, which raised Encoding::CompatibilityError, and it compared the character count (not the byte count) against the 64 byte HMAC block size. Work on the secret's bytes so the response matches HMAC-MD5 per RFC 2195. --- lib/net/smtp/auth_cram_md5.rb | 5 +++-- test/net/smtp/test_smtp.rb | 19 +++++++++++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/lib/net/smtp/auth_cram_md5.rb b/lib/net/smtp/auth_cram_md5.rb index 0490cd65..ab89d9fd 100644 --- a/lib/net/smtp/auth_cram_md5.rb +++ b/lib/net/smtp/auth_cram_md5.rb @@ -20,14 +20,15 @@ def auth(user, secret) # CRAM-MD5: [RFC2195] def cram_md5_response(secret, challenge) - tmp = digest_class::MD5.digest(cram_secret(secret, IMASK) + challenge) + tmp = digest_class::MD5.digest(cram_secret(secret, IMASK) + challenge.b) digest_class::MD5.hexdigest(cram_secret(secret, OMASK) + tmp) end CRAM_BUFSIZE = 64 def cram_secret(secret, mask) - secret = digest_class::MD5.digest(secret) if secret.size > CRAM_BUFSIZE + secret = secret.b + secret = digest_class::MD5.digest(secret) if secret.bytesize > CRAM_BUFSIZE buf = secret.ljust(CRAM_BUFSIZE, "\0") 0.upto(buf.size - 1) do |i| buf[i] = (buf[i].ord ^ mask).chr diff --git a/test/net/smtp/test_smtp.rb b/test/net/smtp/test_smtp.rb index 229b606e..747effe8 100644 --- a/test/net/smtp/test_smtp.rb +++ b/test/net/smtp/test_smtp.rb @@ -523,6 +523,25 @@ def test_start_auth_cram_md5 assert_equal('"openssl" or "digest" library is required', e.message) end + def test_cram_md5_response + omit "openssl library not loaded" unless defined? OpenSSL::HMAC + + auth_cram_md5 = Net::SMTP::AuthCramMD5.new(nil) + challenge = "<1896.697170952@postoffice.reston.mci.net>" + # RFC 2195 example + assert_equal "b913a602c7eda7a495b4e6e7334d3890", auth_cram_md5.cram_md5_response("tanstaaftanstaaf", challenge) + + [ + "p\u00e4ssw\u00f6rd", # non-ASCII secret + "\u00e9" * 40, # 40 characters but 80 bytes, longer than the HMAC block + "\u00e9" * 40 + "a" * 30, # more than 64 characters + ].each do |secret| + assert_equal OpenSSL::HMAC.hexdigest("MD5", secret, challenge), + auth_cram_md5.cram_md5_response(secret, challenge.b), + "secret: #{secret.inspect}" + end + end + def test_start_instance port = fake_server_start smtp = Net::SMTP.new('localhost', port)