digest: add OpenSSL::Digest#squeeze

ext/openssl/extconf.rb

@@ -169,6 +169,9 @@ def find_openssl_library
 have_func("SSL_get0_group_name(NULL)", ssl_h)
 have_func("OSSL_HPKE_CTX_new(0, (OSSL_HPKE_SUITE){0}, 0, NULL, NULL)", "openssl/hpke.h")
 
+# added in 3.3.0
+have_func("EVP_DigestSqueeze(NULL, NULL, 0)", evp_h)
+
 # added in 3.4.0
 have_func("TS_VERIFY_CTX_set0_certs(NULL, NULL)", ts_h)
 

ext/openssl/ossl_digest.c

@@ -283,15 +283,62 @@ ossl_digest_finish(VALUE self)
 {
     EVP_MD_CTX *ctx;
     VALUE str;
+    int size;
 
     GetDigest(self, ctx);
-    str = rb_str_new(NULL, EVP_MD_CTX_size(ctx));
+    size = EVP_MD_CTX_size(ctx);
+    if (size <= 0) {
+#ifdef EVP_MD_FLAG_XOF /* Not in LibreSSL 4.3 (latest) */
+        if (EVP_MD_flags(EVP_MD_CTX_get0_md(ctx)) & EVP_MD_FLAG_XOF)
+            ossl_raise(eDigestError,
+                       "output length not set for XOF; " \
+                       "use OpenSSL::Digest#squeeze instead");
+#endif
+        ossl_raise(eDigestError, "EVP_MD_CTX_size");
+    }
+    str = rb_str_new(NULL, size);
     if (!EVP_DigestFinal_ex(ctx, (unsigned char *)RSTRING_PTR(str), NULL))
         ossl_raise(eDigestError, "EVP_DigestFinal_ex");
 
     return str;
 }
 
+#ifdef HAVE_EVP_DIGESTSQUEEZE
+/*
+ * call-seq:
+ *    digest.squeeze(length) -> string
+ *
+ * Gets the next _length_ bytes of output from the extendable-output function
+ * (XOF). This method can be called multiple times to get more output.
+ *
+ * Unlike #digest, this method mutates the digest instance. Once called, no
+ * further #update or #<< calls are allowed. Use #dup to create a copy before
+ * calling this method if you need intermediate digest values.
+ *
+ * See also the man page EVP_DigestSqueeze(3).
+ * This method is compatible with OpenSSL 3.3 or later.
+ */
+static VALUE
+ossl_digest_squeeze(VALUE self, VALUE length)
+{
+    EVP_MD_CTX *ctx;
+    VALUE str;
+    size_t size;
+
+    GetDigest(self, ctx);
+    size = NUM2SIZET(length);
+    if (size > LONG_MAX)
+        rb_raise(rb_eArgError, "length is negative or too big");
+    str = rb_str_new(NULL, (long)size);
+    if (!EVP_DigestSqueeze(ctx, (unsigned char *)RSTRING_PTR(str), size))
+        ossl_raise(eDigestError, "EVP_DigestSqueeze");
+
+    return str;
+}
+#else
+#define ossl_digest_squeeze rb_f_notimplement
+#endif
+
 /*
  *  call-seq:
  *      digest.name -> string
@@ -466,6 +513,7 @@ Init_ossl_digest(void)
     rb_define_method(cDigest, "update", ossl_digest_update, 1);
     rb_define_alias(cDigest, "<<", "update");
     rb_define_private_method(cDigest, "finish", ossl_digest_finish, 0);
+    rb_define_method(cDigest, "squeeze", ossl_digest_squeeze, 1);
     rb_define_method(cDigest, "digest_length", ossl_digest_size, 0);
     rb_define_method(cDigest, "block_length", ossl_digest_block_length, 0);
 

lib/openssl/digest.rb

@@ -52,6 +52,10 @@ def hexdigest(data)
     class Digest < Digest; end # :nodoc:
     deprecate_constant :Digest
 
+    def inspect
+      "#<#{self.class}: #{name} #{hexdigest rescue $!.inspect}>"
+    end
+
   end # Digest
 
   # Returns a Digest subclass by _name_

test/openssl/test_digest.rb

@@ -139,6 +139,33 @@ def test_fetched_evp_md
     assert_equal(hex, OpenSSL::Digest.hexdigest("KECCAK-256", ""))
   end if openssl?(3, 2, 0)
 
+  def test_xof_squeeze
+    if openssl? && !openssl?(3, 3, 0) || libressl?
+      omit "EVP_DigestSqueeze() is not supported"
+    end
+
+    # NIST CAVP test vectors, "SHA-3 XOF Test Vectors for Byte-Oriented Output"
+    # SHAKE128VariableOut.rsp, COUNT = 9
+    outputlen = 136 # in bits
+    msg = ["f167511ec8864979302237abea4cf7ef"].pack("H*")
+    output = ["20f8938daa54b260860a104f8556278bac"].pack("H*")
+
+    digest = OpenSSL::Digest.new("SHAKE128")
+    digest.update(msg)
+    assert_equal(output, digest.dup.squeeze(17))
+    assert_equal(outputlen / 8, output.bytesize)
+    assert_equal(output, digest.squeeze(8) + digest.squeeze(9))
+  end
+
+  def test_xof_digest
+    # SHAKE had a default output length in OpenSSL < 3.4
+    return unless openssl? && openssl?(3, 4, 0)
+
+    digest = OpenSSL::Digest.new("SHAKE128")
+    assert_raise(OpenSSL::Digest::DigestError) { digest.digest }
+    assert_match(/#<OpenSSL::Digest: SHAKE128/, digest.inspect)
+  end
+
   def test_openssl_digest
     assert_equal OpenSSL::Digest::MD5, OpenSSL::Digest("MD5")