ssl: support IO-like object as the underlying transport

ext/openssl/openssl_missing.h

@@ -156,6 +156,26 @@ IMPL_PKEY_GETTER(EC_KEY, ec)
 #undef IMPL_PKEY_GETTER
 #undef IMPL_KEY_ACCESSOR2
 #undef IMPL_KEY_ACCESSOR3
+
+// BIO
+static inline void *BIO_get_data(BIO *bio) { return bio->ptr; }
+static inline void BIO_set_data(BIO *bio, void *data) { bio->ptr = data; }
+static inline void BIO_set_init(BIO *bio, int init) { bio->init = init; }
+static inline BIO_METHOD *BIO_meth_new(int type, const char *name) {
+    BIO_METHOD *meth = OPENSSL_malloc(sizeof(*meth));
+    if (!meth)
+        return NULL;
+    memset(meth, 0, sizeof(*meth));
+    meth->type = type;
+    meth->name = name;
+    return meth;
+}
+static inline void BIO_meth_free(BIO_METHOD *meth) { OPENSSL_free(meth); }
+static inline int BIO_meth_set_create(BIO_METHOD *meth, int (*f)(BIO *)) { meth->create = f; return 1; }
+static inline int BIO_meth_set_destroy(BIO_METHOD *meth, int (*f)(BIO *)) { meth->destroy = f; return 1; }
+static inline int BIO_meth_set_write(BIO_METHOD *meth, int (*f)(BIO *, const char *, int)) { meth->bwrite = f; return 1; }
+static inline int BIO_meth_set_read(BIO_METHOD *meth, int (*f)(BIO *, char *, int)) { meth->bread = f; return 1; }
+static inline int BIO_meth_set_ctrl(BIO_METHOD *meth, long (*f)(BIO *, int, long, void *)) { meth->ctrl = f; return 1; }
 #endif /* HAVE_OPAQUE_OPENSSL */
 
 #if !defined(EVP_CTRL_AEAD_GET_TAG)

ext/openssl/ossl.c

@@ -1150,6 +1150,7 @@ Init_openssl(void)
     /*
      * Init components
      */
+    Init_ossl_bio();
     Init_ossl_bn();
     Init_ossl_cipher();
     Init_ossl_config();

ext/openssl/ossl_bio.c

@@ -40,3 +40,230 @@ ossl_membio2str(BIO *bio)
 
     return ret;
 }
+
+static BIO_METHOD *ossl_bio_meth;
+static VALUE nonblock_kwargs, sym_wait_readable, sym_wait_writable;
+
+struct ossl_bio_ctx {
+    VALUE io;
+    int state;
+    int eof;
+};
+
+BIO *
+ossl_bio_new(VALUE io)
+{
+    BIO *bio = BIO_new(ossl_bio_meth);
+    if (!bio)
+        ossl_raise(eOSSLError, "BIO_new");
+    struct ossl_bio_ctx *ctx = BIO_get_data(bio);
+    ctx->io = io;
+    BIO_set_init(bio, 1);
+    return bio;
+}
+
+int
+ossl_bio_state(BIO *bio)
+{
+    struct ossl_bio_ctx *ctx = BIO_get_data(bio);
+    int state = ctx->state;
+    ctx->state = 0;
+    return state;
+}
+
+static int
+bio_create(BIO *bio)
+{
+    struct ossl_bio_ctx *ctx = OPENSSL_malloc(sizeof(*ctx));
+    if (!ctx)
+        return 0;
+    memset(ctx, 0, sizeof(*ctx));
+    BIO_set_data(bio, ctx);
+
+    return 1;
+}
+
+static int
+bio_destroy(BIO *bio)
+{
+    struct ossl_bio_ctx *ctx = BIO_get_data(bio);
+    if (ctx) {
+        OPENSSL_free(ctx);
+        BIO_set_data(bio, NULL);
+    }
+
+    return 1;
+}
+
+struct bwrite_args {
+    BIO *bio;
+    struct ossl_bio_ctx *ctx;
+    const char *data;
+    int dlen;
+    int written;
+};
+
+static VALUE
+bio_bwrite0(VALUE args)
+{
+    struct bwrite_args *p = (void *)args;
+    BIO_clear_retry_flags(p->bio);
+
+    VALUE fargs[] = { rb_str_new_static(p->data, p->dlen), nonblock_kwargs };
+    VALUE ret = rb_funcallv_public_kw(p->ctx->io, rb_intern("write_nonblock"),
+                                      2, fargs, RB_PASS_KEYWORDS);
+
+    if (RB_INTEGER_TYPE_P(ret)) {
+        p->written = NUM2INT(ret);
+        return Qtrue;
+    }
+    else if (ret == sym_wait_readable) {
+        BIO_set_retry_read(p->bio);
+        return Qfalse;
+    }
+    else if (ret == sym_wait_writable) {
+        BIO_set_retry_write(p->bio);
+        return Qfalse;
+    }
+    else {
+        rb_raise(rb_eTypeError, "write_nonblock must return an Integer, "
+                 ":wait_readable, or :wait_writable");
+    }
+}
+
+static int
+bio_bwrite(BIO *bio, const char *data, int dlen)
+{
+    struct ossl_bio_ctx *ctx = BIO_get_data(bio);
+    struct bwrite_args args = { bio, ctx, data, dlen, 0 };
+    int state;
+
+    if (ctx->state)
+        return -1;
+
+    VALUE ok = rb_protect(bio_bwrite0, (VALUE)&args, &state);
+    if (state) {
+        ctx->state = state;
+        return -1;
+    }
+    if (RTEST(ok))
+        return args.written;
+    return -1;
+}
+
+struct bread_args {
+    BIO *bio;
+    struct ossl_bio_ctx *ctx;
+    char *data;
+    int dlen;
+    int readbytes;
+};
+
+static VALUE
+bio_bread0(VALUE args)
+{
+    struct bread_args *p = (void *)args;
+    BIO_clear_retry_flags(p->bio);
+
+    VALUE fargs[] = { INT2NUM(p->dlen), nonblock_kwargs };
+    VALUE ret = rb_funcallv_public_kw(p->ctx->io, rb_intern("read_nonblock"),
+                                      2, fargs, RB_PASS_KEYWORDS);
+
+    if (RB_TYPE_P(ret, T_STRING)) {
+        int len = RSTRING_LENINT(ret);
+        if (len > p->dlen)
+            rb_raise(rb_eTypeError, "read_nonblock returned too much data");
+        memcpy(p->data, RSTRING_PTR(ret), len);
+        p->readbytes = len;
+        return Qtrue;
+    }
+    else if (NIL_P(ret)) {
+        // In OpenSSL 3.0 or later: BIO_set_flags(p->bio, BIO_FLAGS_IN_EOF);
+        p->ctx->eof = 1;
+        return Qtrue;
+    }
+    else if (ret == sym_wait_readable) {
+        BIO_set_retry_read(p->bio);
+        return Qfalse;
+    }
+    else if (ret == sym_wait_writable) {
+        BIO_set_retry_write(p->bio);
+        return Qfalse;
+    }
+    else {
+        rb_raise(rb_eTypeError, "write_nonblock must return an Integer, "
+                 ":wait_readable, or :wait_writable");
+    }
+}
+
+static int
+bio_bread(BIO *bio, char *data, int dlen)
+{
+    struct ossl_bio_ctx *ctx = BIO_get_data(bio);
+    struct bread_args args = { bio, ctx, data, dlen, 0 };
+    int state;
+
+    if (ctx->state)
+        return -1;
+
+    VALUE ok = rb_protect(bio_bread0, (VALUE)&args, &state);
+    if (state) {
+        ctx->state = state;
+        return -1;
+    }
+    if (RTEST(ok))
+        return args.readbytes;
+    return -1;
+}
+
+static VALUE
+bio_flush0(VALUE vctx)
+{
+    struct ossl_bio_ctx *ctx = (void *)vctx;
+    return rb_funcallv_public(ctx->io, rb_intern("flush"), 0, NULL);
+}
+
+static long
+bio_ctrl(BIO *bio, int cmd, long larg, void *parg)
+{
+    struct ossl_bio_ctx *ctx = BIO_get_data(bio);
+    int state;
+
+    if (ctx->state)
+        return 0;
+
+    switch (cmd) {
+      case BIO_CTRL_EOF:
+        return ctx->eof;
+      case BIO_CTRL_FLUSH:
+        rb_protect(bio_flush0, (VALUE)ctx, &state);
+        ctx->state = state;
+        return !state;
+      default:
+        return 0;
+    }
+}
+
+void
+Init_ossl_bio(void)
+{
+    ossl_bio_meth = BIO_meth_new(BIO_TYPE_SOURCE_SINK, "Ruby IO-like object");
+    if (!ossl_bio_meth)
+        ossl_raise(eOSSLError, "BIO_meth_new");
+    if (!BIO_meth_set_create(ossl_bio_meth, bio_create) ||
+        !BIO_meth_set_destroy(ossl_bio_meth, bio_destroy) ||
+        !BIO_meth_set_write(ossl_bio_meth, bio_bwrite) ||
+        !BIO_meth_set_read(ossl_bio_meth, bio_bread) ||
+        !BIO_meth_set_ctrl(ossl_bio_meth, bio_ctrl)) {
+        BIO_meth_free(ossl_bio_meth);
+        ossl_bio_meth = NULL;
+        ossl_raise(eOSSLError, "BIO_meth_set_*");
+    }
+
+    nonblock_kwargs = rb_hash_new();
+    rb_hash_aset(nonblock_kwargs, ID2SYM(rb_intern_const("exception")), Qfalse);
+    rb_global_variable(&nonblock_kwargs);
+
+    sym_wait_readable = ID2SYM(rb_intern_const("wait_readable"));
+    sym_wait_writable = ID2SYM(rb_intern_const("wait_writable"));
+}

ext/openssl/ossl_bio.h

@@ -13,4 +13,9 @@
 BIO *ossl_obj2bio(volatile VALUE *);
 VALUE ossl_membio2str(BIO*);
 
+BIO *ossl_bio_new(VALUE io);
+int ossl_bio_state(BIO *bio);
+
+void Init_ossl_bio(void);
+
 #endif