go hs distinguisher also not passing test?

crates/obfs4/src/common/x25519_elligator2.rs

@@ -235,9 +235,11 @@ mod test {
     use super::*;
     use crate::Result;
 
-    use curve25519_elligator2::{MapToPointVariant, Randomized};
+    use curve25519_elligator2::{traits::IsIdentity, MapToPointVariant, MontgomeryPoint, EdwardsPoint, Randomized, RFC9380};
     use hex::FromHex;
 
+    use rand::Rng;
+
     #[test]
     fn representative_match() {
         let repres = <[u8; 32]>::from_hex(
@@ -314,4 +316,105 @@ mod test {
         let pk1 = PublicKey::from(r1);
         assert_eq!(hex::encode(pk.to_bytes()), hex::encode(pk1.to_bytes()));
     }
+
+    const BASEPOINT_ORDER_MINUS_ONE: [u8; 32] = [
+        0xec, 0xd3, 0xf5, 0x5c, 0x1a, 0x63, 0x12, 0x58, 0xd6, 0x9c, 0xf7, 0xa2, 0xde, 0xf9, 0xde,
+        0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0x00, 0x10,
+    ];
+
+    // Generates a new Keypair using, and returns the public key representative
+    // along, with its public key as a newly allocated edwards25519.Point.
+    fn generate<R:RngCore+CryptoRng>(rng: &mut R) -> ([u8; 32], EdwardsPoint) {
+        for _ in 0..63 {
+            let y_sk = rng.gen::<[u8; 32]>();
+
+            let y_repr_bytes = match Randomized::to_representative(&y_sk, 0xff).into() {
+                Some(r) => r,
+                None => continue,
+            };
+            let y_pk = Randomized::mul_base_clamped(y_sk);
+
+            assert_eq!(
+                MontgomeryPoint::from_representative::<Randomized>(&y_repr_bytes)
+                    .expect("failed to re-derive point from representative"),
+                y_pk.to_montgomery()
+            );
+
+            return (y_repr_bytes, y_pk);
+        }
+        panic!("failed to generate a valid keypair");
+    }
+
+    /// Returns a new edwards25519.Point that is v multiplied by the subgroup order.
+    ///
+    /// BASEPOINT_ORDER_MINUS_ONE is the same as scMinusOne in filippo.io/edwards25519.
+    /// https://github.com/FiloSottile/edwards25519/blob/v1.0.0/scalar.go#L34
+    fn scalar_mult_order(v: &EdwardsPoint) -> EdwardsPoint {
+            let order = curve25519_elligator2::Scalar::from_bytes_mod_order(BASEPOINT_ORDER_MINUS_ONE);
+
+            // v * (L - 1) + v => v * L
+            let p = v * order;
+            p + v
+    }
+
+    #[test]
+    fn off_subgroup_check_edw() {
+        let mut rng = rand::thread_rng();
+        for _ in 0..100 {
+            let (repr, pk) = generate(&mut rng);
+
+            // check if the generated public key is off the subgroup
+            let v = scalar_mult_order(&pk);
+            let pk_off = !v.is_identity();
+
+            // ----
+
+            // check if the public key derived from the representative (top bit 0)
+            // is off the subgroup
+            let mut yr_255 = repr.clone();
+            yr_255[31] &= 0xbf;
+            let pk_255 = EdwardsPoint::from_representative::<RFC9380>(&yr_255).expect("from_repr_255, should never fail");
+            let v = scalar_mult_order(&pk_255);
+            let off_255 = !v.is_identity();
+
+            // check if the public key derived from the representative (top two bits 0 - as
+            // our representatives are) is off the subgroup.
+            let mut yr_254 = repr.clone();
+            yr_254[31] &= 0x3f;
+            let pk_254 = EdwardsPoint::from_representative::<RFC9380>(&yr_254).expect("from_repr_254, should never fail");
+            let v = scalar_mult_order(&pk_254);
+            let off_254 = !v.is_identity();
+
+            println!("pk_gen: {pk_off}, pk_255: {off_255}, pk_254: {off_254}");
+        }
+    }
+
+    // #[test]
+    // fn subgroups() {
+    //     let order = curve25519_elligator2::Scalar::from_bytes_mod_order(BASEPOINT_ORDER_MINUS_ONE);
+
+    //     let elligator_direct_map1 = MontgomeryPoint::from_representative::<Randomized>;
+    //     // let elligator_direct_map2 = MontgomeryPoint::from_representative::<RFC9380>;
+
+    //     for _ in 0..100 {
+    //         let e = EphemeralSecret::random();
+    //         let r = PublicRepresentative::from(&e);
+
+    //         let mut yr_255 = r.0.clone();
+    //         yr_255[31] &= 0xbf;
+    //         let y_255 = elligator_direct_map1(&yr_255).expect("to_repr_255, should never fail");
+    //         let z = y_255 * order;
+    //         let off_255 = !z.is_identity();
+
+    //         let mut yr_254 = r.0.clone();
+    //         yr_254[31] &= 0x3f;
+    //         let y_254 = elligator_direct_map1(&yr_254).expect("to_repr_254, should never fail");
+    //         let z = y_254 * order;
+    //         let off_254 = !z.is_identity();
+
+    //         println!("255: {off_255}, 254: {off_254}");
+    //     }
+    // }
+
 }

internal/compatibility/Dockerfile

@@ -8,13 +8,14 @@ RUN go build -o fwd_go .
 
 FROM rust:bullseye as rs-builder
 WORKDIR /usr/src/fwd
-COPY . .
+COPY Cargo.toml Cargo.lock ./
+COPY crates ./crates/
 RUN cargo update
 RUN cd crates/lyrebird && cargo install --path . --force --debug
 
 FROM debian:bullseye-slim
+RUN apt update && apt install -yq python3;
 COPY --from=rs-builder /usr/local/cargo/bin/fwd /usr/local/bin/fwd_rs
 COPY --from=go-builder /usr/src/obfs4-go/fwd_go /usr/local/bin/fwd_go
 COPY internal/compatibility/ell2_bug_check/obfs4-* /usr/bin/
-RUN apt update && apt install -yq python3;
 CMD ["fwd_rs"]

internal/compatibility/docker-compose-ell2.yaml

@@ -5,7 +5,7 @@ services:
     command: bash -c "
       python3 /usr/bin/obfs4-bug-check.py server-proxy:9001 AAAAAAAAAAAAAAAAAAAAAAAAAADTSFvsGKxNFPBcGdOCBSgpEtJInG9zCYZezBPVBuBWag &&
       python3 /usr/bin/obfs4-bug-check-authed.py server-proxy:9001 AAAAAAAAAAAAAAAAAAAAAAAAAADTSFvsGKxNFPBcGdOCBSgpEtJInG9zCYZezBPVBuBWag &&
-      python3 /usr/bin/obfs4-subgroup-check.py server-proxy:9001 AAAAAAAAAAAAAAAAAAAAAAAAAADTSFvsGKxNFPBcGdOCBSgpEtJInG9zCYZezBPVBuBWag"
+      python3 /usr/bin/obfs4-subgroup-check.py server-proxy:9001 AAAAAAAAAAAAAAAAAAAAAAAAAADTSFvsGKxNFPBcGdOCBSgpEtJInG9zCYZezBPVBuBWag -n 100"
     build:
       context: ../../
       dockerfile: Dockerfile
@@ -17,10 +17,9 @@ services:
       server-proxy:
         condition: service_started
 
-
   server-proxy:
     image: obfs4-compat
-    command: fwd_rs -a dev -x server echo
+    command: fwd_go -a dev -x server echo
     tty: true
     build:
       context: ../../
@@ -31,6 +30,19 @@ services:
       - trial
 
 
+        # server-proxy:
+        #   image: obfs4-compat
+        #   command: fwd_rs -a dev -x server echo
+        #   tty: true
+        #   build:
+        #     context: ../../
+        #     dockerfile: Dockerfile
+        #   expose:
+        #     - 9001
+        #   networks:
+        #     - trial
+
+
 networks:
   trial:
     driver: bridge

internal/compatibility/ell2_bug_check/obfs4-subgroup-check.py

@@ -37,6 +37,7 @@
 import sys
 import time
 import traceback
+import random
 
 # Elligator reference implementation by Loup Vaillant
 # https://elligator.org/src/core
@@ -363,6 +364,8 @@ def check(Yrb):
     Y_254 = elligator_dir_map(Yr_254)[0]
     off_subgroup_254 = Mt.scalarmult(Y_254, Mt.order).to_num() != 0
 
+    print(" - 255: %r, 254: %r" % (off_subgroup_255, off_subgroup_254))
+
     return off_subgroup_255 and off_subgroup_254
 
 def trial(addr, server_nodeid, server_pubkey):
@@ -375,7 +378,7 @@ def mac(msg):
     padding = os.urandom(85)
     mark = mac(Xrb)
     epoch_hours = str(int(time.time()) // 3600).encode()
-    s = socket.create_connection((host, port), TIMEOUT)
+    s = socket.create_connection(addr, TIMEOUT)
     try:
         # Client handshake
         # https://gitlab.com/yawning/obfs4/-/blob/obfs4proxy-0.0.13/doc/obfs4-spec.txt#L156-163
@@ -388,35 +391,63 @@ def mac(msg):
         s.close()
     return check(Yrb)
 
-opts, (addr, cert) = getopt.gnu_getopt(sys.argv[1:], "n:t:")
-for o, a in opts:
-    if o == "-n":
-        NUM_TRIALS = int(a)
-    elif o == "-t":
-        TIMEOUT = float(a)
-host, port = re.match(r'^\[?(.*?)\]?:(\d+)$', addr).groups()
-port = int(port)
-cert = base64.b64decode(cert + "==="[:(4-len(cert)%4)%4])
-if len(cert) != 52:
-    raise ValueError(cert)
-server_nodeid = cert[ 0:20]
-server_pubkey = cert[20:52]
-
-num_successes = 0
-err = None
-try:
-    for _ in range(NUM_TRIALS):
-        if trial((host, port), server_nodeid, server_pubkey):
-            dot = f"#"
-            num_successes += 1
-        else:
-            dot = "."
-        print(dot, flush = True, end = "")
-except Exception as e:
-    print("X", flush = True, end = "")
-    print("", addr, "ERROR", str(e))
-    traceback.print_exc()
-    sys.exit(2)
-else:
-    print("", addr, "PASS" if num_successes > 0 else "FAIL", f"{num_successes}/{NUM_TRIALS}")
-    sys.exit(0 if num_successes > 0 else 1)
+
+def main():
+    network_trials()
+    # fixed_trials()
+
+def fixed_trials():
+    low_order_points = [
+        "0100000000000000000000000000000000000000000000000000000000000000",
+        "ecffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff7f",
+        "0000000000000000000000000000000000000000000000000000000000000000",
+        "0000000000000000000000000000000000000000000000000000000000000080",
+        "26e8958fc2b227b045c3f489f2ef98f0d5dfac05d3c63339b13802886d53fc05",
+        "26e8958fc2b227b045c3f489f2ef98f0d5dfac05d3c63339b13802886d53fc85",
+        "c7176a703d4dd84fba3c0b760d10670f2a2053fa2c39ccc64ec7fd7792ac037a",
+        "c7176a703d4dd84fba3c0b760d10670f2a2053fa2c39ccc64ec7fd7792ac03fa",
+    ]
+    for px in low_order_points:
+        p = bytes.fromhex(px)
+        check(p)
+
+    for _ in range(100):
+        p = random.randbytes(32)
+        check(p)
+
+def network_trials():
+    opts, (addr, cert) = getopt.gnu_getopt(sys.argv[1:], "n:t:")
+    for o, a in opts:
+        if o == "-n":
+            NUM_TRIALS = int(a)
+        elif o == "-t":
+            TIMEOUT = float(a)
+    host, port = re.match(r'^\[?(.*?)\]?:(\d+)$', addr).groups()
+    port = int(port)
+    cert = base64.b64decode(cert + "==="[:(4-len(cert)%4)%4])
+    if len(cert) != 52:
+        raise ValueError(cert)
+    server_nodeid = cert[ 0:20]
+    server_pubkey = cert[20:52]
+
+    num_successes = 0
+    err = None
+    try:
+        for _ in range(NUM_TRIALS):
+            if trial((host, port), server_nodeid, server_pubkey):
+                dot = f"#"
+                num_successes += 1
+            else:
+                dot = "."
+            print(dot, flush = True, end = "")
+    except Exception as e:
+        print("X", flush = True, end = "")
+        print("", addr, "ERROR", str(e))
+        traceback.print_exc()
+        sys.exit(2)
+    else:
+        print("", addr, "PASS" if num_successes > 0 else "FAIL", f"{num_successes}/{NUM_TRIALS}")
+        sys.exit(0 if num_successes > 0 else 1)
+
+if __name__ == "__main__":
+    main()

internal/compatibility/obfs4/go.mod

@@ -5,22 +5,22 @@ go 1.21.3
 require (
 	github.com/refraction-networking/obfs4 v0.1.2
 	gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/goptlib v1.5.0
-	gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/lyrebird v0.0.0-20240325194939-a571773a10d8
+	gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/lyrebird v0.0.0-20240620180443-f56896ffb54a
 )
 
 require (
 	filippo.io/edwards25519 v1.1.0 // indirect
-	github.com/andybalholm/brotli v1.0.6 // indirect
-	github.com/cloudflare/circl v1.3.7 // indirect
+	github.com/andybalholm/brotli v1.1.0 // indirect
+	github.com/cloudflare/circl v1.3.9 // indirect
 	github.com/dchest/siphash v1.2.3 // indirect
-	github.com/klauspost/compress v1.17.4 // indirect
-	github.com/quic-go/quic-go v0.42.0 // indirect
-	github.com/refraction-networking/utls v1.6.3 // indirect
+	github.com/klauspost/compress v1.17.9 // indirect
+	github.com/quic-go/quic-go v0.45.1 // indirect
+	github.com/refraction-networking/utls v1.6.6 // indirect
 	gitlab.com/yawning/edwards25519-extra v0.0.0-20231005122941-2149dcafc266 // indirect
-	gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/webtunnel v0.0.0-20240220175603-399c24557a18 // indirect
-	golang.org/x/crypto v0.21.0 // indirect
-	golang.org/x/exp v0.0.0-20221205204356-47842c84f3db // indirect
-	golang.org/x/net v0.23.0 // indirect
-	golang.org/x/sys v0.18.0 // indirect
-	golang.org/x/text v0.14.0 // indirect
+	gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/webtunnel v0.0.0-20240507101150-3b6faa481637 // indirect
+	golang.org/x/crypto v0.24.0 // indirect
+	golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 // indirect
+	golang.org/x/net v0.26.0 // indirect
+	golang.org/x/sys v0.21.0 // indirect
+	golang.org/x/text v0.16.0 // indirect
 )