diff --git a/flake.lock b/flake.lock
index 6100a1e..714d09f 100644
--- a/flake.lock
+++ b/flake.lock
@@ -1,22 +1,5 @@
 {
   "nodes": {
-    "linux-vf2-src": {
-      "flake": false,
-      "locked": {
-        "lastModified": 1705574974,
-        "narHash": "sha256-bwB7Pc+Z+MWXPfWYdgtRGuhqjiNHLDGNCY62e4lBGvE=",
-        "owner": "starfive-tech",
-        "repo": "linux",
-        "rev": "13eb70da2a73187c8c7aece13d23d68928aa8210",
-        "type": "github"
-      },
-      "original": {
-        "owner": "starfive-tech",
-        "ref": "JH7110_VisionFive2_upstream",
-        "repo": "linux",
-        "type": "github"
-      }
-    },
     "nixpkgs": {
       "locked": {
         "lastModified": 1711735929,
@@ -51,44 +34,8 @@
     },
     "root": {
       "inputs": {
-        "linux-vf2-src": "linux-vf2-src",
         "nixpkgs": "nixpkgs",
-        "nixpkgs-native": "nixpkgs-native",
-        "starfive-tools": "starfive-tools",
-        "uboot-vf2-src": "uboot-vf2-src"
-      }
-    },
-    "starfive-tools": {
-      "flake": false,
-      "locked": {
-        "lastModified": 1681470896,
-        "narHash": "sha256-qffCJMA57NZEBEeYYLyvFeoqYnOSRomSob5cVmpFYoo=",
-        "owner": "starfive-tech",
-        "repo": "Tools",
-        "rev": "693661d4ba314424f76c06da1bbb799e9b534c9f",
-        "type": "github"
-      },
-      "original": {
-        "owner": "starfive-tech",
-        "repo": "Tools",
-        "type": "github"
-      }
-    },
-    "uboot-vf2-src": {
-      "flake": false,
-      "locked": {
-        "lastModified": 1712055538,
-        "narHash": "sha256-IlaDdjKq/Pq2orzcU959h93WXRZfvKBGDO/MFw9mZMg=",
-        "owner": "u-boot",
-        "repo": "u-boot",
-        "rev": "25049ad560826f7dc1c4740883b0016014a59789",
-        "type": "github"
-      },
-      "original": {
-        "owner": "u-boot",
-        "ref": "v2024.04",
-        "repo": "u-boot",
-        "type": "github"
+        "nixpkgs-native": "nixpkgs-native"
       }
     }
   },
diff --git a/flake.nix b/flake.nix
index 1b3331c..1819536 100644
--- a/flake.nix
+++ b/flake.nix
@@ -4,31 +4,12 @@
     extra-trusted-public-keys = [ "cache.ztier.link-1:3P5j2ZB9dNgFFFVkCQWT3mh0E+S3rIWtZvoql64UaXM=" ];
   };
 
-  inputs = {
-    nixpkgs = {
-      url = "github:NickCao/nixpkgs/riscv";
-    };
-    nixpkgs-native = {
-      url = "github:NixOS/nixpkgs/nixos-unstable";
-    };
-    linux-vf2-src = {
-      flake = false;
-      url = "github:starfive-tech/linux/JH7110_VisionFive2_upstream";
-    };
-    starfive-tools = {
-      flake = false;
-      url = "github:starfive-tech/Tools";
-    };
-    uboot-vf2-src = {
-      flake = false;
-      url = "github:u-boot/u-boot/v2024.04";
-    };
-  };
+  inputs.nixpkgs.url = "github:NickCao/nixpkgs/riscv";
+  inputs.nixpkgs-native.url = "github:NixOS/nixpkgs/nixos-unstable";
 
   outputs = inputs: {
     overlays.default = self: super: {
       linuxPackages_vf2 = self.linuxPackagesFor (self.callPackage ./linux-vf2.nix {
-        src = inputs.linux-vf2-src;
         kernelPatches = [ ];
       });
     };
@@ -43,9 +24,12 @@
     };
 
     overlays.firmware = self: super: {
-      uboot-vf2 = (super.buildUBoot {
-        version = inputs.uboot-vf2-src.shortRev;
-        src = inputs.uboot-vf2-src;
+      uboot-vf2 = (super.buildUBoot rec {
+        version = "2024.04";
+        src = super.fetchurl {
+          url = "https://ftp.denx.de/pub/u-boot/u-boot-${version}.tar.bz2";
+          hash = "sha256-GKhT/jn6160DqQzC1Cda6u1tppc13vrDSSuAUIhD3Uo=";
+        };
         defconfig = "starfive_visionfive2_defconfig";
         filesToInstall = [
           "u-boot.itb"
@@ -59,7 +43,12 @@
 
       spl-tool = self.stdenv.mkDerivation {
         name = "spl-tool";
-        src = inputs.starfive-tools;
+        src = super.fetchFromGitHub {
+          owner = "starfive-tech";
+          repo = "Tools";
+          rev = "0747c0510e090f69bf7d2884f44903b77b3db5c5";
+          hash = "sha256-up58PtvnZTi6rGZcP2VwZrZBYajrZf4MzILixqaNKbE=";
+        };
         installPhase = ''
           runHook preInstall
 
diff --git a/linux-vf2.nix b/linux-vf2.nix
index 3b17ce1..fa9d6cd 100644
--- a/linux-vf2.nix
+++ b/linux-vf2.nix
@@ -1,5 +1,6 @@
 { lib
 , buildLinux
+, fetchFromGitHub
 , ...
 } @ args:
 
@@ -10,6 +11,13 @@ buildLinux (args // {
   inherit modDirVersion;
   version = "${modDirVersion}-vf2";
 
+  src = fetchFromGitHub {
+    owner = "starfive-tech";
+    repo = "linux";
+    rev = "076ede06c00a4069cd9f90d609eaf35bf1bdc68a";
+    hash = "sha256-oI048iCkvghEIiuDHbxNHtJz/krPwXPB/HB85YUaxL8=";
+  };
+
   structuredExtraConfig = with lib.kernel; {
     CPU_FREQ = yes;
     CPUFREQ_DT = yes;
@@ -42,6 +50,12 @@ buildLinux (args // {
     STARFIVE_HDMI = yes;
 
     PL330_DMA = no;
+
+    # Disable some drivers that we don't need to make the build leaner
+    NET_VENDOR_MELLANOX = no;
+    NET_VENDOR_MARVELL = no;
+    DRM_NOUVEAU = no;
+    DRM_INTEL = no;
   };
 
   preferBuiltin = true;