Skip to content

Commit

Permalink
Support arm binaries
Browse files Browse the repository at this point in the history
  • Loading branch information
willnode committed Aug 24, 2024
1 parent 9ec5d6f commit 4ee88c0
Show file tree
Hide file tree
Showing 5 changed files with 212 additions and 127 deletions.
3 changes: 2 additions & 1 deletion jsconfig.json
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
"module": "ESNext",
"moduleResolution": "Node",
"checkJs": true,
"resolveJsonModule": true
"resolveJsonModule": true,
"target": "ES2015"
}
}
2 changes: 1 addition & 1 deletion src/binaries/metadata.cjs
Original file line number Diff line number Diff line change
@@ -1 +1 @@
module.exports = require('./metadata.json');
module.exports = require('./metadata.json')[process.arch];
134 changes: 92 additions & 42 deletions src/binaries/metadata.json
Original file line number Diff line number Diff line change
@@ -1,46 +1,96 @@
{
"rubyVersionsList": [
"3.1.3",
"3.1.2",
"3.0.5",
"3.0.4",
"3.0.3",
"2.7.7",
"2.7.6",
"2.7.5"
],
"pythonVersionsList": [
"3.12.4",
"3.11.9",
"3.10.14",
"3.9.19"
],
"javaVersionsList": [
"22.0.1+8",
"21.0.3+9.0.LTS",
"20.0.2+9",
"19.0.2+7",
"18.0.2+101",
"17.0.11+9",
"16.0.2+7",
"11.0.23+9",
"8.0.412+8"
],
"pythonVersionsMap": {
"3.10.14": "https://github.com/indygreg/python-build-standalone/releases/download/20240713/cpython-3.10.14+20240713-x86_64_v3-unknown-linux-gnu-pgo+lto-full.tar.zst",
"3.11.9": "https://github.com/indygreg/python-build-standalone/releases/download/20240713/cpython-3.11.9+20240713-x86_64_v3-unknown-linux-gnu-pgo+lto-full.tar.zst",
"3.12.4": "https://github.com/indygreg/python-build-standalone/releases/download/20240713/cpython-3.12.4+20240713-x86_64_v3-unknown-linux-gnu-pgo+lto-full.tar.zst",
"3.9.19": "https://github.com/indygreg/python-build-standalone/releases/download/20240713/cpython-3.9.19+20240713-x86_64_v3-unknown-linux-gnu-pgo+lto-full.tar.zst"
"x64": {
"rubyVersionsList": [
"3.1.3",
"3.1.2",
"3.0.5",
"3.0.4",
"3.0.3",
"2.7.7",
"2.7.6",
"2.7.5"
],
"pythonVersionsList": [
"3.12.5",
"3.11.9",
"3.10.14",
"3.9.19"
],
"javaVersionsList": [
"22.0.2+9",
"21.0.4+7.0.LTS",
"20.0.2+9",
"19.0.2+7",
"18.0.2+101",
"17.0.12+7",
"16.0.2+7",
"11.0.24+8",
"8.0.422+5"
],
"pythonVersionsMap": {
"3.10.14": "https://github.com/indygreg/python-build-standalone/releases/download/20240814/cpython-3.10.14+20240814-x86_64_v3-unknown-linux-gnu-pgo+lto-full.tar.zst",
"3.11.9": "https://github.com/indygreg/python-build-standalone/releases/download/20240814/cpython-3.11.9+20240814-x86_64_v3-unknown-linux-gnu-pgo+lto-full.tar.zst",
"3.12.5": "https://github.com/indygreg/python-build-standalone/releases/download/20240814/cpython-3.12.5+20240814-x86_64_v3-unknown-linux-gnu-pgo+lto-full.tar.zst",
"3.9.19": "https://github.com/indygreg/python-build-standalone/releases/download/20240814/cpython-3.9.19+20240814-x86_64_v3-unknown-linux-gnu-pgo+lto-full.tar.zst"
},
"javaVersionsMap": {
"8.0.422+5": "https://github.com/adoptium/temurin8-binaries/releases/download/jdk8u422-b05/OpenJDK8U-jdk_x64_linux_hotspot_8u422b05.tar.gz",
"11.0.24+8": "https://github.com/adoptium/temurin11-binaries/releases/download/jdk-11.0.24%2B8/OpenJDK11U-jdk_x64_linux_hotspot_11.0.24_8.tar.gz",
"16.0.2+7": "https://github.com/adoptium/temurin16-binaries/releases/download/jdk-16.0.2%2B7/OpenJDK16U-jdk_x64_linux_hotspot_16.0.2_7.tar.gz",
"17.0.12+7": "https://github.com/adoptium/temurin17-binaries/releases/download/jdk-17.0.12%2B7/OpenJDK17U-jdk_x64_linux_hotspot_17.0.12_7.tar.gz",
"18.0.2+101": "https://github.com/adoptium/temurin18-binaries/releases/download/jdk-18.0.2.1%2B1/OpenJDK18U-jdk_x64_linux_hotspot_18.0.2.1_1.tar.gz",
"19.0.2+7": "https://github.com/adoptium/temurin19-binaries/releases/download/jdk-19.0.2%2B7/OpenJDK19U-jdk_x64_linux_hotspot_19.0.2_7.tar.gz",
"20.0.2+9": "https://github.com/adoptium/temurin20-binaries/releases/download/jdk-20.0.2%2B9/OpenJDK20U-jdk_x64_linux_hotspot_20.0.2_9.tar.gz",
"21.0.4+7.0.LTS": "https://github.com/adoptium/temurin21-binaries/releases/download/jdk-21.0.4%2B7/OpenJDK21U-jdk_x64_linux_hotspot_21.0.4_7.tar.gz",
"22.0.2+9": "https://github.com/adoptium/temurin22-binaries/releases/download/jdk-22.0.2%2B9/OpenJDK22U-jdk_x64_linux_hotspot_22.0.2_9.tar.gz"
}
},
"javaVersionsMap": {
"8.0.412+8": "https://github.com/adoptium/temurin8-binaries/releases/download/jdk8u412-b08/OpenJDK8U-jdk_x64_linux_hotspot_8u412b08.tar.gz",
"11.0.23+9": "https://github.com/adoptium/temurin11-binaries/releases/download/jdk-11.0.23%2B9/OpenJDK11U-jdk_x64_linux_hotspot_11.0.23_9.tar.gz",
"16.0.2+7": "https://github.com/adoptium/temurin16-binaries/releases/download/jdk-16.0.2%2B7/OpenJDK16U-jdk_x64_linux_hotspot_16.0.2_7.tar.gz",
"17.0.11+9": "https://github.com/adoptium/temurin17-binaries/releases/download/jdk-17.0.11%2B9/OpenJDK17U-jdk_x64_linux_hotspot_17.0.11_9.tar.gz",
"18.0.2+101": "https://github.com/adoptium/temurin18-binaries/releases/download/jdk-18.0.2.1%2B1/OpenJDK18U-jdk_x64_linux_hotspot_18.0.2.1_1.tar.gz",
"19.0.2+7": "https://github.com/adoptium/temurin19-binaries/releases/download/jdk-19.0.2%2B7/OpenJDK19U-jdk_x64_linux_hotspot_19.0.2_7.tar.gz",
"20.0.2+9": "https://github.com/adoptium/temurin20-binaries/releases/download/jdk-20.0.2%2B9/OpenJDK20U-jdk_x64_linux_hotspot_20.0.2_9.tar.gz",
"21.0.3+9.0.LTS": "https://github.com/adoptium/temurin21-binaries/releases/download/jdk-21.0.3%2B9/OpenJDK21U-jdk_x64_linux_hotspot_21.0.3_9.tar.gz",
"22.0.1+8": "https://github.com/adoptium/temurin22-binaries/releases/download/jdk-22.0.1%2B8/OpenJDK22U-jdk_x64_linux_hotspot_22.0.1_8.tar.gz"
"arm64": {
"rubyVersionsList": [
"3.1.3",
"3.1.2",
"3.0.5",
"3.0.4",
"3.0.3",
"2.7.7",
"2.7.6",
"2.7.5"
],
"pythonVersionsList": [
"3.12.5",
"3.11.9",
"3.10.14",
"3.9.19",
"3.8.19"
],
"javaVersionsList": [
"22.0.2+9",
"21.0.4+7.0.LTS",
"20.0.2+9",
"19.0.2+7",
"18.0.2+101",
"17.0.12+7",
"16.0.2+7",
"11.0.24+8",
"8.0.422+5"
],
"pythonVersionsMap": {
"3.10.14": "https://github.com/indygreg/python-build-standalone/releases/download/20240814/cpython-3.10.14+20240814-aarch64-unknown-linux-gnu-lto-full.tar.zst",
"3.11.9": "https://github.com/indygreg/python-build-standalone/releases/download/20240814/cpython-3.11.9+20240814-aarch64-unknown-linux-gnu-lto-full.tar.zst",
"3.12.5": "https://github.com/indygreg/python-build-standalone/releases/download/20240814/cpython-3.12.5+20240814-aarch64-unknown-linux-gnu-lto-full.tar.zst",
"3.8.19": "https://github.com/indygreg/python-build-standalone/releases/download/20240814/cpython-3.8.19+20240814-aarch64-unknown-linux-gnu-lto-full.tar.zst",
"3.9.19": "https://github.com/indygreg/python-build-standalone/releases/download/20240814/cpython-3.9.19+20240814-aarch64-unknown-linux-gnu-lto-full.tar.zst"
},
"javaVersionsMap": {
"8.0.422+5": "https://github.com/adoptium/temurin8-binaries/releases/download/jdk8u422-b05/OpenJDK8U-jdk_aarch64_linux_hotspot_8u422b05.tar.gz",
"11.0.24+8": "https://github.com/adoptium/temurin11-binaries/releases/download/jdk-11.0.24%2B8/OpenJDK11U-jdk_aarch64_linux_hotspot_11.0.24_8.tar.gz",
"16.0.2+7": "https://github.com/adoptium/temurin16-binaries/releases/download/jdk-16.0.2%2B7/OpenJDK16U-jdk_aarch64_linux_hotspot_16.0.2_7.tar.gz",
"17.0.12+7": "https://github.com/adoptium/temurin17-binaries/releases/download/jdk-17.0.12%2B7/OpenJDK17U-jdk_aarch64_linux_hotspot_17.0.12_7.tar.gz",
"18.0.2+101": "https://github.com/adoptium/temurin18-binaries/releases/download/jdk-18.0.2.1%2B1/OpenJDK18U-jdk_aarch64_linux_hotspot_18.0.2.1_1.tar.gz",
"19.0.2+7": "https://github.com/adoptium/temurin19-binaries/releases/download/jdk-19.0.2%2B7/OpenJDK19U-jdk_aarch64_linux_hotspot_19.0.2_7.tar.gz",
"20.0.2+9": "https://github.com/adoptium/temurin20-binaries/releases/download/jdk-20.0.2%2B9/OpenJDK20U-jdk_aarch64_linux_hotspot_20.0.2_9.tar.gz",
"21.0.4+7.0.LTS": "https://github.com/adoptium/temurin21-binaries/releases/download/jdk-21.0.4%2B7/OpenJDK21U-jdk_aarch64_linux_hotspot_21.0.4_7.tar.gz",
"22.0.2+9": "https://github.com/adoptium/temurin22-binaries/releases/download/jdk-22.0.2%2B9/OpenJDK22U-jdk_aarch64_linux_hotspot_22.0.2_9.tar.gz"
}
}
}
188 changes: 111 additions & 77 deletions src/binaries/update.js
Original file line number Diff line number Diff line change
@@ -1,104 +1,138 @@
import axios from 'axios';
import fs from 'fs';
import { dirname } from 'path';
import { fileURLToPath } from 'url';
import axios from "axios";
import fs from "fs";
import { dirname } from "path";
import { fileURLToPath } from "url";
const __dirname = dirname(fileURLToPath(import.meta.url));

// https://packagist.org/php-statistics
let rubyVersionsList = [];
let pythonVersionsList = [];
let javaVersionsList = [];
/**
* @type {Record<string, string>}
*/
let pythonVersionsMap = {};
/**
* @type {Record<string, string>}
*/
let javaVersionsMap = {};
const pythonConstants = {
// https://raw.githubusercontent.com/indygreg/python-build-standalone/latest-release/latest-release.json
tag: "20240107",
// NOTE: x86_64_v3 requires AVX2 CPU support
match: /cpython-(\d+\.\d+\.\d+)\+\d+-x86_64_v3-unknown-linux-gnu-pgo\+lto-full\.tar\.zst/g,
index() {
return `https://github.com/indygreg/python-build-standalone/releases/expanded_assets/${this.tag}`
},
latestTagUrl() {
return 'https://raw.githubusercontent.com/indygreg/python-build-standalone/latest-release/latest-release.json';
},
// https://raw.githubusercontent.com/indygreg/python-build-standalone/latest-release/latest-release.json
tag: "20240107",
// NOTE: x86_64_v3 requires AVX2 CPU support
match: {
x64: /cpython-(\d+\.\d+\.\d+)\+\d+-x86_64_v3-unknown-linux-gnu-pgo\+lto-full\.tar\.zst/g,
arm64:
/cpython-(\d+\.\d+\.\d+)\+\d+-aarch64-unknown-linux-gnu-lto-full\.tar\.zst/g,
},
index() {
return `https://github.com/indygreg/python-build-standalone/releases/expanded_assets/${this.tag}`;
},
latestTagUrl() {
return "https://raw.githubusercontent.com/indygreg/python-build-standalone/latest-release/latest-release.json";
},
/**
* @param {string} filename
*/
asset_url(filename) {
return `https://github.com/indygreg/python-build-standalone/releases/download/${this.tag}/${filename}`;
},
};
export const initUtils = async () => {
// TODO: detect OS/arch?

const result = {};

for (const arch of ["x64", "arm64"]) {
const archLinux = {
x64: "x64",
arm64: "aarch64",
};
// https://packagist.org/php-statistics
let rubyVersionsList = [];
let pythonVersionsList = [];
let javaVersionsList = [];
/**
* @param {string} filename
* @type {Record<string, string>}
*/
asset_url(filename) {
return `https://github.com/indygreg/python-build-standalone/releases/download/${this.tag}/${filename}`;
},
}
export const initUtils = async () => {
// TODO: detect OS/arch?
await axios.get('https://rvm_io.global.ssl.fastly.net/binaries/centos/9/x86_64/').then(res => {
let pythonVersionsMap = {};
/**
* @type {Record<string, string>}
*/
let javaVersionsMap = {};
await axios
.get("https://rvm_io.global.ssl.fastly.net/binaries/centos/9/x86_64/") // currently aarch64 have to rebuilt
.then((res) => {
// @ts-ignore
var matches = [...("" + res.data).matchAll(/href="ruby-([.\d]+).tar.bz2"/g)]
var matches = [
...("" + res.data).matchAll(/href="ruby-([.\d]+).tar.bz2"/g),
];
for (const match of matches) {
if (!rubyVersionsList.includes(match[1])) {
rubyVersionsList.push(match[1]);
}
if (!rubyVersionsList.includes(match[1])) {
rubyVersionsList.push(match[1]);
}
}
rubyVersionsList = sortSemver(rubyVersionsList).reverse();
}).catch(err => {
console.error('error fetching Ruby releases', err);
});
})
.catch((err) => {
console.error("error fetching Ruby releases", err.message);
});

await axios.get(pythonConstants.latestTagUrl()).then(res => {
if (res.data && res.data.tag) {
pythonConstants.tag = res.data.tag;
} else {
console.warn('unable get latest python tag');
}
})
await axios.get(pythonConstants.index()).then(res => {
await axios.get(pythonConstants.latestTagUrl()).then((res) => {
if (res.data && res.data.tag) {
pythonConstants.tag = res.data.tag;
} else {
console.warn("unable get latest python tag");
}
});
await axios
.get(pythonConstants.index())
.then((res) => {
// @ts-ignore
var matches = [...("" + res.data).matchAll(pythonConstants.match)]
var matches = [
...("" + res.data).matchAll(pythonConstants.match[arch]),
];
for (const match of matches) {
if (!pythonVersionsMap[match[1]]) {
pythonVersionsMap[match[1]] = pythonConstants.asset_url(match[0]);
pythonVersionsList.push(match[1]);
}
if (!pythonVersionsMap[match[1]]) {
pythonVersionsMap[match[1]] = pythonConstants.asset_url(match[0]);
pythonVersionsList.push(match[1]);
}
}
pythonVersionsList = sortSemver(pythonVersionsList).reverse();
}).catch(err => {
console.error('error fetching Python releases', err);
});
await axios.get('https://api.adoptium.net/v3/info/available_releases').then(async res => {
})
.catch((err) => {
console.error("error fetching Python releases", err.message);
});
await axios
.get("https://api.adoptium.net/v3/info/available_releases")
.then(async (res) => {
for (const ver of res.data.available_releases) {
await axios.get(`https://api.adoptium.net/v3/assets/latest/${ver}/hotspot?architecture=x64&image_type=jdk&os=linux&vendor=eclipse`).then(x => {
for (const binary of x.data) {
javaVersionsMap[binary.version.semver] = binary.binary.package.link;
}
})
await axios
.get(
`https://api.adoptium.net/v3/assets/latest/${ver}/hotspot?architecture=${archLinux[arch]}&image_type=jdk&os=linux&vendor=eclipse`
)
.then((x) => {
for (const binary of x.data) {
javaVersionsMap[binary.version.semver] =
binary.binary.package.link;
}
});
}
javaVersionsList = sortSemver(Object.keys(javaVersionsMap)).reverse();
})

});

fs.writeFileSync(__dirname + '/metadata.json', JSON.stringify({
rubyVersionsList,
pythonVersionsList,
javaVersionsList,
pythonVersionsMap,
javaVersionsMap,
}, null, 2))
}
result[arch] = {
rubyVersionsList,
pythonVersionsList,
javaVersionsList,
pythonVersionsMap,
javaVersionsMap,
};
}

fs.writeFileSync(
__dirname + "/metadata.json",
JSON.stringify(result, null, 2)
);
};

// https://stackoverflow.com/a/40201629/3908409
/**
* @param {string[]} arr
*/
export function sortSemver(arr) {
return arr.map(a => a.replace(/\d+/g, n => +n + 100000 + '')).sort()
.map(a => a.replace(/\d+/g, n => +n - 100000 + ''));
return arr
.map((a) => a.replace(/\d+/g, (n) => +n + 100000 + ""))
.sort()
.map((a) => a.replace(/\d+/g, (n) => +n - 100000 + ""));
}


initUtils().then(() => console.log('metadata written'));
initUtils().then(() => console.log("metadata written"));
Loading

0 comments on commit 4ee88c0

Please sign in to comment.