Skip to content

Commit

Permalink
Introduce nativelink-bridge
Browse files Browse the repository at this point in the history 
This is a fast prototype for subscribing to the redis/dragonflydb "build_events" channel and
decode them properly via protobuf and send them via websocket to the browser.

It's a plain prototype without any error handling and formatting on the browser.
  • Loading branch information
@SchahinRohani
SchahinRohani committed Sep 2, 2024
1 parent 40e954c commit c78cc97
Show file tree
Hide file tree
Showing 14 changed files with 1,010 additions and 0 deletions.
178 changes: 178 additions & 0 deletions nativelink-config/examples/basic_cas_test_conf.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,178 @@
{
"stores": {
"AC_MAIN_STORE": {
"filesystem": {
"content_path": "/tmp/nativelink/data-worker-test/content_path-ac",
"temp_path": "/tmp/nativelink/data-worker-test/tmp_path-ac",
"eviction_policy": {
"max_bytes": 100000000000
}
}
},
"BEP_STORE": {

"redis_store": {
"addresses": [
"redis://@localhost:6379/0"
],
"response_timeout_s": 42,
"connection_timeout_s": 42,
"experimental_pub_sub_channel": "build_event",
"key_prefix": "nativelink.",
"mode": "standard"
}
},
"WORKER_FAST_SLOW_STORE": {
"fast_slow": {
"fast": {
"filesystem": {
"content_path": "/tmp/nativelink/data-worker-test/content_path-cas",
"temp_path": "/tmp/nativelink/data-worker-test/tmp_path-cas",
"eviction_policy": {
"max_bytes": 100000000000
}
}
},
"slow": {
"noop": {}
}
}
}
},
"schedulers": {
"MAIN_SCHEDULER": {
"simple": {
"supported_platform_properties": {
"cpu_count": "minimum",
"memory_kb": "minimum",
"network_kbps": "minimum",
"disk_read_iops": "minimum",
"disk_read_bps": "minimum",
"disk_write_iops": "minimum",
"disk_write_bps": "minimum",
"shm_size": "minimum",
"gpu_count": "minimum",
"gpu_model": "exact",
"cpu_vendor": "exact",
"cpu_arch": "exact",
"cpu_model": "exact",
"kernel_version": "exact",
"OSFamily": "priority",
"container-image": "priority"
}
}
}
},
"workers": [
{
"local": {
"worker_api_endpoint": {
"uri": "grpc://127.0.0.1:50061"
},
"cas_fast_slow_store": "WORKER_FAST_SLOW_STORE",
"upload_action_result": {
"ac_store": "AC_MAIN_STORE"
},
"work_directory": "/tmp/nativelink/work",
"platform_properties": {
"cpu_count": {
"values": [
"16"
]
},
"memory_kb": {
"values": [
"500000"
]
},
"network_kbps": {
"values": [
"100000"
]
},
"cpu_arch": {
"values": [
"x86_64"
]
},
"OSFamily": {
"values": [
""
]
},
"container-image": {
"values": [
""
]
}
}
}
}
],
"servers": [
{
"name": "public",
"listener": {
"http": {
"socket_address": "0.0.0.0:50051"
}
},
"services": {
"cas": {
"main": {
"cas_store": "WORKER_FAST_SLOW_STORE"
}
},
"ac": {
"main": {
"ac_store": "AC_MAIN_STORE"
}
},
"execution": {
"main": {
"cas_store": "WORKER_FAST_SLOW_STORE",
"scheduler": "MAIN_SCHEDULER"
}
},
"capabilities": {
"main": {
"remote_execution": {
"scheduler": "MAIN_SCHEDULER"
}
}
},
"bytestream": {
"cas_stores": {
"main": "WORKER_FAST_SLOW_STORE"
}
}
}
},
{
"name": "private_workers_servers",
"listener": {
"http": {
"socket_address": "0.0.0.0:50061"
}
},
"services": {
"experimental_prometheus": {
"path": "/metrics"
},
"experimental_bep": {
"store": "BEP_STORE"
},
"worker_api": {
"scheduler": "MAIN_SCHEDULER"
},
"admin": {},
"health": {
"path": "/status"
}
}
}
],
"global": {
"max_open_files": 512
}
}
1 change: 1 addition & 0 deletions tools/pre-commit-hooks.nix
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,7 @@ in {

# Bun binary lockfile
"web/app/bun.lockb"
"web/bridge/bun.lockb"
];
enable = true;
types = ["binary"];
Expand Down
121 changes: 121 additions & 0 deletions web/app/src/pages/app/index.astro
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,121 @@
---
import Layout from "../../layouts/Layout.astro";
---

<Layout title="NativeLink">
<div class="w-screen h-full py-24 flex justify-center items-center bg-black">

<div id="output" class="px-4 text-white text-[10px]"></div>
</div>

<script>
const socket = new WebSocket('ws://localhost:8080');
socket.binaryType = "arraybuffer"
const outputDiv = document.getElementById('output');

function wrapAnsiCodesWithTailwind(text: string) {
// Define a map of ANSI color codes to Tailwind CSS classes
const ansiToTailwind = {
'30': 'text-black', // ANSI black
'31': 'text-red-500', // ANSI red
'32': 'text-green-500', // ANSI green
'33': 'text-yellow-500', // ANSI yellow
'34': 'text-blue-500', // ANSI blue
'35': 'text-purple-500', // ANSI magenta
'36': 'text-cyan-500', // ANSI cyan
'37': 'text-white', // ANSI white
'90': 'text-gray-500', // ANSI bright black (gray)
'91': 'text-red-700', // ANSI bright red
'92': 'text-green-700', // ANSI bright green
'93': 'text-yellow-700', // ANSI bright yellow
'94': 'text-blue-700', // ANSI bright blue
'95': 'text-purple-700', // ANSI bright magenta
'96': 'text-cyan-700', // ANSI bright cyan
'97': 'text-white', // ANSI bright white
};

// Regular expression to match ANSI escape codes
const ansiRegex = /\x1B\[(\d+)[;0-9]*m/g;

// Replace ANSI codes with corresponding Tailwind-wrapped text
let result = '';
let currentIndex = 0;
let match;

while ((match = ansiRegex.exec(text)) !== null) {
const ansiCode = match[1];
let tailwindClass;
if (ansiCode) {
if (ansiCode in ansiToTailwind) {
tailwindClass = ansiToTailwind[ansiCode as keyof typeof ansiToTailwind] || '';
}
}
if (tailwindClass) {
// Add the text before the ANSI code
result += text.substring(currentIndex, match.index);

// Find the text affected by this ANSI code (until the next ANSI code or the end of the string)
const nextIndex = text.indexOf('\x1B[', ansiRegex.lastIndex);
const endIndex = nextIndex !== -1 ? nextIndex : text.length;
const wrappedText = text.substring(ansiRegex.lastIndex, endIndex);

// Wrap the affected text with a div and the Tailwind class
result += `<span class="${tailwindClass}">${wrappedText}</span>`;

// Move the currentIndex to after the ANSI code
currentIndex = endIndex;

// Move the regex index to after the wrapped text
ansiRegex.lastIndex = endIndex;
} else {
// If no valid ANSI code, just append the raw text
result += text.substring(currentIndex, match.index);
currentIndex = ansiRegex.lastIndex;
}
}

// Append any remaining text after the last ANSI code
result += text.substring(currentIndex);

return result;
}

function formatTextForHtml(text: string) {
// Step 1: Remove ANSI escape codes related to cursor movement
let cleanedText = text.replace(/\x1B\[[0-9;]*[a-zA-Z]/g, '');

// console.log(cleanedText)

// Step 5: Replace newlines with <br /> to retain line breaks in HTML
cleanedText= cleanedText.replace(/<span[^>]*>(INFO:|Loading:|Analyzing:)\s*<\/span><br \/>/g, '');
let formattedText = cleanedText.replace(/\n/g, '<br />').replace(/\r/g, '');

return formattedText;
}

socket.onmessage = function(event) {
if (event.data instanceof ArrayBuffer) {
const text = new TextDecoder().decode(new Uint8Array(event.data)); // Decode the ArrayBuffer
const cleanedText = wrapAnsiCodesWithTailwind(text);
const formattedText = formatTextForHtml(cleanedText);
if (outputDiv) {
outputDiv.innerHTML += formattedText + '<br />';
outputDiv.scrollTop = outputDiv.scrollHeight; // Scroll to the bottom
}
}
};

socket.onopen = function() {
console.log('WebSocket connection established');
};

socket.onclose = function() {
console.log('WebSocket connection closed');
};

socket.onerror = function(error) {
console.error('WebSocket error:', error);
};
</script>
</Layout>
Loading

0 comments on commit c78cc97

Please sign in to comment.