Skip to content

Commit

Permalink
Update documentation
Browse files Browse the repository at this point in the history
  • Loading branch information
aloftus23 committed Mar 14, 2024
1 parent eedf3a7 commit c0af290
Show file tree
Hide file tree
Showing 7 changed files with 38 additions and 66 deletions.
5 changes: 3 additions & 2 deletions backend/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -112,8 +112,9 @@
"deploy-worker-prod": "./tools/deploy-worker.sh crossfeed-prod-worker",
"syncdb": "docker-compose exec -T backend npx ts-node src/tools/run-syncdb.ts",
"pesyncdb": "docker-compose exec -T backend npx ts-node src/tools/run-pesyncdb.ts",
"control-queue": "docker-compose exec -T backend npx ts-node src/tools/consumeControlQueue.ts"
"scan-exec": "docker-compose exec -T backend npx ts-node src/tools/run-scanExecution.ts",
"send-message": "node sendMessage.js"
},
"author": "",
"license": "ISC"
}
}
20 changes: 12 additions & 8 deletions backend/sendMessage.js
Original file line number Diff line number Diff line change
@@ -1,19 +1,18 @@
// sendMessage.js
const amqp = require('amqplib');

async function sendMessageToControlQueue(message) {
async function sendMessageToQueue(message, queue) {
const connection = await amqp.connect('amqp://localhost');
const channel = await connection.createChannel();
const controlQueue = 'ControlQueue';

await channel.assertQueue(controlQueue, { durable: true });
await channel.assertQueue(queue, { durable: true });

// Simulate sending a message to the ControlQueue
channel.sendToQueue(controlQueue, Buffer.from(JSON.stringify(message)), {
// Simulate sending a message to the queue
channel.sendToQueue(queue, Buffer.from(JSON.stringify(message)), {
persistent: true
});

console.log('Message sent to ControlQueue:', message);
console.log('Message sent:', message);

setTimeout(() => {
connection.close();
Expand All @@ -22,7 +21,12 @@ async function sendMessageToControlQueue(message) {

// Simulate sending a message
const message = {
scriptType: 'shodan',
scriptType: 'dnstwist',
org: 'DHS'
};
sendMessageToControlQueue(message);
const queue = 'dnstwistQueue';
sendMessageToQueue(message, queue);
sendMessageToQueue(message, queue);
sendMessageToQueue(message, queue);
sendMessageToQueue(message, queue);
sendMessageToQueue(message, queue);
2 changes: 1 addition & 1 deletion backend/src/tasks/ecs-client.ts
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ class ECSClient {
// In order to use the host name "db" to access the database from the
// crossfeed-worker image, we must launch the Docker container with
// the Crossfeed backend network.
NetworkMode: 'crossfeed_backend',
NetworkMode: 'xfd_backend',
Memory: 4000000000 // Limit memory to 4 GB. We do this locally to better emulate fargate memory conditions. TODO: In the future, we could read the exact memory from SCAN_SCHEMA to better emulate memory requirements for each scan.
},
Env: [
Expand Down
12 changes: 5 additions & 7 deletions backend/src/tasks/scanExecution.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,11 @@ import * as AWS from 'aws-sdk';
import { integer } from 'aws-sdk/clients/cloudfront';

const ecs = new AWS.ECS();
let docker;
let docker: any;

if (process.env.IS_LOCAL) {
docker = require('dockerode');
const Docker = require('dockerode');
docker = new Docker();
}

const toSnakeCase = (input) => input.replace(/ /g, '-');
Expand Down Expand Up @@ -90,7 +92,7 @@ async function startLocalContainers(
// In order to use the host name "db" to access the database from the
// crossfeed-worker image, we must launch the Docker container with
// the Crossfeed backend network.
NetworkMode: 'crossfeed_backend',
NetworkMode: 'xfd_backend',
Memory: 4000000000 // Limit memory to 4 GB. We do this locally to better emulate fargate memory conditions. TODO: In the future, we could read the exact memory from SCAN_SCHEMA to better emulate memory requirements for each scan.
},
Env: [
Expand Down Expand Up @@ -158,28 +160,24 @@ export const handler: Handler = async (event) => {
process.env.SHODAN_QUEUE_URL!
);
} else if (scanType === 'dnstwist') {
desiredCount = 30;
await startDesiredTasks(
scanType,
desiredCount,
process.env.DNSTWIST_QUEUE_URL!
);
} else if (scanType === 'hibp') {
desiredCount = 20;
await startDesiredTasks(
scanType,
desiredCount,
process.env.HIBP_QUEUE_URL!
);
} else if (scanType === 'intelx') {
desiredCount = 10;
await startDesiredTasks(
scanType,
desiredCount,
process.env.INTELX_QUEUE_URL!
);
} else if (scanType === 'cybersixgill') {
desiredCount = 10;
await startDesiredTasks(
scanType,
desiredCount,
Expand Down
42 changes: 0 additions & 42 deletions backend/src/tools/consumeControlQueue.ts

This file was deleted.

10 changes: 10 additions & 0 deletions backend/src/tools/run-scanExecution.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
// Script to execute the scanExecution function
import { handler as scanExecution } from '../tasks/scanExecution';

async function localScanExecution() {
console.log('Starting...');
const payload = { scanType: 'dnstwist', desiredCount: 3 };
scanExecution(payload, {} as any, () => null);

Check warning

Code scanning / CodeQL

Superfluous trailing arguments Warning

Superfluous arguments passed to
function handler
.
}

localScanExecution();
13 changes: 7 additions & 6 deletions docs/src/documentation-pages/dev/quickstart.md
Original file line number Diff line number Diff line change
Expand Up @@ -62,22 +62,23 @@ This quickstart describes the initial setup required to run an instance of Cross
npm run pesyncdb
```

4. Start the RabbitMQ listener. This will listen for any messages sent to the queue and
trigger the scanExecution.ts function. This will stay running with this message: "Waiting for messages from ControlQueue..."
4. Send messages to RabbitMQ queue. First, edit backend/nodeMessage.js to run the desired scan and
organization. Then run below:"

```bash
cd backend
npm run control-queue
npm run send-message
```

5. Run sendMessage.js to send a sample message to the queue. Feel free to edit this file
while testing.
4. Invoke scans by running below. You can edit the backend/src/tools/run-scanExecution.ts to run the desired scan type."

```bash
cd backend
node sendMessage.js
npm run scan-exec
```

5. Observe logs in docker containers.

### Running tests

To run tests, first make sure you have already started Crossfeed with `npm start` (or, at bare minimum, that the database container is running). Then run:
Expand Down

0 comments on commit c0af290

Please sign in to comment.