diff --git a/backend/sskai-ddb-data-api/index.mjs b/backend/sskai-ddb-data-api/index.mjs index c958c6bdc1..c57cf03fae 100644 --- a/backend/sskai-ddb-data-api/index.mjs +++ b/backend/sskai-ddb-data-api/index.mjs @@ -9,11 +9,12 @@ import { import { DeleteObjectCommand, DeleteObjectsCommand, S3Client } from "@aws-sdk/client-s3"; import { randomUUID } from 'crypto'; -const client = new DynamoDBClient({}); -const clientS3 = new S3Client({}); +const region = process.env.AWS_REGION; +const client = new DynamoDBClient({ region }); +const clientS3 = new S3Client({ region }); const dynamo = DynamoDBDocumentClient.from(client); const TableName = "sskai-data"; -const Bucket = "sskai-model-storage"; +const Bucket = process.env.BUCKET_NAME; const REQUIRED_FIELDS = ["name", "user"]; export const handler = async (event) => { @@ -137,7 +138,7 @@ export const handler = async (event) => { await clientS3.send(deleteFileCommand); await clientS3.send(deletedDirCommand); - body = { message: "Data deleted", uid: event.pathParameters.id }; + body = { message: "Data deleted", uid: event.pathParameters.id, data: deleted.Attributes}; break; } } catch (err) { diff --git a/backend/sskai-ddb-inferences-api/index.mjs b/backend/sskai-ddb-inferences-api/index.mjs index 409153706f..e27cd1a9b2 100644 --- a/backend/sskai-ddb-inferences-api/index.mjs +++ b/backend/sskai-ddb-inferences-api/index.mjs @@ -8,7 +8,8 @@ import { } from "@aws-sdk/lib-dynamodb"; import { randomUUID } from 'crypto'; -const client = new DynamoDBClient({}); +const region = process.env.AWS_REGION; +const client = new DynamoDBClient({ region }); const dynamo = DynamoDBDocumentClient.from(client); const TableName = "sskai-inferences"; const REQUIRED_FIELDS = ["name", "model", "type"]; diff --git a/backend/sskai-ddb-logs-api/index.mjs b/backend/sskai-ddb-logs-api/index.mjs index 82278a2d0c..2105e9758f 100644 --- a/backend/sskai-ddb-logs-api/index.mjs +++ b/backend/sskai-ddb-logs-api/index.mjs @@ -2,11 +2,12 @@ import { DynamoDBClient } from '@aws-sdk/client-dynamodb'; import { DynamoDBDocumentClient, PutCommand, - ScanCommand, + QueryCommand, } from "@aws-sdk/lib-dynamodb"; import { randomUUID } from 'crypto'; -const client = new DynamoDBClient({}); +const region = process.env.AWS_REGION; +const client = new DynamoDBClient({ region }); const dynamo = DynamoDBDocumentClient.from(client); const TableName = "sskai-logs" @@ -28,7 +29,7 @@ export const handler = async (event) => { user: data.user, kind_of_job: data.kind_of_job, job: data.job, - type: data.type, + name: data.name, created_at: new Date().getTime(), } }; @@ -36,18 +37,18 @@ export const handler = async (event) => { body = { message: "Log created", log: command }; break; - case "GET /logs/{id}": - command = { + case "GET /logs": + body = await dynamo.send(new QueryCommand({ TableName, - FilterExpression: '#user = :user', + IndexName: "user-index", + KeyConditionExpression: "#user = :user", ExpressionAttributeNames: { "#user": "user" }, ExpressionAttributeValues: { - ':user': event.pathParameters.id, - } - }; - body = await dynamo.send(new ScanCommand(command)); + ':user': event.headers.user, + }, + })); body = body.Items; break; } diff --git a/backend/sskai-ddb-models-api/index.mjs b/backend/sskai-ddb-models-api/index.mjs index 330b580920..f1dfb3a139 100644 --- a/backend/sskai-ddb-models-api/index.mjs +++ b/backend/sskai-ddb-models-api/index.mjs @@ -9,11 +9,12 @@ import { import { DeleteObjectCommand, DeleteObjectsCommand, S3Client } from "@aws-sdk/client-s3"; import { randomUUID } from 'crypto'; -const client = new DynamoDBClient({}); -const clientS3 = new S3Client({}); +const region = process.env.AWS_REGION; +const client = new DynamoDBClient({ region }); +const clientS3 = new S3Client({ region }); const dynamo = DynamoDBDocumentClient.from(client); const TableName = "sskai-models"; -const Bucket = "sskai-model-storage"; +const Bucket = process.env.BUCKET_NAME; const REQUIRED_FIELDS = ["name", "type", "user"]; export const handler = async (event) => { @@ -150,7 +151,7 @@ export const handler = async (event) => { await clientS3.send(deleteFileCommand); await clientS3.send(deletedDirCommand); - body = { message: "Model deleted", uid: event.pathParameters.id }; + body = { message: "Model deleted", uid: event.pathParameters.id, model: deleted.Attributes }; break; } } catch (err) { diff --git a/backend/sskai-ddb-trains-api/index.mjs b/backend/sskai-ddb-trains-api/index.mjs index 72844d4411..8c057494f7 100644 --- a/backend/sskai-ddb-trains-api/index.mjs +++ b/backend/sskai-ddb-trains-api/index.mjs @@ -8,7 +8,8 @@ import { } from "@aws-sdk/lib-dynamodb"; import { randomUUID } from 'crypto'; -const client = new DynamoDBClient({}); +const region = process.env.AWS_REGION; +const client = new DynamoDBClient({ region }); const dynamo = DynamoDBDocumentClient.from(client); const TableName = "sskai-trains"; const REQUIRED_FIELDS = ["name", "data", "user"]; diff --git a/backend/sskai-ddb-users-api/index.mjs b/backend/sskai-ddb-users-api/index.mjs index e61241863c..b464a8432b 100644 --- a/backend/sskai-ddb-users-api/index.mjs +++ b/backend/sskai-ddb-users-api/index.mjs @@ -7,7 +7,8 @@ import { } from "@aws-sdk/lib-dynamodb"; import { randomUUID } from 'crypto'; -const client = new DynamoDBClient({}); +const region = process.env.AWS_REGION; +const client = new DynamoDBClient({ region }); const dynamo = DynamoDBDocumentClient.from(client); const TableName = "sskai-users" const REQUIRED_FIELDS = ["email", "name"]; diff --git a/backend/sskai-s3-multipart-presigned-url/index.mjs b/backend/sskai-s3-multipart-presigned-url/index.mjs index 2b07178b5b..d0aec460d9 100644 --- a/backend/sskai-s3-multipart-presigned-url/index.mjs +++ b/backend/sskai-s3-multipart-presigned-url/index.mjs @@ -6,8 +6,9 @@ import { } from "@aws-sdk/client-s3"; import { getSignedUrl } from "@aws-sdk/s3-request-presigner"; -const Bucket = 'sskai-model-storage'; -const client = new S3Client({ region: 'ap-northeast-2' }); +const region = process.env.AWS_REGION; +const Bucket = process.env.BUCKET_NAME; +const client = new S3Client({ region }); export const handler = async (event) => { let body, statusCode = 200; diff --git a/backend/sskai-s3-presigned-url-api/index.mjs b/backend/sskai-s3-presigned-url-api/index.mjs index 90ac8ac11d..0ee8813ba1 100644 --- a/backend/sskai-s3-presigned-url-api/index.mjs +++ b/backend/sskai-s3-presigned-url-api/index.mjs @@ -1,6 +1,10 @@ import { PutObjectCommand, S3Client } from "@aws-sdk/client-s3"; import { getSignedUrl } from "@aws-sdk/s3-request-presigner"; +const region = process.env.AWS_REGION; +const Bucket = process.env.BUCKET_NAME; +const client = new S3Client({ region }); + export const handler = async (event) => { const headers = { 'Content-Type': "application/json" @@ -37,8 +41,7 @@ export const handler = async (event) => { headers }; - const client = new S3Client({ region: 'ap-northeast-2' }); - const command = new PutObjectCommand({ Bucket: 'sskai-model-storage', Key: `${user_uid}/${upload_type}/${uid}/${upload_type}.zip` }); + const command = new PutObjectCommand({ Bucket, Key: `${user_uid}/${upload_type}/${uid}/${upload_type}.zip` }); const url = await getSignedUrl(client, command, { expiresIn: 3600 }); return { diff --git a/frontend/sskai-console/src/api/index.jsx b/frontend/sskai-console/src/api/index.jsx index b9c2d866c6..79ac6c00e8 100644 --- a/frontend/sskai-console/src/api/index.jsx +++ b/frontend/sskai-console/src/api/index.jsx @@ -11,6 +11,13 @@ const STREAMLIT_API = import.meta.env.VITE_STREAMLIT_API_URL; // Model export const createModel = async (args) => { const res = await axios.post(`${DB_API}/models`, args).catch((err) => err); + if (res?.data) + await createLog({ + user: args.user, + name: args.name, + kind_of_job: 'model', + job: 'Model Created' + }); return res?.data?.model; }; @@ -18,6 +25,15 @@ export const updateModel = async (uid, args) => { const res = await axios .put(`${DB_API}/models/${uid}`, args) .catch((err) => err); + if (res?.data && args?.name) { + const { model } = res.data; + await createLog({ + user: model.user, + name: model.name, + kind_of_job: 'model', + job: 'Model Updated' + }); + } return res?.data?.model; }; @@ -38,7 +54,16 @@ export const getModels = async (user_uid) => { }; export const deleteModel = async (uid) => { - const res = await axios.delete(`${DB_API}/models/${uid}`); + const res = await axios.delete(`${DB_API}/models/${uid}`).catch((err) => err); + if (res?.data) { + const { model } = res.data; + await createLog({ + user: model.user, + name: model.name, + kind_of_job: 'model', + job: 'Model Deleted' + }); + } return res?.data; }; @@ -64,6 +89,13 @@ export const getData = async (user_uid) => { export const createData = async (args) => { const res = await axios.post(`${DB_API}/data`, args).catch((err) => err); + if (res?.data) + await createLog({ + user: args.user, + name: args.name, + kind_of_job: 'data', + job: 'Data Created' + }); return res?.data?.data; }; @@ -71,11 +103,29 @@ export const updateData = async (uid, args) => { const res = await axios .put(`${DB_API}/data/${uid}`, args) .catch((err) => err); + if (res?.data && args?.name) { + const { data } = res.data; + await createLog({ + user: data.user, + name: data.name, + kind_of_job: 'data', + job: 'Data Updated' + }); + } return res?.data?.data; }; export const deleteData = async (uid) => { const res = await axios.delete(`${DB_API}/data/${uid}`).catch((err) => err); + if (res?.data) { + const { data } = res.data; + await createLog({ + user: data.user, + name: data.name, + kind_of_job: 'data', + job: 'Data Deleted' + }); + } return res?.data; }; @@ -165,10 +215,17 @@ export const createUserTrain = async (args) => { }) .catch((err) => err); + await createLog({ + user: args.user, + name: args.name, + kind_of_job: 'train', + job: 'Train Created' + }); + return model; }; -export const deleteTrain = async (uid, status) => { +export const deleteTrain = async (uid, status, user, name) => { if (status !== 'Completed') await axios .post(USER_TRAIN_API, { @@ -178,6 +235,13 @@ export const deleteTrain = async (uid, status) => { .catch((err) => err); await axios.delete(`${DB_API}/trains/${uid}`); + + await createLog({ + user, + name, + kind_of_job: 'train', + job: 'Train Deleted' + }); }; // Inferences @@ -213,6 +277,13 @@ export const createSpotInference = async (args) => { return false; } + await createLog({ + user: args.user, + name: args.name, + kind_of_job: 'inference', + job: 'Endpoint (using Spot) Created' + }); + return Item; }; @@ -225,6 +296,13 @@ export const deleteSpotInference = async (args) => { }) .catch((err) => err); + await createLog({ + user: args.user, + name: args.name, + kind_of_job: 'inference', + job: 'Endpoint (using Spot) Deleted' + }); + return spot.status === 200; }; @@ -232,7 +310,15 @@ export const updateInference = async (uid, args) => { const res = await axios .put(`${DB_API}/inferences/${uid}`, args) .catch((err) => err); - + if (res?.data) { + const { inference } = res.data; + await createLog({ + user: inference.user, + name: inference.name, + kind_of_job: 'inference', + job: 'Endpoint Updated' + }); + } return res?.data; }; @@ -268,6 +354,13 @@ export const createServerlessInference = async (args) => { return false; } + await createLog({ + user: args.user, + name: args.name, + kind_of_job: 'inference', + job: 'Endpoint (using Serverless) Created' + }); + return Item; }; @@ -286,6 +379,13 @@ export const deleteServerlessInference = async (args) => { }) .catch((err) => err); + await createLog({ + user: args.user, + name: args.name, + kind_of_job: 'inference', + job: 'Endpoint (using Serverless) Deleted' + }); + return serverless.status === 200; }; @@ -305,7 +405,8 @@ export const manageStreamlit = async ({ uid, model_type, endpoint_url, - action + action, + name }) => { const res = await axios .post(STREAMLIT_API, { @@ -316,6 +417,14 @@ export const manageStreamlit = async ({ endpoint_url }) .catch((err) => err); + + await createLog({ + user, + name, + kind_of_job: 'inference', + job: `Streamlit ${action === 'create' ? 'Deployed' : 'Un-Deployed'}` + }); + return res.status === 200; }; @@ -421,3 +530,28 @@ export const uploadS3Multipart = async (upload_type, user_uid, uid, file) => { return true; }; + +// Logs + +export const getLogs = async (user_uid) => { + const res = await axios + .get(`${DB_API}/logs`, { + headers: { + user: user_uid + } + }) + .catch((err) => err); + return res?.data; +}; + +const createLog = async ({ user, name, kind_of_job, job }) => { + const capitalize = (str) => str.charAt(0).toUpperCase() + str.slice(1); + await axios + .post(`${DB_API}/logs`, { + user, + name, + kind_of_job, + job + }) + .catch((err) => err); +}; diff --git a/frontend/sskai-console/src/pages/Dashboard/index.jsx b/frontend/sskai-console/src/pages/Dashboard/index.jsx index 786a2b0fd1..04c6a05611 100644 --- a/frontend/sskai-console/src/pages/Dashboard/index.jsx +++ b/frontend/sskai-console/src/pages/Dashboard/index.jsx @@ -1,6 +1,6 @@ import { PageLayout } from '../styles.jsx'; import styled from 'styled-components'; -import { Flex, Progress, Space, Table } from 'antd'; +import { Flex, Progress, Table, Tag } from 'antd'; import { QuestionCircleOutlined } from '@ant-design/icons'; import { useEffect, useState } from 'react'; import { Section } from '../../components/Section/index.jsx'; @@ -16,6 +16,8 @@ import { YAxis } from 'recharts'; import CountUp from 'react-countup'; +import { formatTimestamp } from '../../utils/index.jsx'; +import { getLogs } from '../../api/index.jsx'; const Title = styled.div` display: flex; @@ -48,37 +50,46 @@ const Cost = styled.div` font-weight: 600; `; +const TAG_COLOR = { + data: 'red', + model: 'orange', + train: 'green', + inference: 'geekblue' +}; + const LOG_TABLE_COLUMNS = [ { title: 'Name', dataIndex: 'name', - key: 'name' + key: 'name', + width: 250 }, { - title: 'Status', - dataIndex: 'status', - key: 'status' + title: 'Type', + dataIndex: 'kind_of_job', + key: 'type', + width: 150, + render: (type) => {type.toUpperCase()} }, { title: 'Recent Job', dataIndex: 'job', - key: 'job' + key: 'job', + width: 300 }, { - title: 'Last Time', - dataIndex: 'time', - key: 'time' + title: 'Time', + dataIndex: 'created_at', + key: 'time', + width: 350, + render: (timestamp) => formatTimestamp(timestamp) }, { title: 'Action', + dataIndex: 'kind_of_job', key: 'action', - render: () => ( - - Data - Train - Inference - - ) + width: 150, + render: (kind) => {kind.toUpperCase()} } ]; @@ -136,17 +147,32 @@ const EXAMPLE_COST = [ } ]; -export default function Dashboard(props) { +export default function Dashboard() { const [savingsPercent, setSavingsPercent] = useState({ time: 0, cost: 0 }); + const [logs, setLogs] = useState([]); + const [loading, setLoading] = useState(false); + const fetchData = async () => { + setLoading(true); + // TODO: User UID Value Storing in Storage (browser's) + const user = import.meta.env.VITE_TMP_USER_UID; + const logs = await getLogs(user); + + if (!logs) return; + + logs.sort((a, b) => b.created_at - a.created_at); + setLogs(logs); + setLoading(false); + }; useEffect(() => { setSavingsPercent({ time: 95, cost: 95 }); + fetchData(); }, []); return ( @@ -227,7 +253,12 @@ export default function Dashboard(props) { Progress - +
); diff --git a/frontend/sskai-console/src/pages/Data/index.jsx b/frontend/sskai-console/src/pages/Data/index.jsx index f10aa5f0d7..a2a556d690 100644 --- a/frontend/sskai-console/src/pages/Data/index.jsx +++ b/frontend/sskai-console/src/pages/Data/index.jsx @@ -28,8 +28,6 @@ import { } from '../../api/index.jsx'; import { SearchOutlined, PlusOutlined, InboxOutlined } from '@ant-design/icons'; import { formatTimestamp } from '../../utils/index.jsx'; -import { useNavigate } from 'react-router-dom'; -import styled from 'styled-components'; const DATA_TABLE_COLUMNS = [ { @@ -45,7 +43,7 @@ const DATA_TABLE_COLUMNS = [ } ]; -export default function Data(props) { +export default function Data() { const [data, setData] = useState([]); const [selected, setSelected] = useState(''); const [filterInput, setFilterInput] = useState(''); diff --git a/frontend/sskai-console/src/pages/Inference/index.jsx b/frontend/sskai-console/src/pages/Inference/index.jsx index 26101314ec..d8fe052609 100644 --- a/frontend/sskai-console/src/pages/Inference/index.jsx +++ b/frontend/sskai-console/src/pages/Inference/index.jsx @@ -259,12 +259,14 @@ export default function Inference(props) { target.type === 'Spot' ? await deleteSpotInference({ uid: target.uid, - user: target.user + user: target.user, + name: target.name }) : await deleteServerlessInference({ uid: target.uid, user: target.user, - model: target.model + model: target.model, + name: target.name }); await fetchData(); messageApi.open({ @@ -279,6 +281,7 @@ export default function Inference(props) { const isCompleted = await manageStreamlit({ user: selectedDetail[0].user, uid: selectedDetail[0].uid, + name: selectedDetail[0].name, model_type: selectedDetail[0].model_type, endpoint_url: selectedDetail[0].endpoint, action diff --git a/frontend/sskai-console/src/pages/NewModel/index.jsx b/frontend/sskai-console/src/pages/NewModel/index.jsx index 58219225da..11266293d3 100644 --- a/frontend/sskai-console/src/pages/NewModel/index.jsx +++ b/frontend/sskai-console/src/pages/NewModel/index.jsx @@ -144,8 +144,8 @@ export default function NewModel(props) { const uploaded = modelFile[0].size < 5 * 1024 * 1024 * 1024 // 5GB - ? await uploadS3('model', user, modelFile.uid, modelFile[0]) - : await uploadS3Multipart('model', user, modelFile.uid, modelFile[0]); + ? await uploadS3('model', user, model.uid, modelFile[0]) + : await uploadS3Multipart('model', user, model.uid, modelFile[0]); if (!uploaded) { await deleteModel(model.uid); diff --git a/frontend/sskai-console/src/pages/NewModel/TrainCreateModal.jsx b/frontend/sskai-console/src/pages/Train/TrainCreateModal.jsx similarity index 100% rename from frontend/sskai-console/src/pages/NewModel/TrainCreateModal.jsx rename to frontend/sskai-console/src/pages/Train/TrainCreateModal.jsx diff --git a/frontend/sskai-console/src/pages/Train/index.jsx b/frontend/sskai-console/src/pages/Train/index.jsx index a9167e0a61..1d71dbdb62 100644 --- a/frontend/sskai-console/src/pages/Train/index.jsx +++ b/frontend/sskai-console/src/pages/Train/index.jsx @@ -1,10 +1,4 @@ -import { - ErrorMessage, - InputTitle, - PageLayout, - TableToolbox, - Title -} from '../styles.jsx'; +import { PageLayout, TableToolbox } from '../styles.jsx'; import { Section } from '../../components/Section/index.jsx'; import { Badge, @@ -26,7 +20,7 @@ import { QuestionCircleOutlined } from '@ant-design/icons'; import { calculateDuration } from '../../utils/index.jsx'; -import TrainCreateModal from '../NewModel/TrainCreateModal.jsx'; +import TrainCreateModal from './TrainCreateModal.jsx'; const STATUS_BADGE_MAPPER = { Running: 'processing', @@ -94,15 +88,23 @@ export default function Train(props) { setFetchLoading(true); setNow(Date.now()); // TODO: User UID Value Storing in Storage (browser's) - const trains = await getTrains(import.meta.env.VITE_TMP_USER_UID); + const user = import.meta.env.VITE_TMP_USER_UID; + const trains = await getTrains(user); trains.sort((a, b) => b.created_at - a.created_at); setTrains(trains.map((train) => ({ ...train, key: train.uid }))); setFetchLoading(false); }; const handleDeleteTrain = async () => { + // TODO: User UID Value Storing in Storage (browser's) + const user = import.meta.env.VITE_TMP_USER_UID; if (!selected.length) return; - await deleteTrain(selected[0], selectedDetail[0]?.status); + await deleteTrain( + selected[0], + selectedDetail[0]?.status, + user, + selectedDetail[0]?.name + ); await fetchData(); messageApi.open({ type: 'success',