Merge pull request #209 from getmaxun/develop

chore: release v0.0.3
getmaxun · Dec 4, 2024 · da9c90f · da9c90f
2 parents 36ebff4 + ee7d7b9
commit da9c90f
Show file tree

Hide file tree

Showing 27 changed files with 485 additions and 113 deletions.
diff --git a/.github/.github/COMMIT_CONVENTION.md b/.github/.github/COMMIT_CONVENTION.md
@@ -0,0 +1,130 @@
+## Git Commit Message Convention
+
+> This is adapted from [Conventional Commits 1.0.0](https://www.conventionalcommits.org/en/v1.0.0/).
+
+## Summary
+
+The Conventional Commits specification is a lightweight convention on top of commit messages.
+It provides an easy set of rules for creating an explicit commit history;
+which makes it easier to write automated tools on top of.
+This convention dovetails with [SemVer](http://semver.org),
+by describing the features, fixes, and breaking changes made in commit messages.
+
+The commit message should be structured as follows:
+
+---
+
+```
+<type>[optional scope]: <description>
+[optional body]
+[optional footer(s)]
+```
+---
+
+<br />
+The commit contains the following structural elements, to communicate intent to the
+consumers of your library:
+
+1. **fix:** a commit of the _type_ `fix` patches a bug in your codebase (this correlates with [`PATCH`](http://semver.org/#summary) in Semantic Versioning).
+1. **feat:** a commit of the _type_ `feat` introduces a new feature to the codebase (this correlates with [`MINOR`](http://semver.org/#summary) in Semantic Versioning).
+1. **BREAKING CHANGE:** a commit that has a footer `BREAKING CHANGE:`, or appends a `!` after the type/scope, introduces a breaking API change (correlating with [`MAJOR`](http://semver.org/#summary) in Semantic Versioning).
+A BREAKING CHANGE can be part of commits of any _type_.
+1. _types_ other than `fix:` and `feat:` are allowed, for example [@commitlint/config-conventional](https://github.com/conventional-changelog/commitlint/tree/master/%40commitlint/config-conventional) (based on the [the Angular convention](https://github.com/angular/angular/blob/22b96b9/CONTRIBUTING.md#-commit-message-guidelines)) recommends `build:`, `chore:`,
+  `ci:`, `docs:`, `style:`, `refactor:`, `perf:`, `test:`, and others.
+1. _footers_ other than `BREAKING CHANGE: <description>` may be provided and follow a convention similar to
+  [git trailer format](https://git-scm.com/docs/git-interpret-trailers).
+
+Additional types are not mandated by the Conventional Commits specification, and have no implicit effect in Semantic Versioning (unless they include a BREAKING CHANGE).
+<br /><br />
+A scope may be provided to a commit's type, to provide additional contextual information and is contained within parenthesis, e.g., `feat(parser): add ability to parse arrays`.
+
+## Examples
+
+### Commit message with description and breaking change footer
+```
+feat: allow provided config object to extend other configs
+
+BREAKING CHANGE: `extends` key in config file is now used for extending other config files
+```
+
+### Commit message with `!` to draw attention to breaking change
+```
+feat!: send an email to the customer when a product is shipped
+```
+
+### Commit message with scope and `!` to draw attention to breaking change
+```
+feat(api)!: send an email to the customer when a product is shipped
+```
+
+### Commit message with both `!` and BREAKING CHANGE footer
+```
+chore!: drop support for Node 6
+
+BREAKING CHANGE: use JavaScript features not available in Node 6.
+```
+
+### Commit message with no body
+```
+docs: correct spelling of CHANGELOG
+```
+
+### Commit message with scope
+```
+feat(lang): add polish language
+```
+
+### Commit message with multi-paragraph body and multiple footers
+```
+fix: prevent racing of requests
+
+Introduce a request id and a reference to latest request. Dismiss
+incoming responses other than from latest request.
+
+Remove timeouts which were used to mitigate the racing issue but are
+obsolete now.
+
+Reviewed-by: Z
+Refs: #123
+```
+
+## Specification
+
+The key words “MUST”, “MUST NOT”, “REQUIRED”, “SHALL”, “SHALL NOT”, “SHOULD”, “SHOULD NOT”, “RECOMMENDED”, “MAY”, and “OPTIONAL” in this document are to be interpreted as described in [RFC 2119](https://www.ietf.org/rfc/rfc2119.txt).
+
+1. Commits MUST be prefixed with a type, which consists of a noun, `feat`, `fix`, etc., followed
+  by the OPTIONAL scope, OPTIONAL `!`, and REQUIRED terminal colon and space.
+1. The type `feat` MUST be used when a commit adds a new feature to your application or library.
+1. The type `fix` MUST be used when a commit represents a bug fix for your application.
+1. A scope MAY be provided after a type. A scope MUST consist of a noun describing a
+  section of the codebase surrounded by parenthesis, e.g., `fix(parser):`
+1. A description MUST immediately follow the colon and space after the type/scope prefix.
+The description is a short summary of the code changes, e.g., _fix: array parsing issue when multiple spaces were contained in string_.
+1. A longer commit body MAY be provided after the short description, providing additional contextual information about the code changes. The body MUST begin one blank line after the description.
+1. A commit body is free-form and MAY consist of any number of newline separated paragraphs.
+1. One or more footers MAY be provided one blank line after the body. Each footer MUST consist of
+ a word token, followed by either a `:<space>` or `<space>#` separator, followed by a string value (this is inspired by the
+  [git trailer convention](https://git-scm.com/docs/git-interpret-trailers)).
+1. A footer's token MUST use `-` in place of whitespace characters, e.g., `Acked-by` (this helps differentiate
+  the footer section from a multi-paragraph body). An exception is made for `BREAKING CHANGE`, which MAY also be used as a token.
+1. A footer's value MAY contain spaces and newlines, and parsing MUST terminate when the next valid footer
+  token/separator pair is observed.
+1. Breaking changes MUST be indicated in the type/scope prefix of a commit, or as an entry in the
+  footer.
+1. If included as a footer, a breaking change MUST consist of the uppercase text BREAKING CHANGE, followed by a colon, space, and description, e.g.,
+_BREAKING CHANGE: environment variables now take precedence over config files_.
+1. If included in the type/scope prefix, breaking changes MUST be indicated by a
+  `!` immediately before the `:`. If `!` is used, `BREAKING CHANGE:` MAY be omitted from the footer section,
+  and the commit description SHALL be used to describe the breaking change.
+1. Types other than `feat` and `fix` MAY be used in your commit messages, e.g., _docs: updated ref docs._
+1. The units of information that make up Conventional Commits MUST NOT be treated as case sensitive by implementors, with the exception of BREAKING CHANGE which MUST be uppercase.
+1. BREAKING-CHANGE MUST be synonymous with BREAKING CHANGE, when used as a token in a footer.
+
+## Why Use Conventional Commits
+
+* Automatically generating CHANGELOGs.
+* Automatically determining a semantic version bump (based on the types of commits landed).
+* Communicating the nature of changes to teammates, the public, and other stakeholders.
+* Triggering build and publish processes.
+* Making it easier for people to contribute to your projects, by allowing them to explore
+  a more structured commit history.
diff --git a/.github/CODE_OF_CONDUCT.md b/.github/CODE_OF_CONDUCT.md
@@ -0,0 +1,12 @@
+# Contributor Code of Conduct
+As contributors and maintainers of this project, we pledge to respect all people who contribute through reporting issues, posting feature requests, updating documentation, submitting pull requests or patches, and other activities.
+
+We are committed to making participation in this project a harassment-free experience for everyone, regardless of the level of experience, gender, gender identity and expression, sexual orientation, disability, personal appearance, body size, race, age, or religion.
+
+Examples of unacceptable behavior by participants include the use of sexual language or imagery, derogatory comments or personal attacks, trolling, public or private harassment, insults, or other unprofessional conduct.
+
+Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct. Project maintainers who do not follow the Code of Conduct may be removed from the project team.
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by opening an issue or contacting one or more of the project maintainers.
+
+This Code of Conduct is adapted from the Contributor Covenant, version 1.0.0, available at http://contributor-covenant.org/version/1/0/0/
diff --git a/ENVEXAMPLE b/ENVEXAMPLE
@@ -9,14 +9,19 @@ DB_PORT=5432                            # Port for PostgreSQL (default: 5432)
 ENCRYPTION_KEY=f4d5e6a7b8c9d0e1f23456789abcdef01234567890abcdef123456789abcdef0      # Key for encrypting sensitive data (passwords and proxies)
 MINIO_ENDPOINT=minio                    # MinIO endpoint in Docker
 MINIO_PORT=9000                         # Port for MinIO (default: 9000)
+MINIO_CONSOLE_PORT=9001                 # Web UI Port for MinIO (default: 9001)
 MINIO_ACCESS_KEY=minio_access_key       # MinIO access key
 MINIO_SECRET_KEY=minio_secret_key       # MinIO secret key
 REDIS_HOST=redis                        # Redis host in Docker
 REDIS_PORT=6379                         # Redis port (default: 6379)
 
-# Backend URLs
-BACKEND_URL=http://localhost:8080       # Internal URL for backend service
-VITE_BACKEND_URL=http://localhost:8080  # URL used by frontend to connect to backend
+# Backend and Frontend URLs and Ports
+BACKEND_PORT=8080 # Port to run backend on. Needed for Docker setup 
+FRONTEND_PORT=5173 # Port to run frontend on. Needed for Docker setup 
+BACKEND_URL=http://localhost:8080       # URL on which the backend runs. You can change it based on your needs. 
+PUBLIC_URL=http://localhost:5173        # URL on which the frontend runs. You can change it based on your needs. 
+VITE_BACKEND_URL=http://localhost:8080  # URL used by frontend to connect to backend. It should always have the same value as BACKEND_URL
+VITE_PUBLIC_URL=http://localhost:5173   # URL used by backend to connect to frontend. It should always have the same value as PUBLIC_URL
 
 # Optional Google OAuth settings for Google Sheet Integration
 GOOGLE_CLIENT_ID=your_google_client_id

diff --git a/README.md b/README.md
@@ -29,14 +29,15 @@ Maxun lets you train a robot in 2 minutes and scrape the web on auto-pilot. Web
 
 <img src="https://static.scarf.sh/a.png?x-pxid=c12a77cc-855e-4602-8a0f-614b2d0da56a" />
 
-> Note: We are in early stages of development and do not support self hosting yet. You can run Maxun locally.
+> Note: Maxun is in its early stages of development and currently does not support self-hosting. However, you can run Maxun locally. Self-hosting capabilities are planned for a future release and will be available soon.
 
-# Local Setup
+# Local Installation
 ### Docker Compose
 ```
 git clone https://github.com/getmaxun/maxun
-docker-compose up -d --build
+docker-compose up -d
 ```
+You can access the frontend at http://localhost:5173/ and backend at http://localhost:8080/
 
 ### Without Docker
 1. Ensure you have Node.js, PostgreSQL, MinIO and Redis installed on your system.
@@ -76,8 +77,12 @@ You can access the frontend at http://localhost:5173/ and backend at http://loca
 
 | Variable              | Mandatory | Description                                                                                  | If Not Set                                                   |
 |-----------------------|-----------|----------------------------------------------------------------------------------------------|--------------------------------------------------------------|
+| `BACKEND_PORT`            | Yes       | Port to run backend on. Needed for Docker setup                                          | Default value: 8080 |
+| `FRONTEND_PORT`            | Yes       | Port to run frontend on. Needed for Docker setup                                        | Default value: 5173 |
 | `BACKEND_URL`            | Yes       | URL to run backend on.                                                                    | Default value: http://localhost:8080 |
 | `VITE_BACKEND_URL`            | Yes       | URL used by frontend to connect to backend                                           | Default value: http://localhost:8080 |
+| `PUBLIC_URL`            | Yes       | URL to run frontend on.                                                                    | Default value: http://localhost:5173 |
+| `VITE_PUBLIC_URL`            | Yes       | URL used by backend to connect to frontend                                           | Default value: http://localhost:5173 |
 | `JWT_SECRET`          | Yes       | Secret key used to sign and verify JSON Web Tokens (JWTs) for authentication.                | JWT authentication will not work.                            |
 | `DB_NAME`             | Yes       | Name of the Postgres database to connect to.                                                 | Database connection will fail.                               |
 | `DB_USER`             | Yes       | Username for Postgres database authentication.                                               | Database connection will fail.                               |
@@ -87,6 +92,7 @@ You can access the frontend at http://localhost:5173/ and backend at http://loca
 | `ENCRYPTION_KEY`      | Yes       | Key used for encrypting sensitive data (proxies, passwords).                                 | Encryption functionality will not work.                      |
 | `MINIO_ENDPOINT`      | Yes       | Endpoint URL for MinIO, to store Robot Run Screenshots.                                      | Connection to MinIO storage will fail.                       |
 | `MINIO_PORT`          | Yes       | Port number for MinIO service.                                                               | Connection to MinIO storage will fail.                       |
+| `MINIO_CONSOLE_PORT`          | No       | Port number for MinIO WebUI service. Needed for Docker setup.                         | Cannot access MinIO Web UI. |
 | `MINIO_ACCESS_KEY`    | Yes       | Access key for authenticating with MinIO.                                                    | MinIO authentication will fail.                              |
 | `GOOGLE_CLIENT_ID`    | No       | Client ID for Google OAuth, used for Google Sheet integration authentication.                 | Google login will not work.                                  |
 | `GOOGLE_CLIENT_SECRET`| No       | Client Secret for Google OAuth.                                                              | Google login will not work.                                  |

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -8,7 +8,7 @@ services:
       POSTGRES_PASSWORD: ${DB_PASSWORD}
       POSTGRES_DB: ${DB_NAME}
     ports:
-      - "5432:5432"
+      - "${DB_PORT:-5432}:${DB_PORT:-5432}"
     volumes:
       - postgres_data:/var/lib/postgresql/data
     healthcheck:
@@ -23,7 +23,7 @@ services:
       REDIS_HOST: ${REDIS_HOST}
       REDIS_PORT: ${REDIS_PORT}
     ports:
-      - "6379:6379"
+      - "${REDIS_PORT:-6379}:${REDIS_PORT:-6379}"
     volumes:
       - redis_data:/data
 
@@ -32,22 +32,23 @@ services:
     environment:
       MINIO_ROOT_USER: ${MINIO_ACCESS_KEY}
       MINIO_ROOT_PASSWORD: ${MINIO_SECRET_KEY}
-    command: server /data --console-address :9001
+    command: server /data --console-address :${MINIO_CONSOLE_PORT:-9001}
     ports:
-      - "9000:9000"  # API port
-      - "9001:9001"  # WebUI port
+      - "${MINIO_PORT:-9000}:${MINIO_PORT:-9000}"  # API port
+      - "${MINIO_CONSOLE_PORT:-9001}:${MINIO_CONSOLE_PORT:-9001}"  # WebUI port
     volumes:
       - minio_data:/data
 
   backend:
     #build:
       #context: .
       #dockerfile: server/Dockerfile
-    image: getmaxun/maxun-backend:v0.0.2
+    image: getmaxun/maxun-backend:v0.0.3
     ports:
-      - "8080:8080"
+      - "${BACKEND_PORT:-8080}:${BACKEND_PORT:-8080}"
     env_file: .env
     environment:
+      BACKEND_URL: ${BACKEND_URL}
       # to ensure Playwright works in Docker
       PLAYWRIGHT_BROWSERS_PATH: /ms-playwright
       PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 0
@@ -56,9 +57,8 @@ services:
       CHROMIUM_FLAGS: '--disable-gpu --no-sandbox --headless=new'
     security_opt:
     - seccomp=unconfined  # This might help with browser sandbox issues
-    # Increase shared memory size for Chromium
-    shm_size: '2gb'
-    mem_limit: 2g  # Set a 2GB memory limit
+    shm_size: '2gb' # Increase shared memory size for Chromium
+    mem_limit: 2g   # Set a 2GB memory limit
     depends_on:
       - postgres
       - redis
@@ -72,13 +72,16 @@ services:
       #build:
       #context: .
       #dockerfile: Dockerfile
-    image: getmaxun/maxun-frontend:v0.0.1
+    image: getmaxun/maxun-frontend:v0.0.2
     ports:
-      - "5173:5173"
+      - "${FRONTEND_PORT:-5173}:${FRONTEND_PORT:-5173}"
     env_file: .env
+    environment:
+      PUBLIC_URL: ${PUBLIC_URL}
+      BACKEND_URL: ${BACKEND_URL}
     volumes:
       - ./:/app             # Mount entire frontend app directory for hot reloading
-      - /app/node_modules    # Anonymous volume to prevent overwriting node_modules
+      - /app/node_modules   # Anonymous volume to prevent overwriting node_modules
     depends_on:
       - backend
 

diff --git a/esbuild.config.js b/esbuild.config.js
diff --git a/maxun-core/src/browserSide/scraper.js b/maxun-core/src/browserSide/scraper.js
@@ -249,7 +249,7 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
         }
       },
       (key) => key // Use the original key in the output
-    ));
+    )) || [];
   }
 
   /**
-Original file line number
+Diff line change
@@ Expand Up @@
             }
           },
           (key) => key // Use the original key in the output
-        ));
+        )) || [];
       }
       /**
@@ Expand Down @@