Skip to content

Commit

Permalink
Merge branch 'main' into feat/dynamic_scrapper
Browse files Browse the repository at this point in the history
  • Loading branch information
clatapie authored Apr 29, 2024
2 parents 27d1822 + f6ab2b8 commit 1a96668
Show file tree
Hide file tree
Showing 8 changed files with 199 additions and 73 deletions.
45 changes: 45 additions & 0 deletions .ci/collect_mapdl_logs.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
#!/bin/bash
if [[ $MAPDL_VERSION == *"ubuntu"* ]] ; then
echo "It is an ubuntu based image"
export FILE=/jobs/file
export WDIR='/jobs/'

else
echo "It is a CentOS based image"
export FILE=file
export WDIR=""

fi;


mkdir "$LOG_NAMES" && echo "Successfully generated directory $LOG_NAMES"

####
echo "Collecting MAPDL logs..."

(docker exec "$MAPDL_INSTANCE" /bin/bash -c "mkdir -p /mapdl_logs && echo 'Successfully created directory inside docker container'") || echo "Failed to create a directory inside docker container for logs."
(docker exec "$MAPDL_INSTANCE" /bin/bash -c "if compgen -G '$FILE*.out' > /dev/null ;then cp -f /file*.out /mapdl_logs && echo 'Successfully copied out files.'; fi") || echo "Failed to copy the 'out' files into a local file"
(docker exec "$MAPDL_INSTANCE" /bin/bash -c "if compgen -G '$FILE*.err' > /dev/null ;then cp -f /file*.err /mapdl_logs && echo 'Successfully copied err files.'; fi") || echo "Failed to copy the 'err' files into a local file"
(docker exec "$MAPDL_INSTANCE" /bin/bash -c "if compgen -G '$FILE*.log' > /dev/null ;then cp -f /file*.log /mapdl_logs && echo 'Successfully copied log files.'; fi") || echo "Failed to copy the 'log' files into a local file"
(docker exec "$MAPDL_INSTANCE" /bin/bash -c "if compgen -G '$WDIR*.crash' > /dev/null ;then cp -f /*.crash /mapdl_logs && echo 'Successfully copied crash files.'; fi") || echo "Failed to copy the 'crash' files into a local file"

docker cp "$MAPDL_INSTANCE":/mapdl_logs/. ./"$LOG_NAMES"/. || echo "Failed to copy the 'log-build-docs' files into a local directory"

####
echo "Collecting local build logs..."

echo "Collecting docker run log..."
cp log.txt ./"$LOG_NAMES"/log.txt || echo "MAPDL run docker log not found."

echo "Copying docker launch log..."
cp mapdl_launch.log ./"$LOG_NAMES"/mapdl_launch.log || echo "MAPDL launch docker log not found."
# cp mapdl_launch_1.log ./"$LOG_NAMES"/mapdl_launch_1.log || echo "MAPDL launch docker log not found."

echo "Collecting file structure..."
ls -R > ./"$LOG_NAMES"/files_structure.txt || echo "Failed to copy file structure to a file"

echo "Collecting docker file structure..."
docker exec "$MAPDL_INSTANCE" /bin/bash -c "ls -R" > ./"$LOG_NAMES"/docker_files_structure.txt || echo "Failed to copy the docker structure into a local file"

echo "Tar files..."
tar cvzf ./"$LOG_NAMES".tgz ./"$LOG_NAMES" || echo "Failed to compress"
20 changes: 20 additions & 0 deletions .ci/display_logs.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
#!/bin/bash

echo "::group:: Display files structure" && ls -R && echo "::endgroup::"


echo "::group:: Display files structure" && docker exec "$MAPDL_INSTANCE" /bin/bash -c "ls -R" && echo "::endgroup::" || echo "Failed to display the docker structure."


echo "::group:: Display docker run log" && (cat log.txt | echo "The file 'log.txt' has not been generated") && echo "::endgroup::"

# Displaying MAPDL files
FILE_PAT=./"$LOG_NAMES"/*.err
if compgen -G "$FILE_PAT" > /dev/null ;then for f in "$FILE_PAT"; do echo "::group:: Error file $f" && cat "$f" && echo "::endgroup::" ; done; fi || echo "Failed to display the 'out' files."

FILE_PAT=./"$LOG_NAMES"/*.log
if compgen -G "$FILE_PAT" > /dev/null ;then for f in "$FILE_PAT"; do echo "::group:: Log file $f" && cat "$f" && echo "::endgroup::" ; done; fi || echo "Failed to display the 'err' files."

FILE_PAT=./"$LOG_NAMES"/*.out
if compgen -G "$FILE_PAT" > /dev/null ;then for f in "$FILE_PAT"; do echo "::group:: Output file $f" && cat "$f" && echo "::endgroup::" ; done; fi || echo "Failed to display the 'log' files."

50 changes: 42 additions & 8 deletions .ci/start_mapdl.sh
Original file line number Diff line number Diff line change
@@ -1,7 +1,40 @@
#!/bin/bash
echo "MAPDL Instance name: $INSTANCE_NAME"
echo "MAPDL_VERSION: $MAPDL_VERSION"

export MAPDL_IMAGE="$MAPDL_PACKAGE:$MAPDL_VERSION"
echo "MAPDL_IMAGE: $MAPDL_IMAGE"
docker pull "$MAPDL_IMAGE"

export MAJOR=$(echo "$MAPDL_VERSION" | head -c 3 | tail -c 2)
export MINOR=$(echo "$MAPDL_VERSION" | head -c 5 | tail -c 1)

export VERSION="$MAJOR$MINOR"
echo "MAPDL VERSION: $VERSION"


if [[ $MAPDL_VERSION == *"latest-ubuntu"* ]]; then
echo "It is latest-ubuntu. Using 'ansys' script to launch"
export EXEC_PATH=ansys
# export P_SCHEMA=/ansys_inc/ansys/ac4/schema

elif [[ $MAPDL_VERSION == *"ubuntu"* ]] ; then
echo "It is an ubuntu based image"
export EXEC_PATH=/ansys_inc/v$VERSION/ansys/bin/mapdl
export P_SCHEMA=/ansys_inc/v$VERSION/ansys/ac4/schema

else
echo "It is a CentOS based image"
export EXEC_PATH=/ansys_inc/ansys/bin/mapdl
export P_SCHEMA=/ansys_inc/ansys/ac4/schema
fi;

echo "EXEC_PATH: $EXEC_PATH"
echo "P_SCHEMA: $P_SCHEMA"

docker run \
--name mapdl \
--entrypoint "/bin/bash" \
--name "$INSTANCE_NAME" \
--restart always \
--health-cmd="ps aux | grep \"[/]ansys_inc/.*ansys\.e.*grpc\" -q && echo 0 || echo 1" \
--health-interval=0.5s \
Expand All @@ -12,12 +45,13 @@ docker run \
-e ANSYS_LOCK="OFF" \
-p "$PYMAPDL_PORT":50052 \
-p "$PYMAPDL_DB_PORT":50055 \
--shm-size=1gb \
--shm-size=2gb \
-e I_MPI_SHM_LMT=shm \
-e P_SCHEMA=/ansys_inc/ansys/ac4/schema \
--oom-kill-disable \
--memory=6656MB \
-e P_SCHEMA="$P_SCHEMA" \
-w /jobs \
-u=0:0 \
--memory=6656MB \
--memory-swap=16896MB \
"$MAPDL_IMAGE" \
-"$DISTRIBUTED_MODE" -np 2 > log.txt &
grep -q 'Server listening on' <(timeout 60 tail -f log.txt)
"$MAPDL_IMAGE" "$EXEC_PATH" -grpc -dir /jobs -"$DISTRIBUTED_MODE" -np 2 > log.txt &

# grep -q 'Server listening on' <(timeout 60 tail -f log.txt)
19 changes: 19 additions & 0 deletions .ci/waiting_services.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
#!/bin/bash
echo "Waiting for the MAPDL service to be up..."
nc -v -z localhost "$PYMAPDL_PORT"
echo "::group:: ps aux Output" && ps aux && echo "::endgroup::"

echo "Waiting for MAPDL port is open..."
echo "::group:: Waiting for the MAPDL port to be open..."
while ! nc -z localhost "$PYMAPDL_PORT"; do
sleep 0.1
done
echo "::endgroup::"
echo "MAPDL service is up!"

echo "::group:: Waiting for the DPF port to be open..."
while ! nc -z localhost "$DPF_PORT"; do
sleep 0.1
done
echo "::endgroup::"
echo "DPF service is up!"
119 changes: 66 additions & 53 deletions .github/workflows/ci_cd.yml
Original file line number Diff line number Diff line change
Expand Up @@ -12,10 +12,16 @@ on:
- main

env:
DPF_PORT: 21002
DPF_START_SERVER: False
MAIN_PYTHON_VERSION: '3.10'
MAPDL_PACKAGE: ghcr.io/ansys/mapdl
MAPDL_IMAGE_VERSION_DOCS_BUILD: v24.1-ubuntu-student
ON_CI: True
PYANSYS_OFF_SCREEN: True
RESET_EXAMPLES_CACHE: 1
RESET_DOC_BUILD_CACHE: 1
USE_CACHE: true
USE_CACHE: False

concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
Expand Down Expand Up @@ -59,88 +65,101 @@ jobs:
doc-build:
name: "Documentation building"
needs: [style, doc-style]
timeout-minutes: 60
outputs:
PYMAPDL_VERSION: ${{ steps.version.outputs.PYMAPDL_VERSION }}
runs-on: ubuntu-latest
env:
PYMAPDL_PORT: 21000 # default won't work on GitHub runners
PYMAPDL_DB_PORT: 21001 # default won't work on GitHub runners
PYMAPDL_START_INSTANCE: FALSE
DOCKER_PACKAGE: ghcr.io/ansys/mapdl
DOCKER_IMAGE_VERSION_DOCS_BUILD: v23.1.0
DPF_PORT: 21002

ON_DOCUMENTATION: TRUE

steps:
- name: "Install Git and checkout project"
uses: actions/checkout@v4

- name: "Login in Github Container registry"
uses: docker/[email protected]
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}

- name: "Pull, launch, and validate MAPDL service"
id: start_mapdl
env:
LICENSE_SERVER: ${{ secrets.LICENSE_SERVER }}
MAPDL_VERSION: ${{ env.MAPDL_IMAGE_VERSION_DOCS_BUILD }}
DISTRIBUTED_MODE: "dmp"
run: |
export INSTANCE_NAME=MAPDL_0
.ci/start_mapdl.sh & export DOCKER_PID=$!
echo "Launching MAPDL service at PID: $DOCKER_PID"
echo "DOCKER_PID=$(echo $DOCKER_PID)" >> $GITHUB_OUTPUT
- name: "DPF Server Activation"
run: |
$(docker pull ghcr.io/ansys/dpf-core:22.2dev && docker run -d --name dpfserver -p ${{ env.DPF_PORT }}:50052 ghcr.io/ansys/dpf-core:22.2dev && echo "DPF Server active on port ${{ env.DPF_PORT }}.") &
- name: "Install OS packages"
run: |
sudo apt-get update
sudo apt install zip pandoc libgl1-mesa-glx xvfb texlive-latex-extra latexmk graphviz texlive-xetex libgomp1
- name: "Set up Python using cache"
uses: actions/setup-python@v5
with:
python-version: ${{ env.MAIN_PYTHON_VERSION }}
cache: 'pip'
cache-dependency-path: 'requirements/requirements_doc.txt'

- name: "Install system dependencies"
run: |
sudo apt-get update
sudo apt install zip pandoc libgl1-mesa-glx xvfb texlive-latex-extra latexmk graphviz texlive-xetex libgomp1

- name: "Install Python requirements"
run: pip install -r requirements/requirements_doc.txt
run: |
pip uninstall ansys-mapdl-core
pip install -r requirements/requirements_doc.txt
- name: "Test virtual framebuffer"
run: |
xvfb-run python .ci/display_test.py
- name: "Login in Github Container registry"
uses: docker/[email protected]
with:
registry: ghcr.io
username: ${{ secrets.GH_USERNAME }}
password: ${{ secrets.GITHUB_TOKEN }}

- name: "Pull, launch, and validate MAPDL service"
env:
LICENSE_SERVER: ${{ secrets.LICENSE_SERVER }}
MAPDL_IMAGE: '${{ env.DOCKER_PACKAGE }}:${{ env.DOCKER_IMAGE_VERSION_DOCS_BUILD }}'
run: .ci/start_mapdl.sh
- name: "Retrieve PyMAPDL version"
id: version
run: |
echo "PYMAPDL_VERSION=$(python -c 'from ansys.mapdl.core import __version__; print(__version__)')" >> $GITHUB_ENV
echo "PYMAPDL_VERSION=$(python -c 'from ansys.mapdl.core import __version__; print(__version__)')" >> $GITHUB_OUTPUT
echo "PyMAPDL version is: $(python -c "from ansys.mapdl.core import __version__; print(__version__)")"
id: version
- name: "DPF Server Activation"
run: |
docker pull ghcr.io/ansys/dpf-core:22.2dev
docker run -d --name dpfserver -p ${{ env.DPF_PORT }}:50052 ghcr.io/ansys/dpf-core:22.2dev && echo "DPF Server active on port ${{ env.DPF_PORT }}."
- name: "Cache Verification Manual examples"
uses: actions/cache@v4
if: ${{ env.USE_CACHE }} == 'true'
if: env.USE_CACHE == true
with:
path: doc/source/verif-manual
key: Examples-v${{ env.RESET_EXAMPLES_CACHE }}-${{ env.PYMAPDL_VERSION }}-${{ github.sha }}
key: Examples-v${{ env.RESET_EXAMPLES_CACHE }}-${{ steps.version.outputs.PYMAPDL_VERSION }}-${{ github.sha }}
restore-keys: |
Examples-v${{ env.RESET_EXAMPLES_CACHE }}-${{ env.PYMAPDL_VERSION }}
Examples-v${{ env.RESET_EXAMPLES_CACHE }}-${{ steps.version.outputs.PYMAPDL_VERSION }}
- name: "Cache docs build directory"
uses: actions/cache@v4
if: ${{ env.USE_CACHE }} == 'true'
if: env.USE_CACHE == true
with:
path: doc/_build
key: doc-build-v${{ env.RESET_DOC_BUILD_CACHE }}-${{ env.PYMAPDL_VERSION }}-${{ github.sha }}
restore-keys: |
doc-build-v${{ env.RESET_DOC_BUILD_CACHE }}-${{ env.PYMAPDL_VERSION }}
- name: "Waiting for the services to be up"
timeout-minutes: 15
run: |
.ci/waiting_services.sh
- name: "Run Ansys documentation building action"
uses: ansys/actions/doc-build@v5
with:
requires-xvfb: true
python-version: ${{ env.MAIN_PYTHON_VERSION }}
checkout: false
skip-install: true
sphinxopts: -j auto
sphinxopts: -j auto -W --keep-going
use-python-cache: False
check-links: False

Expand All @@ -153,29 +172,23 @@ jobs:
folder: doc/_build/html
clean: true
single-commit: true

- name: "Display files structure"
if: always()
run: |
mkdir logs-build-docs
echo "::group:: Display files structure" && ls -R && echo "::endgroup::"
ls -R > ./logs-build-docs/files_structure.txt

- name: "Display docker files structures"
- name: "Display Docker files structures"
if: always()
env:
MAPDL_INSTANCE: MAPDL_0
LOG_NAMES: logs-build-docs
run: |
echo "::group:: Display files structure" && docker exec mapdl /bin/bash -c "ls -R" && echo "::endgroup::"
docker exec mapdl /bin/bash -c "ls -R" > ./logs-build-docs/docker_files_structure.txt
.ci/display_logs.sh
- name: "Collect MAPDL logs on failure"
if: always()
env:
MAPDL_VERSION: ${{ env.MAPDL_IMAGE_VERSION_DOCS_BUILD }}
MAPDL_INSTANCE: MAPDL_0
LOG_NAMES: logs-build-docs
run: |
docker exec mapdl /bin/bash -c "mkdir -p /mapdl_logs && echo 'Successfully created directory inside docker container'"
docker exec mapdl /bin/bash -c "if compgen -G 'file*.out' > /dev/null ;then cp -f /file*.out /mapdl_logs && echo 'Successfully copied out files.'; fi"
docker exec mapdl /bin/bash -c "if compgen -G 'file*.err' > /dev/null ;then cp -f /file*.err /mapdl_logs && echo 'Successfully copied err files.'; fi"
docker exec mapdl /bin/bash -c "if compgen -G 'file*.log' > /dev/null ;then cp -f /file*.log /mapdl_logs && echo 'Successfully copied log files.'; fi"
docker exec mapdl /bin/bash -c "if compgen -G '*.crash' > /dev/null ;then cp -f /*.crash /mapdl_logs && echo 'Successfully copied crash files.'; fi"
docker cp mapdl:/mapdl_logs/. ./logs-build-docs/.
.ci/collect_mapdl_logs.sh
- name: "Tar logs"
if: always()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ Starting MAPDL as a service and importing an external model
pext = 0.24 # Differential external pressure (MPa)
# start MAPDL as a service
mapdl = launch_mapdl(run_location="D:\PyAnsys\Examples\Buckling_PostBuckling_TD21")
mapdl = launch_mapdl()
print(mapdl)
mapdl.filname("buckling") # change filename
Expand Down
Loading

0 comments on commit 1a96668

Please sign in to comment.