diff --git a/.buildkite/Buildkite.csproj b/.buildkite/Buildkite.csproj new file mode 100644 index 00000000000..212bdcfce59 --- /dev/null +++ b/.buildkite/Buildkite.csproj @@ -0,0 +1,13 @@ + + + + net5.0 + + + + + .dockerignore + + + + diff --git a/.buildkite/DockerFile b/.buildkite/DockerFile new file mode 100644 index 00000000000..9b34f4d228c --- /dev/null +++ b/.buildkite/DockerFile @@ -0,0 +1,51 @@ +ARG DOTNET_VERSION=6.0.403 +FROM mcr.microsoft.com/dotnet/sdk:${DOTNET_VERSION} AS elasticsearch-net-build + +ARG USER_ID +ARG GROUP_ID + +RUN echo addgroup --gid $GROUP_ID user +RUN addgroup --gid $GROUP_ID user +RUN adduser --disabled-password --gecos '' --uid $USER_ID --gid $GROUP_ID user + +WORKDIR /sln +RUN chown $GROUP_ID:$USER_ID $(pwd) +COPY ./*.sln ./nuget.config ./*.Build.props ./*.Build.targets ./ + +COPY ./dotnet-tools.json ./ + +# todo standardize on Build.props as Directory.Build.props needs that form +COPY ./src/*.Build.props ./src/ +COPY ./tests/*.Build.props ./tests/ + +# Copy the main source project files +COPY src/*/*.?sproj ./src/ +COPY tests/*/*.?sproj ./tests/ +COPY benchmarks/*/*.?sproj ./benchmarks/ + +# this puts the project files back into original location since COPY flattens +RUN for file in $(find . -name "*.?sproj"); do echo mkdir -p $(dirname $file)/$(basename ${file%.*})/ && echo mv $file $(dirname $file)/$(basename ${file%.*})/; done +RUN for file in $(find . -name "*.?sproj"); do mkdir -p $(dirname $file)/$(basename ${file%.*})/ && mv $file $(dirname $file)/$(basename ${file%.*})/; done + +# copy these manually since these do not follow a pattern under src or tests +COPY build/scripts/scripts.fsproj ./build/scripts/ +COPY .buildkite/Buildkite.csproj ./.buildkite/ + +# Install app dependencies +RUN dotnet restore +RUN dotnet tool restore + +RUN chown -R $GROUP_ID:$USER_ID $(pwd) +RUN chown -R $GROUP_ID:$USER_ID /tmp/NuGetScratch + +# copy relevant files (see .dockerignore) +COPY . . + +# making sure enough git info is available inside the container +RUN git rev-parse HEAD . + +# USER user + + + + diff --git a/.buildkite/certs/ca.crt b/.buildkite/certs/ca.crt new file mode 100755 index 00000000000..71f9bfc818a --- /dev/null +++ b/.buildkite/certs/ca.crt @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDSjCCAjKgAwIBAgIVAJQLm8V2LcaCTHUcoIfO+KL63nG3MA0GCSqGSIb3DQEB +CwUAMDQxMjAwBgNVBAMTKUVsYXN0aWMgQ2VydGlmaWNhdGUgVG9vbCBBdXRvZ2Vu +ZXJhdGVkIENBMB4XDTIwMDIyNjA1NTA1N1oXDTIzMDIyNTA1NTA1N1owNDEyMDAG +A1UEAxMpRWxhc3RpYyBDZXJ0aWZpY2F0ZSBUb29sIEF1dG9nZW5lcmF0ZWQgQ0Ew +ggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDYyajkPvGtUOE5M1OowQfB +kWVrWjo1+LIxzgCeRHp0YztLtdVJ0sk2xoSrt2uZpxcPepdyOseLTjFJex1D2yCR +AEniIqcFif4G72nDih2LlbhpUe/+/MTryj8ZTkFTzI+eMmbQi5FFMaH+kwufmdt/ +5/w8YazO18SxxJUlzMqzfNUrhM8vvvVdxgboU7PWhk28wZHCMHQovomHmzclhRpF +N0FMktA98vHHeRjH19P7rNhifSd7hZzoH3H148HVAKoPgqnZ6vW2O2YfAWOP6ulq +cyszr57p8fS9B2wSdlWW7nVHU1JuKcYD67CxbBS23BeGFgCj4tiNrmxO8S5Yf85v +AgMBAAGjUzBRMB0GA1UdDgQWBBSWAlip9eoPmnG4p4OFZeOUBlAbNDAfBgNVHSME +GDAWgBSWAlip9eoPmnG4p4OFZeOUBlAbNDAPBgNVHRMBAf8EBTADAQH/MA0GCSqG +SIb3DQEBCwUAA4IBAQA19qqrMTWl7YyId+LR/QIHDrP4jfxmrEELrAL58q5Epc1k +XxZLzOBSXoBfBrPdv+3XklWqXrZjKWfdkux0Xmjnl4qul+srrZDLJVZG3I7IrITh +AmQUmL9MuPiMnAcxoGZp1xpijtW8Qmd2qnambbljWfkuVaa4hcVRfrAX6TciIQ21 +bS5aeLGrPqR14h30YzDp0RMmTujEa1o6ExN0+RSTkE9m89Q6WdM69az8JW7YkWqm +I+UCG3TcLd3TXmN1zNQkq4y2ObDK4Sxy/2p6yFPI1Fds5w/zLfBOvvPQY61vEqs8 +SCCcQIe7f6NDpIRIBlty1C9IaEHj7edyHjF6rtYb +-----END CERTIFICATE----- diff --git a/.buildkite/certs/ca.key b/.buildkite/certs/ca.key new file mode 100644 index 00000000000..dfc41b558d1 --- /dev/null +++ b/.buildkite/certs/ca.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpgIBAAKCAQEA2Mmo5D7xrVDhOTNTqMEHwZFla1o6NfiyMc4AnkR6dGM7S7XV +SdLJNsaEq7drmacXD3qXcjrHi04xSXsdQ9sgkQBJ4iKnBYn+Bu9pw4odi5W4aVHv +/vzE68o/GU5BU8yPnjJm0IuRRTGh/pMLn5nbf+f8PGGsztfEscSVJczKs3zVK4TP +L771XcYG6FOz1oZNvMGRwjB0KL6Jh5s3JYUaRTdBTJLQPfLxx3kYx9fT+6zYYn0n +e4Wc6B9x9ePB1QCqD4Kp2er1tjtmHwFjj+rpanMrM6+e6fH0vQdsEnZVlu51R1NS +binGA+uwsWwUttwXhhYAo+LYja5sTvEuWH/ObwIDAQABAoIBAQC8QDGnMnmPdWJ+ +13FYY3cmwel+FXXjFDk5QpgK15A2rUz6a8XxO1d7d1wR+U84uH4v9Na6XQyWjaoD +EyPQnuJiyAtgkZLUHoY244PGR5NsePEQlBSCKmGeF5w/j1LvP/2e9EmP4wKdQYJY +nLxFNcgEBCFnFbKIU5n8fKa/klybCrwlBokenyBro02tqH4LL7h1YMRRrl97fv1V +e/y/0WcMN+KnMglfz6haimBRV2yamCCHHmBImC+wzOgT/quqlxPfI+a3ScHxuA65 +3QyCavaqlPh+T3lXnN/Na4UWqFtzMmwgJX2x1zM5qiln46/JoDiXtagvV43L3rNs +LhPRFeIRAoGBAPhEB7nNpEDNjIRUL6WpebWS9brKAVY7gYn7YQrKGhhCyftyaiBZ +zYgxPaJdqYXf+DmkWlANGoYiwEs40QwkR/FZrvO4+Xh3n3dgtl59ZmieuoQvDsG+ +RYIj+TfBaqhewhZNMMl7dxz7DeyQhyRCdsvl3VqJM0RuOsIrzrhCIEItAoGBAN+K +lgWI7swDpOEaLmu+IWMkGImh1LswXoZqIgi/ywZ7htZjPzidOIeUsMi+lrYsKojG +uU3sBxASsf9kYXDnuUuUbGT5M/N2ipXERt7klUAA/f5sg1IKlTrabaN/HGs/uNtf +Efa8v/h2VyTurdPCJ17TNpbOMDwX1qGM62tyt2CLAoGBAIHCnP8iWq18QeuQTO8b +a3/Z9hHRL22w4H4MI6aOB6GSlxuTq6CJD4IVqo9IwSg17fnCy2l3z9s4IqWuZqUf ++XJOW8ELd2jdrT2qEOfGR1Z7UCVyqxXcq1vgDYx0zZh/HpalddB5dcJx/c8do2Ty +UEE2PcHqYB9uNcvzNbLc7RtpAoGBALbuU0yePUTI6qGnajuTcQEPpeDjhRHWSFRZ +ABcG1N8uMS66Mx9iUcNp462zgeP8iqY5caUZtMHreqxT+gWKK7F0+as7386pwElF +QPXgO18QMMqHBIQb0vlBjJ1SRPBjSiSDTVEML1DljvTTOX7kEJHh6HdKrmBO5b54 +cqMQUo53AoGBAPVWRPUXCqlBz914xKna0ZUh2aesRBg5BvOoq9ey9c52EIU5PXL5 +0Isk8sWSsvhl3tjDPBH5WuL5piKgnCTqkVbEHmWu9s1T57Mw6NuxlPMLBWvyv4c6 +tB9brOxv0ui3qGMuBsBoDKbkNnwXyOXLyFg7O+H4l016A3mLQzJM+NGV +-----END RSA PRIVATE KEY----- diff --git a/.buildkite/certs/testnode.crt b/.buildkite/certs/testnode.crt new file mode 100755 index 00000000000..a49dfd77507 --- /dev/null +++ b/.buildkite/certs/testnode.crt @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDIzCCAgugAwIBAgIVAMTO6uVx9dLox2t0lY4IcBKZXb5WMA0GCSqGSIb3DQEB +CwUAMDQxMjAwBgNVBAMTKUVsYXN0aWMgQ2VydGlmaWNhdGUgVG9vbCBBdXRvZ2Vu +ZXJhdGVkIENBMB4XDTIwMDIyNjA1NTA1OVoXDTIzMDIyNTA1NTA1OVowEzERMA8G +A1UEAxMIaW5zdGFuY2UwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDK +YLTOikVENiN/qYupOsoXd7VYYnryyfCC/dK4FC2aozkbqjFzBdvPGAasoc4yEiH5 +CGeXMgJuOjk1maqetmdIsw00j4oHJviYsnGXzxxS5swhD7spcW4Uk4V4tAUzrbfT +vW/2WW/yYCLe5phVb2chz0jL+WYb4bBmdfs/t6RtP9RqsplYAmVp3gZ6lt2YNtvE +k9gz0TVk3DuO1TquIClfRYUjuywS6xDSvxJ8Jl91EfDWM8QU+9F+YAtiv74xl2U3 +P0wwMqNvMxf9/3ak3lTQGsgO4L6cwbKpVLMMzxSVunZz/sgl19xy3qHHz1Qr2MjJ +/2c2J7vahUL4NPRkjJClAgMBAAGjTTBLMB0GA1UdDgQWBBS2Wn8E2VZv4oenY+pR +O8G3zfQXhzAfBgNVHSMEGDAWgBSWAlip9eoPmnG4p4OFZeOUBlAbNDAJBgNVHRME +AjAAMA0GCSqGSIb3DQEBCwUAA4IBAQAvwPvCiJJ6v9jYcyvYY8I3gP0oCwrylpRL +n91UlgRSHUmuAObyOoVN5518gSV/bTU2SDrstcLkLFxHvnfpoGJoxsQEHuGxwDRI +nhYNd62EKLerehNM/F9ILKmvTh8f6QPCzjUuExTXv+63l2Sr6dBS7FHsGs6UKUYO +llM/y9wMZ1LCuZuBg9RhtgpFXRSgDM9Z7Begu0d/BPX9od/qAeZg9Arz4rwUiCN4 +IJOMEBEPi5q1tgeS0Fb1Grpqd0Uz5tZKtEHNKzLG+zSMmkneL62Nk2HsmEFZKwzg +u2pU42UaUE596G6o78s1aLn9ICcElPHTjiuZNSiyuu9IzvFDjGQw +-----END CERTIFICATE----- diff --git a/.buildkite/certs/testnode.key b/.buildkite/certs/testnode.key new file mode 100755 index 00000000000..82efeecb949 --- /dev/null +++ b/.buildkite/certs/testnode.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEogIBAAKCAQEAymC0zopFRDYjf6mLqTrKF3e1WGJ68snwgv3SuBQtmqM5G6ox +cwXbzxgGrKHOMhIh+QhnlzICbjo5NZmqnrZnSLMNNI+KByb4mLJxl88cUubMIQ+7 +KXFuFJOFeLQFM623071v9llv8mAi3uaYVW9nIc9Iy/lmG+GwZnX7P7ekbT/UarKZ +WAJlad4GepbdmDbbxJPYM9E1ZNw7jtU6riApX0WFI7ssEusQ0r8SfCZfdRHw1jPE +FPvRfmALYr++MZdlNz9MMDKjbzMX/f92pN5U0BrIDuC+nMGyqVSzDM8Ulbp2c/7I +Jdfcct6hx89UK9jIyf9nNie72oVC+DT0ZIyQpQIDAQABAoIBADAh7f7NjgnaInlD +ds8KB3SraPsbeQhzlPtiqRJU4j/MIFH/GYG03AGWQkget67a9y+GmzSvlTpoKKEh +6h2TXl9BDpv4o6ht0WRn1HJ5tM/Wyqf2WNpTew3zxCPgFPikkXsPrChYPzLTQJfp +GkP/mfTFmxfAOlPZSp4j41zVLYs53eDkAegFPVfKSr1XNNJ3QODLPcIBfxBYsiC9 +oU+jRW8xYuj31cEl5k5UqrChJ1rm3mt6cguqXKbISuoSvi13gXI6DccqhuLAU+Kr +ib2XYrRP+pWocZo/pM9WUVoNGtFxfY88sAQtvG6gDKo2AURtFyq84Ow0h9mdixV/ +gRIDPcECgYEA5nEqE3OKuG9WuUFGXvjtn4C0F6JjflYWh7AbX51S4F6LKrW6/XHL +Rg4BtF+XReT7OQ6llsV8kZeUxsUckkgDLzSaA8lysNDV5KkhAWHfRqH//QKFbqZi +JL9t3x63Qt81US8s2hQk3khPYTRM8ZB3xHiXvZYSGC/0x/DxfEO3QJECgYEA4NK5 +sxtrat8sFz6SK9nWEKimPjDVzxJ0hxdX4tRq/JdOO5RncawVqt6TNP9gTuxfBvhW +MhJYEsQj8iUoL1dxo9d1eP8HEANNV0iX5OBvJNmgBp+2OyRSyr+PA55+wAxYuAE7 +QKaitOjW57fpArNRt2hQyiSzTuqUFRWTWJHCWNUCgYAEurPTXF6vdFGCUc2g61jt +GhYYGhQSpq+lrz6Qksj9o9MVWE9zHh++21C7o+6V16I0RJGva3QoBMVf4vG4KtQt +5tV2WG8LI+4P2Ey+G4UajP6U8bVNVQrUmD0oBBhcvfn5JY+1Fg6/pRpD82/U0VMz +7AmpMWhDqNBMPiymkTk0kQKBgCuWb05cSI0ly4SOKwS5bRk5uVFhYnKNH255hh6C +FGP4acB/WzbcqC7CjEPAJ0nl5d6SExQOHmk1AcsWjR3wlCWxxiK5PwNJwJrlhh1n +reS1FKN0H36D4lFQpkeLWQOe4Sx7gKNeKzlr0w6Fx3Uwku0+Gju2tdTdAey8jB6l +08opAoGAEe1AuR/OFp2xw6V8TH9UHkkpGxy+OrXI6PX6tgk29PgB+uiMu4RwbjVz +1di1KKq2XecAilVbnyqY+edADxYGbSnci9x5wQRIebfMi3VXKtV8NQBv2as6qwtW +JDcQUWotOHjpdvmfJWWkcBhbAKrgX8ukww00ZI/lC3/rmkGnBBg= +-----END RSA PRIVATE KEY----- diff --git a/.buildkite/certs/testnode_san.crt b/.buildkite/certs/testnode_san.crt new file mode 100644 index 00000000000..8abba55b597 --- /dev/null +++ b/.buildkite/certs/testnode_san.crt @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDVjCCAj6gAwIBAgIULh42yRefYlRRl1hvt055LrUH0HwwDQYJKoZIhvcNAQEL +BQAwNDEyMDAGA1UEAxMpRWxhc3RpYyBDZXJ0aWZpY2F0ZSBUb29sIEF1dG9nZW5l +cmF0ZWQgQ0EwHhcNMjAwMjI4MDMzNzIwWhcNMjMwMjI3MDMzNzIwWjATMREwDwYD +VQQDEwhpbnN0YW5jZTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAIUP +t267NN21z+3ukajej8eojSXwP6zHxy7CUAp+sQ7bTq2XCKxkYX3CW9ThcS4cV9mL +ayYdWEYnbEDGYPQDo7Wk3Ih5OEXTMZb/yNEx5D4S2lGMOS5bCDdYx6GvwCMG4jNx +aMktosaxpprAJiHh2oLgQk0hQc/a9JfMo6kJKtuhjxsxjxLwcOHhuaUD7NS0Pjop +CJkSYcrL+nnQPQjKe4uLhAbSyiX914h4QX0CJ0e4z1ccdDX2PFWTrwaIf//vQhCR +wP2YKdfjR0JB4oDAlu85GsIs2cFLPysM5ufuNZO4fCr8uOwloKI8zZ2HhlIfBEcY +Gcy4g9N/9epmxMXZlGcCAwEAAaOBgDB+MB0GA1UdDgQWBBRefYm8DHHDdkTPHhS1 +HEUwTb2uiDAfBgNVHSMEGDAWgBSWAlip9eoPmnG4p4OFZeOUBlAbNDAxBgNVHREE +KjAogglsb2NhbGhvc3SHBH8AAAGHEAAAAAAAAAAAAAAAAAAAAAGCA2VzMTAJBgNV +HRMEAjAAMA0GCSqGSIb3DQEBCwUAA4IBAQC+pauqM2wJjQaHyHu+kIm59P4b/5Oj +IH1cYCQfMB7Y2UMLxp0ew+f7o7zzE2DA52YYFDWy6J5DVWtSBPyeFGgX+RH+aA+9 +Iv4cc9QpAs6aFjncorHrzNOrWLgCHIeRAxTR0CAkeP2dUZfDBuMpRyP6rAsYzyLH +Rb3/BfYJSI5vxgt5Ke49Y/ljDKFJTyDmAVrHQ4JWrseYE1UZ2eDkBXeiRlYE/QtB +YsrUSqdL6zvFZyUcilxDUUabNcA+GgeGZ2lAEA90F8vwi62QwRXo3Iv1Hz+6xc43 +nFofDK9D8/qkrUD9iuhpx1974QwPhwWyjn9RZRpbZA4ngRL+szdRXR4N +-----END CERTIFICATE----- diff --git a/.buildkite/certs/testnode_san.key b/.buildkite/certs/testnode_san.key new file mode 100644 index 00000000000..75d19539e69 --- /dev/null +++ b/.buildkite/certs/testnode_san.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEogIBAAKCAQEAhQ+3brs03bXP7e6RqN6Px6iNJfA/rMfHLsJQCn6xDttOrZcI +rGRhfcJb1OFxLhxX2YtrJh1YRidsQMZg9AOjtaTciHk4RdMxlv/I0THkPhLaUYw5 +LlsIN1jHoa/AIwbiM3FoyS2ixrGmmsAmIeHaguBCTSFBz9r0l8yjqQkq26GPGzGP +EvBw4eG5pQPs1LQ+OikImRJhysv6edA9CMp7i4uEBtLKJf3XiHhBfQInR7jPVxx0 +NfY8VZOvBoh//+9CEJHA/Zgp1+NHQkHigMCW7zkawizZwUs/Kwzm5+41k7h8Kvy4 +7CWgojzNnYeGUh8ERxgZzLiD03/16mbExdmUZwIDAQABAoIBAEwhjulLMVc9JEfV +PP/qv0cUOBYh3LzF3T/yq4slq7Z9YgnOJYdFM8aZgqNNjc09KEJvE5JOLeiNu9Ff +768Nugg+2HM5MCo7SN9FYCfZLOcbMFCCM2FDcnMAV9A512vzD08xryuT8dNPZ6yZ +DfhK2hQRrb2lrpr3gwSrcGRRu3THqvq7X1RIjpLV3teDMeP8rQPAlpj8fmP+kdVV +5y1ihiDIo87McihG9FMavJtBDXQkUEuVw6eIeir8L/zHHD/ZwhYjNHZGWbrB88sz +CkJkfWh/FlA63tCVdJzkmnERALLTVy9mR0Sq6sUlnFhFNO2BRdWgYLrcp9McfTJC +e8+WsSECgYEAuwQ3nAaFL0jqYu1AREyKT/f3WUenf2UsX7dwwV2/yFtQvkzW7ji4 +uZLnfUnZBojtHf35dRo+hDgtvhZhgZNAuPPsbOl/EIMTcbChEqV/3CSTFlhLFM1d +hfM9PoM+Bt/pyUNabjD1sWM0X7WeUhzcddshY3S4daBsNsLuOzweRRcCgYEAtiSS +4qiiGafYsY7gOHuAlOhs/00+1uWIFEHKgoHM9vzCxDN3LCmBdynHk8ZE2TAdhw+l +7xpu6LUxKQDfGmVZa9Epg0kQmVq9c54oQP57pJ3tR+68++insEkfnaZH8jblfq2s +sSkFrY3pdS19edq60nuft64kswKRUUkamCXTXTECgYBdoSfiMpV9bekC7DsPtq5M +iR3KEgi2zEViCmomNTRuL+GF1NyKWdWJ+xVwcYd5MRZdvKimyyPfeGzWTUg14i42 +KtEEWgZmkukqMz8BIeCYq6sENeIpIQQgqv3PjU+Bi5r1S4Y7wsFPNRakkD4aaB6r +1rCppWcwZMeoxwEUoO2aswKBgBdDIIdWJi3EpAY5SyWrkEZ0UMdiZC4p7nE33ddB +IJ5CtdU9BXFcc652ZYjX/58FaCABvZ2F8LhDu92SwOusGfmNIxIjWL1dO2jywA1c +8wmZKd7P/M7nbdMz45fMzs9+d1zwbWfK53C8+R4AC1BuwQF0zHc3BHTgVRLelUjt +O8thAoGAdO2gHIqEsZzTgbvLbsh52eVbumjfNGnrnEv1fjb+o+/wAol8dymcmzbL +bZCRzoyA0qwU9kdPFgX46H6so6o1tUM2GQtVFoT6kDnPv7EkLQK0C4cDh6OOHxDU +NPvr/9fHhQd9EDWDvS1JnVMAdKDO6ELp3SoKGGmCXR2QplnqWAk= +-----END RSA PRIVATE KEY----- diff --git a/.buildkite/functions/cleanup.sh b/.buildkite/functions/cleanup.sh new file mode 100644 index 00000000000..98dfe4e1412 --- /dev/null +++ b/.buildkite/functions/cleanup.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +# +# Shared cleanup routines between different steps +# +# Please source .buildkite/functions/imports.sh as a whole not just this file +# +# Version 1.0.0 +# - Initial version after refactor + +function cleanup_volume { + if [[ "$(docker volume ls -q -f name=$1)" ]]; then + echo -e "\033[34;1mINFO:\033[0m Removing volume $1\033[0m" + (docker volume rm "$1") || true + fi +} +function container_running { + if [[ "$(docker ps -q -f name=$1)" ]]; then + return 0; + else return 1; + fi +} +function cleanup_node { + if container_running "$1"; then + echo -e "\033[34;1mINFO:\033[0m Removing container $1\033[0m" + (docker container rm --force --volumes "$1") || true + fi + if [[ -n "$1" ]]; then + echo -e "\033[34;1mINFO:\033[0m Removing volume $1-${suffix}-data\033[0m" + cleanup_volume "$1-${suffix}-data" + fi +} +function cleanup_network { + if [[ "$(docker network ls -q -f name=$1)" ]]; then + echo -e "\033[34;1mINFO:\033[0m Removing network $1\033[0m" + (docker network rm "$1") || true + fi +} + +function cleanup_trap { + status=$? + set +x + if [[ "$DETACH" != "true" ]]; then + echo -e "\033[34;1mINFO:\033[0m clean the network if not detached (start and exit)\033[0m" + cleanup_all_in_network "$1" + fi + # status is 0 or SIGINT + if [[ "$status" == "0" || "$status" == "130" ]]; then + echo -e "\n\033[32;1mSUCCESS run-tests\033[0m" + exit 0 + else + echo -e "\n\033[31;1mFAILURE during run-tests\033[0m" + exit ${status} + fi +}; +function cleanup_all_in_network { + + if [[ -z "$(docker network ls -q -f name="^$1\$")" ]]; then + echo -e "\033[34;1mINFO:\033[0m $1 is already deleted\033[0m" + return 0 + fi + containers=$(docker network inspect -f '{{ range $key, $value := .Containers }}{{ printf "%s\n" .Name}}{{ end }}' $1) + while read -r container; do + cleanup_node "$container" + done <<< "$containers" + cleanup_network $1 + echo -e "\033[32;1mSUCCESS:\033[0m Cleaned up and exiting\033[0m" +}; diff --git a/.buildkite/functions/imports.sh b/.buildkite/functions/imports.sh new file mode 100644 index 00000000000..c05f3682695 --- /dev/null +++ b/.buildkite/functions/imports.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +# +# Sets up all the common variables and imports relevant functions +# +# Version 1.0.1 +# - Initial version after refactor +# - Validate STACK_VERSION asap + +function require_stack_version() { + if [[ -z $STACK_VERSION ]]; then + echo -e "\033[31;1mERROR:\033[0m Required environment variable [STACK_VERSION] not set\033[0m" + exit 1 + fi +} + +require_stack_version + +if [[ -z $es_node_name ]]; then + # only set these once + set -euo pipefail + export TEST_SUITE=${TEST_SUITE-free} + export RUNSCRIPTS=${RUNSCRIPTS-} + export DETACH=${DETACH-false} + export CLEANUP=${CLEANUP-false} + + export es_node_name=instance + export elastic_password=changeme + export elasticsearch_image=elasticsearch + export elasticsearch_scheme="https" + if [[ $TEST_SUITE != "platinum" ]]; then + export elasticsearch_scheme="http" + fi + export elasticsearch_url=${elasticsearch_scheme}://elastic:${elastic_password}@${es_node_name}:9200 + export external_elasticsearch_url=${elasticsearch_url/$es_node_name/localhost} + export elasticsearch_container="${elasticsearch_image}:${STACK_VERSION}" + + export suffix=rest-test + export moniker=$(echo "$elasticsearch_container" | tr -C "[:alnum:]" '-') + export network_name=${moniker}${suffix} + + export ssl_cert="${script_path}/certs/testnode.crt" + export ssl_key="${script_path}/certs/testnode.key" + export ssl_ca="${script_path}/certs/ca.crt" + +fi + + export script_path=$(dirname $(realpath -s $0)) + source $script_path/functions/cleanup.sh + source $script_path/functions/wait-for-container.sh + trap "cleanup_trap ${network_name}" EXIT + + +if [[ "$CLEANUP" == "true" ]]; then + cleanup_all_in_network $network_name + exit 0 +fi + +echo -e "\033[34;1mINFO:\033[0m Creating network $network_name if it does not exist already \033[0m" +docker network inspect "$network_name" > /dev/null 2>&1 || docker network create "$network_name" + diff --git a/.buildkite/functions/wait-for-container.sh b/.buildkite/functions/wait-for-container.sh new file mode 100644 index 00000000000..82e7394fb08 --- /dev/null +++ b/.buildkite/functions/wait-for-container.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# +# Exposes a routine scripts can call to wait for a container if that container set up a health command +# +# Please source .buildkite/functions/imports.sh as a whole not just this file +# +# Version 1.0.1 +# - Initial version after refactor +# - Make sure wait_for_contiainer is silent + +function wait_for_container { + set +x + until ! container_running "$1" || (container_running "$1" && [[ "$(docker inspect -f "{{.State.Health.Status}}" ${1})" != "starting" ]]); do + echo "" + docker inspect -f "{{range .State.Health.Log}}{{.Output}}{{end}}" ${1} + echo -e "\033[34;1mINFO:\033[0m waiting for node $1 to be up\033[0m" + sleep 10; + done; + + # Always show logs if the container is running, this is very useful both on CI as well as while developing + if container_running $1; then + docker logs $1 + fi + + if ! container_running $1 || [[ "$(docker inspect -f "{{.State.Health.Status}}" ${1})" != "healthy" ]]; then + cleanup_all_in_network $2 + echo + echo -e "\033[31;1mERROR:\033[0m Failed to start $1 in detached mode beyond health checks\033[0m" + echo -e "\033[31;1mERROR:\033[0m dumped the docker log before shutting the node down\033[0m" + return 1 + else + echo + echo -e "\033[32;1mSUCCESS:\033[0m Detached and healthy: ${1} on docker network: ${network_name}\033[0m" + return 0 + fi +} diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 90230f464f7..a1c83db4429 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -1,3 +1,14 @@ steps: - - label: ":dotnet: Greetings" - command: "echo 'Hello, world!'" + - label: ":dotnet: Elasticsearch-Net {{ matrix.suite }}" + agents: + provider: "gcp" + env: + TEST_SUITE: "{{ matrix.suite }}" + STACK_VERSION: master-SNAPSHOT + DOTNET_VERSION: 6.0.403 + matrix: + setup: + suite: + - "free" + - "platinum" + command: ./.buildkite/run-tests diff --git a/.buildkite/run-elasticsearch.ps1 b/.buildkite/run-elasticsearch.ps1 new file mode 100644 index 00000000000..f319ea3f322 --- /dev/null +++ b/.buildkite/run-elasticsearch.ps1 @@ -0,0 +1,253 @@ +# Launch one or more Elasticsearch nodes via the Docker image, +# to form a cluster suitable for running the REST API tests. +# +# Export the ELASTICSEARCH_VERSION variable, eg. 'elasticsearch:8.0.0-SNAPSHOT'. + +param( + [string] + $NODE_NAME, + + [string] + $MASTER_NODE_NAME, + + [string] + $CLUSTER_NAME, + + [int] + $HTTP_PORT = 9200, + + [string] + $ELASTIC_PASSWORD = "changeme", + + [string] + $SSL_CERT = "$PWD/certs/testnode.crt", + + [string] + $SSL_KEY = "$PWD/certs/testnode.key", + + [string] + $SSL_CA = "$PWD/certs/ca.crt", + + [switch] + $DETACH, + + [switch] + $CLEANUP, + + [string] + $NETWORK_NAME + +) + +trap { + cleanup +} + +$ESC = [char]27 + +if ($null -eq $env:ELASTICSEARCH_VERSION) { + Write-Error "ERROR: Required environment variable [ELASTICSEARCH_VERSION] not set" + exit 1 +} + +$moniker = $env:ELASTICSEARCH_VERSION -replace "[^a-zA-Z\d]", "-" +$suffix = "rest-test" + +if (!$NODE_NAME) { + $NODE_NAME = "${moniker}node1" +} + +if (!$MASTER_NODE_NAME) { + $MASTER_NODE_NAME = $NODE_NAME +} + +if (!$CLUSTER_NAME) { + $CLUSTER_NAME = "${moniker}${suffix}" +} + +$volume_name = "${NODE_NAME}-${suffix}-data" + +if (!$NETWORK_NAME) { + $NETWORK_NAME= "${moniker}${suffix}" +} + + +function cleanup_volume { + param( + [string] + $Name + ) + + if ("$(docker volume ls --quiet --filter name="$Name")") { + Write-Output "$ESC[34;1mINFO:$ESC[0m Removing volume $Name$ESC[0m" + docker volume rm "$Name" + } +} +function cleanup_node { + param( + [string] + $Name + ) + + if ("$(docker ps --quiet --filter name="$Name")") { + Write-Output "$ESC[34;1mINFO:$ESC[0m Removing container $Name$ESC[0m" + docker container rm --force --volumes "$Name" + cleanup_volume "$Name-${suffix}-data" + } +} +function cleanup_network { + param( + [string] + $Name + ) + + if ("$(docker network ls --quiet --filter name="$Name")") { + Write-Output "$ESC[34;1mINFO:$ESC[0m Removing network ${Name}$ESC[0m" + docker network rm "$Name" + } +} + +function cleanup { + param( + [switch] + $Cleanup + ) + + if ((-not $DETACH) -or $Cleanup) { + Write-Output "$ESC[34;1mINFO:$ESC[0m clean the node and volume on startup (1) OR on exit if not detached$ESC[0m" + cleanup_node "$NODE_NAME" + } + if (-not $DETACH) { + Write-Output "$ESC[34;1mINFO:$ESC[0m clean the network if not detached (start and exit)$ESC[0m" + cleanup_network "$NETWORK_NAME" + } +} + +if ($CLEANUP) { + #trap - EXIT + if ("$(docker network ls --quiet --filter name=${NETWORK_NAME})" -eq "") { + Write-Output "$ESC[34;1mINFO:$ESC[0m $NETWORK_NAME is already deleted$ESC[0m" + exit 0 + } + $containers = $(docker network inspect --format '{{ range $key, $value := .Containers }}{{ println .Name}}{{ end }}' ${NETWORK_NAME}) + + foreach($container in $containers) { + cleanup_node "$container" + } + + cleanup_network "$NETWORK_NAME" + Write-Output "$ESC[32;1mSUCCESS:$ESC[0m Cleaned up and exiting$ESC[0m" + exit 0 +} + +Write-Output "$ESC[34;1mINFO:$ESC[0m Making sure previous run leftover infrastructure is removed$ESC[0m" +cleanup -Cleanup + +Write-Output "$ESC[34;1mINFO:$ESC[0m Creating network $NETWORK_NAME if it does not exist already$ESC[0m" + +docker network inspect "$NETWORK_NAME" | Out-Null +if ($LASTEXITCODE -ne 0) { + docker network create "$NETWORK_NAME" +} + +$environment = @( + "--env", "node.name=`"$NODE_NAME`"", + "--env", "cluster.name=`"$CLUSTER_NAME`"", + "--env", "cluster.initial_master_nodes=`"$MASTER_NODE_NAME`"", + "--env", "discovery.seed_hosts=`"$MASTER_NODE_NAME`"", + "--env", "cluster.routing.allocation.disk.threshold_enabled=false", + "--env", "bootstrap.memory_lock=true", + "--env", "node.attr.testattr=test", + "--env", "path.repo=/tmp", + "--env", "repositories.url.allowed_urls=http://snapshot.test*", + "--env", "ingest.geoip.downloader.enabled=false" +) + +$volumes = @( + "--volume", "${volume_name}:/usr/share/elasticsearch/data" +) + +if (-not $env:ELASTICSEARCH_VERSION -contains "free") { + $environment += @( + "--env", "ELASTIC_PASSWORD=`"$ELASTIC_PASSWORD`"", + "--env", "xpack.license.self_generated.type=trial", + "--env", "xpack.security.enabled=true", + "--env", "xpack.security.http.ssl.enabled=true", + "--env", "xpack.security.http.ssl.verification_mode=certificate", + "--env", "xpack.security.http.ssl.key=certs/testnode.key", + "--env", "xpack.security.http.ssl.certificate=certs/testnode.crt", + "--env", "xpack.security.http.ssl.certificate_authorities=certs/ca.crt", + "--env", "xpack.security.transport.ssl.enabled=true", + "--env", "xpack.security.transport.ssl.key=certs/testnode.key", + "--env", "xpack.security.transport.ssl.certificate=certs/testnode.crt", + "--env", "xpack.security.transport.ssl.certificate_authorities=certs/ca.crt" + ) + + $volumes += @( + "--volume", "`"${SSL_CERT}`":/usr/share/elasticsearch/config/certs/testnode.crt", + "--volume", "`"${SSL_KEY}`":/usr/share/elasticsearch/config/certs/testnode.key", + "--volume", "`"${SSL_CA}`":/usr/share/elasticsearch/config/certs/ca.crt" + ) +} + +$url="http://$NODE_NAME" +if (-not $env:ELASTICSEARCH_VERSION -contains "free") { + $url="https://elastic:$ELASTIC_PASSWORD@$NODE_NAME" +} + +Write-Output "$ESC[34;1mINFO:$ESC[0m Starting container $NODE_NAME $ESC[0m" + +if ($DETACH) { + $d = "true" +} else { + $d = "false" +} + +docker run ` + --name "`"$NODE_NAME`"" ` + --network "`"$NETWORK_NAME`"" ` + --env ES_JAVA_OPTS=-"`"Xms1g -Xmx1g`"" ` + $environment ` + $volumes ` + --publish ${HTTP_PORT}:9200 ` + --ulimit nofile=65536:65536 ` + --ulimit memlock=-1:-1 ` + --detach="$d" ` + --health-cmd="`"curl --silent --fail ${url}:9200/_cluster/health || exit 1`"" ` + --health-interval=2s ` + --health-retries=20 ` + --health-timeout=2s ` + --rm docker.elastic.co/elasticsearch/"$($env:ELASTICSEARCH_VERSION)" + + +if ($DETACH) { + while("$(docker inspect -f '{{.State.Health.Status}}' ${NODE_NAME})" -eq "starting") { + Start-Sleep 2; + Write-Output "" + Write-Output "$ESC[34;1mINFO:$ESC[0m waiting for node $NODE_NAME to be up$ESC[0m" + } + + # Always show the node getting started logs, this is very useful both on CI as well as while developing + docker logs "$NODE_NAME" + + if ("$(docker inspect -f '{{.State.Health.Status}}' ${NODE_NAME})" -ne "healthy") { + cleanup -Cleanup + Write-Output "" + Write-Output "$ESC[31;1mERROR:$ESC[0m Failed to start $($env:ELASTICSEARCH_VERSION) in detached mode beyond health checks$ESC[0m" + Write-Output "$ESC[31;1mERROR:$ESC[0m dumped the docker log before shutting the node down$ESC[0m" + exit 1 + } else { + Write-Output "" + Write-Output "$ESC[32;1mSUCCESS:$ESC[0m Detached and healthy: ${NODE_NAME} on docker network: ${NETWORK_NAME}$ESC[0m" + $es_host = $url + if (!$es_host) { + $es_host = $NODE_NAME + } + if (!$es_host) { + $es_host = "localhost" + } + + Write-Output "$ESC[32;1mSUCCESS:$ESC[0m Running on: ${es_host}:${HTTP_PORT}$ESC[0m" + exit 0 + } +} diff --git a/.buildkite/run-elasticsearch.sh b/.buildkite/run-elasticsearch.sh new file mode 100755 index 00000000000..7ed65d12a2f --- /dev/null +++ b/.buildkite/run-elasticsearch.sh @@ -0,0 +1,135 @@ +#!/usr/bin/env bash +# +# Launch one or more Elasticsearch nodes via the Docker image, +# to form a cluster suitable for running the REST API tests. +# +# Export the STACK_VERSION variable, eg. '8.0.0-SNAPSHOT'. +# Export the TEST_SUITE variable, eg. 'free' or 'platinum' defaults to 'free'. +# Export the NUMBER_OF_NODES variable to start more than 1 node + +# Version 1.4.0 +# - Initial version of the run-elasticsearch.sh script +# - Deleting the volume should not dependent on the container still running +# - Fixed `ES_JAVA_OPTS` config +# - Moved to STACK_VERSION and TEST_VERSION +# - Refactored into functions and imports +# - Support NUMBER_OF_NODES +# - Added 5 retries on docker pull for fixing transient network errors +# - Added flags to make local CCR configurations work +# - Added action.destructive_requires_name=false as the default will be true in v8 +# - Added ingest.geoip.downloader.enabled=false as it causes false positives in testing + +# Version 1.2.0 +# - Added action.destructive_requires_name=false as the default will be true in v8 + +# Version 1.4.0 +# - Added ingest.geoip.downloader.enabled=false as it causes false positives in testing + +# Version 1.5.0 +# - Moved ELASTIC_PASSWORD to the base arguments for "Security On by default" + +script_path=$(dirname $(realpath -s $0)) +source $script_path/functions/imports.sh +set -euo pipefail + +echo -e "\033[34;1mINFO:\033[0m Take down node if called twice with the same arguments (DETACH=true) or on seperate terminals \033[0m" +cleanup_node $es_node_name + +master_node_name=${es_node_name} +cluster_name=${moniker}${suffix} + +BUILDKITE=${BUILDKITE-false} + +# Set vm.max_map_count kernel setting to 262144 if we're in CI +if [[ "$BUILDKITE" == "true" ]]; then + sudo sysctl -w vm.max_map_count=262144 +fi + +declare -a volumes +environment=($(cat <<-END + --env ELASTIC_PASSWORD=$elastic_password + --env node.name=$es_node_name + --env cluster.name=$cluster_name + --env cluster.initial_master_nodes=$master_node_name + --env discovery.seed_hosts=$master_node_name + --env cluster.routing.allocation.disk.threshold_enabled=false + --env bootstrap.memory_lock=true + --env node.attr.testattr=test + --env path.repo=/tmp + --env repositories.url.allowed_urls=http://snapshot.test* + --env ingest.geoip.downloader.enabled=false + --env cluster.deprecation_indexing.enabled=false +END +)) +if [[ "$TEST_SUITE" == "platinum" ]]; then + environment+=($(cat <<-END + --env xpack.license.self_generated.type=trial + --env xpack.security.enabled=true + --env xpack.security.http.ssl.enabled=true + --env xpack.security.http.ssl.verification_mode=certificate + --env xpack.security.http.ssl.key=certs/testnode.key + --env xpack.security.http.ssl.certificate=certs/testnode.crt + --env xpack.security.http.ssl.certificate_authorities=certs/ca.crt + --env xpack.security.transport.ssl.enabled=true + --env xpack.security.transport.ssl.key=certs/testnode.key + --env xpack.security.transport.ssl.certificate=certs/testnode.crt + --env xpack.security.transport.ssl.certificate_authorities=certs/ca.crt +END +)) + volumes+=($(cat <<-END + --volume $ssl_cert:/usr/share/elasticsearch/config/certs/testnode.crt + --volume $ssl_key:/usr/share/elasticsearch/config/certs/testnode.key + --volume $ssl_ca:/usr/share/elasticsearch/config/certs/ca.crt +END +)) +fi + +cert_validation_flags="" +if [[ "$TEST_SUITE" == "platinum" ]]; then + cert_validation_flags="--insecure --cacert /usr/share/elasticsearch/config/certs/ca.crt --resolve ${es_node_name}:443:127.0.0.1" +fi + +NUMBER_OF_NODES=${NUMBER_OF_NODES-1} +http_port=9200 +for (( i=0; i<$NUMBER_OF_NODES; i++, http_port++ )); do + node_name=${es_node_name}$i + node_url=${external_elasticsearch_url/9200/${http_port}}$i + if [[ "$i" == "0" ]]; then node_name=$es_node_name; fi + environment+=($(cat <<-END + --env node.name=$node_name +END +)) + echo "$i: $http_port $node_url " + volume_name=${node_name}-${suffix}-data + volumes+=($(cat <<-END + --volume $volume_name:/usr/share/elasticsearch/data${i} +END +)) + echo "--- :elasticsearch: Running container" + # make sure we detach for all but the last node if DETACH=false (default) so all nodes are started + local_detach="true" + if [[ "$i" == "$((NUMBER_OF_NODES-1))" ]]; then local_detach=$DETACH; fi + echo -e "\033[34;1mINFO:\033[0m Starting container $node_name \033[0m" + set -x + docker run \ + --name "$node_name" \ + --network "$network_name" \ + --env "ES_JAVA_OPTS=-Xms1g -Xmx1g" \ + "${environment[@]}" \ + "${volumes[@]}" \ + --publish "$http_port":9200 \ + --ulimit nofile=65536:65536 \ + --ulimit memlock=-1:-1 \ + --detach="$local_detach" \ + --health-cmd="curl $cert_validation_flags --fail $elasticsearch_url/_cluster/health || exit 1" \ + --health-interval=2s \ + --health-retries=20 \ + --health-timeout=2s \ + --rm \ + docker.elastic.co/elasticsearch/"$elasticsearch_container"; + + set +x + if wait_for_container "$es_node_name" "$network_name"; then + echo -e "\033[32;1mSUCCESS:\033[0m Running on: $node_url\033[0m" + fi +done diff --git a/.buildkite/run-repository.ps1 b/.buildkite/run-repository.ps1 new file mode 100644 index 00000000000..33c2798c862 --- /dev/null +++ b/.buildkite/run-repository.ps1 @@ -0,0 +1,42 @@ +# ELASTICSEARCH_URL -- The url at which elasticsearch is reachable +# NETWORK_NAME -- The docker network name +# NODE_NAME -- The docker container name also used as Elasticsearch node name +# DOTNET_VERSION -- SDK version (defined in test-matrix.yml, a default is hardcoded here) + +param( + [System.Uri] + $ELASTICSEARCH_URL, + + [string] + $NETWORK_NAME, + + [string] + $NODE_NAME, + + [string] + $DOTNET_VERSION = "6.0.403" +) + +$ESC = [char]27 + +Write-Output "$ESC[34;1mINFO:$ESC[0m URL ${ELASTICSEARCH_URL}$ESC[0m" +Write-Output "$ESC[34;1mINFO:$ESC[0m DOTNET_VERSION ${DOTNET_VERSION}$ESC[0m" + +Write-Output "$ESC[1m>>>>> Build [elastic/elasticsearch-net container] >>>>>>>>>>>>>>>>>>>>>>>>>>>>>$ESC[0m" + +docker build --file .buildkite/DockerFile --tag elastic/elasticsearch-net . + +Write-Output "$ESC[1m>>>>> Run [elastic/elasticsearch-net container] >>>>>>>>>>>>>>>>>>>>>>>>>>>>>$ESC[0m" + +mkdir -p build/output -ErrorAction Ignore + +$repo = Get-Location + +docker run ` + --network="`"$NETWORK_NAME`"" ` + --env "DOTNET_VERSION=$DOTNET_VERSION" ` + --name test-runner ` + --volume $repo/build/output:/sln/build/output ` + --rm ` + elastic/elasticsearch-net ` + ./build.sh rest-spec-tests -f count -e $ELASTICSEARCH_URL -o /sln/build/output/rest-spec-junit.xml diff --git a/.buildkite/run-repository.sh b/.buildkite/run-repository.sh new file mode 100755 index 00000000000..0604642fc39 --- /dev/null +++ b/.buildkite/run-repository.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +# parameters are available to this script + +# STACK_VERSION -- version e.g Major.Minor.Patch(-Prelease) +# TEST_SUITE -- which test suite to run: free or platinum +# ELASTICSEARCH_URL -- The url at which elasticsearch is reachable, a default is composed based on STACK_VERSION and TEST_SUITE +# DOTNET_VERSION -- SDK version (defined in test-matrix.yml, a default is hardcoded here) +script_path=$(dirname $(realpath -s $0)) +source $script_path/functions/imports.sh +set -euo pipefail + +DOTNET_VERSION=${DOTNET_VERSION-6.0.403} +ELASTICSEARCH_URL=${ELASTICSEARCH_URL-"$elasticsearch_url"} +elasticsearch_container=${elasticsearch_container-} + +TEST_FOLDER=${TEST_FOLDER-} +TEST_FILE=${TEST_FILE-} +TEST_SECTION=${TEST_SECTION-} + +run_script_args="" +if [[ "$TEST_FOLDER" != "" ]]; then + run_script_args="-f ${TEST_FOLDER}" +fi +if [[ "$TEST_FILE" != "" ]]; then + run_script_args="${run_script_args} -t ${TEST_FILE}" +fi +if [[ "$TEST_SECTION" != "" ]]; then + run_script_args="${run_script_args} -s ${TEST_SECTION}" +fi + +echo -e "--- :computer: Environment variables" +echo -e "\033[34;1mINFO:\033[0m VERSION ${STACK_VERSION}\033[0m" +echo -e "\033[34;1mINFO:\033[0m TEST_SUITE ${TEST_SUITE}\033[0m" +echo -e "\033[34;1mINFO:\033[0m URL ${ELASTICSEARCH_URL}\033[0m" +echo -e "\033[34;1mINFO:\033[0m CONTAINER ${elasticsearch_container}\033[0m" +echo -e "\033[34;1mINFO:\033[0m DOTNET_VERSION ${DOTNET_VERSION}\033[0m" + +echo -e "--- :dotnet: :docker: Build [elastic/elasticsearch-net container]" +docker build --file .buildkite/DockerFile --tag elastic/elasticsearch-net \ + --build-arg USER_ID=$(id -u) \ + --build-arg GROUP_ID=$(id -g) . +echo -e "--- :dotnet: :docker: Run [elastic/elasticsearch-net container]" +mkdir -p build/output + +repo=$(realpath $(dirname $(realpath -s $0))/../) + +docker run \ + --network=${network_name} \ + --env "DOTNET_VERSION" \ + --name test-runner \ + --volume ${repo}/build/output:/sln/build/output \ + --rm \ + elastic/elasticsearch-net \ + ./build.sh rest-spec-tests $TEST_SUITE -e ${ELASTICSEARCH_URL} -o /sln/build/output/rest-spec-junit.xml ${run_script_args} diff --git a/.buildkite/run-tests b/.buildkite/run-tests new file mode 100755 index 00000000000..87a68216fa9 --- /dev/null +++ b/.buildkite/run-tests @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# +# Version 1.1 +# - Moved to .ci folder and seperated out `run-repository.sh` +# - Add `$RUNSCRIPTS` env var for running Elasticsearch dependent products +script_path=$(dirname $(realpath -s $0)) +source $script_path/functions/imports.sh +set -euo pipefail + +echo "--- :elasticsearch: Starting Elasticsearch" +DETACH=true bash .buildkite/run-elasticsearch.sh + +if [[ -n "$RUNSCRIPTS" ]]; then + for RUNSCRIPT in ${RUNSCRIPTS//,/ } ; do + echo -e "\033[1m>>>>> Running run-$RUNSCRIPT.sh >>>>>>>>>>>>>>>>>>>>>>>>>>>>>\033[0m" + CONTAINER_NAME=${RUNSCRIPT} \ + DETACH=true \ + bash .buildkite/run-${RUNSCRIPT}.sh + done +fi + +echo -e "\033[1m>>>>> Repository specific tests >>>>>>>>>>>>>>>>>>>>>>>>>>>>>\033[0m" +bash .buildkite/run-repository.sh diff --git a/.buildkite/run-tests.ps1 b/.buildkite/run-tests.ps1 new file mode 100644 index 00000000000..b7d8eef1242 --- /dev/null +++ b/.buildkite/run-tests.ps1 @@ -0,0 +1,76 @@ +param ( + [string] + [Parameter(Mandatory = $true)] + $ELASTICSEARCH_VERSION, + + [string] + [ValidateSet("free", "platinum")] + $TEST_SUITE = "free", + + [string] + $DOTNET_VERSION = "6.0.403" +) + +$ESC = [char]27 +$NODE_NAME = "es1" +$elasticsearch_image= "elasticsearch" +$elasticsearch_url = "https://elastic:changeme@${NODE_NAME}:9200" + +if ($TEST_SUITE -ne "platinum") { + $elasticsearch_url = "http://${NODE_NAME}:9200" +} + +# set for run-elasticsearch.ps1 +$env:ELASTICSEARCH_VERSION = "${elasticsearch_image}:$ELASTICSEARCH_VERSION" + +function cleanup { + $status=$? + + $runParams = @{ + NODE_NAME= $NODE_NAME + NETWORK_NAME = "elasticsearch" + CLEANUP = $true + } + + ./.buildkite/run-elasticsearch.ps1 @runParams + + # Report status and exit + if ($status -eq 0) { + Write-Output "SUCCESS run-tests" + exit 0 + } else { + Write-Output "FAILURE during run-tests" + exit $status + } +} + + +try { + Write-Output "$ESC[1m>>>>> Start [$env:ELASTICSEARCH_VERSION container] >>>>>>>>>>>>>>>>>>>>>>>>>>>>>$ESC[0m" + + $runParams = @{ + NODE_NAME= "$NODE_NAME" + NETWORK_NAME = "elasticsearch" + DETACH = $true + } + + ./.buildkite/run-elasticsearch.ps1 @runParams + + Write-Output "$ESC[1m>>>>> Repository specific tests >>>>>>>>>>>>>>>>>>>>>>>>>>>>>$ESC[0m" + + $runParams = @{ + ELASTICSEARCH_VERSION = "$env:ELASTICSEARCH_VERSION" + ELASTICSEARCH_CONTAINER = "$env:ELASTICSEARCH_VERSION" + NETWORK_NAME = "elasticsearch" + NODE_NAME= "$NODE_NAME" + ELASTICSEARCH_URL = $elasticsearch_url + DOTNET_VERSION = $DOTNET_VERSION + } + + ./.buildkite/run-repository.ps1 @runParams + + cleanup +} +catch { + cleanup +}