Merge pull request #238 from telefonicaid/task/upgrade_antelope

Task/upgrade antelope

ChangeLog

@@ -1,3 +1,9 @@
+1.19.0
+
+Upgrade to use Keystone Antelope
+Upgrade docker to centos 9 Stream
+Upgrade to support python 3.9
+
 1.18.0
 
 Fix: policy.json file from stein for docker (#234)

README.md

@@ -23,6 +23,7 @@ a recover procedure password, a second factor authentication (2FA)  and so on.
 - 1.9.0 uses keystone Rocky
 - 1.10.0 to 1.17.0 uses keystone Stein
 - 1.18.0 uses keystone Xena
+- 1.19.0 uses keystone antelope
 
 
 ## Installing and Configuration
@@ -173,6 +174,12 @@ Restart again keystone container
 ##### Upgrade from 1.17.0 to 1.18.0
 -> no workaround needed
 
+##### Upgrade from 1.17.0 to 1.18.0
+-> no workaround needed
+
+##### Upgrade from 1.18.0 to 1.19.0
+To upgrade to 1.19.0 version make sure upgrade firts to 1.18.0 version before.
+
 
 ## Usage
 

docker/Dockerfile

@@ -1,56 +1,48 @@
-FROM centos:8.4.2105
+FROM quay.io/centos/centos:stream9
 
 MAINTAINER IoT team
 
 ENV DB_HOST localhost
 ENV KEYSTONE_ADMIN_PASSWORD 4pass1w0rd
-ENV KEYSTONE_SCIM_VERSION 1.7.0
-ENV KEYSTONE_SPASSWORD_VERSION 1.18.0
+ENV KEYSTONE_SCIM_VERSION 1.8.0
+ENV KEYSTONE_SPASSWORD_VERSION 1.19.0
 
 COPY ./*.sh /opt/keystone/
 COPY ./*.patch /opt/keystone/
 COPY ./*.rpm /opt/keystone/
 COPY ./*.json /opt/keystone/
+COPY ./*.py /opt/keystone/
+COPY ./*.cfg /opt/keystone/
 COPY ./keystone-all /opt/keystone/
 
 WORKDIR /opt/keystone
 
 RUN \
-    # Install dependencie
-    sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-Linux-* && \
-    sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-Linux-* && \
-    yum update -y && yum install -y curl && \
+    yum update -y && \
+    #yum install -y curl && \
+    dnf install curl -y --allowerasing && \
     # rpm -qa && \
     yum install -y epel-release && yum update -y epel-release && \
     # Install MySQL client
     yum -y install mysql git && \
     # Install keystone dependencies
-    yum -y install rpm-build tar findutils && \
+    yum -y install rpm-build tar findutils procps-ng chkconfig && \
     yum -y install python3 cronie && \
-    curl -s --insecure -L 'https://repos.fedorapeople.org/openstack/archived/openstack-xena/rdo-release-xena-2.el8.noarch.rpm' > rdo-release-xena-2.el8.noarch.rpm && \
-    yum localinstall -y --nogpgcheck rdo-release-xena-2.el8.noarch.rpm && \
+    curl -s --insecure -L 'https://repos.fedorapeople.org/openstack/archived/openstack-antelope/rdo-release-antelope-2.el9s.noarch.rpm' > rdo-release-antelope-2.el9s.noarch.rpm && \
+    yum localinstall -y --nogpgcheck rdo-release-antelope-2.el9s.noarch.rpm && \
     # Set Centos mirror to ensure openstack version
     sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/rdo-release.repo && \
     sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://buildlogs.centos.org|g' /etc/yum.repos.d/rdo-release.repo && \
     sed -i 's/gpgcheck=1/gpgcheck=0/g' /etc/yum.repos.d/rdo-release.repo && \
-    sed -i 's/$stream/8-stream/g' /etc/yum.repos.d/rdo-release.repo && \
     sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/messaging.repo && \
     sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/messaging.repo && \
     sed -i 's/gpgcheck=1/gpgcheck=0/g' /etc/yum.repos.d/messaging.repo && \
-    sed -i 's/$stream/8.4.2105/g' /etc/yum.repos.d/messaging.repo && \
-    sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/advanced-virtualization.repo && \
-    sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/advanced-virtualization.repo && \
-    sed -i 's|advancedvirt-common|advanced-virtualization|g' /etc/yum.repos.d/advanced-virtualization.repo && \
-    sed -i 's/gpgcheck=1/gpgcheck=0/g' /etc/yum.repos.d/advanced-virtualization.repo && \
-    sed -i 's/$stream/8.4.2105/g' /etc/yum.repos.d/advanced-virtualization.repo && \
     sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/nfv-openvswitch.repo && \
     sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/nfv-openvswitch.repo && \
     sed -i 's/gpgcheck=1/gpgcheck=0/g' /etc/yum.repos.d/nfv-openvswitch.repo && \
-    sed -i 's/$stream/8.4.2105/g' /etc/yum.repos.d/nfv-openvswitch.repo && \
-    sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/ceph-pacific.repo && \
-    sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/ceph-pacific.repo && \
-    sed -i 's/gpgcheck=1/gpgcheck=0/g' /etc/yum.repos.d/ceph-pacific.repo && \
-    sed -i 's/$stream/8.4.2105/g' /etc/yum.repos.d/ceph-pacific.repo && \
+    sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/ceph-quincy.repo && \
+    sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/ceph-quincy.repo && \
+    sed -i 's/gpgcheck=1/gpgcheck=0/g' /etc/yum.repos.d/ceph-quincy.repo && \
     yum update -y && \
     # Install keystone dependencies
     yum -y install crudini openstack-keystone python3-keystoneclient python3-openstackclient && \
@@ -63,10 +55,10 @@ RUN \
     chmod 755 /usr/bin/keystone-all && \
     cp /opt/keystone/openstack-config.sh /usr/bin/openstack-config && \
     chmod 755 /usr/bin/openstack-config && \
-    yum -y install unzip tcping jq python3-redis && \
+    yum -y install unzip nmap-ncat jq python3-redis && \
     echo "INFO: install cracklib...\n" && \
     yum -y install cracklib gcc python3-devel && \
-    yum localinstall -y --nogpgcheck cracklib-devel-2.9.6-15.el8.x86_64.rpm && \
+    yum localinstall -y --nogpgcheck cracklib-devel-2.9.6-27.el9.x86_64.rpm && \
     pip3 install cracklib && \
     # Set keystone configuration
     openstack-config --set /etc/keystone/keystone.conf \
@@ -94,8 +86,8 @@ RUN \
     # Keystone postconfig file
     chmod 755 /opt/keystone/postlaunchconfig.sh /opt/keystone/postlaunchconfig_update.sh /opt/keystone/keystone-entrypoint.sh && \
     ln -s /etc/keystone/keystone-paste.ini /usr/share/keystone/keystone-dist-paste.ini && \
-    mkdir -p /usr/lib/python3.6/site-packages/keystone/contrib && \
-    touch /usr/lib/python3.6/site-packages/keystone/contrib/__init__.py && \    
+    mkdir -p /usr/lib/python3.9/site-packages/keystone/contrib && \
+    touch /usr/lib/python3.9/site-packages/keystone/contrib/__init__.py && \
     # Install Keystone-SCIM
     tag=$KEYSTONE_SCIM_VERSION && \
     user="telefonicaid" && \
@@ -106,7 +98,7 @@ RUN \
     rm -fR $dir && mkdir -p $dir && \
     curl -s --insecure -L "${url_scim}" | tar zxvf - -C ${dir} && \
     cd ${dir}/${repo_scim}-${tag} && \
-    source ./${pack_scim} --with-python36 --with-version $KEYSTONE_SCIM_VERSION --with-release 0 && \
+    source ./${pack_scim} --with-python39 --with-version $KEYSTONE_SCIM_VERSION --with-release 0 && \
     find . -name "*.rpm" -exec rpm -Uvh {} \; && \
     # Install Keystone-SPASSWORD
     cd /opt/keystone && \
@@ -119,7 +111,7 @@ RUN \
     rm -fR $dir && mkdir -p $dir && \
     curl -s --insecure -L "${url_spassword}" | tar zxvf - -C ${dir} && \
     cd ${dir}/${repo_spassword}-${tag} && \
-    source ./${pack_spassword} --with-python36 --with-version $KEYSTONE_SPASSWORD_VERSION --with-release 0 && \
+    source ./${pack_spassword} --with-python39 --with-version $KEYSTONE_SPASSWORD_VERSION --with-release 0 && \
     find . -name "*.rpm" -exec rpm -Uvh {} \; && \
     # LDAP pre-support
     yum -y install openldap-clients && \
@@ -134,38 +126,40 @@ RUN \
     # Federation by SAML2 pre-support
     yum install -y xmlsec1 && \
     # Patching ...
-    ln -s /usr/lib/python3.6/site-packages/keystone/contrib/scim/scim.py  /usr/lib/python3.6/site-packages/keystone/api && \
-    ln -s /usr/lib/python3.6/site-packages/keystone/contrib/spassword/spassword.py  /usr/lib/python3.6/site-packages/keystone/api && \
-    rm -f /usr/lib/python3.6/site-packages/keystone/contrib/spassword/controllers.* && \
-    rm -f /usr/lib/python3.6/site-packages/keystone/contrib/spassword/routers.* && \
-    rm -f /usr/lib/python3.6/site-packages/keystone/contrib/scim/controllers.* && \
-    rm -f /usr/lib/python3.6/site-packages/keystone/contrib/scim/routers.* && \
-    cd /usr/lib/python3.6/site-packages/keystone/identity && \
+    ln -s /usr/lib/python3.9/site-packages/keystone/contrib/scim/scim.py  /usr/lib/python3.9/site-packages/keystone/api && \
+    ln -s /usr/lib/python3.9/site-packages/keystone/contrib/spassword/spassword.py  /usr/lib/python3.9/site-packages/keystone/api && \
+    rm -f /usr/lib/python3.9/site-packages/keystone/contrib/spassword/controllers.* && \
+    rm -f /usr/lib/python3.9/site-packages/keystone/contrib/spassword/routers.* && \
+    rm -f /usr/lib/python3.9/site-packages/keystone/contrib/scim/controllers.* && \
+    rm -f /usr/lib/python3.9/site-packages/keystone/contrib/scim/routers.* && \
+    cd /usr/lib/python3.9/site-packages/keystone/identity && \
     patch -f -p0 < /opt/keystone/core.patch && \
-    cd /usr/lib/python3.6/site-packages/keystone/common && \
+    cd /usr/lib/python3.9/site-packages/keystone/common && \
     patch -f -p0 < /opt/keystone/manager.patch && \
-    cd /usr/lib/python3.6/site-packages/keystone/auth && \
+    cd /usr/lib/python3.9/site-packages/keystone/auth && \
     patch -f -p0 < /opt/keystone/core_auth.patch && \
-    cd /usr/lib/python3.6/site-packages/keystone/server && \
+    cd /usr/lib/python3.9/site-packages/keystone/server && \
     patch -f -p0 < /opt/keystone/backends.patch && \
-    cd /usr/lib/python3.6/site-packages/keystone/auth/plugins && \
+    cd /usr/lib/python3.9/site-packages/keystone/auth/plugins && \
     patch -f -p0 < /opt/keystone/auth_plugins_token.patch && \
-    cd /usr/lib/python3.6/site-packages/keystone/common && \
+    cd /usr/lib/python3.9/site-packages/keystone/common && \
     patch -f -p0 < /opt/keystone/common_render_token.patch && \
-    cd /usr/lib/python3.6/site-packages/keystone/models && \
+    cd /usr/lib/python3.9/site-packages/keystone/models && \
     patch -f -p0 < /opt/keystone/token_model.patch && \
-    cd /usr/lib/python3.6/site-packages/keystone/token && \
+    cd /usr/lib/python3.9/site-packages/keystone/token && \
     patch -f -p0 < /opt/keystone/token_provider.patch && \
-    cd /usr/lib/python3.6/site-packages/keystone/api && \
+    cd /usr/lib/python3.9/site-packages/keystone/api && \
     patch -f -p0 < /opt/keystone/apis.patch && \
-    cd /usr/lib/python3.6/site-packages/keystone/api && \
+    cd /usr/lib/python3.9/site-packages/keystone/api && \
     patch -f -p0 < /opt/keystone/api_role_assignments.patch && \
-    cd /usr/lib/python3.6/site-packages/keystone/api && \
+    cd /usr/lib/python3.9/site-packages/keystone/api && \
     patch -f -p0 < /opt/keystone/projects.patch && \
-    cd /usr/lib/python3.6/site-packages/keystone/api/_shared && \
+    cd /usr/lib/python3.9/site-packages/keystone/api/_shared && \
     patch -f -p0 < /opt/keystone/api_auth.patch && \
-    find /usr/lib/python3.6/site-packages/keystone -name "*.pyc" -delete && \
-    find /usr/lib/python3.6/site-packages/keystone -name "*.pyo" -delete && \
+    cd /usr/lib/python3.9/site-packages/keystone/common/sql/legacy_migrations/expand_repo/versions && \
+    patch -f -p0 < /opt/keystone/expand_initial_migration.patch && \
+    find /usr/lib/python3.9/site-packages/keystone -name "*.pyc" -delete && \
+    find /usr/lib/python3.9/site-packages/keystone -name "*.pyo" -delete && \
     sed -ri \
               -e 's!^(\s*CustomLog)\s+\S+!\1 /proc/1/fd/1!g' \
               -e 's!^(\s*ErrorLog)\s+\S+!\1 /proc/1/fd/2!g' \

docker/expand_initial_migration.patch

@@ -0,0 +1,37 @@
+--- 073_expand_initial_migration.py
++++ 073_expand_initial_migration.py_new
+@@ -533,6 +533,26 @@
+         mysql_charset='utf8',
+     )
+
++    spassword = sql.Table(
++        'spassword',
++        meta,        
++        sql.Column('user_id', sql.String(64), primary_key=True),
++        sql.Column('user_name', sql.String(255)),
++        sql.Column('domain_id', sql.String(64)),
++        sql.Column('creation_time', sql.DateTime()),
++        sql.Column('login_attempts', sql.Integer),
++        sql.Column('last_login_attempt_time', sql.DateTime()),
++        sql.Column('extra', sql.Text()),
++        sql.Column('sndfa', sql.Boolean, default=False),
++        sql.Column('sndfa_last', sql.DateTime(), default=None),
++        sql.Column('sndfa_code', sql.String(32), default=None),
++        sql.Column('sndfa_time_code', sql.DateTime(), default=None),
++        sql.Column('sndfa_email', sql.Boolean, default=False),
++        sql.Column('sndfa_email_code', sql.String(32), default=None),
++        mysql_engine='InnoDB',
++        mysql_charset='utf8',
++    )
++
+     system_assignment = sql.Table(
+         'system_assignment',
+         meta,
+@@ -1045,6 +1065,7 @@
+         sensitive_config,
+         service,
+         service_provider,
++        spassword,
+         system_assignment,
+         token,
+         trust,

docker/keystone-all

@@ -1,4 +1,4 @@
 #!/bin/bash
 echo "[ keystone-all ] "
 sleep 5
-while ! tcping -t 1 localhost 5001 ; do echo "[ keystone-all ] forcing restart "; /usr/sbin/httpd -k restart; sleep 10; done
+while ! nc -zvw10 localhost 5001 ; do echo "[ keystone-all ] forcing restart "; /usr/sbin/httpd -k restart; sleep 10; done

docker/manage.py

@@ -0,0 +1,18 @@
+#!/usr/bin/env python
+
+#  Licensed under the Apache License, Version 2.0 (the "License"); you may
+#  not use this file except in compliance with the License. You may obtain
+#  a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#  WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#  License for the specific language governing permissions and limitations
+#  under the License.
+
+from migrate.versioning.shell import main
+
+if __name__ == '__main__':
+    main(debug='False')

keystone-spassword.spec

@@ -22,6 +22,9 @@ BuildArch: noarch
 %if 0%{?with_python36}
 %define python_lib /usr/lib/python3.6/site-packages
 %endif # if with_python36
+%if 0%{?with_python39}
+%define python_lib /usr/lib/python3.9/site-packages
+%endif # if with_python39
 
 %define check_paste %(test -e /etc/keystone/keystone-paste.ini && echo 1 || echo 0)
 %if %{check_paste}
@@ -88,8 +91,6 @@ sndfa_time_window=24
 fi
 
 ln -fs %{python_lib}/keystone_spassword/contrib/spassword %{python_lib}/keystone/contrib
-ln -s %{python_lib}/keystone_spassword/contrib/spassword/migrate_repo/versions/001_spassword_table.py %{python_lib}/keystone/common/sql/migrate_repo/versions/110_spassword_table.py
-ln -s %{python_lib}/keystone_spassword/contrib/spassword/migrate_repo/versions/002_add_sndfa_spassword_table.py %{python_lib}/keystone/common/sql/migrate_repo/versions/111_add_sndfa_spassword_table.py
 
 echo "SPASSWORD extension installed successfully. Restart Keystone daemon to take effect."
 

keystone_spassword/contrib/spassword/backends/sql.py

@@ -345,8 +345,8 @@ def authenticate(self, user_id, password):
                     expiration_date = spassword_ref['creation_time'] + \
                         datetime.timedelta(days=CONF.spassword.pwd_exp_days)
                     res['extras'] = {
-                        "password_creation_time": timeutils.isotime(spassword['creation_time']),
-                        "password_expiration_time": timeutils.isotime(expiration_date),
+                        "password_creation_time": datetime.datetime.isoformat(spassword['creation_time']),
+                        "password_expiration_time": datetime.datetime.isoformat(expiration_date),
                         "pwd_user_in_blacklist": user_id in CONF.spassword.pwd_user_blacklist,
                         "last_login_attempt_time": spassword['last_login_attempt_time'],
                         "previous_login_attempts": previous_login_attempts
@@ -424,8 +424,8 @@ def authenticate(self, user_id, password):
                     expiration_date = data_user['creation_time'] + \
                         datetime.timedelta(days=CONF.spassword.pwd_exp_days)
                     res['extras'] = {
-                        "password_creation_time": timeutils.isotime(data_user['creation_time']),
-                        "password_expiration_time": timeutils.isotime(expiration_date),
+                        "password_creation_time": datetime.datetime.isoformat(data_user['creation_time']),
+                        "password_expiration_time": datetime.datetime.isoformat(expiration_date),
                         "pwd_user_in_blacklist": user_id in CONF.spassword.pwd_user_blacklist,
                         "sndfa" : False,
                         "sndfa_email" : False,

package-keystone-spassword.sh

@@ -12,6 +12,7 @@ VERSION_VALUE=${string% *}
 RELEASE_VALUE=${string#* }
 PYTHON27_VALUE=0
 PYTHON36_VALUE=0
+PYTHON39_VALUE=0
 
 args=("$@")
 ELEMENTS=${#args[@]}
@@ -24,6 +25,9 @@ for (( i=0;i<$ELEMENTS;i++)); do
     if [ "$arg" == "--with-python36" ]; then
         PYTHON36_VALUE=1
     fi
+    if [ "$arg" == "--with-python39" ]; then
+        PYTHON39_VALUE=1
+    fi
     if [ "$arg" == "--with-version" ]; then
         VERSION_VALUE=${args[${i}+1]}
     fi
@@ -41,4 +45,5 @@ rpmbuild -bb keystone-spassword.spec \
   --define "_version $VERSION_VALUE"\
   --define "_release $RELEASE_VALUE"\
   --define "with_python27 $PYTHON27_VALUE"\
-  --define "with_python36 $PYTHON36_VALUE"
+  --define "with_python36 $PYTHON36_VALUE"\
+  --define "with_python39 $PYTHON39_VALUE"

setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = keystone_spassword
-version = 1.17.0
+version = 1.19.0
 summary = Keystone SPASSWORD extension
 description-file =
     README.md